User-Agent: *# Disallow all crawlers access to certain pages.
Disallow: /anon_ftp
Disallow: /cgi-bin
Disallow: /error_docs
Disallow: /httpsdocs
Disallow: /private
Disallow: /statistics
Disallow: /subdomains
Disallow: /vault_scripts
Disallow: /web_users
Allow: /