robots.txt

#
# This file is used to prevent web robots from flooding
# the web server. This file should be named
# "/robots.txt" and should appear in the root web directory.
#
# For more info, see 
# https://web.nexor.co.uk/users/mak/doc/robots/norobots.html
# https://info.webcrawler.com/mak/projects/robots/norobots.html
#

# Wild card is *, these directives apply to all robots

User-agent: *                  

# We don't want robots walking through the executables

Disallow: /cgi-bin/
Disallow: /lxr/
Disallow: /mirrors/

# Don't let the docs for old versions show up in Google anymore
Disallow: /docs/v1.6/
Disallow: /docs/v1.8/
Disallow: /docs/v2.0/
Disallow: /docs/v2.2/
Disallow: /docs/v2.4/
Disallow: /docs/v2.6/
Disallow: /docs/v3/
Disallow: /docs/v4/

# Keep robots away from the beta site

Disallow: /beta/