# This file should be retrieved by web crawlers before they start
# exploring our server.  They shouldn't try to go into the local
# pages (they wouldn't get in anyway) or anything generated by
# running a script, but they can look at everything else.
# htdig is our crawler to build the search database, and gets
# access to things other crawlers don't.
# Netscape entry must be first - bug in crawler won't identify
# entry properly otherwise.

User-agent: Netscape-Catalog-Robot/1.01 
Disallow: /ee-only/
Disallow: /ed-only/
Disallow: /ams/
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /bin/wwwuser/
Disallow: /teaching
Disallow: /ICMTS

User-agent: EUCS-Ultraseek
Disallow: /ee-only/
Disallow: /ed-only/
Disallow: /ams/
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /bin/wwwuser/
Disallow: /teaching
Disallow: /ICMTS

User-agent: EUCS-Ultraseek-Others
Disallow: /ee-only/
Disallow: /ed-only/
Disallow: /ams/
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /bin/wwwuser/
Disallow: /teaching
Disallow: /ICMTS

User-agent: htdig
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /bin/wwwuser/
Disallow: /teaching
Disallow: /ICMTS

User-agent: *
Disallow: /ee-only/
Disallow: /ed-only/
Disallow: /ams/
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /bin/wwwuser/
Disallow: /teaching