# robots.txt for /

User-agent: ia_archiver
Disallow: /

User-agent:  NaverBot
Disallow: /

User-agent: Zbot
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: oBot
Disallow: /

User-agent: lib/clsHTTP
Disallow: /

User-agent: Openfind
Disallow: /

User-agent: FAST-WebCrawler
Disallow: /

# Ask Jeeves has a bad crawler, and generates a ton of not founds
User-agent: Ask Jeeves
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: Zao
Disallow: /

User-agent: msnbot
Disallow: /

User-agent: kashmir 
Disallow: /

User-agent: e-SocietyRobot
Disallow: /

User-agent: vspider
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: OmniExplorer_Bot
Disallow: /

User-agent: * 

# these files are not to be indexed 

Disallow: /_fpclass
Disallow: /_notes
Disallow: /cgi-bin/
Disallow: /js/
Disallow: /css/
Disallow: /testpage
Disallow: /mainindex_files
Disallow: /mainindex1_files
Disallow: /fence_files
Disallow: /_vti_cnf
Disallow: /animations
Disallow: /awnData-horzmenu
Disallow: /_derived
Disallow: /_overlay
Disallow: /_vti_cnf