#  See http://www.robotstxt.org/wc/norobots.html for documentation on
#  how to use the robots.txt file 
User-Agent: Slurp
Crawl-delay: 1000

User-agent: dotbot 
Disallow: /