# This is for every spider!
User-Agent: *
#
# Hello ... now I won't get 404 errors in my logs
#
# stay away from this folder
Disallow: /cgi-bin/
#
# Some crawlers observe crawl-delay, but not all. If they do, then it's 30 seconds.
#
Crawl-Delay: 30
#