User-agent: crawler@alexa.com
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver/1.6
Disallow: /

User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: IAArchiver-1.0
Disallow: /

User-agent: psbot
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: Yahoo-MMAudVid/1.0
Disallow: /

User-agent: asterias/2.0
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.8
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.11
Disallow: /

User-agent: Slurp
Crawl-delay: 10

User-agent: *
Disallow: /cgi-bin/
Disallow: /nobots/
Disallow: /nyetbots/