# Tells Scanning Robots Where They Are And Are Not Welcome
# User-agent:	can also specify by name; "*" is for everyone
# Disallow:	if this matches first part of requested path,
#			forget it

#	EXCITE
User-agent:  ArchitextSpider 
#	INFOSEEK
User-agent:  InfoSeek Sidewinder/0.9
User-agent:  Mozilla/3.01 (Win95; I)  
User-agent:  Mozilla/4.5 [en] (Win95; I)
User-agent:  Mozilla/4.0 (compatible; MSIE 4.01; Windows NT)
User-agent:  Ultraseek
User-agent:  Mozilla/4.5 (Macintosh; I; PPC)
User-agent:  Mozilla/4.0 (compatible; MSIE 5.0; Windows NT)
User-agent:  
#	LYCOS
User-agent:  Lycos_Spider_(T-Rex)
#	NORTHERN LIGHT
User-agent:  Gulliver/1.2  
User-agent:  Gulliver/1.3
# 	GOOGLE
User-agent:  Googlebot/1.0 (googlebot@googlebot.com http://googlebot.com/) 
User-agent:  Googlebot/2.1w (+http://googlebot.com/bot.html)
User-agent:  Googlebot/2.0 beta (googlebot(at)googlebot.com)
User-agent:  Googlebot/1.0 (googlebot(at)googlebot.com)
#	INKTOMI
User-agent:  Slurp.so/1.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/2.0 
User-agent:  Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/cat (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/2.0-Boobook (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Mozilla/4.72 [en] (X11; U; NetBSD 1.4.2 i386; Nav)
User-agent:  Slurp/2.0-KiteHourly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/2.0-GreatWhiteCrawl (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/2.0-QA (slurp@inktomi.com;
User-agent:  Slurp/2.0-DN (david.nadler@inktomi.com;
User-agent:  Slurp/3.0-AU (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/2.0-Owl_Weekly_Temp (spider@aeneid.com; http://www.inktomi.com/slurp.html)
User-agent:  LC_catalog (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/3.0-c (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:  Slurp/2.0-Pre_WTC (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
#	ALTAVISTA
User-agent:  scooter  
User-agent:  Scooter/2.0 G.R.A.B. X2.0
User-agent:  Scooter/1.0
User-agent:  Mercator-1.0
User-agent:  Scooter/2.0 G.R.A.B. V1.1.0
User-agent:  Scooter/2.0 G.R.A.B V1.0
#	FAST AND WEBCRAWLER
User-agent:  FAST-WebCrawler/2.1-pre11 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
User-agent:  FAST-WebCrawler/2.1-pre14 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
User-agent:  FAST-WebCrawler/2.1-pre10 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
User-agent:  FAST-WebCrawler/2.2-pre9 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
#  	NO IDEA BUT IT LOOKED GOOD
User-agent:  ia_archiver
Disallow: /dwy.htm   # disallow indexing of doorway.htm