User-agent: *
Disallow: /bestilling.html
Disallow: /modlogan/
Disallow: /webalizer/
Disallow: /images/
Disallow: /cgi-bin/
Disallow: /fm/
Disallow: /cp/
# Danske søgemaskiner
User-agent: Google
Disallow:
User-agent: *
Disallow:
User-agent: The Jubii Indexing Robot (cybernet.dk)
Disallow:
User-agent: JubiiRobot/version
Disallow:
User-agent: jubii
Disallow:
User-agent: FastCrawler 3.0.X (crawler@1klik.dk) - http://www.1klik.dk
Disallow:
User-agent: I Robot 0.4 (irobot@chaos.dk)
Disallow:
User-agent: Solbot/1.0 LWP/5.07
Disallow:
User-agent: Solbot (kvasir)
Disallow:
User-agent: w3index
Disallow:
User-agent: Denmex
Disallow:
User-agent: AltaVista (DK) (http://altavista.telia.com)
Disallow:
User-agent: Thor (http://www.danielsen.com/thor/)
Disallow:
User-agent: AltaVista Intranet V2.0 www.auu.dk webmaster@opasia.dk (TDC)
Disallow:
# Nedenstående er Internationale robots, spider, crawler, email-grapper og spam-bots.
# A
User-agent: ABCdatos BotLink
Disallow: /
User-agent: ABCDee
Disallow: /
User-agent: Accessdiver
Disallow: /
User-agent: Acme.Spider
Disallow: /
User-agent: Ahoy! The Homepage Finder
Disallow: /
User-agent: Alexa
Disallow: /
User-agent: Alexibot
Disallow: /
User-agent: Alkaline
Disallow: /
User-agent: ananzi
Disallow: /
User-agent: Anthill
Disallow: /
User-agent: Aqua_Products
Disallow: /
User-agent: Arachnophilia
Disallow: /
User-agent: Arale
Disallow: /
User-agent: Araneo
Disallow: /
User-agent: AraybOt
Disallow: /
User-agent: Archive
Disallow: /
User-agent: Aretha
Disallow: /
User-agent: ARIADNE
Disallow: /
User-agent: arks
Disallow: /
User-agent: AskJeeves
Disallow: /
User-agent: ASpider (Associative Spider)
Disallow: /
User-agent: asterias
Disallow: /
User-agent: ATN Worldwide
Disallow: /
User-agent: Atomz
Disallow: /
User-agent: Atomz.com Search Robot
Disallow: /
User-agent: ATV
Disallow: /
User-agent: AURESYS
Disallow: /
User-agent: A-Online Search
Disallow: /
User-agent: AbachoBOT
Disallow: /
User-agent: AbachoBOT (Mozilla compatible)
Disallow: /
User-agent: ABCdatos BotLink/5.xx.xxx
Disallow: /
User-agent: Aberja Checkomat
Disallow: /
User-agent: abot/0.1 (abot; http://www.abot.com; abot@abot.com)
Disallow: /
User-agent: About/0.1libwww-perl/5.47
Disallow: /
User-agent: Ace Explorer
Disallow: /
User-agent: AcoiRobot
Disallow: /
User-agent: Acoon Robot v1.50.001
Disallow: /
User-agent: Acoon Robot v1.52 (http://www.acoon.de)
Disallow: /
User-agent: accoona
Disallow: /
User-agent: ActiveBookmark 1.0
Disallow: /
User-agent: ActiveWorlds/3.xx (xxx)
Disallow: /
User-agent: Ad Muncher v4.xx.x
Disallow: /
User-agent: AESOP_com_SpiderMan
Disallow: /
User-agent: agadine/1.x.x (+http://www.agada.de)
Disallow: /
User-agent: Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)
Disallow: /
User-agent: AgentName/0.1 libwww-perl/5.48
Disallow: /
User-agent: Aladin/3.324
Disallow: /
User-agent: AlkalineBOT/1.3
Disallow: /
User-agent: AlkalineBOT/1.4 (1.4.0326.0 RTM)
Disallow: /
User-agent: Aleksika Spider/1.0 (+http://www.aleksika.com/)
Disallow: /
User-agent: Allesklar/0.1 libwww-perl/5.46
Disallow: /
User-agent: Alligator 1.31 (www.nearsoftware.com)
Disallow: /
User-agent: AltaVista Intranet V2.0 AVS EVAL search@freeit.com
Disallow:
User-agent: AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net
Disallow:
User-agent: AltaVista Intranet V2.0 evreka.com crawler@evreka.com
Disallow:
User-agent: AltaVista V2.0B crawler@evreka.com
Disallow:
User-agent: AmfibiBOT
Disallow: /
User-agent: Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)
Disallow: /
User-agent: amibot
Disallow: /
User-agent: Amiga-AWeb/3.4.167SE
Disallow: /
User-agent: AmigaVoyager/3.4.4 (MorphOS/PPC native)
Disallow: /
User-agent: AnnoMille spider 0.1 alpha - http://www.annomille.it
Disallow: /
User-agent: Anonymized by ProxyOS: http://www.megaproxy.com
Disallow: /
User-agent: Anonymizer/1.1
Disallow: /
User-agent: AnswerChase x.0
Disallow: /
User-agent: AnswerChase PROve x.0
Disallow: /
User-agent: ANTFresco/x.xx
Disallow: /
User-agent: AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)
Disallow: /
User-agent: antibot-V1.1.5/i586-linux-2.2
Disallow: /
User-agent: Aplix HTTP/1.0.1
Disallow: /
User-agent: Aplix_SANYO_browser/1.x (Japanese)
Disallow: /
User-agent: Aplix_SEGASATURN_browser/1.x (Japanese)
Disallow: /
User-agent: appie 1.1 (www.walhello.com)
Disallow: /
User-agent: Aport
Disallow: /
User-agent: Arachnoidea (arachnoidea@euroseek.com)
Disallow: /
User-agent: ArchitextSpider
Disallow: /
User-agent: archive.org_bot
Disallow: /
User-agent: Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)
Disallow: /
User-agent: Asahina-Antenna/1.x
Disallow: /
User-agent: Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)
Disallow: /
User-agent: ask.24x.info
Disallow: /
User-agent: ASPSeek/1.2.x
Disallow: /
User-agent: ASPseek/1.2.xx
Disallow: /
User-agent: ASPSeek/1.2.xxpre
Disallow: /
User-agent: ASPSeek/1.2.xa
Disallow: /
User-agent: ASPSeek/1.2.5
Disallow: /
User-agent: ASPseek/1.2.9d
Disallow: /
User-agent: ASSORT/0.10
Disallow: /
User-agent: asterias/2.0
Disallow: /
User-agent: AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)
Disallow: /
User-agent: Atomz/1.0
Disallow: /
User-agent: atSpider/1.0
Disallow: /
User-agent: augurfind
Disallow: /
User-agent: augurnfind V-1.x
Disallow: /
User-agent: autoemailspider
Disallow: /
User-agent: autohttp
Disallow: /
User-agent: AV Fetch 1.0
Disallow: /
User-agent: AVSearch-1.0(peter.turney@nrc.ca)
Disallow: /
User-agent: AVSearch-2.0-fusionIdx-14-CompetitorWebSites
Disallow: /
User-agent: AVSearch-3.0(AltaVista/AVC)
Disallow: /
User-agent: AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.
Disallow: /
# B
User-agent: b2w/0.1
Disallow: /
User-agent: BackDoorBot/1.0
Disallow: /
User-agent: BackRub
Disallow: /
User-agent: Baldgorilla
Disallow: /
User-agent: BBot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: Big Brother
Disallow: /
User-agent: Bjaaland
Disallow: /
User-agent: Black Hole
Disallow: /
User-agent: Bloodhound
Disallow: /
User-agent: BlowFish/1.0
Disallow: /
User-agent: BoardViewer
Disallow: /
User-agent: Bookmark search tool
Disallow: /
User-agent: Borg-Bot
Disallow: /
User-agent: BotALot
Disallow: /
User-agent: BotRightHere
Disallow: /
User-agent: BoxSeaBot
Disallow: /
User-agent: bright.net caching robot
Disallow: /
User-agent: Browse
Disallow: /
User-agent: BSpider
Disallow: /
User-agent: BuiltBotTough
Disallow: /
User-agent: Bullseye/1.0
Disallow: /
User-agent: Burner
Disallow: /
User-agent: BaboomBot/1.x.x (+http://www.baboom.us)
Disallow: /
User-agent: BaiDuSpider
Disallow: /
User-agent: Baiduspider+ (+http://www.baidu.com/search/spider.htm)
Disallow: /
User-agent: BanBots/1.2 (spider@banbots.com)
Disallow: /
User-agent: BarraHomeCrawler (albertof@barrahome.org)
Disallow: /
User-agent: bdcindexer_2.6.2 (research@bdc)
Disallow: /
User-agent: BDFetch
Disallow: /
User-agent: BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)
Disallow: /
User-agent: beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)
Disallow: /
User-agent: BeebwareDirectory/v0.01 - http://directory.beebware.co.uk Linkchecker
Disallow: /
User-agent: Big Brother (http://pauillac.inria.fr/~fpottier/)
Disallow: /
User-agent: BigBrother/1.6e
Disallow: /
User-agent: BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)
Disallow: /
User-agent: Bilbo/2.3b-UNIX
Disallow: /
User-agent: billbot wjj@cs.cmu.edu
Disallow: /
User-agent: Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)
Disallow: /
User-agent: BlackWidow
Disallow: /
User-agent: Blaiz-Bee/1.0 (+http://www.blaiz.net)
Disallow: /
User-agent: BlitzBOT@tricus.net
Disallow: /
User-agent: BlitzBOT@tricus.net (Mozilla compatible)
Disallow: /
User-agent: BlogBot/1.x
Disallow: /
User-agent: Bloglines Title Fetch/1.0 (http://www.bloglines.com)
Disallow: /
User-agent: blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)
Disallow: /
User-agent: bluefish 0.6 HTML editor
Disallow: /
User-agent: BMCLIENT
Disallow: /
User-agent: Bobby/4.0.x RPT-HTTPClient/0.3-3E
Disallow: /
User-agent: boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)
Disallow: /
User-agent: boitho.com-robot/1.x
Disallow: /
User-agent: boitho.com-robot/1.x (http://www.boitho.com/bot.html)
Disallow: /
User-agent: Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]
Disallow: /
User-agent: Bot mailto:craftbot@yahoo.com
Disallow: /
User-agent: BravoBrian bstop.bravobrian.it
Disallow: /
User-agent: BravoBrian SpiderEngine MarcoPolo
Disallow: /
User-agent: BSDSeek/1.0
Disallow: /
User-agent: BStop.BravoBrian.it Agent Detector
Disallow: /
User-agent: BullsEye
Disallow: /
User-agent: bumblebee@relevare.com
Disallow: /
User-agent: BunnySlippers
Disallow: /
User-agent: Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)
Disallow: /
# C
User-agent: CACTVS Chemistry Spider
Disallow: /
User-agent: Calif
Disallow: /
User-agent: Cassandra
Disallow: /
User-agent: Catalog
Disallow: /
User-agent: Cegbfeieh
Disallow: /
User-agent: Cerb
Disallow: /
User-agent: Checkbot
Disallow: /
User-agent: CheeseBot
Disallow: /
User-agent: CherryPicker
Disallow: /
User-agent: CherryPicker /1.0
Disallow: /
User-agent: CherryPickerElite/1.0
Disallow: /
User-agent: CherryPickerSE/1.0
Disallow: /
User-agent: ChinaClaw
Disallow: /
User-agent: ChristCrawler.com
Disallow: /
User-agent: churl
Disallow: /
User-agent: cIeNcIaFiCcIoN.nEt
Disallow: /
User-agent: Clickgarden
Disallow: /
User-agent: cmc
Disallow: /
User-agent: CMC/0.01
Disallow: /
User-agent: Collective
Disallow: /
User-agent: Collector
Disallow: /
User-agent: Combine System
Disallow: /
User-agent: ComputingSite Robi/1.0
Disallow: /
User-agent: ConfuzzledBot
Disallow: /
User-agent: CoolBot
Disallow: /
User-agent: Copernic
Disallow: /
User-agent: CopyRightCheck
Disallow: /
User-agent: cosmos
Disallow: /
User-agent: Crawl
Disallow: /
User-agent: Crescent
Disallow: /
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /
User-agent: CtPhotoBook
Disallow: /
User-agent: Cusco
Disallow: /
User-agent: CyberSpyder Link Test
Disallow: /
User-agent: CydralSpider
Disallow: /
User-agent: Cancer Information and Support International;
Disallow: /
User-agent: Carnegie_Mellon_University_WebCrawler,http://www.andrew.cmu.edu/~brgordon/webbot/index.html
Disallow: /
User-agent: Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html
Disallow: /
User-agent: Ceramic Tile Installation Guide (http://www.floorstransformed.com)
Disallow: /
User-agent: cg-eye interactive
Disallow: /
User-agent: Charon/1.x (Amiga)
Disallow: /
User-agent: Checkbot/1.xx LWP/5.xx
Disallow: /
User-agent: CheckLinks/1.x.x
Disallow: /
User-agent: CheckUrl
Disallow: /
User-agent: CheckWeb
Disallow: /
User-agent: ChristCRAWLER 2.0
Disallow: /
User-agent: CipinetBot (http://www.cipinet.com/bot.html)
Disallow: /
User-agent: CJ Spider/
Disallow: /
User-agent: ClariaBot/1.0
Disallow: /
User-agent: Claymont.com
Disallow: /
User-agent: Clushbot/2.1 (+http://www.clush.com/bot.html)
Disallow: /
User-agent: Clushbot/3.3-BinaryFury (+http://www.clush.com/bot.html)
Disallow: /
User-agent: ColdFusion
Disallow: /
User-agent: ColdFusion (BookmarkTracker.com)
Disallow: /
User-agent: collage.cgi/1.82
Disallow: /
User-agent: combine/0.0
Disallow: /
User-agent: Combine/x.0
Disallow: /
User-agent: Commerce Browser Center
Disallow: /
User-agent: Contact
Disallow: /
User-agent: contype
Disallow: /
User-agent: ConveraCrawler/0.2
Disallow: /
User-agent: Convera Internet Spider V6.x
Disallow: /
User-agent: ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)
Disallow: /
User-agent: cosmos/0.8_(robot@xyleme.com)
Disallow: /
User-agent: cosmos/0.9_(robot@xyleme.com)
Disallow: /
User-agent: Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)
Disallow: /
User-agent: Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)
Disallow: /
User-agent: CrawlConvera0.1 (CrawlConvera@yahoo.com)
Disallow: /
User-agent: Crawler
Disallow: /
User-agent: Crawler (cometsearch@cometsystems.com)
Disallow: /
User-agent: Crawler admin@crawler.de
Disallow: /
User-agent: Crawler V 0.2.x admin@crawler.de
Disallow: /
User-agent: crawler@alexa.com
Disallow: /
User-agent: CrawlerBoy Pinpoint.com
Disallow: /
User-agent: CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)
Disallow: /
User-agent: CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)
Disallow: /
User-agent: Cuam Ver0.050bx
Disallow: /
User-agent: Cuasarbot/0.9b http://www.cuasar.com/spider_beta/
Disallow: /
User-agent: curl/7.7.1 (i386--freebsd4.3) libcurl 7.7.1 (SSL 0.9.6) (ipv6 enabled)
Disallow: /
User-agent: curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6)
Disallow: /
User-agent: curl/7.9.x (win32) libcurl 7.9.x
Disallow: /
User-agent: CurryGuide SiteScan 1.1
Disallow: /
User-agent: Custo x.x (www.netwu.com)
Disallow: /
User-agent: Custom Spider www.bisnisseek.com /1.0
Disallow: /
User-agent: Cyberdog/2.0 (Macintosh; 68k)
Disallow: /
User-agent: CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)
Disallow: /
User-agent: CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)
Disallow: /
# D
User-agent: DA
Disallow: /
User-agent: Desert Realm Spider
Disallow: /
User-agent: DeWeb(c) Katalog/Index
Disallow: /
User-agent: Die Blinde Kuh
Disallow: /
User-agent: DienstSpider
Disallow: /
User-agent: Digger
Disallow: /
User-agent: Digimarc MarcSpider
Disallow: /
User-agent: Digimarc Marcspider/CGI
Disallow: /
User-agent: Digital Integrity Robot
Disallow: /
User-agent: DIIbot
Disallow: /
User-agent: Direct Hit Grabber
Disallow: /
User-agent: DISCoFinder
Disallow: /
User-agent: DNAbot
Disallow: /
User-agent: DownLoad Express
Disallow: /
User-agent: Downloader
Disallow: /
User-agent: Downloadit
Disallow: /
User-agent: DragonBot
Disallow: /
User-agent: DWCP (Dridus' Web Cataloging Project)
Disallow: /
User-agent: DA 3.5 (www.lidan.com)
Disallow: /
User-agent: DA 4.0
Disallow: /
User-agent: DA 4.0 (www.downloadaccelerator.com)
Disallow: /
User-agent: DA 5.0
Disallow: /
User-agent: DA 7.0
Disallow: /
User-agent: Dart Communications PowerTCP
Disallow: /
User-agent: DataparkSearch/4.xx (http://www.dataparksearch.org/)
Disallow: /
User-agent: DaviesBot/1.7 (www.wholeweb.net)
Disallow: /
User-agent: dbDig(http://www.prairielandconsulting.com)
Disallow: /
User-agent: DBrowse 1.4b
Disallow: /
User-agent: DBrowse 1.4d
Disallow: /
User-agent: DC-Sakura/x.xx
Disallow: /
User-agent: dCSbot/1.1
Disallow: /
User-agent: DDD
Disallow: /
User-agent: dds explorer v1.0 beta
Disallow: /
User-agent: DeadLinkCheck/0.4.0 libwww-perl/5.xx
Disallow: /
User-agent: deepak-USC/ISI
Disallow: /
User-agent: DeepIndex
Disallow: /
User-agent: DeepIndex ( http://www.zetbot.com )
Disallow: /
User-agent: DeepIndex (www.en.deepindex.com)
Disallow: /
User-agent: DeepIndexer.ca
Disallow: /
User-agent: Demo Bot DOT 16b
Disallow: /
User-agent: Demo Bot Z 16b
Disallow: /
User-agent: Denmex websearch (http://search.denmex.com)
Disallow: /
User-agent: DevComponents.com HtmlDocument Object
Disallow: /
User-agent: DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)
Disallow: /
User-agent: Diamond/1.0
Disallow: /
User-agent: DiamondBot
Disallow: /
User-agent: Digger/1.0 JDK/1.3.0rc3
Disallow: /
User-agent: DigOut4U
Disallow: /
User-agent: Dillo/0.x.x
Disallow: /
User-agent: DittoSpyder
Disallow: /
User-agent: DIIbot/1.2
Disallow: /
User-agent: dlman
Disallow: /
User-agent: dloader(NaverRobot)/1.0
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10/TB
Disallow: /
User-agent: DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)
Disallow:
User-agent: DoCoMo/2.0 P900iV(c100;TB;W24H11)
Disallow: /
User-agent: DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804
Disallow: /
User-agent: Download Demon/3.x.x.x
Disallow: /
User-agent: Download Express 1.0
Disallow: /
User-agent: Download Master
Disallow: /
User-agent: Download Wonder
Disallow: /
User-agent: Download-Tipp Linkcheck (http://download-tipp.de/)
Disallow: /
User-agent: Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)
Disallow: /
User-agent: Drecombot/1.0 (http://career.drecom.jp/bot.html)
Disallow: /
User-agent: DSurf15a 01
Disallow: /
User-agent: DSurf15a 71
Disallow: /
User-agent: DSurf15a 81
Disallow: /
User-agent: DSurf15a VA
Disallow: /
User-agent: dtSearchSpider
Disallow: /
User-agent: Dual Proxy
Disallow: /
User-agent: Dumbot(version 0.1 beta)
Disallow: /
User-agent: Dumbot(version 0.1 beta - dumbfind.com)
Disallow: /
# E
User-agent: EbiNess
Disallow: /
User-agent: eCatch
Disallow: /
User-agent: e-collector
Disallow: /
User-agent: Educate
Disallow: /
User-agent: efp@gmx\.net
Disallow: /
User-agent: EirGrabber
Disallow: /
User-agent: EIT Link Verifier Robot
Disallow: /
User-agent: ELFINBOT
Disallow: /
User-agent: Emacs-w3 Search Engine
Disallow: /
User-agent: EmailCollector
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: esculapio
Disallow: /
User-agent: Esther
Disallow: /
User-agent: Evliya Celebi
Disallow: /
User-agent: EyeNetIE
Disallow: /
User-agent: e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)
Disallow: /
User-agent: EARTHCOM.info/1.x [www.earthcom.info]
Disallow: /
User-agent: EARTHCOM.info/1.xbeta [www.earthcom.info]
Disallow: /
User-agent: EasyDL/3.xx
Disallow: /
User-agent: EasyDL/3.xx http://keywen.com/Encyclopedia/Bot
Disallow: /
User-agent: EBrowse 1.4b
Disallow: /
User-agent: eCatch/3.0
Disallow: /
User-agent: Educate Search VxB
Disallow: /
User-agent: EchO!/2.0
Disallow: /
User-agent: egothor/3.0a (+http://www.xdefine.org/robot.html)
Disallow: /
User-agent: EgotoBot/4.8 (+http://www.egoto.com/about.htm)
Disallow: /
User-agent: EldoS TimelyWeb/3.x
Disallow: /
User-agent: elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)
Disallow: /
User-agent: ELinks (0.x.x; Linux 2.4.20 i586; 132x60)
Disallow: /
User-agent: ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: EmailWolf 1.00
Disallow: /
User-agent: EmeraldShield.com WebBot
Disallow: /
User-agent: EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)
Disallow: /
User-agent: Enfish Tracker
Disallow: /
User-agent: Enterprise_Search/1.0
Disallow: /
User-agent: Enterprise_Search/1.0.xxx
Disallow: /
User-agent: Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)
Disallow: /
User-agent: EroCrawler
Disallow: /
User-agent: ES.NET_Crawler/2.0 (http://search.innerprise.net/)
Disallow: /
User-agent: eseek-larbin_2.6.2 (crawler@exactseek.com)
Disallow: /
User-agent: ESISmartSpider
Disallow: /
User-agent: eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)
Disallow: /
User-agent: ESurf15a 15
Disallow: /
User-agent: Exalead NG/MimeLive Client (convert/http/0.120)
Disallow: /
User-agent: ExactSeek Crawler/0.1
Disallow: /
User-agent: exactseek-crawler-2.63 (crawler@exactseek.com)
Disallow: /
User-agent: Excalibur Internet Spider V6.5.4
Disallow: /
User-agent: ExperimentalHenrytheMiragoRobot
Disallow: /
User-agent: Expired Domain Sleuth
Disallow: /
User-agent: Express WebPictures (www.express-soft.com)
Disallow: /
User-agent: ExtractorPro
Disallow: /
User-agent: EyeCatcher (Download-tipp.de)/1.0
Disallow: /
# F
User-agent: FastCrawler
Disallow: /
User-agent: Felix IDE
Disallow: /
User-agent: FetchRover
Disallow: /
User-agent: fido
Disallow: /
User-agent: Fish search
Disallow: /
User-agent: Flaming AttackBot
Disallow: /
User-agent: Flashget
Disallow: /
User-agent: FlickBot
Disallow: /
User-agent: Fluid Dynamics Search Engine robot
Disallow: /
User-agent: Foobot
Disallow: /
User-agent: Fouineur
Disallow: /
User-agent: Franklin
Disallow: /
User-agent: Freecrawl
Disallow: /
User-agent: Full
Disallow: /
User-agent: FunnelWeb
Disallow: /
User-agent: FairAd Client
Disallow: /
User-agent: FastBug http://www.ay-up.com
Disallow: /
User-agent: Fast Crawler Gold Edition
Disallow: /
User-agent: FAST Enterprise Crawler 6 (Experimental)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by FAST (FAST)
Disallow: /
User-agent: FAST Enterprise Crawler/6 (www.fastsearch.com)
Disallow: /
User-agent: FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)
Disallow: /
User-agent: Fast PartnerSite Crawler
Disallow: /
User-agent: FastCrawler 3.0.1 (crawler@1klik.dk)
Disallow: /
User-agent: FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
Disallow: /
User-agent: FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
Disallow: /
User-agent: FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
Disallow: /
User-agent: FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)
Disallow: /
User-agent: FAST-WebCrawler/3.2 test
Disallow: /
User-agent: FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)
Disallow: /
User-agent: FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)
Disallow: /
User-agent: FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)
Disallow: /
User-agent: Favorites Sweeper v.2.03
Disallow: /
User-agent: Faxobot/1.0
Disallow: /
User-agent: Felix - Mixcat Crawler (+http://mixcat.com)
Disallow: /
User-agent: FFC Trap Door Spider
Disallow: /
User-agent: Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)
Disallow: /
User-agent: FineBot
Disallow: /
User-agent: FindAnISP.com_ISP_Finder_v99a
Disallow: /
User-agent: findlinks/0.xxx (+http://wortschatz.uni-leipzig.de/findlinks/)
Disallow: /
User-agent: Firefly/1.0
Disallow: /
User-agent: Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)
Disallow: /
User-agent: FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov
Disallow: /
User-agent: firstsbot
Disallow: /
User-agent: FlickBot 2.0 RPT-HTTPClient/0.3-3
Disallow: /
User-agent: flunky
Disallow: /
User-agent: fly/6.01 libwww/4.0D
Disallow: /
User-agent: FocusedSampler/1.0
Disallow: /
User-agent: Francis/1.0 (francis@neomo.de http://www.neomo.de/)
Disallow: /
User-agent: Franklin Locator 1.8
Disallow: /
User-agent: FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)
Disallow: /
User-agent: FSurf15a 01
Disallow: /
User-agent: Full Web Bot 0416B
Disallow: /
User-agent: Full Web Bot 0516B
Disallow: /
User-agent: Full Web Bot 2816B
Disallow: /
User-agent: FyberSpider (+http://www.fybersearch.com/fyberspider.php)
Disallow: /
# G
User-agent: Gaisbot
Disallow: /
User-agent: gammaSpider, FocusedCrawler
Disallow: /
User-agent: gazz
Disallow: /
User-agent: GCreep
Disallow: /
User-agent: GetRight
Disallow: /
User-agent: GetRight/4.2
Disallow: /
User-agent: GetterroboPlus Puu
Disallow: /
User-agent: GetURL
Disallow: /
User-agent: GetWebPage
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Golem
Disallow: /
User-agent: Gozilla
Disallow: /
User-agent: grab
Disallow: /
User-agent: Grafula
Disallow: /
User-agent: Grapnel/0.01 Experiment
Disallow: /
User-agent: Green
Disallow: /
User-agent: Griffon
Disallow: /
User-agent: Gromit
Disallow: /
User-agent: grub
Disallow: /
User-agent: Grub.org
Disallow: /
User-agent: Gulper Bot
Disallow: /
User-agent: Gagglebot
Disallow: /
User-agent: GAIS Robot/1.0B2
Disallow: /
User-agent: Gaisbot/3.0+(indexer@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)
Disallow: /
User-agent: GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)
Disallow: /
User-agent: GammaSpider/1.0
Disallow: /
User-agent: Gamespy_Arcade, GameSpyHTTP/1.0
Disallow: /
User-agent: gazz/x.x (gazz@nttrd.com)
Disallow: /
User-agent: geckobot
Disallow: /
User-agent: generic_crawler/01.0217/
Disallow: /
User-agent: GeonaBot 1.x; http://www.geona.com/
Disallow: /
User-agent: GetBot
Disallow: /
User-agent: GetRight/3.3.3
Disallow: /
User-agent: GetRight/4.x
Disallow: /
User-agent: GetRight/4.x[a-e]
Disallow: /
User-agent: GetRight/4.5xx
Disallow: /
User-agent: GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)
Disallow: /
User-agent: gigabaz/3.14 (baz@gigabaz.com; http://gigabaz.com/gigabaz/)
Disallow: /
User-agent: Gigabot/x.0
Disallow: /
User-agent: Gigabot/2.0 (gigablast.com)
Disallow: /
User-agent: GigabotSiteSearch/2.0 (sitesearch.gigablast.com)
Disallow: /
User-agent: GNODSPIDER (www.gnod.net)
Disallow: /
User-agent: Go!Zilla 3.x (www.gozilla.com)
Disallow: /
User-agent: Go!Zilla/4.x.x.xx
Disallow: /
User-agent: Go-Ahead-Got-It/1.1
Disallow: /
User-agent: Goblin/0.9 (http://www.goguides.org/)
Disallow: /
User-agent: Goblin/0.9.x (http://www.goguides.org/goblin-info.html)
Disallow: /
User-agent: GoForIt.com
Disallow: /
User-agent: Goofer/0.2
Disallow: /
# GOOGLE
User-agent: Googlebot/Test
Disallow:
User-agent: Mediapartners-Google/2.1
Disallow:
User-agent: Mediapartners-Google*
Disallow:
User-agent: Googlebot
Disallow:
User-agent: Googlebot-Image
Disallow:
User-agent: googlebot (larbin2.6.0@unspecified.mail)
Disallow:
User-agent: Googlebot/2.1 (+http://www.google.com/bot.html)
Disallow:
User-agent: Googlebot/2.1 (+http://www.googlebot.com/bot.html)
Disallow:
User-agent: Googlebot 3
Disallow:
User-agent: Googlebot/Test (+http://www.googlebot.com/bot.html)
Disallow:
User-agent: Googlebot-Image/1.0 (+http://www.googlebot.com/bot.html)
Disallow:
# NOTE: FRESHBOT - DEEPBOT - FULLBOT er også googlebots.
User-agent: GrigorBot 0.8 (http://www.grigor.biz/bot.html)
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: grub crawler(http://www.grub.org)
Disallow: /
User-agent: gsa-crawler (Enterprise; GID-01422; jplastiras@google.com)
Disallow: /
User-agent: gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com)
Disallow: /
User-agent: Gulliver/1.3
Disallow: /
User-agent: Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)
Disallow: /
# H
User-agent: HamBot
Disallow: /
User-agent: Harvest
Disallow: /
User-agent: Harvest/1.5
Disallow: /
User-agent: havIndex
Disallow: /
User-agent: HI (HTML Index) Search
Disallow: /
User-agent: HKU WWW Octopus
Disallow: /
User-agent: HLoad
Disallow: /
User-agent: hloader
Disallow: /
User-agent: HMView
Disallow: /
User-agent: Hometown Spider Pro
Disallow: /
User-agent: ht://Dig
Disallow: /
User-agent: html_analyzer
Disallow: /
User-agent: HTMLgobble
Disallow: /
User-agent: HttpDownload/2.0
Disallow: /
User-agent: httplib
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: humanlinks
Disallow: /
User-agent: Hyper-Decontextualizer
Disallow: /
User-agent: Hämähäkki
Disallow: /
User-agent: Harvest-NG/1.0.2
Disallow: /
User-agent: Haste/0.12 (HOME: http://haste.kytoon.com/)
Disallow: /
User-agent: Hatena Antenna/0.4 (http://a.hatena.ne.jp/help/robot)
Disallow: /
User-agent: HeinrichderMiragoRobot
Disallow: /
User-agent: HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)
Disallow: /
User-agent: HenrytheMiragoRobot
Disallow: /
User-agent: Hippias/0.9 Beta
Disallow: /
User-agent: HitList
Disallow: /
User-agent: Hitwise Spider v1.0 http://www.hitwise.com
Disallow: /
User-agent: HLoader
Disallow: /
User-agent: holmes/x.x
Disallow: /
User-agent: HomePageSearch(hpsearch.uni-trier.de)
Disallow: /
User-agent: Homerbot: www.homerweb.com
Disallow: /
User-agent: HotJava/1.0.1/JRE1.1.x
Disallow: /
User-agent: Hotzonu/x.0
Disallow: /
User-agent: HooWWWer/2.1.0 (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-infohiit.fi)
Disallow: /
User-agent: htdig/3.1.x (root@localhost)
Disallow: /
User-agent: htdig/3.1.6 (http://computerorgs.com)
Disallow: /
User-agent: Html Link Validator (www.lithopssoft.com)
Disallow: /
User-agent: http://Ask.24x.Info/ (http://narres.it/)
Disallow: /
User-agent: http://www.almaden.ibm.com/cs/crawler
Disallow: /
User-agent: http://www.almaden.ibm.com/cs/crawler [wf216]
Disallow: /
User-agent: HTTPResume v. 1.x
Disallow: /
# I
User-agent: I, Robot
Disallow: /
User-agent: iajaBot
Disallow: /
User-agent: IBM_Planetwide
Disallow: /
User-agent: ICB
Disallow: /
User-agent: image.kapsi.net
Disallow: /
User-agent: imagefetch
Disallow: /
User-agent: Imagelock
Disallow: /
User-agent: IncyWincy
Disallow: /
User-agent: Industry
Disallow: /
User-agent: Indy
Disallow: /
User-agent: InfoNaviRobot
Disallow: /
User-agent: Informant
Disallow: /
User-agent: InfoSeek Robot 1.0
Disallow: /
User-agent: Infoseek Sidewinder
Disallow: /
User-agent: InfoSpiders
Disallow: /
User-agent: Ingrid
Disallow: /
User-agent: Inktomi Slurp
Disallow: /
User-agent: Inspector Web
Disallow: /
User-agent: IntelliAgent
Disallow: /
User-agent: InterGET
Disallow: /
User-agent: Internet Cruiser Robot
Disallow: /
User-agent: Internet Shinchakubin
Disallow: /
User-agent: Iron33
Disallow: /
User-agent: Iron33/1.0.2
Disallow: /
User-agent: iseeker
Disallow: /
User-agent: Israeli-search
Disallow: /
User-agent: IU
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: ia_archiver-web.archive.org
Disallow: /
User-agent: ia_archiver/1.6
Disallow: /
User-agent: IAArchiver-1.0
Disallow: /
User-agent: IBrowse/2.2 (AmigaOS 3.5)
Disallow: /
User-agent: IBrowse/2.2 (Windows 3.1)
Disallow: /
User-agent: iCab/2.5.2 (Macintosh; I; PPC)
Disallow: /
User-agent: ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)
Disallow: /
User-agent: ichiro/1.0 (ichiro@nttr.co.jp)
Disallow: /
User-agent: IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)
Disallow: /
User-agent: IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)
Disallow: /
User-agent: ICOO Loader v.x.x.x
Disallow: /
User-agent: ICRA_label_spider/x.0
Disallow: /
User-agent: icsbot-0.1
Disallow: /
User-agent: ideare - SignSite/1.x
Disallow: /
User-agent: IEFav172Free
Disallow: /
User-agent: iGetter/1.x (Macintosh;G;PPC)
Disallow: /
User-agent: IlTrovatore-Setaccio (+http://www.iltrovatore.it)
Disallow: /
User-agent: Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)
Disallow: /
User-agent: IlTrovatore-Setaccio/1.2 (+http://www.iltrovatore.it/aiuto/faq.html)
Disallow: /
User-agent: Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)
Disallow: /
User-agent: iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....)
Disallow: /
User-agent: IncyWincy(http://www.look.com)
Disallow: /
User-agent: IncyWincy(http://www.loopimprovements.com/robot.html)
Disallow: /
User-agent: IncyWincy/2.1(loopimprovements.com/robot.html)
Disallow: /
User-agent: IncyWincy data gatherer(webmaster@loopimprovements.com,http://www.loopimprovements.com/robot.html)
Disallow: /
User-agent: IncyWincy page crawler(webmaster@loopimprovements.com,http://www.loopimprovements.com/robot.html)
Disallow: /
User-agent: IndexTheWeb.com Crawler7
Disallow: /
User-agent: Industry Program 1.0.x
Disallow: /
User-agent: InelaBot/0.2 (+http://inelegant.org/bot)
Disallow: /
User-agent: Inet library
Disallow: /
User-agent: InfoFly/1.0 (http://www.versions-project.org/)
Disallow: /
User-agent: InfoLink/1.x
Disallow: /
User-agent: INFOMINE/8.0 Adders
Disallow: /
User-agent: INFOMINE/8.0 RemoteServices
Disallow: /
User-agent: INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)
Disallow: /
User-agent: InfoNaviRobot(F107)
Disallow: /
User-agent: InfoSeek Sidewinder/0.9
Disallow: /
User-agent: InfoSeek Sidewinder/1.0A
Disallow: /
User-agent: InfoSeek Sidewinder/1.1A
Disallow: /
User-agent: Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)
Disallow: /
User-agent: Infoseek SideWinder/2.0B (Linux 2.4 i686)
Disallow: /
User-agent: Inktomi Search
Disallow: /
User-agent: InnerpriseBot/1.0 (http://www.innerprise.com/)
Disallow: /
User-agent: InstallShield DigitalWizard
Disallow: /
User-agent: InternetLinkAgent/3.1
Disallow: /
User-agent: InternetSeer.com
Disallow: /
User-agent: Internet Ninja 6.0
Disallow: /
User-agent: intraVnews/1.x
Disallow: /
User-agent: IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com
Disallow: /
User-agent: IPiumBot laurion(dot)com
Disallow: /
User-agent: Iria/1.xxa
Disallow: /
User-agent: IRLbot/1.0 (+http://irl.cs.tamu.edu/crawler)
Disallow: /
User-agent: Irvine/1.x.x
Disallow: /
User-agent: isurf (tszhu@canada.com)
Disallow: /
User-agent: IUPUI Research Bot v 1.9a
Disallow: /
# J
User-agent: Java
Disallow: /
User-agent: JavaBee
Disallow: /
User-agent: JBot Java Web Robot
Disallow: /
User-agent: JCrawler
Disallow: /
User-agent: JennyBot
Disallow: /
User-agent: JoBo Java Web Robot
Disallow: /
User-agent: Jobot
Disallow: /
User-agent: JoeBot
Disallow: /
User-agent: JumpStation
Disallow: /
User-agent: J-PHONE/3.0/J-SH07
Disallow: /
User-agent: Jabot/7.x.x (http://odin.ingrid.org/)
Disallow: /
User-agent: Jabot/6.x (http://odin.ingrid.org/)
Disallow: /
User-agent: Jack
Disallow: /
User-agent: Jakarta Commons-HttpClient/2.0xxx
Disallow: /
User-agent: Java/1.4.1_01
Disallow: /
User-agent: Java1.0.21.0
Disallow: /
User-agent: Java 1.1
Disallow: /
User-agent: Java1.1.xx.x
Disallow: /
User-agent: Java1.3.x
Disallow: /
User-agent: Java1.3.0rc1
Disallow: /
User-agent: Java1.4.0
Disallow: /
User-agent: JBH Agent 2.0
Disallow: /
User-agent: jBrowser/J2ME Profile/MIDP-1.0 Configuration/CLDC-1.0 (Google WAP Proxy/1.0)
Disallow: /
User-agent: JDK/1.1
Disallow: /
User-agent: JCheckLinks/0.1 RPT-HTTPClient/0.3-1
Disallow: /
User-agent: Jeode/1.x.x
Disallow: /
User-agent: JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)
Disallow: /
User-agent: JetCar
Disallow: /
User-agent: Jetbot/1.0
Disallow: /
User-agent: Jigsaw/2.2.0 W3C_CSS_Validator_JFouffa/2.0
Disallow: /
User-agent: JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)
Disallow: /
User-agent: JoBo/1.x (http://www.matuschek.net/jobo.html)
Disallow: /
User-agent: JOC Web Spider
Disallow: /
User-agent: JRTS Check Favorites Utility
Disallow: /
User-agent: JRTwine Software Check Favorites Utility
Disallow: /
User-agent: Jyxobot/x
Disallow: /
# K
User-agent: Katipo
Disallow: /
User-agent: KDD-Explorer
Disallow: /
User-agent: Keyword Density/0.9
Disallow: /
User-agent: Kilroy
Disallow: /
User-agent: KIT-Fireball
Disallow: /
User-agent: KO_Yappo_Robot
Disallow: /
User-agent: K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011
Disallow: /
User-agent: k2spider
Disallow: /
User-agent: Kapere (http://www.kapere.com)
Disallow: /
User-agent: KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)
Disallow: /
User-agent: KE_1.0/2.0 libwww/5.2.8
Disallow: /
User-agent: Kenjin Spider
Disallow: /
User-agent: Kevin http://dznet.com/kevin/
Disallow: /
User-agent: Kevin http://websitealert.net/kevin/
Disallow: /
User-agent: KIT-Fireball/2.0
Disallow: /
User-agent: KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)
Disallow: /
User-agent: Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)
Disallow: /
User-agent: Knowledge.com/0.x
Disallow: /
User-agent: Kontiki Client x.xx
Disallow: /
User-agent: kulokobot www.kuloko.com kuloko@backweave.com
Disallow: /
User-agent: kuloko-bot/0.x
Disallow: /
User-agent: kulturarw3/0.1
Disallow: /
# L
User-agent: LabelGrabber
Disallow: /
User-agent: larbin
Disallow: /
User-agent: LeechGet
Disallow: /
User-agent: legs
Disallow: /
User-agent: LexiBot
Disallow: /
User-agent: libfetch
Disallow: /
User-agent: libWeb/clsHTTP
Disallow: /
User-agent: Lincoln
Disallow: /
User-agent: Link Validator
Disallow: /
User-agent: LinkextractorPro
Disallow: /
User-agent: LINKS ARoMATIZED
Disallow: /
User-agent: LinkScan
Disallow: /
User-agent: LinkScan/8.1a Unix
Disallow: /
User-agent: Lockon
Disallow: /
User-agent: logo.gif Crawler
Disallow: /
User-agent: lwp
Disallow: /
User-agent: LWP::Simple
Disallow: /
User-agent: lwp-trivial
Disallow: /
User-agent: Lachesis
Disallow: /
# NOTE larbin er googlebots
User-agent: LARBIN-EXPERIMENTAL (efp@gmx.net)
Disallow:
User-agent: larbin_2.1.1 larbin2.1.1@somewhere.com
Disallow:
User-agent: larbin_2.2.0 (crawl@compete.com)
Disallow:
User-agent: larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr)
Disallow:
User-agent: larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr)
Disallow:
User-agent: larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp)
Disallow: /
User-agent: larbin_2.6_basileocaml (basile.starynkevitch@cea.fr)
Disallow:
User-agent: larbin_2.6.0 (larbin2.6.0@unspecified.mail)
Disallow: /
User-agent: larbin_2.6.1 (larbin2.6.1@unspecified.mail)
Disallow: /
User-agent: larbin_2.6.2 (hamasaki@grad.nii.ac.jp)
Disallow: /
User-agent: larbin_2.6.2 (larbin2.6.2@unspecified.mail)
Disallow: /
User-agent: larbin_2.6.2 (listonATccDOTgatechDOTedu)
Disallow: /
User-agent: larbin_2.6.2 (pimenas@systems.tuc.gr)
Disallow: /
User-agent: larbin_2.6.2 (tom@lemurconsulting.com)
Disallow: /
User-agent: larbin_2.6.2 (vitalbox1@hotmail.com)
Disallow: /
User-agent: larbin_2.6.3 (wgao@genieknows.com)
Disallow: /
User-agent: larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi
Disallow:
User-agent: larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/)
Disallow:
User-agent: larbin (samualt9@bigfoot.com)
Disallow:
User-agent: lcabotAccept: */*
Disallow: /
User-agent: LECodeChecker/3.0 libgetdoc/1.0
Disallow: /
User-agent: LeechGet 200x (www.leechget.de)
Disallow: /
User-agent: LEIA/2.90
Disallow: /
User-agent: LEIA/3.01pr (LEIAcrawler; [SNIP])
Disallow: /
User-agent: LexiBot/1.00
Disallow: /
User-agent: LG/U8138/v1.0
Disallow: /
User-agent: Libby_1.1/libwww-perl/5.47
Disallow: /
User-agent: libWeb/clsHTTP -- hiongun@kt.co.kr
Disallow: /
User-agent: libwww/5.3.2
Disallow: /
User-agent: libwww-perl/5.41
Disallow: /
User-agent: libwww-perl/5.45
Disallow: /
User-agent: libwww-perl/5.48
Disallow: /
User-agent: libwww-perl/5.50
Disallow: /
User-agent: libwww-perl/5.52 FP/2.1
Disallow: /
User-agent: libwww-perl/5.52 FP/4.0
Disallow: /
User-agent: libwww-perl/5.53
Disallow: /
User-agent: libwww-perl/5.63
Disallow: /
User-agent: libwww-perl/5.64
Disallow: /
User-agent: libwww-perl/5.65
Disallow: /
User-agent: LightningDownload/1.x.x
Disallow: /
User-agent: LightningDownload/1.0beta2
Disallow: /
User-agent: LightningDownload/1.x.x [Accelerated x]
Disallow: /
User-agent: Lincoln State Web Browser
Disallow: /
User-agent: LinkAlarm/2.x
Disallow: /
User-agent: Link Valet Online 1.x
Disallow: /
User-agent: linkbot
Disallow: /
User-agent: Linkbot
Disallow: /
User-agent: Linkbot x.0
Disallow: /
User-agent: LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)
Disallow: /
User-agent: LinkLint-checkonly/2.x.x
Disallow: /
User-agent: LinkLint-spider/2.x.x
Disallow: /
User-agent: LinkProver 2.1
Disallow: /
User-agent: Links (0.9x; Linux 2.4.7-10 i686)
Disallow: /
User-agent: Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)
Disallow: /
User-agent: Links (2.xpre7; Linux 2.4.18 i586; x)
Disallow: /
User-agent: Links - http://gossamer-threads.com/scripts/links/
Disallow: /
User-agent: Links 2.0 (http://gossamer-threads.com/scripts/links/)
Disallow: /
User-agent: Links SQL (http://gossamer-threads.com/scripts/links-sql/)
Disallow: /
User-agent: LinkScan/x.x Unix
Disallow: /
User-agent: LinkScan/9.0g Unix
Disallow: /
User-agent: LinkScan/11.0beta2 Unix
Disallow: /
User-agent: LinksManager.com (http://linksmanager.com/linkchecker.html)
Disallow: /
User-agent: LinkSweeper/1.x
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: linknzbot
Disallow: /
User-agent: ListBidBot (freelance job spider http://listbid.com)Freelance
Disallow: /
User-agent: Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)
Disallow: /
User-agent: lmspider (lmspider@scansoft.com)
Disallow: /
User-agent: LNSpiderguy
Disallow: /
User-agent: Lockstep Spider/1.0
Disallow: /
User-agent: Look.com
Disallow: /
User-agent: Lotus-Notes/4.5 ( Windows-NT )
Disallow: /
User-agent: LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)
Disallow: /
User-agent: LWP::Simple/5.22
Disallow: /
User-agent: LWP::Simple/5.36
Disallow: /
User-agent: LWP::Simple/5.48
Disallow: /
User-agent: LWP::Simple/5.50
Disallow: /
User-agent: LWP::Simple/5.51
Disallow: /
User-agent: LWP::Simple/5.53
Disallow: /
User-agent: LWP::Simple/5.63
Disallow: /
User-agent: lwp-trivial/1.32
Disallow: /
User-agent: lwp-trivial/1.34
Disallow: /
User-agent: lwp-trivial/1.35
Disallow: /
User-agent: Lycos
Disallow:
User-agent: T-Rex (lycos)
Disallow:
User-agent: Lycos_Spider_(T-Rex)
Disallow:
User-agent: Lycos_Spider_(modspider)
Disallow: /
User-agent: Lynx/2.6 libwww-FM/2.14
Disallow: /
User-agent: Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6
Disallow: /
User-agent: Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)
Disallow: /
# M
User-agent: MMCrawler
Disallow: /
User-agent: Mac Finder
Disallow: /
User-agent: Mac WWWWorm
Disallow: /
User-agent: Magpie
Disallow: /
User-agent: marvin/infoseek
Disallow: /
User-agent: Mass
Disallow: /
User-agent: Mass Downloader
Disallow: /
User-agent: Mata Hari
Disallow: /
User-agent: Mattie
Disallow: /
User-agent: MD
Disallow: /
User-agent: MediaFox
Disallow: /
User-agent: Mega Moose
Disallow: /
User-agent: MemoWeb
Disallow: /
User-agent: MerzScope
Disallow: /
User-agent: MFC_Tear
Disallow: /
User-agent: Microsoft URL Control
Disallow:
User-agent: Microsoft URL Control - 5.01.4511
Disallow:
User-agent: Microsoft URL Control - 6.00.8169
Disallow:
User-agent: Microsoft.URL
Disallow:
User-agent: MIIxpc
Disallow: /
User-agent: MindCrawler
Disallow: /
User-agent: Mirago
Disallow: /
User-agent: mirror
Disallow: /
User-agent: Missauga
Disallow: /
User-agent: Missigua
Disallow: /
User-agent: Mister PiX
Disallow: /
User-agent: Miz
Disallow: /
User-agent: MM3
Disallow: /
User-agent: MnoGo
Disallow: /
User-agent: mnoGoSearch search engine software
Disallow: /
User-agent: moget
Disallow: /
User-agent: moget/2.1
Disallow: /
User-agent: MOMspider
Disallow: /
User-agent: Monster
Disallow: /
User-agent: Mooz
Disallow: /
User-agent: Motor
Disallow: /
User-agent: Mozilla/[0-9]\.[0-9]+ \(compatible; Win32; WinHttp\.WinHttpRequest\.[0-9]+\)
Disallow:
User-agent: Mozilla/3\.0 \(compatible\)
Disallow:
User-agent: mozilla/4
Disallow:
User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
Disallow:
User-agent: mozilla/5
Disallow:
User-agent: MSIECrawler
Disallow:
User-agent: MSNBot
Disallow:
User-agent: MSNBot/0.1
Disallow:
User-agent: MSNBot/0.11
Disallow:
User-agent: MSNBot/0.3+
Disallow:
User-agent: MSNBot/1.0+
Disallow:
User-agent: MSNBOT/0.xx (http://search.msn.com/msnbot.htm)
Disallow:
User-agent: msnbot/x.xx (+http://search.msn.com/msnbot.htm)
Disallow:
User-agent: Muncher
Disallow: /
User-agent: Muninn
Disallow: /
User-agent: Muscat Ferret
Disallow: /
User-agent: Mwd.Search
Disallow: /
User-agent: MyGetRight
Disallow: /
User-agent: Mac Finder 1.0.xx
Disallow: /
User-agent: Mackster( http://www.ukwizz.com )
Disallow: /
User-agent: Mag-Net
Disallow: /
User-agent: mailto:webcraft@bea.com
Disallow: /
User-agent: mammoth/1.0 (+http://www.sli-systems.com/)
Disallow: /
User-agent: MantraAgent
Disallow: /
User-agent: Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)
Disallow: /
User-agent: Marketwave Hit List
Disallow: /
User-agent: Martini
Disallow: /
User-agent: Marvin v0.3
Disallow: /
User-agent: Mass Downloader 2.x
Disallow: /
User-agent: Mata Hari/2.00
Disallow: /
User-agent: maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com)
Disallow: /
User-agent: MediaCrawler-1.0 (Experimental)
Disallow: /
User-agent: Mediapartners-Google/2.1 (+http://www.googlebot.com/bot.html)
Disallow:
User-agent: MediaSearch/0.1
Disallow: /
User-agent: MegaSheep v1.0 (www.searchuk.com internet sheep)
Disallow: /
User-agent: Mercator-1.x
Disallow: /
User-agent: Mercator-2.0
Disallow: /
User-agent: Mercator-Scrub-1.1
Disallow: /
User-agent: MetaGer-LinkChecker
Disallow: /
User-agent: MetaProducts Download Express/1.x
Disallow: /
User-agent: Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)
Disallow: /
User-agent: MFC Foundation Class Library 4.0
Disallow: /
User-agent: MFC_Tear_Sample
Disallow: /
User-agent: MFHttpScan
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider Protocol Discovery
Disallow: /
User-agent: MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)
Disallow: /
User-agent: Microsoft URL Control - 6.00.8xxx
Disallow:
User-agent: MIIxpc/4.2
Disallow: /
User-agent: minibot
Disallow: /
User-agent: MiracleAlphaTest
Disallow: /
User-agent: Missauga Locate 1.0.0
Disallow: /
User-agent: Missigua Locator 1.9
Disallow: /
User-agent: Missouri College Browse
Disallow: /
User-agent: Mister PiX version.dll
Disallow: /
User-agent: Mister Pix II 2.02a
Disallow: /
User-agent: MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)
Disallow: /
User-agent: MnogoSearch/3.2.xx
Disallow: /
User-agent: Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)
Disallow: /
User-agent: moget/x.x (moget@goo.ne.jp)
Disallow: /
User-agent: mogimogi/1.0
Disallow: /
User-agent: moiNAG 0.02
Disallow: /
User-agent: MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)
Disallow: /
User-agent: Morris - Mixcat Crawler (+http://mixcat.com)
Disallow: /
User-agent: MovableType/x.x
Disallow: /
User-agent: mozDex/0.04-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com)
Disallow: /
User-agent: Mozi!
Disallow: /
User-agent: Mozilla
Disallow:
User-agent: Mozilla (Mozilla@somewhere.com)
Disallow:
User-agent: Mozilla/1.1 (compatible; MSPIE 2.0; Windows CE)
Disallow:
User-agent: Mozilla/1.10 [en] (Compatible; RISC OS 3.70; Oregano 1.10)
Disallow:
User-agent: Mozilla/1.22 (compatible; MSIE 2.0d; Windows NT)
Disallow:
User-agent: Mozilla/1.22 (compatible; MSIE 5.01; PalmOS 3.0) EudoraWeb 2
Disallow:
User-agent: Mozilla/2.0 (compatible; AOL 3.0; Mac_PowerPC)
Disallow:
User-agent: Mozilla/2.0 (Compatible; AOL-IWENG 3.0; Win16)
Disallow:
User-agent: Mozilla/2.0 (compatible; Ask Jeeves)
Disallow:
User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
Disallow:
User-agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)
Disallow:
User-agent: Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)
Disallow:
User-agent: Mozilla/2.0 (compatible; MS FrontPage x.0)
Disallow:
User-agent: Mozilla/2.0 (compatible; MSIE 2.1; Mac_PowerPC)
Disallow:
User-agent: Mozilla/2.0 (compatible; MSIE 3.0; AK; Windows 95)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.02; Update a; AK; Windows NT)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.02; Update a; AOL 3.0; Windows 95)
Disallow: /
User-agent: Mozilla/2.0 (compatible; MSIE 3.0B; Win32)
Disallow: /
User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32)
Disallow: /
User-agent: Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)
Disallow: /
User-agent: Mozilla/2.0 compatible; Check&Get 1.1x (Windows 98)
Disallow:
User-agent: Mozilla/2.01 (Win16; I)
Disallow: /
User-agent: Mozilla/2.02Gold (Win95; I)
Disallow: /
User-agent: Mozilla/3.0 (compatible)
Disallow: /
User-agent: Mozilla/3.0 (compatible; AvantGo 3.2)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; HP Web PrintSmart 04b0 1.0.1.34)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Indy Library)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Linkman)
Disallow: /
User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)
Disallow: /
User-agent: Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)
Disallow: /
User-agent: Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64)
Disallow: /
User-agent: Mozilla/3.0 (compatible; NetPositive/2.2)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Opera/3.0; Windows 3.1) v3.1
Disallow: /
User-agent: Mozilla/3.0 (compatible; Opera/3.0; Windows 95/NT4) 3.2
Disallow: /
User-agent: Mozilla/3.0 (compatible; PerMan Surfer 3.0; Win95)
Disallow: /
User-agent: Mozilla/3.0 (compatible; scan4mail (advanced version), http://www.peterspages.net/?scan4mail)
Disallow: /
User-agent: Mozilla/3.0 (compatible; REL Software, Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/3.0 (Compatible;Viking/1.8)
Disallow: /
User-agent: Mozilla/3.0 (compatible; WebCapture x.x; Auto; Windows)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)
Disallow: /
User-agent: Mozilla/3.0 (compatible; Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/3.0 (DreamPassport/3.0)
Disallow: /
User-agent: Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)
Disallow: /
User-agent: Mozilla/3.0 (Liberate DTV 1.1)
Disallow: /
User-agent: Mozilla/3.0 (Planetweb/2.100 JS SSL US; Dreamcast US)
Disallow: /
User-agent: Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)
Disallow: /
User-agent: Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)
Disallow: /
User-agent: Mozilla/3.0 (Win16; I)
Disallow: /
User-agent: Mozilla/3.0 (Win95; I)
Disallow: /
User-agent: Mozilla/3.0 (WinNT; I)
Disallow: /
User-agent: Mozilla/3.0 (WorldGate Gazelle 3.5.1 build 11; FreeBSD2.2.8-STABLE)
Disallow: /
User-agent: Mozilla/3.0 (X11; I; OSF1 V4.0 alpha)
Disallow: /
User-agent: Mozilla/3.0 NAVIO_AOLTV (11; 13; Philips; PH200; 1; R2.0C36_AOL.0110OPTIK; R2.0.0139d_OPTIK)
Disallow: /
User-agent: Mozilla/3.0 WebTV/1.2 (compatible; MSIE 2.0)
Disallow: /
User-agent: Mozilla/3.01 (compatible; AmigaVoyager/2.95; AmigaOS/MC680x0)
Disallow: /
User-agent: Mozilla/3.01 (Compatible; Links2Go Similarity Engine)
Disallow: /
User-agent: Mozilla/3.01 (compatible; Netbox/3.5 R92; Linux 2.2)
Disallow: /
User-agent: Mozilla/3.01-C-MACOS8 (Macintosh; I; PPC)
Disallow: /
User-agent: Mozilla/3.01Gold (X11; I; Linux 2.0.32 i486)
Disallow: /
User-agent: Mozilla/3.01Gold (X11; I; SunOS 5.5.1 sun4m)
Disallow: /
User-agent: Mozilla/3.01SGoldC-SGI (X11; I; IRIX 6.3 IP32)
Disallow: /
User-agent: Mozilla/3.04 (compatible; ANTFresco/2.13; RISC OS 4.02)
Disallow: /
User-agent: Mozilla/3.04 (compatible; NCBrowser/2.35; ANTFresco/2.17; RISC OS-NC 5.13 Laz1UK1309)
Disallow: /
User-agent: Mozilla/3.x (I-Opener 1.1; Netpliance)
Disallow: /
User-agent: Mozilla/4.0 (agadine3.0) www.agada.de
Disallow: /
User-agent: Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Arachmo)
Disallow: /
User-agent: Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)
Disallow: /
User-agent: Mozilla/4.0 (compatible; BorderManager 3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; ChristCrawler.com, ChristCrawler@ChristCENTRAL.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; DnloadMage 1.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; FastCrawler3, support-fastcrawler3@fast.no)
Disallow: /
User-agent: Mozilla/4.0 (compatible; FDSE robot)
Disallow: /
User-agent: Mozilla/4.0 (compatible; www.galaxy.com)
Disallow:
User-agent: Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)
Disallow:
User-agent: Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)
Disallow: /
User-agent: Mozilla/4.0 (compatible; grub-client-2.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; ICS 1.2.xxx)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Iplexx Spider/1.0, http://www.iplexx.at)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Lotus-Notes/5.0; Windows-NT)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) WebQuest Designs
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; MSIECrawler; Windows 95)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; MSN Companion 2.0; 800x600; Compaq)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT Windows CE)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; AOL 5.0; Windows 95; DigExt; Gateway2000; sureseeker.com)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC; AtHome021)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Win32) via proxy gateway CERN-HTTPD/3.0 libwww/2.17
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 5.11 [en]
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows ME; Link Checker 2.x.xx, http://www.kyosoft.com)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; +http://www.galaxy.com/info/crawler.html)
Disallow:
User-agent: Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)
Disallow: /
User-agent: Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8
Disallow: /
User-agent: Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows 95) via Avirt Gateway Server v4.0
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; NetCaptor 6.5.0RC1)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; AOL 4.0; Windows 98; GoBeez (www.gobeez.com))
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 95; Transmission Segment; Hotbar 2.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Crazy Browser 1.x.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; KITV4.7 Wanadoo)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SAFEXPLORER TL)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SYMPA; Katiesoft 7; SimulBrowse 3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; BTinternet V8.1)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; MSIECrawler)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Active Cache Request
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AIRF)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; N_o_k_i_a)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; T312461) RPT-HTTPClient/0.3-3E
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Win32) WebWasher 3.0
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Net M@nager V3.02 - www.vinn.com.au)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) REL Software, Web Link Validator 2.x)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90; http://www.Abolimba.de)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) (dns_admin@c-a-s-h.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 +http://www.cs.washington.edu/research/networking/websys/)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Hotbar 3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iRider 2.21.1108; FDM)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman3.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; PeoplePal 3.0; MSIECrawler)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; IOpener Release 1.1.04)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; StumbleUpon.com 1.760; .NET CLR 1.1.4322)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; DX-Browser 5.0.0.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Opera/3.0; Windows 4.10) 3.51 [en]
Disallow: /
User-agent: Mozilla/4.0 (compatible; Powermarks/3.5; Windows 95/98/2000/NT)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SiteKiosk 4.0; MSIE 5.0; Windows 98; SiteCoach 1.0)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SPENG)
Disallow: /
User-agent: Mozilla/4.0 (compatible; SuperCleaner 2.xx; Windows 98)
Disallow: /
User-agent: Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)
Disallow: /
User-agent: Mozilla/4.0 (compatible; www.linkguard.com Linkguard Online 1.0; Windows NT)
Disallow: /
User-agent: Mozilla/4.0(compatible; Zealbot 1.0)
Disallow: /
User-agent: Mozilla/4.0 (fantomBrowser)
Disallow: /
User-agent: Mozilla/4.0 (fantomCrew Browser)
Disallow: /
User-agent: Mozilla/4.0 (hhjhj@yahoo.com)
Disallow: /
User-agent: Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)
Disallow: /
User-agent: Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0
Disallow: /
User-agent: Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
Disallow:
User-agent: Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
Disallow:
User-agent: Mozilla/4.0 (Sleek Spider/1.2)
Disallow: /
User-agent: Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)
Disallow: /
User-agent: Mozilla/4.0 efp@gmx.net
Disallow: /
User-agent: Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0)
Disallow: /
User-agent: Mozilla/4.01 [en](Win95;I)
Disallow: /
User-agent: Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u)
Disallow: /
User-agent: Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712)
Disallow: /
User-agent: Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)
Disallow: /
User-agent: Mozilla/4.05 (Macintosh; I; 68K, Nav)
Disallow: /
User-agent: Mozilla/4.05 (Macintosh; I; PPC, Nav)
Disallow: /
User-agent: Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m)
Disallow: /
User-agent: Mozilla/4.08 [en] (Win98; U ;Nav)" [Spaces are not mistakes]Help->About: "Version 4.08 [en]-98306"
Disallow: /
User-agent: Mozilla/4.08 [en] (WinNT; U)
Disallow: /
User-agent: Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)
Disallow: /
User-agent: Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC)
Disallow: /
User-agent: Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC)
Disallow: /
User-agent: Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I)
Disallow: /
User-agent: Mozilla/4.5 RPT-HTTPClient/0.3-2
Disallow: /
User-agent: Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)
Disallow: /
User-agent: Mozilla/4.61 [de] (OS/2; I)
Disallow: /
User-agent: Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows)
Disallow: /
User-agent: Mozilla/4.7
Disallow: /
User-agent: Mozilla/4.7 (compatible; http://eidetica.com/spider)
Disallow: /
User-agent: Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)
Disallow: /
User-agent: Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2
Disallow: /
User-agent: Mozilla/4.7 (compatible; Whizbang)
Disallow: /
User-agent: Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)
Disallow: /
User-agent: Mozilla/4.7 [en](BecomeBot@exava.com)
Disallow: /
User-agent: Mozilla/4.7 [en](Exabot@exava.com)
Disallow: /
User-agent: Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I)
Disallow: /
User-agent: Mozilla/4.72 [en] (BACS http://www.ba.be)
Disallow: /
User-agent: Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686)
Disallow: /
User-agent: Mozilla/4.75C-ja [ja] (X11; U; OSF1 V5.1 alpha)
Disallow: /
User-agent: Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386)
Disallow: /
User-agent: Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u)
Disallow: /
User-agent: Mozilla/4.76 (Windows 98; U) Opera 5.12 [en]
Disallow: /
User-agent: Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32)
Disallow: /
User-agent: Mozilla/5.0
Disallow: /
User-agent: Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/1.23; +http://www.become.com/webmasters.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/1.68; MSIE 6.0 compatible; +http://www.become.com/webmasters.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible; BecomeBot/2.0beta; +http://www.become.com/webmasters.html)
Disallow: /
User-agent: Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
Disallow:
User-agent: mozilla/5.0 (compatible; heritrix/1.3.0 +http://archive.crawler.org)
Disallow:
User-agent: Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding
Disallow:
User-agent: Mozilla/5.0 (compatible; Konqueror/2.1.1; X11)
Disallow:
User-agent: Mozilla/5.0 (compatible; Konqueror/2.2.2)
Disallow:
User-agent: Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)
Disallow:
User-agent: Mozilla/5.0 (compatible; LemSpider 0.1)
Disallow:
User-agent: Mozilla/5.0 (compatible; SpurlBot/0.2)
Disallow:
User-agent: Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;)
Disallow:
User-agent: Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)
Disallow:
User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
Disallow:
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7
Disallow:
User-agent: Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1
Disallow:
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML, like Gecko) Safari/12x.x
Disallow:
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML, like Gecko) OmniWeb/v5xx.xx
Disallow:
User-agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6+
Disallow:
User-agent: Mozilla/5.0 (research@mediatrec.com)
Disallow: /
User-agent: Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow:
User-agent: Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow:
User-agent: Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en]
Disallow:
User-agent: Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1
Disallow:
User-agent: Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6
Disallow:
User-agent: Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable
Disallow:
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6
Disallow:
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104
Disallow:
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x
Disallow:
User-agent: Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01
Disallow:
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408
Disallow:
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801
Disallow:
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3
Disallow:
User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1
Disallow:
User-agent: Mozilla/5.0 Galeon/1.0.2 (X11; Linux i686; U;) Gecko/20011224
Disallow:
User-agent: Mozilla/5.0 gURLChecker/0.x.x (Linux)
Disallow:
User-agent: Mozilla/5.0 URL-Spider
Disallow:
User-agent: Mozilla/5.0 usww.com-Spider-for-w8.net
Disallow:
User-agent: Mozilla/5.0 wgao@genieknows.com
Disallow: /
User-agent: MSFrontPage/4.0
Disallow:
User-agent: MSProxy/2.0
Disallow: /
User-agent: MSIE 4.0 (Win95)
Disallow:
User-agent: MSIE-5.13 (larbin@unspecified.mail)
Disallow: /
User-agent: MSNPTC/1.0
Disallow: /
User-agent: multiBlocker browser
Disallow: /
User-agent: MultiText/0.1
Disallow: /
User-agent: MusicWalker2.0 (+http://www.somusical.com)
Disallow: /
User-agent: myDaemon
Disallow: /
User-agent: MyGetRight/1.0b
Disallow: /
User-agent: MyGetRight/1.0.0
Disallow: /
User-agent: Mylinea.com Crawler 2.0
Disallow: /
# N
User-agent: NaverBot
Disallow: /
User-agent: NavRoad
Disallow: /
User-agent: Navroad
Disallow: /
User-agent: NDSpider
Disallow: /
User-agent: NearSite
Disallow: /
User-agent: NEC-MeshExplorer
Disallow: /
User-agent: Nederland.zoek
Disallow: /
User-agent: Net M@nager
Disallow: /
User-agent: Net\ Vampire
Disallow: /
User-agent: NetAnts
Disallow: /
User-agent: NetCarta WebMap Engine
Disallow: /
User-agent: NetMechanic
Disallow: /
User-agent: NetScoop
Disallow: /
User-agent: NetSpider
Disallow: /
User-agent: NetstreamRecorder
Disallow: /
User-agent: newscan-online
Disallow: /
User-agent: NHSE Web Forager
Disallow: /
User-agent: NICErsPRO
Disallow: /
User-agent: Nomad
Disallow: /
User-agent: Northern Light Gulliver
Disallow: /
User-agent: NP
Disallow: /
User-agent: nzexplorer
Disallow: /
User-agent: nabot_1.0
Disallow: /
User-agent: NABOT/5.0
Disallow: /
User-agent: NationalDirectoryAddURL/1.0
Disallow: /
User-agent: NationalDirectory-WebSpider/1.3
Disallow: /
User-agent: NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)
Disallow: /
User-agent: NaverBot_dloader/1.5
Disallow: /
User-agent: NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)
Disallow: /
User-agent: NEC-Hayek/1.0
Disallow: /
User-agent: NEC Research Agent -- compuman at research.nj.nec.com
Disallow: /
User-agent: NetAnts/1.2x
Disallow: /
User-agent: NETCOMplete/x.xx
Disallow: /
User-agent: NetLookout/2.24
Disallow: /
User-agent: NetMechanic Vx.0
Disallow: /
User-agent: NetNoseCrawler/v1.0
Disallow: /
User-agent: Netprospector JavaCrawler
Disallow: /
User-agent: NetPumper/x.xx
Disallow: /
User-agent: NetResearchServer(http://www.look.com)
Disallow: /
User-agent: NetResearchServer/x.x(loopimprovements.com/robot.html)
Disallow: /
User-agent: NetZippy
Disallow: /
User-agent: NeuralBot/0.2
Disallow: /
User-agent: NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)
Disallow: /
User-agent: NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x
Disallow: /
User-agent: newsearchengine (ThisUser@unspecified.mail)
Disallow: /
User-agent: NG/1.0
Disallow: /
User-agent: NICO/1.0
Disallow: /
User-agent: NITLE Blog Spider/0.01
Disallow: /
User-agent: Nitro Downloader 1.x (www.klsofttools.com)
Disallow: /
User-agent: Noago Spider
Disallow: /
User-agent: Nocilla/1.0
Disallow: /
User-agent: Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com
Disallow:
User-agent: Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0)
Disallow:
User-agent: NokodoBot/1.x (+http://nokodo.com/bot.htm)
Disallow: /
User-agent: NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)
Disallow: /
User-agent: NPBot (http://www.nameprotect.com/botinfo.html)
Disallow: /
User-agent: NPBot-1/2.0
Disallow: /
User-agent: nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp)
Disallow: /
User-agent: nuSearch Spider www.nusearch.com (compatible; MSIE 4.01)
Disallow: /
User-agent: NuSearch Spider www.nusearch.com
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)
Disallow: /
User-agent: NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)
Disallow: /
# O
User-agent: ObjectsSearch
Disallow: /
User-agent: oBot
Disallow: /
User-agent: Occam
Disallow: /
User-agent: offline
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Offline\ Navigator
Disallow: /
User-agent: OntoSpider
Disallow: /
User-agent: Openbot
Disallow: /
User-agent: Openfind
Disallow: /
User-agent: Openfind data gathere
Disallow: /
User-agent: Openfind data gatherer
Disallow: /
User-agent: Orb Search
Disallow: /
User-agent: obidos-bot (just looking for books.)
Disallow: /
User-agent: ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)
Disallow: /
User-agent: oBot ((compatible;Win32))
Disallow: /
User-agent: Ocelli/1.x (http://www.globalspec.com/Ocelli)
Disallow: /
User-agent: Octopus
Disallow: /
User-agent: Offline Explorer 1.*
Disallow: /
User-agent: OliverPerry
Disallow: /
User-agent: OmniExplorer_Bot/1.07 (+http://www.omni-explorer.com) Internet Categorizer
Disallow:
User-agent: OmniExplorer_Bot/1.07 (+http://www.omni-explorer.com) Job Crawler
Disallow:
User-agent: OmniExplorer_Bot/1.18 (+http://www.omni-explorer.com) Torrent Crawler
Disallow: /
User-agent: online link validator (http://www.dead-links.com/)
Disallow: /
User-agent: OntoSpider/1.0 libwww-perl/5.65
Disallow: /
User-agent: Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)
Disallow: /
User-agent: Openfind data gatherer, Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)
Disallow: /
User-agent: Openfind Robot/1.1A2
Disallow: /
User-agent: OpenTextSiteCrawler/2.9.2
Disallow: /
User-agent: OpenWebSpider/x
Disallow: /
User-agent: Opera/5.0 (Linux 2.0.38 i386; U) [en]
Disallow:
User-agent: Opera/5.11 (Windows ME; U) [ru]
Disallow:
User-agent: Opera/5.12 (Windows 98; U) [en]
Disallow:
User-agent: Opera/6.x (Linux 2.4.8-26mdk i686; U) [en]
Disallow:
User-agent: Opera/6.x (Windows NT 4.0; U) [de]
Disallow:
User-agent: Opera/6.01 (larbin@unspecified.mail)
Disallow:
User-agent: Opera/7.x (Windows NT 5.1; U) [en]
Disallow:
User-agent: Opera/8.xx (Windows NT 5.1; U; en)
Disallow:
User-agent: OpidooBOT (larbin2.6.3@unspecified.mail)
Disallow:
User-agent: Oracle iMTCrawler
Disallow: /
User-agent: Oracle Ultra Search
Disallow: /
User-agent: Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
Disallow: /
User-agent: OWR_Crawler 0.1
Disallow: /
# P
User-agent: Pack Rat
User-agent: PageBoy
Disallow: /
User-agent: PageDown
Disallow: /
User-agent: PageRipper
Disallow: /
User-agent: PageSucker
Disallow: /
User-agent: ParaSite
Disallow: /
User-agent: Patric
Disallow: /
User-agent: Patwebbot
Disallow: /
User-agent: pcBrowser
Disallow: /
User-agent: pegasus
Disallow: /
User-agent: PerlCrawler 1.0
Disallow: /
User-agent: Personal Internet Engine
Disallow: /
User-agent: PersonaPilot
Disallow: /
User-agent: PGP Key Agent
Disallow: /
User-agent: Phantom
Disallow: /
User-agent: PhpDig
Disallow: /
User-agent: PiltdownMan
Disallow: /
User-agent: Pimptrain.com's robot
Disallow: /
User-agent: Pioneer
Disallow: /
User-agent: PlumtreeWebAccessor
Disallow: /
User-agent: Poppi
Disallow: /
User-agent: Popular Iconoclast
Disallow: /
User-agent: Portal Juice Spider
Disallow: /
User-agent: PortalB Spider
Disallow: /
User-agent: Program
Disallow: /
User-agent: Project
Disallow: /
User-agent: ProPowerBot/2.14
Disallow: /
User-agent: ProWebWalker
Disallow: /
User-agent: psbot
Disallow: /
User-agent: Python-urllib
Disallow: /
User-agent: PagmIEDownload
Disallow: /
User-agent: parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1
Disallow: /
User-agent: ParaSite/1.0b (http://www.ianett.com/parasite/)
Disallow: /
User-agent: Patwebbot (http://www.herz-power.de/technik.html)
Disallow: /
User-agent: pavuk/0.9pl29b i686-pc-linux-gnu
Disallow: /
User-agent: PBrowse 1.4b
Disallow: /
User-agent: pd02_1.0.0 pd02_1.0.0@dzimi@post.sk
Disallow: /
User-agent: PEAR HTTP_Request class ( http://pear.php.net/ )
Disallow: /
User-agent: PEERbot www.peerbot.com
Disallow: /
User-agent: PeopleChat/Search_Engine
Disallow: /
User-agent: PEval 1.4b
Disallow: /
User-agent: PHP/3.x.xx
Disallow: /
User-agent: PHP/4.0.4pl1
Disallow: /
User-agent: PHP/4.0.6
Disallow: /
User-agent: PHP/4.1.1
Disallow: /
User-agent: PHP/4.1.2
Disallow: /
User-agent: PicoSearch/1.0
Disallow: /
User-agent: PingALink Monitoring Services 1.0
Disallow: /
User-agent: PingALink Monitoring Services 1.0 (http://www.pingalink.com)
Disallow: /
User-agent: pipeLiner/0.10 (PipeLine Spider; http://www.pipeline-search.com/webmaster.html)
Disallow: /
User-agent: pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com)
Disallow: /
User-agent: Pita
Disallow: /
User-agent: Pizilla++ ver 2.45
Disallow: /
User-agent: PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)
Disallow: /
User-agent: PlagiarBot/1.0
Disallow: /
User-agent: PlantyNet_WebRobot_V1.9 dhkang@plantynet.com
Disallow: /
User-agent: Pockey/x.x.x
Disallow: /
User-agent: Pockey7.x.x(WIN32GUI)
Disallow: /
User-agent: Pockey-GetHTML/x.xx
Disallow: /
User-agent: Pockey-GetHTML/4.12.0 (Win32; GUI; ix86)
Disallow: /
User-agent: POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000)
Disallow: /
User-agent: polybot 1.0 (http://cis.poly.edu/polybot/)
Disallow: /
User-agent: Pompos/1.x pompos@iliad.fr
Disallow: /
User-agent: Pompos/1.x http://dir.com/pompos.html
Disallow: /
User-agent: Popdexter/1.0
Disallow: /
User-agent: PortalBSpider/2.0 (spider@portalb.com)
Disallow: /
User-agent: potbot 1.0
Disallow: /
User-agent: Privoxy/3.0 (Anonymous)
Disallow: /
User-agent: Production Bot 0116B
Disallow: /
User-agent: Production Bot 2016B
Disallow: /
User-agent: Production Bot DOT 3016B
Disallow: /
User-agent: Program Shareware 1.0.2
Disallow: /
User-agent: Progressive Download
Disallow: /
User-agent: Progressive Download HTTP check
Disallow: /
User-agent: Project XP5 [2.03.07-111203]
Disallow: /
User-agent: PROve AnswerBot 4.0
Disallow: /
User-agent: ProWebGuide Link Checker (http://www.prowebguide.com)
Disallow: /
User-agent: psbot/0.1 (+http://www.picsearch.com/bot.html)
Disallow: /
User-agent: PSurf15a 11
Disallow: /
User-agent: PSurf15a 51
Disallow: /
User-agent: PSurf15a VA
Disallow: /
User-agent: PubCrawl (pubcrawl.stanford.edu)
Disallow: /
User-agent: puf/0.91beta6a (Linux 2.2.18; i686)
Disallow: /
User-agent: PureSight
Disallow: /
User-agent: PuxaRapido v1.0
Disallow: /
User-agent: PycURL
Disallow: /
User-agent: Python-urllib/1.15
Disallow: /
User-agent: Python-urllib/2.0a1
Disallow: /
# Q
User-agent: Que
Disallow: /
User-agent: Qango.com Web Directory (http://www.qango.com/)
Disallow: /
User-agent: QPCreep Test Rig ( We are not indexing, just testing )
Disallow: /
User-agent: QuepasaCreep ( crawler@quepasacorp.com )
Disallow: /
User-agent: QuepasaCreep v0.9.1x
Disallow: /
User-agent: QueryN Metasearch
Disallow: /
# R
User-agent: Radiation Retriever 1.1
Disallow: /
User-agent: RafaBot
Disallow: /
User-agent: Raven Search
Disallow: /
User-agent: RBSE Spider
Disallow: /
User-agent: RepoMonkey
Disallow: /
User-agent: Resume Robot
Disallow: /
User-agent: River
Disallow: /
User-agent: RixBot
Disallow: /
User-agent: RMA
Disallow: /
User-agent: Road Runner: The ImageScape Robot
Disallow: /
User-agent: RoadHouse Crawling System
Disallow: /
User-agent: roadrunner
Disallow: /
User-agent: Robbie the Robot
Disallow: /
User-agent: RoboCrawl Spider
Disallow: /
User-agent: RoboFox
Disallow: /
User-agent: Robot Francoroute
Disallow: /
User-agent: Robozilla
Disallow: /
User-agent: Roverbot
Disallow: /
User-agent: RPT
Disallow: /
User-agent: RuLeS
Disallow: /
User-agent: rabaz (rabaz at gigabaz dot com)
Disallow: /
User-agent: RaBot/1.0 Agent-admin/phortse@hanmail.net
Disallow: /
User-agent: Rainbot1.1
Disallow: /
User-agent: ramBot xtreme x.x
Disallow: /
User-agent: Rational SiteCheck (Windows NT)
Disallow: /
User-agent: RealDownload/4.0.0.4x
Disallow: /
User-agent: Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)
Disallow: /
User-agent: Reaper/2.0x (+http://www.sitesearch.ca/reaper)
Disallow: /
User-agent: REBOL Core 2.x.x.x.x
Disallow: /
User-agent: REL Link Checker Lite x.x
Disallow: /
User-agent: RepoMonkey Bait & Tackle/v1.01
Disallow: /
User-agent: Rewebber/1.2 libwww-perl/5.41
Disallow: /
User-agent: rico/0.1
Disallow: /
User-agent: RMA/1.0 (compatible; RealMedia)
Disallow: /
User-agent: RoboCrawl (www.canadiancontent.net)
Disallow: /
User-agent: RoboPal (http://www.findpal.com/)
Disallow: /
User-agent: Robot@SuperSnooper.Com
Disallow: /
User-agent: Robot/www.pj-search.com
Disallow: /
User-agent: Robot: NutchCrawler, Owner: wdavies@acm.org
Disallow: /
User-agent: Robozilla/1.0
Disallow: /
User-agent: Rotondo/3.1 libwww/5.3.1
Disallow: /
User-agent: RPT-HTTPClient/0.3-x
Disallow: /
User-agent: RRC (crawler_admin@bigfoot.com)
Disallow: /
User-agent: RSurf15a 41
Disallow: /
User-agent: RSurf15a 51
Disallow: /
User-agent: RSurf15a 81
Disallow: /
User-agent: Rumours-Agent
Disallow: /
# S
User-agent: SafetyNet Robot
Disallow: /
User-agent: Scooter
Disallow: /
User-agent: Scrubby
Disallow: /
User-agent: Search.Aus-AU.COM
Disallow: /
User-agent: SearchProcess
Disallow: /
User-agent: Secret
Disallow: /
User-agent: Selfseek
Disallow: /
User-agent: Senrigan
Disallow: /
User-agent: SG-Scout
Disallow: /
User-agent: ShagSeeker
Disallow: /
User-agent: Shai'Hulud
Disallow: /
User-agent: Sift
Disallow: /
User-agent: Simmany Robot Ver1.0
Disallow: /
User-agent: Simple/
Disallow: /
User-agent: Site Searcher
Disallow: /
User-agent: Site Valet
Disallow: /
User-agent: SiteMapper
Disallow: /
User-agent: SiteTech-Rover
Disallow: /
User-agent: Skymob.com
Disallow: /
User-agent: SLCrawler
Disallow: /
User-agent: Sleek
Disallow: /
User-agent: Slurp
Disallow: /
User-agent: Smart Spider
Disallow: /
User-agent: SnapItUp
Disallow: /
User-agent: Snoop
Disallow: /
User-agent: Snooper
Disallow: /
User-agent: Solbot
Disallow: /
User-agent: SpankBot
Disallow: /
User-agent: spanner
Disallow: /
User-agent: Speedy Spider
Disallow: /
User-agent: spider_monkey
Disallow: /
User-agent: SpiderBot
Disallow: /
User-agent: Spiderline Crawler
Disallow: /
User-agent: SpiderView(tm)
Disallow: /
User-agent: Spry Wizard Robot
Disallow: /
User-agent: sqworm
Disallow: /
User-agent: Sqworm
Disallow: /
User-agent: StackRambler
Disallow: /
User-agent: Star
Disallow: /
User-agent: Suke
Disallow: /
User-agent: Sundoh
Disallow: /
User-agent: suntek search engine
Disallow: /
User-agent: SuperBot
Disallow: /
User-agent: SuperHTTP
Disallow: /
User-agent: Surfbot
Disallow: /
User-agent: SurfSaver
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: Sven
Disallow: /
User-agent: Sygol
Disallow: /
User-agent: Szukacz/1.4
Disallow: /
User-agent: S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx)
Disallow: /
User-agent: SafariBookmarkChecker (+http://www.coriolis.ch/)
Disallow: /
User-agent: SBL-BOT (http://sbl.net)
Disallow: /
User-agent: ScanWeb
Disallow: /
User-agent: schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp)
Disallow: /
User-agent: Science Traveller International 1X/1.0
Disallow: /
User-agent: Scooter/1.0
Disallow: /
User-agent: Scooter/1.0 scooter@pa.dec.com
Disallow: /
User-agent: Scooter/1.1 (custom)
Disallow: /
User-agent: Scooter/2.0 G.R.A.B. V1.1.0
Disallow: /
User-agent: Scooter/2.0 G.R.A.B. X2.0
Disallow: /
User-agent: Scooter-3.0.EU
Disallow: /
User-agent: Scooter-3.0.FS
Disallow: /
User-agent: Scooter-3.0.HD
Disallow: /
User-agent: Scooter-3.0.VNS
Disallow: /
User-agent: Scooter-3.0QI
Disallow: /
User-agent: Scooter-3.2
Disallow: /
User-agent: Scooter-3.2.BT
Disallow: /
User-agent: Scooter-3.2.DIL
Disallow: /
User-agent: Scooter-3.2.EX
Disallow: /
User-agent: Scooter-3.2.JT
Disallow: /
User-agent: Scooter-3.2.NIV
Disallow: /
User-agent: Scooter-3.2.SF0
Disallow: /
User-agent: Scooter-3.2.snippet
Disallow: /
User-agent: Scooter/3.3
Disallow: /
User-agent: Scooter/3.3.QA.pczukor
Disallow: /
User-agent: Scooter/3.3_SF
Disallow: /
User-agent: Scooter/3.3.vscooter
Disallow: /
User-agent: Scooter-3.3dev
Disallow: /
User-agent: Scooter-ARS-1.1
Disallow: /
User-agent: Scooter-ARS-1.1-ih
Disallow: /
User-agent: Scooter_bh0-3.0.3
Disallow: /
User-agent: Scooter_trk3-3.0.3
Disallow: /
User-agent: scooter-venus-3.0.vns
Disallow: /
User-agent: Scooter-W3-1.0
Disallow: /
User-agent: Scooter-W3.1.2
Disallow: /
User-agent: Scooter2_Mercator_x-x.0
Disallow: /
User-agent: ScoutAbout
Disallow: /
User-agent: Scrubby/2.x (http://www.scrubtheweb.com/)
Disallow: /
User-agent: Search+
Disallow: /
User-agent: Search/1.0 (http://www.innerprise.net/es-spider.asp)
Disallow: /
User-agent: search.ch V1.4
Disallow: /
User-agent: search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch)
Disallow: /
User-agent: SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)
Disallow: /
User-agent: SearchExpress Spider0.99
Disallow: /
User-agent: SearchGuild_DMOZ_Experiment (chris@searchguild.com)
Disallow: /
User-agent: Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)
Disallow: /
User-agent: SearchSpider.com/1.1
Disallow: /
User-agent: Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)
Disallow: /
User-agent: SearchTone2.0 - IDEARE
Disallow: /
User-agent: Seeker.lookseek.com
Disallow: /
User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3
Disallow: /
User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)
Disallow: /
User-agent: semanticdiscovery/0.x
Disallow: /
User-agent: Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)
Disallow: /
User-agent: SeznamBot/1.0
Disallow: /
User-agent: SeznamBot/1.0 (+http://fulltext.seznam.cz/)
Disallow: /
User-agent: Shareaza v1.x.x.xx
Disallow: /
User-agent: SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com)
Disallow: /
User-agent: Simpy 1.x; http://www.simpy.com/
Disallow: /
User-agent: Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)
Disallow: /
User-agent: SiteBar/x.x
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com)
Disallow: /
User-agent: SiteRecon+(xx)
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: SiteSpider +(http://www.SiteSpider.com/)
Disallow: /
User-agent: SiteWinder
Disallow: /
User-agent: SiteXpert
Disallow: /
User-agent: Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)
Disallow: /
User-agent: Skimpy/0.x (http://www.skaffe.com/skampy-info.html)
Disallow: /
User-agent: Slarp/0.1
Disallow: /
User-agent: Sleipnir Version 1.xx
Disallow: /
User-agent: Slider_Search_v1-de
Disallow: /
User-agent: SlimBrowser
Disallow: /
User-agent: Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow: /
User-agent: SlySearch (slysearch@slysearch.com)
Disallow: /
User-agent: SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html
Disallow: /
User-agent: SlySearch/1.x http://www.slysearch.com
Disallow: /
User-agent: SmartDownload/1.2.67 (Win32; Jan 12 1999)
Disallow: /
User-agent: SmartDownload/1.2.77 (Win32; Feb 1 2000)
Disallow: /
User-agent: SmartDownload/1.2.77 (Win32; Jun 19 2001)
Disallow: /
User-agent: smartwit.com
Disallow: /
User-agent: sna-0.0.1 mikeelliott@hotmail.com
Disallow: /
User-agent: SnoopRob/x.x
Disallow: /
User-agent: Snoopy_v0.xx
Disallow: /
User-agent: Snoopy v1.xx
Disallow: /
User-agent: Snoopy v1.xx, : User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2)
Disallow: /
User-agent: SnykeBot/0.6 (http://www.snyke.com)
Disallow: /
User-agent: SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)
Disallow: /
User-agent: sohu-search
Disallow: /
User-agent: SpeedDownload/1.x
Disallow: /
User-agent: speedfind ramBot xtreme 8.1
Disallow: /
User-agent: Speedy Spider (Beta/x.x; speedy@entireweb.com)
Disallow: /
User-agent: Speedy_Spider (http://www.entireweb.com)
Disallow: /
User-agent: Spida/0.1
Disallow: /
User-agent: Spider/maxbot.com admin@maxbot.com
Disallow: /
User-agent: spider.batsch.com
Disallow: /
User-agent: spider.yellopet.com - www.yellopet.com
Disallow: /
User-agent: Spider-Sleek/2.0 (+http://search-info.com/linktous.html)
Disallow: /
User-agent: SpiderKU/0.x
Disallow: /
User-agent: SpiderMan
Disallow: /
User-agent: SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)
Disallow: /
User-agent: Spinne/2.0
Disallow: /
User-agent: Spinne/2.0 med
Disallow: /
User-agent: Spinne/2.0 med_AH
Disallow: /
User-agent: SQ Webscanner
Disallow: /
User-agent: Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)
Disallow: /
User-agent: Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)
Disallow: /
User-agent: Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu)
Disallow: /
User-agent: SSurf15a 11
Disallow: /
User-agent: StackRambler/x.x
Disallow: /
User-agent: Stamina/1.4
Disallow: /
User-agent: Star Downloader
Disallow: /
User-agent: stat statcrawler@gmail.com
Disallow: /
User-agent: Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)
Disallow: /
User-agent: Submission Spider at surfsafely.com
Disallow: /
User-agent: suchbot
Disallow: /
User-agent: Suchknecht.at-Robot
Disallow: /
User-agent: SuperBot/x.x (Win32)
Disallow: /
User-agent: SURF
Disallow: /
User-agent: SurferF3 1/0
Disallow: /
User-agent: SurfMaster
Disallow: /
User-agent: SurveyBot/2.2 Whois Source
Disallow: /
User-agent: SurveyBot/2.3 (Whois Source)
Disallow: /
User-agent: suzuran
Disallow: /
User-agent: SWB/V1.4 (HP)
Disallow: /
User-agent: swbot/0.9c libwww/5.3.1
Disallow: /
User-agent: http://www.sygol.com
Disallow: /
User-agent: Sylera/1.2.x
Disallow: /
User-agent: SyncBot
Disallow: /
User-agent: SyncIT/x.x
Disallow: /
User-agent: SynoBot
Disallow: /
User-agent: Szukacz/1.x
Disallow: /
User-agent: Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)
Disallow: /
# T
User-agent: TACH Black Widow
Disallow: /
User-agent: tAkeOut
Disallow: /
User-agent: Tarantula
Disallow: /
User-agent: tarspider
Disallow: /
User-agent: Tcl W3 Robot
Disallow: /
User-agent: TechBOT
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: Teleport Pro
Disallow: /
User-agent: Teleport\ Pro
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: Telesoft
Disallow: /
User-agent: Templeton
Disallow: /
User-agent: Teoma
Disallow: /
User-agent: Terrapin
Disallow: /
User-agent: The Intraformant
Disallow: /
User-agent: The NorthStar Robot
Disallow: /
User-agent: The NWI Robot
Disallow: /
User-agent: The Peregrinator
Disallow: /
User-agent: The Python Robot
Disallow: /
User-agent: The TkWWW Robot
Disallow: /
User-agent: The Web Moose
Disallow: /
User-agent: The Web Wombat
Disallow: /
User-agent: The Webfoot Robot
Disallow: /
User-agent: the World Wide Web Wanderer
Disallow: /
User-agent: The World Wide Web Worm
Disallow: /
User-agent: TheNomad
Disallow: /
User-agent: TightTwatBot
Disallow: /
User-agent: TITAN
Disallow: /
User-agent: Titan
Disallow: /
User-agent: TitIn
Disallow: /
User-agent: TLSpider
Disallow: /
User-agent: True_Robot
Disallow: /
User-agent: True_Robot/1.0
Disallow: /
User-agent: turingos
Disallow: /
User-agent: Turn It In
Disallow: /
User-agent: Turnitin
Disallow: /
User-agent: Turnitinbot
Disallow: /
User-agent: Tagword (http://tagword.com/dmoz_survey.php)
Disallow: /
User-agent: Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com, Home: http://222.122.15.190/webshot)
Disallow: /
User-agent: TAMU_CS_IRL_CRAWLER/1.0
Disallow: /
User-agent: targetblaster.com/0.9k
Disallow: /
User-agent: TE
Disallow: /
User-agent: TECOMAC-Crawler/0.x
Disallow: /
User-agent: TeamSoft WinInet Component
Disallow: /
User-agent: Teleport Pro/1.2x(.1xxx)
Disallow: /
User-agent: Teoma MP
Disallow: /
User-agent: teomaagent crawler-admin@teoma.com
Disallow: /
User-agent: teomaagent1 [crawler-admin@teoma.com]
Disallow: /
User-agent: teoma_agent1
Disallow: /
User-agent: Teradex Mapper; mapper@teradex.com; http://www.teradex.com
Disallow: /
User-agent: Test spider
Disallow: /
User-agent: The Expert HTML Source Viewer (http://www.expert-html.com)
Disallow: /
User-agent: TheSuBot/0.1 (www.thesubot.de)
Disallow: /
User-agent: tivraSpider/1.0 (crawler@tivra.com)
Disallow: /
User-agent: TJG/Spider
Disallow: /
User-agent: TJvMultiHttpGrabber Component
Disallow: /
User-agent: Tkensaku/x.x(http://www.tkensaku.com/q.html)
Disallow: /
User-agent: toCrawl/UrlDispatcher
Disallow: /
User-agent: Toutatis x.x-x
Disallow: /
User-agent: Toutatis x.x (hoppa.com)
Disallow: /
User-agent: Toutatis x-xx.x (hoppa.com)
Disallow: /
User-agent: Trampelpfad-Spider
Disallow: /
User-agent: Trampelpfad-Spider-v0.1
Disallow: /
User-agent: tricosMetaCheck 1.2216-08-1999 (http://www.tricos.com/metacheck)
Disallow: /
User-agent: TSurf15a 11
Disallow: /
User-agent: TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17
Disallow: /
User-agent: TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8
Disallow: /
User-agent: TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)
Disallow: /
User-agent: Turnpike Emporium LinkChecker/0.1
Disallow: /
User-agent: TutorGig/1.5 (+http://www.tutorgig.com/crawler)
Disallow: /
User-agent: Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)
Disallow: /
User-agent: Twotrees Reactive Filter V2.0
Disallow: /
User-agent: TygoBot
Disallow: /
User-agent: TygoProwler
Disallow: /
# U
User-agent: UnChaos (From Chaos To Order, Hybrid Web Search Engine. (vadim_gonchar@unchaos.com))
Disallow: /
User-agent: UnChaos Bot (Hybrid Web Search Engine.(vadim_gonchar@unchaos.com))
Disallow: /
User-agent: UnChaosBot (From Chaos To Order, UnChaos Hybrid Web Search Engine at www.unchaos.com (info@unchaos.com))
Disallow: /
User-agent: UCSD Crawl
Disallow: /
User-agent: UdmSearch
Disallow: /
User-agent: unnamed
Disallow: /
User-agent: UptimeBot
Disallow: /
User-agent: URL Check
Disallow: /
User-agent: URL Control
Disallow: /
User-agent: URL Spider Pro
Disallow: /
User-agent: URL_Spider_Pro
Disallow: /
User-agent: URLy Warning
Disallow: /
User-agent: UCmore
Disallow: /
User-agent: UCMore Crawler App
Disallow: /
User-agent: UDM
Disallow: /
User-agent: UdmSearch/3.1.x
Disallow: /
User-agent: UIowaCrawler/1.0
Disallow: /
User-agent: Ultraseek
Disallow: /
User-agent: Under the Rainbow 2.2
Disallow: /
User-agent: UP.Browser/3.01-IG01 UP.Link/3.2.3.4
Disallow: /
User-agent: updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om)
Disallow: /
User-agent: UPG1 UP/4.0 (compatible; Blazer 1.0)
Disallow: /
User-agent: Uptimebot
Disallow: /
User-agent: UptimeBot(www.uptimebot.com)
Disallow: /
User-agent: URLBlaze
Disallow: /
User-agent: URLGetFile
Disallow: /
User-agent: URL Spider Pro/x.xx (innerprise.net)
Disallow: /
User-agent: URL_Spider_Pro/x.x
Disallow: /
User-agent: URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)
Disallow: /
User-agent: user-agent=Mozilla/3.01Gold
Disallow: /
User-agent: User-Agent: FileHeap! file downloader (http://www.fileheap.com)
Disallow: /
User-agent: User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)
Disallow: /
User-agent: USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)
Disallow: /
User-agent: UtilMind HTTPGet
Disallow: /
User-agent: Utopia WebWasher 3.0
Disallow: /
# V
User-agent: Valkyrie
Disallow: /
User-agent: vayala
Disallow: /
User-agent: VCI
Disallow: /
User-agent: VCI WebViewer VCI WebViewer Win32
Disallow: /
User-agent: Verticrawl
Disallow: /
User-agent: Victoria
Disallow: /
User-agent: vision-search
Disallow: /
User-agent: Visual Web Task
Disallow: /
User-agent: void-bot
Disallow: /
User-agent: VoidEYE
Disallow: /
User-agent: Voyager
Disallow: /
User-agent: VWbot
Disallow: /
User-agent: Vagabondo/1.x MT (webagent@wise-guys.nl)
Disallow: /
User-agent: Vagabondo/2.0 MT
Disallow: /
User-agent: Vagabondo/2.0 MT (webagent at wise-guys dot nl)
Disallow: /
User-agent: Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)
Disallow: /
User-agent: Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)
Disallow: /
User-agent: Vayala|Creep-v0.0.1 (codepoet@wildties.com)
Disallow: /
User-agent: VayalaCreep-v0.0.1 (haploid@haploid.com)
Disallow: /
User-agent: vb wininet
Disallow: /
User-agent: versus 0.2 (+http://versus.integis.ch)
Disallow: /
User-agent: VeryGoodSearch.com.DaddyLongLegs
Disallow: /
User-agent: verzamelgids.nl - Networking4all Bot/x.x
Disallow: /
User-agent: vspider
Disallow: /
User-agent: vspider/3.x
Disallow: /
# W
User-agent: w@pSpider by wap4.com
Disallow: /
User-agent: W3M2
Disallow: /
User-agent: w3mir
Disallow: /
User-agent: Walhello appie
Disallow: /
User-agent: WallPaper (alias crawlpaper)
Disallow: /
User-agent: WEB
Disallow: /
User-agent: Web Core / Roots
Disallow: /
User-agent: Web Downloader
Disallow: /
User-agent: Web Image
Disallow: /
User-agent: Web Page Finder
Disallow: /
User-agent: Web Sucker
Disallow: /
User-agent: Web2Map
Disallow: /
User-agent: WebAuto
Disallow: /
User-agent: WebBandit
Disallow: /
User-agent: WebBandit Web Spider
Disallow: /
User-agent: WebBandit/3.50
Disallow: /
User-agent: webcapture
Disallow: /
User-agent: WebCatcher
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebCopy
Disallow: /
User-agent: Webdupe
Disallow: /
User-agent: WebEMailExtrac.*
Disallow: /
User-agent: WebEnhancer
Disallow: /
User-agent: webfetcher
Disallow: /
User-agent: WebFountain
Disallow: /
User-agent: WebHook
Disallow: /
User-agent: Webinator
Disallow: /
User-agent: weblayers
Disallow: /
User-agent: WebLinker
Disallow: /
User-agent: Weblog Monitor
Disallow: /
User-agent: WebmasterWorldForumBot
Disallow: /
User-agent: WebMiner
Disallow: /
User-agent: WebMirror
Disallow: /
User-agent: webpictureboss
Disallow: /
User-agent: WebPictures
Disallow: /
User-agent: WebQuest
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: webs
Disallow: /
User-agent: WebSauger
Disallow: /
User-agent: Website Quester
Disallow: /
User-agent: WebSlinky
Disallow: /
User-agent: Websnarf
Disallow: /
User-agent: WebSpider
Disallow: /
User-agent: Webster
Disallow: /
User-agent: Webster Pro
Disallow: /
User-agent: WebStolperer
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebVac
Disallow: /
User-agent: WebVCR
Disallow: /
User-agent: webwalk
Disallow: /
User-agent: WebWalker
Disallow: /
User-agent: WebWatch
Disallow: /
User-agent: WebWhacker
Disallow: /
User-agent: WebZinger
Disallow: /
User-agent: WebZip
Disallow: /
User-agent: WebZip/4.0
Disallow: /
User-agent: WEP
Disallow: /
User-agent: Wget
Disallow: /
User-agent: Wget/1.5.3
Disallow: /
User-agent: Wget/1.6
Disallow: /
User-agent: whatUseek Winona
Disallow: /
User-agent: Whizg
Disallow: /
User-agent: WhoWhere Robot
Disallow: /
User-agent: Widow
Disallow: /
User-agent: Wild Ferret Web Hopper
Disallow: /
User-agent: Wired Digital
Disallow: /
User-agent: Wisenutbot
Disallow: /
User-agent: worm
Disallow: /
User-agent: WWWC Ver 0.2.5
Disallow: /
User-agent: WWW-Collector-E
Disallow: /
User-agent: [Ww]eb[Bb]andit
Disallow: /
User-agent: W3C-checklink/3.x.x.x libwww-perl/5.xx
Disallow: /
User-agent: W3C-checklink/4.x [4.xx] libwww-perl/5.xxx
Disallow: /
User-agent: W3C_Validator/1.xxx libwww-perl/5.xx
Disallow: /
User-agent: W3CLineMode/5.4.0 libwww/5.x.x
Disallow: /
User-agent: W3CRobot/5.4.0 libwww/5.4.0
Disallow: /
User-agent: w3m/0.x.xx
Disallow: /