# robots.txt file for anon.com ### # Email for more site information contact mailto:anon@anon.com ### # Email for more site information contact mailto:net-abuse@nocs.insp.irs.gov ### # # For information on the Robots Exclusion Protocol, please visit: ### # http://info.webcrawler.com/mak/projects/robots/norobots.html ### # # /robots.txt for info.webcrawler.com ### # /robots.txt for www.google.com ### # /robots.txt for sportsillustrated.cnn.com ### # /robots.txt for www.robotstxt.org ### # Allow All Spiders - by using only the next two lines ### # User-agent: * # Disallow: User-agent: Wget/1.6 Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Crescent Disallow: / User-agent: CherryPicker Disallow: / User-agent: [Ww]eb[Bb]andit Disallow: / User-agent: WebEMailExtrac.* Disallow: / User-agent: NICErsPRO Disallow: / User-agent: Telesoft Disallow: / User-agent: Zeus.*Webster Disallow: / User-agent: Microsoft.URL Disallow: / User-agent: EmailCollector Disallow: / #User-Agent: InternetSeer.com #Disallow: #User-Agent: InfoSeek Sidewinder/0.9 #Disallow: #User-Agent: Microsoft URL Control #Disallow: #User-Agent: Firefly/1.0 #Disallow: #User-Agent: Googlebot/2.0 #Disallow: #User-Agent: Googlebot/2.1 #Disallow: #User-Agent: Scooter-ARS-1.1 #Disallow: #User-Agent: Scooter/3.2 #Disallow: #User-Agent: Scooter/3.3 #Disallow: #User-Agent: ia_archiver #Disallow: #User-Agent: www.webwombat.com.au #Disallow: #User-Agent: ZyBorg/1.0 #Disallow: #User-Agent: Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com) #Disallow: #User-Agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma) #Disallow: #User-Agent: Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org) #Disallow: #User-Agent: 63.148.99.229 #Disallow: #User-Agent: Microsoft URL Control - 6.00.8169 #Disallow: User-Agent: Googlebot Disallow: /*.ico$ Disallow: /*.tif$ Disallow: /*.pict$ Disallow: /*.png$ Disallow: /*.gif$ Disallow: /*.jpg$ Disallow: /*.doc$ Disallow: /*.xls$ Disallow: /*.pps$ Disallow: /*.ppt$ Disallow: /*.eml$ Disallow: /*.url$ Disallow: /*.log$ Disallow: /*.txt$ Disallow: /*.pdf$ Disallow: /*.js$ Disallow: /*.pac$ Disallow: /*.css$ Disallow: /*.csv$ Disallow: /*.class$ Disallow: /*.c$ Disallow: /*.wav$ Disallow: /*.avi$ Disallow: /*.dat$ Disallow: /*.mov$ Disallow: /*.mpeg$ Disallow: /*.mpg$ Disallow: /*.swf$ Disallow: /*.bin$ Disallow: /*.hqx$ Disallow: /*.sea$ Disallow: /*.sit$ Disallow: /*.exe$ Disallow: /*.zip$ Disallow: /*.tar$ Disallow: /*.gz$ Disallow: /*.gzip$ User-agent: * Disallow: /email_addresses/ Disallow: /mailto_addresses/ Disallow: /htaccess/ Disallow: /cgi-bin/ Disallow: /zgi-bin/ Disallow: /cgi-sys/ Disallow: /zgi-sys/ Disallow: /cgi-win/ Disallow: /zgi-win/ Disallow: /test/robots/all/ Disallow: /test/robots/noindex/ Disallow: /test/robots/indexfollow/ Disallow: /test/robots/disal/ Disallow: /test/robots/partial Disallow: /info/meetings/ Disallow: /info/articles/ Disallow: /st/ Disallow: /ST/ Disallow: /St/ Disallow: /asp/ Disallow: /bin/ Disallow: /cgi/ Disallow: /class/ Disallow: /clock/ Disallow: /count/ Disallow: /css/ Disallow: /data/ Disallow: /dev/ Disallow: /docs/ Disallow: /download/ Disallow: /ecom/ Disallow: /etc/ Disallow: /ezicart/ Disallow: /icons/ Disallow: /icons/small/ Disallow: /images/ Disallow: /img/ Disallow: /incoming/ Disallow: /java/ Disallow: /javascript/ Disallow: /js/ Disallow: /lib/ Disallow: /logs/ Disallow: /passwd/ Disallow: /password/ Disallow: /public_html/ Disallow: /pub/ Disallow: /redir/ Disallow: /search/ Disallow: /secure/ Disallow: /stats/ Disallow: /usr/ Disallow: /.search/ Disallow: /_private/ Disallow: /_vti_bin/ Disallow: /_vti_cnf/ Disallow: /_vti_inf/ Disallow: /_vti_log/ Disallow: /_vti_pvt/ Disallow: /_vti_txt/