#
#  Production robots.txt file http://teq.queensland.com
#  CORPORATE
#  
# Major Search Engines and Known Friendly Spiders 
User-agent: Googlebot
User-agent: googlebot-image
User-agent: googlebot-mobile
User-agent: MSNBot
User-agent: psbot
User-agent: Slurp
User-agent: yahoo-mmcrawler
User-agent: yahoo-blogs/v3.9
User-agent: baiduspider
User-agent: Baiduspider-image
User-agent: Yandex
User-agent: Teoma
User-agent: Twiceler
User-agent: Gigabot
User-agent: Scrubby
User-agent: Robozilla
Disallow: /App_Browsers/
Disallow: /App_Config/
Disallow: /App_Data/
Disallow: /bin/
Disallow: /ClearScript.V8/
Disallow: /Content/
Disallow: /data/
Disallow: /DeployItems/
Disallow: /layouts/
Disallow: /Properties/
Disallow: /Scripts/
Disallow: /SearchIndexes/
Disallow: /sitecore/ #this one stops indexing of all global content and content in other site tree
Disallow: /sitecore%20modules/
Disallow: /sitecore_files/
Disallow: /upload/
Disallow: /Utils/
Disallow: /Views/
Disallow: /xsl/

# Known Unfriendly Spiders 

# MegaIndex crawler
User-agent: MegaIndex.ru
Disallow: /
User-agent: MegaIndex.com
Disallow: /


#all others.  same same, only crawl-delay
User-agent: *
Disallow: /App_Browsers/
Disallow: /App_Config/
Disallow: /App_Data/
Disallow: /bin/
Disallow: /ClearScript.V8/
Disallow: /Content/
Disallow: /data/
Disallow: /DeployItems/
Disallow: /layouts/
Disallow: /Properties/
Disallow: /Scripts/
Disallow: /SearchIndexes/
Disallow: /sitecore/ #this one stops indexing of all global content and content in other site tree
Disallow: /sitecore%20modules/
Disallow: /sitecore_files/
Disallow: /upload/
Disallow: /Utils/
Disallow: /Views/
Disallow: /xsl/
Crawl-delay: 5

#sitemap entry
#Sitemap: http://teq.queensland.com/site-map
Host: teq.queensland.com