#Crawlers that we thank kindly for playing, but we are not interested so please go away
User-agent: BaiDuSpider
User-agent: beijingcrawler
User-agent: ClariaBot
User-agent: e-SocietyRobot
User-agent: FANGCrawl
User-agent: nicebot
User-agent: SBIder
User-agent: T8Abot
User-agent: Xenu
User-agent: Zealbot
User-agent: ZipppBot
User-agent: ZyBORG
User-agent: Bitacle bot
User-agent: brilliant shopper bot
User-agent: BySpider
User-agent: Dulance Bot
User-agent: FAST-WebCrawler
User-agent: GAIS Robot
User-agent: Girafabot
User-agent: GomezAgent
User-agent: HenryTheMiragoRobot
User-agent: infomine
User-agent: IRLbot
User-agent: MojeekBot
User-agent: Naverbot
User-agent: noxtrumbot
User-agent: nusearch
User-agent: Searchmee! Spider
User-agent: Sensis Web Crawler
User-agent: SeznamBot
User-agent: Snapbot
User-agent: VMBot
User-agent: VoilaBot
User-agent: WeRelateBot
User-agent: Missigua Locator 1.9
Disallow: /

# Allow all
User-agent: *
User-agent: Mediapartners-Google*
User-agent: Googlebot
Allow: /
Disallow: /*-link_off.xhtml
Disallow: /*main=ajax
Disallow: /*pfdbk
Disallow: /*ajax_price_alert
Disallow: /*ajaxreview
Disallow: /*ajax_lists
Disallow: /*ccn_send_page
Disallow: /*?r=

User-agent: adidxbot
Crawl-Delay: 1