# For all robots User-agent: * # Block access to specific groups of pages Disallow: /cart Disallow: /checkout Disallow: /my-account Request-rate: 1/10 # maximum rate is one page every 10 seconds Crawl-delay: 10 # 10 seconds between page requests Visit-time: 0400-0845 # only visit between 04:00 and 08:45 UTC # Allow search crawlers to discover the sitemap Sitemap: /sitemap.xml # Block CazoodleBot as it does not present correct accept content headers User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: / # Block targeted promo pages User-agent: * Disallow: /b/Keurig-K475-Coffee-Maker-Starter-Bundle Disallow: /janship20 Disallow: /beverages/perfect-cup/c/ # 2015-10-15 added by Remi for SEO purposes on k.com Disallow: /write-power-review Disallow: /_ui Disallow: /powerreviews Disallow: relevance Sitemap: /sitemap_xdomain_kcom.xmlSitemap: /images_sitemap_kcom.xml # end added by Remi for SEO purposes on k.com #01-08-2016 changes to block PR and OrderConfirmation calls # Block AhrefsBot power-review call User-agent: AhrefsBot Disallow: /write-power-review # Block Googlebot power-review call User-agent: Googlebot Disallow: /write-power-review # Block bingbot power-review call User-agent: bingbot Disallow: /write-power-review # Block YandexBot power-review call User-agent: YandexBot Disallow: /write-power-review # Block Wotbox power-review call User-agent: Wotbox Disallow: /write-power-review # Block AdsBot-Google OrderConfirmation call User-agent: AdsBot-Google Disallow: /checkout/orderConfirmation