# https://www.robotstxt.org/ # generated by www.seoptimer.com User-agent: * Crawl-delay: 3 Disallow: /.well-known/ Disallow: /.wp-cli/ Disallow: /cgi-bin/ Disallow: /wp-admin/ Disallow: /wp-content/plugins/ Disallow: /wp-content/themes/ Disallow: /wp-content/upgrade/ Disallow: /wp-includes/ Disallow: /wp-json/ Disallow: /author/ Disallow: /page/ Disallow: /tag/*/page/ Disallow: /category/*/page/ Disallow: /*? Disallow: /xmlrpc.php Disallow: /wp-login.php Sitemap: /sitemap.xml # http://www.semrush.com/bot.html User-agent: SiteAuditBot Crawl-delay: 30 # https://opensiteexplorer.org/dotbot User-agent: dotbot Crawl-delay: 30 # https://moz.com/help/moz-procedures/crawlers/rogerbot User-agent: rogerbot Crawl-delay: 30 # https://webmaster.petalsearch.com/site/petalbot User-agent: PetalBot Crawl-delay: 30 # https://developer.amazon.com/support/amazonbot User-agent: Amazonbot Crawl-delay: 30 # http://webmeup-crawler.com/ User-agent: BLEXBot Crawl-delay: 30 # https://serpstatbot.com/ User-agent: serpstatbot Crawl-delay: 30 # http://yandex.com/bots User-agent: YandexBot Crawl-delay: 30 # http://ahrefs.com/robot/ User-agent: AhrefsBot Crawl-delay: 30 # https://www.comscore.com/Web-Crawler User-agent: proximic Crawl-delay: 30 # http://help.coccoc.com/searchengine User-agent: coccocbot-web Crawl-delay: 30 # https://app.hypefactors.com/media-monitoring/about.html User-agent: Buck Crawl-delay: 30 # https://www.zoominfo.com/about-zoominfo/zoominfobot User-agent: ZoominfoBot Crawl-delay: 30 # http://suite.seozoom.it/bot.html User-agent: ZoomBot Crawl-delay: 30 # https://babbar.tech/crawler User-agent: Barkrowler Crawl-delay: 30 # https://www.mojeek.com/bot.html User-agent: MojeekBot Crawl-delay: 30 # https://velen.io User-agent: VelenPublicWebCrawler Crawl-delay: 30 # https://valassis.com User-agent: MaxPointCrawler Crawl-delay: 30 # https://well-known.dev/about/#bot User-agent: WellKnownBot Crawl-delay: 30 # https://www.seokicks.de/robot.html User-agent: SEOkicks Crawl-delay: 30 # https://webpros.com User-agent: webprosbot Crawl-delay: 30 # https://commoncrawl.org/faq/ User-agent: CCBot Crawl-delay: 30