{ "tool": "check-sources/1", "run": "2026-10-22T13:44:34.604Z", "input": "sites/crawlguide/data/crawlers.json", "summary": { "sources": 166, "passed": 166, "failed": 0, "pages": 31, "pages_failed": 0 }, "pages": [ { "url": "https://support.apple.com/en-us/119829", "fetched": "https://support.apple.com/en-us/119829", "status": 200, "final_url": "https://support.apple.com/en-us/119829" }, { "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "fetched": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "status": 200, "final_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler" }, { "url": "https://developer.amazon.com/en/amazonbot", "fetched": "https://developer.amazon.com/en/amazonbot", "status": 200, "final_url": "https://developer.amazon.com/en/amazonbot" }, { "url": "https://commoncrawl.org/ccbot", "fetched": "https://commoncrawl.org/ccbot", "status": 200, "final_url": "https://commoncrawl.org/ccbot" }, { "url": "https://commoncrawl.org/faq", "fetched": "https://commoncrawl.org/faq", "status": 200, "final_url": "https://commoncrawl.org/faq" }, { "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "fetched": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "status": 200, "final_url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996" }, { "url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "fetched": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "status": 200, "final_url": "https://www.diffbot.com/docs/crawl/faq/robots-txt" }, { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "fetched": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "status": 200, "final_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot" }, { "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "fetched": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "status": 200, "final_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot" }, { "url": "https://metacpan.org/pod/WWW::RobotRules", "fetched": "https://metacpan.org/pod/WWW::RobotRules", "status": 200, "final_url": "https://metacpan.org/pod/WWW::RobotRules" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "fetched": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers?hl=en", "status": 200, "final_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers?hl=en" }, { "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "fetched": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "status": 200, "final_url": "https://ziyuan.baidu.com/college/articleinfo?id=1516" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "fetched": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers?hl=en", "status": 200, "final_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers?hl=en" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/apis-user-agent", "fetched": "https://developers.google.com/crawling/docs/crawlers-fetchers/apis-user-agent?hl=en", "status": 200, "final_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/apis-user-agent?hl=en" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent", "fetched": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent?hl=en", "status": 200, "final_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent?hl=en" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "fetched": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers?hl=en", "status": 200, "final_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers?hl=en" }, { "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/feedfetcher", "fetched": "https://developers.google.com/crawling/docs/crawlers-fetchers/feedfetcher?hl=en", "status": 200, "final_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/feedfetcher?hl=en" }, { "url": "https://developers.google.com/search/docs/appearance/ai-features", "fetched": "https://developers.google.com/search/docs/appearance/ai-features?hl=en", "status": 200, "final_url": "https://developers.google.com/search/docs/appearance/ai-features?hl=en" }, { "url": "https://developers.google.com/search/docs/crawling-indexing/googlebot", "fetched": "https://developers.google.com/search/docs/crawling-indexing/googlebot?hl=en", "status": 200, "final_url": "https://developers.google.com/search/docs/crawling-indexing/googlebot?hl=en" }, { "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "fetched": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0", "status": 200, "final_url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=8c184ec0" }, { "url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec", "fetched": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=cb7c31ec", "status": 200, "final_url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=cb7c31ec" }, { "url": "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a", "fetched": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=30fba23a", "status": 200, "final_url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=30fba23a" }, { "url": "https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26", "fetched": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=3905dc26", "status": 200, "final_url": "https://www.bing.com/webmasters/api/help/htmlcontent?ArticleId=3905dc26" }, { "url": "https://developers.openai.com/api/docs/bots", "fetched": "https://developers.openai.com/api/docs/bots", "status": 200, "final_url": "https://developers.openai.com/api/docs/bots" }, { "url": "https://docs.mistral.ai/robots", "fetched": "https://docs.mistral.ai/robots", "status": 200, "final_url": "https://docs.mistral.ai/robots" }, { "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "fetched": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "status": 200, "final_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers" }, { "url": "https://search.brave.com/help/brave-search-crawler", "fetched": "https://search.brave.com/help/brave-search-crawler", "status": 200, "final_url": "https://search.brave.com/help/brave-search-crawler", "retried": true }, { "url": "https://yandex.com/support/webmaster/en/yandex-ai", "fetched": "https://yandex.com/support/webmaster/en/yandex-ai", "status": 200, "final_url": "https://yandex.com/support/webmaster/en/yandex-ai" }, { "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "fetched": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "status": 200, "final_url": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers" }, { "url": "https://yandex.com/support/webmaster/en/robot-workings/user-agent", "fetched": "https://yandex.com/support/webmaster/en/robot-workings/user-agent", "status": 200, "final_url": "https://yandex.com/support/webmaster/en/robot-workings/user-agent" }, { "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "fetched": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "status": 200, "final_url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots" } ], "results": [ { "label": "amazonbot", "url": "https://developer.amazon.com/en/amazonbot", "quote": "Amazonbot is used to improve our products and services. This helps us provide more accurate information to customers and may be used to train Amazon AI models.", "status": 200, "pass": true }, { "label": "amazonbot", "url": "https://developer.amazon.com/en/amazonbot", "quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol, honoring the user-agent and the allow/disallow directives.", "status": 200, "pass": true }, { "label": "amzn-searchbot", "url": "https://developer.amazon.com/en/amazonbot", "quote": "If robots.txt files don’t mention Amzn-SearchBot but allow other search bots, Amzn-SearchBot will crawl in accordance with the robots.txt directives given to other search bots.", "status": 200, "pass": true }, { "label": "amzn-searchbot", "url": "https://developer.amazon.com/en/amazonbot", "quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol, honoring the user-agent and the allow/disallow directives.", "status": 200, "pass": true }, { "label": "amzn-user", "url": "https://developer.amazon.com/en/amazonbot", "quote": "Because actions taken by Amzn-User can be initiated by a user, it may not follow all robots.txt directives.", "status": 200, "pass": true }, { "label": "claude-searchbot", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "Claude-SearchBot navigates the web to improve search result quality for users.", "status": 200, "pass": true }, { "label": "claude-searchbot", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt.", "status": 200, "pass": true }, { "label": "claude-searchbot", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic.", "status": 200, "pass": true }, { "label": "claude-user", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "When individuals ask questions to Claude, it may access websites using a Claude-User agent.", "status": 200, "pass": true }, { "label": "claude-user", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt.", "status": 200, "pass": true }, { "label": "claude-user", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic.", "status": 200, "pass": true }, { "label": "claudebot", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "ClaudeBot helps enhance the utility and safety of our generative AI models by collecting web content that could potentially contribute to their training.", "status": 200, "pass": true }, { "label": "claudebot", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt.", "status": 200, "pass": true }, { "label": "claudebot", "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic.", "status": 200, "pass": true }, { "label": "applebot", "url": "https://support.apple.com/en-us/119829", "quote": "Applebot respects standard robots.txt directives in general search crawls that are targeted at Applebot.", "status": 200, "pass": true }, { "label": "applebot", "url": "https://support.apple.com/en-us/119829", "quote": "If robots instructions don't mention Applebot but mention Googlebot, the Apple robot will follow Googlebot instructions.", "status": 200, "pass": true }, { "label": "applebot", "url": "https://support.apple.com/en-us/119829", "quote": "Traffic coming from Applebot is generally identified by using reverse DNS in the *.applebot.apple.com domain.", "status": 200, "pass": true }, { "label": "applebot", "url": "https://support.apple.com/en-us/119829", "quote": "In this example, Applebot doesn't try to crawl documents that are under /private/ or /not-allowed/", "status": 200, "pass": true }, { "label": "applebot-extended", "url": "https://support.apple.com/en-us/119829", "quote": "Applebot-Extended does not crawl webpages. Webpages that disallow Applebot-Extended can still be included in search results.", "status": 200, "pass": true }, { "label": "applebot-extended", "url": "https://support.apple.com/en-us/119829", "quote": "Applebot-Extended is only used to determine how to use the data crawled by the Applebot user agent.", "status": 200, "pass": true }, { "label": "applebot-extended", "url": "https://support.apple.com/en-us/119829", "quote": "Web publishers can opt out of their content being used in these broad world knowledge answers by applying the nosnippet meta tag to specific content.", "status": 200, "pass": true }, { "label": "itms", "url": "https://support.apple.com/en-us/119829", "quote": "The iTMS user agent does not follow robots.txt, as it is not a general search crawler. The iTMS user agent only crawls URLs associated with registered content on Apple Podcasts.", "status": 200, "pass": true }, { "label": "baiduspider", "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "quote": "Baiduspider是百度搜索引擎的一个自动程序，它的作用是访问互联网上的网页，建立索引数据库", "status": 200, "pass": true }, { "label": "baiduspider", "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "quote": "Baiduspider遵守互联网robots协议。", "status": 200, "pass": true }, { "label": "baiduspider", "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "quote": "Baiduspider的hostname以 *.baidu.com 或 *.baidu.jp 的格式命名，非 *.baidu.com 或 *.baidu.jp 即为冒充。", "status": 200, "pass": true }, { "label": "baiduspider", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "robot会根据第一个匹配成功的Allow或Disallow行确定是否访问某个URL", "status": 200, "pass": true }, { "label": "baiduspider", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "\"Disallow:\"说明允许robot访问该网站的所有url", "status": 200, "pass": true }, { "label": "baiduspider", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "文件包含一条或更多的记录，这些记录通过空行分开", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "quote": "Baiduspider遵守互联网robots协议。", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "quote": "如果您想完全禁止百度所有的产品收录，可以直接对Baiduspider设置禁止抓取。", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://help.baidu.com/question?prod_id=99&class=476&id=2996", "quote": "以下robots实现禁止所有来自百度的抓取但允许图片搜索抓取/image/目录", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "robot会根据第一个匹配成功的Allow或Disallow行确定是否访问某个URL", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "\"Disallow:\"说明允许robot访问该网站的所有url", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "文件包含一条或更多的记录，这些记录通过空行分开", "status": 200, "pass": true }, { "label": "baiduspider-image", "url": "https://ziyuan.baidu.com/college/articleinfo?id=1516", "quote": "禁止Baiduspider抓取网站上所有图片", "status": 200, "pass": true }, { "label": "brave-search", "url": "https://search.brave.com/help/brave-search-crawler", "quote": "The Brave Search crawler does not advertise a differentiated user agent because we must avoid discrimination from websites that allow only Google to crawl them.", "status": 200, "pass": true }, { "label": "brave-search", "url": "https://search.brave.com/help/brave-search-crawler", "quote": "if a domain or page is not crawlable by Googlebot, then Brave Search’s bot will not crawl it either.", "status": 200, "pass": true }, { "label": "ccbot", "url": "https://commoncrawl.org/ccbot", "quote": "To prevent Common Crawl from crawling your website, include the following in your robots.txt:", "status": 200, "pass": true }, { "label": "ccbot", "url": "https://commoncrawl.org/faq", "quote": "CCBot is an automated crawler, checking first the robots.txt, and if crawling a page is allowed, fetches pages using HTTP GET requests.", "status": 200, "pass": true }, { "label": "ccbot", "url": "https://commoncrawl.org/ccbot", "quote": "These IP ranges (v4 and v6) are also provided as JSON at https://index.commoncrawl.org/ccbot.json.", "status": 200, "pass": true }, { "label": "diffbot", "url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "quote": "By default Diffbot's web crawls adhere to a site’s robots.txt instructions, including the disallow and crawl-delay directives.", "status": 200, "pass": true }, { "label": "diffbot", "url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "quote": "General, proactive web crawling for building a general search engine.", "status": 200, "pass": true }, { "label": "diffbot-user", "url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "quote": "This is used by requests originating on behalf of a human user browsing a URL using Diffbot software, in response to their input.", "status": 200, "pass": true }, { "label": "diffbot-user", "url": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "quote": "By default Diffbot's web crawls adhere to a site’s robots.txt instructions", "status": 200, "pass": true }, { "label": "duckassistbot", "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "quote": "DuckAssistBot is a web crawler for DuckDuckGo Search that crawls pages in real-time for our AI-assisted answers, which prominently cite their sources.", "status": 200, "pass": true }, { "label": "duckassistbot", "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "quote": "Once you’ve updated the file to disallow the DuckAssistBot user agent, the change will take effect after 72 hours and DuckAssistBot will stop crawling your site.", "status": 200, "pass": true }, { "label": "duckduckbot", "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot", "quote": "The crawler's user agent will appear as DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html). It respects WWW::RobotRules", "status": 200, "pass": true }, { "label": "duckduckbot", "url": "https://metacpan.org/pod/WWW::RobotRules", "quote": "any URL that starts with this value will not be retrieved", "status": 200, "pass": true }, { "label": "adsbot-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "For example, AdsBot ignores the global robots.txt user agent (*) with the ad publisher's permission.", "status": 200, "pass": true }, { "label": "adsbot-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "Crawling preferences addressed to the AdsBot-Google user agent affect Google Ads' ability to check web page ad quality.", "status": 200, "pass": true }, { "label": "adsbot-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The global user agent (*) is ignored.", "status": 200, "pass": true }, { "label": "adsbot-google-mobile", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "status": 200, "pass": true }, { "label": "adsbot-google-mobile", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "Crawling preferences addressed to the AdsBot-Google-Mobile user agent affect Google Ads' ability to check web page ad quality.", "status": 200, "pass": true }, { "label": "adsbot-google-mobile", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The global user agent (*) is ignored.", "status": 200, "pass": true }, { "label": "apis-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "status": 200, "pass": true }, { "label": "apis-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "Crawling preferences addressed to the APIs-Google user agent affect the delivery of push notification messages by Google APIs.", "status": 200, "pass": true }, { "label": "apis-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The global user agent (*) is ignored.", "status": 200, "pass": true }, { "label": "apis-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/apis-user-agent", "quote": "a reverse DNS lookup shows the googlebot.com or google.com domain.", "status": 200, "pass": true }, { "label": "feedfetcher-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "feedfetcher-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Feedfetcher is used for crawling RSS or Atom feeds for Google News and WebSub.", "status": 200, "pass": true }, { "label": "feedfetcher-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/feedfetcher", "quote": "Since Feedfetcher requests come from explicit action by human users, and not from automated crawlers, Feedfetcher ignores robots.txt rules.", "status": 200, "pass": true }, { "label": "google-agent", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "google-agent", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Google-Agent is used by agents hosted on Google infrastructure to navigate the web and perform actions upon user request.", "status": 200, "pass": true }, { "label": "google-agent", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "It uses IP ranges from user-triggered-agents.json.", "status": 200, "pass": true }, { "label": "google-cloudvertexbot", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "google-cloudvertexbot", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "affect crawls requested by the site owners' for building Vertex AI Agents. It has no effect on Google Search or other products.", "status": 200, "pass": true }, { "label": "google-cws", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "google-cws", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "The Chrome Web Store fetcher requests URLs that developers provide in the metadata of their Chrome extensions and themes.", "status": 200, "pass": true }, { "label": "google-extended", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Google-Extended doesn't have a separate HTTP request user agent string. Crawling is done with existing Google user agent strings; the robots.txt user-agent token is used in a control capacity.", "status": 200, "pass": true }, { "label": "google-extended", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Google-Extended does not impact a site's inclusion in Google Search nor is it used as a ranking signal in Google Search.", "status": 200, "pass": true }, { "label": "google-gemininotebook", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "google-gemininotebook", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "The Gemini Notebook fetcher requests individual URLs that Gemini Notebook users have provided as sources for their projects.", "status": 200, "pass": true }, { "label": "google-inspectiontool", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "google-inspectiontool", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "affect Search testing tools such as the Rich Result Test and URL inspection in Search Console. It has no effect on Google Search or other products.", "status": 200, "pass": true }, { "label": "google-pinpoint", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "google-pinpoint", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "The Google-Pinpoint fetcher requests individual URLs that Pinpoint users specified as sources for their personal collections of documents.", "status": 200, "pass": true }, { "label": "google-read-aloud", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "google-read-aloud", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Upon user request, Google Read Aloud fetches and reads out web pages using text-to-speech (TTS).", "status": 200, "pass": true }, { "label": "google-read-aloud", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent", "quote": "Since Google Read Aloud is initiated by a user, not a result of automated web crawl, you can't opt out by using a robots.txt file.", "status": 200, "pass": true }, { "label": "google-safety", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The Google-Safety user agent ignores robots.txt rules.", "status": 200, "pass": true }, { "label": "google-safety", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The Google-Safety user agent handles abuse-specific crawling, such as malware discovery for publicly posted links on Google properties.", "status": 200, "pass": true }, { "label": "google-site-verification", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "google-site-verification", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Google Site Verifier fetches Search Console verification tokens.", "status": 200, "pass": true }, { "label": "googlebot", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "They always obey robots.txt rules when crawling automatically.", "status": 200, "pass": true }, { "label": "googlebot", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "The common crawlers generally crawl from the IP ranges published in the common-crawlers.json object, and the reverse DNS mask of their hostname matches crawl-***-***-***-***.googlebot.com", "status": 200, "pass": true }, { "label": "googlebot", "url": "https://developers.google.com/search/docs/crawling-indexing/googlebot", "quote": "both crawler types obey the same product token (user agent token) in robots.txt, and so you cannot selectively target either Googlebot Smartphone or Googlebot Desktop using robots.txt.", "status": 200, "pass": true }, { "label": "googlebot", "url": "https://developers.google.com/search/docs/appearance/ai-features", "quote": "AI is built into Search and integral to how Search functions, which is why robots.txt directives for Googlebot is the control for site owners to manage access to how their sites are crawled for Search.", "status": 200, "pass": true }, { "label": "googlebot-image", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "They always obey robots.txt rules when crawling automatically.", "status": 200, "pass": true }, { "label": "googlebot-image", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "googlebot-image", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Crawling preferences addressed to the Googlebot-Image user agent affect Google Images, Discover, Video mode, and all features in Google Search where images, logos, and favicons are presented.", "status": 200, "pass": true }, { "label": "googlebot-news", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "googlebot-news", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Googlebot-news doesn't have a separate HTTP request user agent string. Crawling is done with various Googlebot user agent strings.", "status": 200, "pass": true }, { "label": "googlebot-video", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "They always obey robots.txt rules when crawling automatically.", "status": 200, "pass": true }, { "label": "googlebot-video", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "googlebot-video", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Crawling preferences addressed to the Googlebot-Video user agent affect video-related Google Search features and other products dependent on videos.", "status": 200, "pass": true }, { "label": "googlemessages", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "googlemessages", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Google Messages fetcher is used to generate link previews for URLs sent in chat messages.", "status": 200, "pass": true }, { "label": "googleother", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "They always obey robots.txt rules when crawling automatically.", "status": 200, "pass": true }, { "label": "googleother", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "GoogleOther is the generic crawler that may be used by various product teams for fetching publicly accessible content from sites.", "status": 200, "pass": true }, { "label": "googleother-image", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "googleother-image", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "GoogleOther-Image is the version of GoogleOther optimized for fetching publicly accessible image URLs.", "status": 200, "pass": true }, { "label": "googleother-video", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Some crawlers have more than one user agent token; you need to match only one crawler token for a rule to apply.", "status": 200, "pass": true }, { "label": "googleother-video", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "GoogleOther-Video is the version of GoogleOther optimized for fetching publicly accessible video URLs.", "status": 200, "pass": true }, { "label": "googleproducer", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules.", "status": 200, "pass": true }, { "label": "googleproducer", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "quote": "Google Publisher Center fetches and processes feeds that publishers explicitly supplied for use in Google News landing pages.", "status": 200, "pass": true }, { "label": "mediapartners-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The special-case crawlers may ignore robots.txt rules and so they operate from a different IP range than the common crawlers.", "status": 200, "pass": true }, { "label": "mediapartners-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The Mediapartners-Google crawler visits participating sites in order to provide them with relevant ads.", "status": 200, "pass": true }, { "label": "mediapartners-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers", "quote": "The global user agent (*) is ignored.", "status": 200, "pass": true }, { "label": "storebot-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "They always obey robots.txt rules when crawling automatically.", "status": 200, "pass": true }, { "label": "storebot-google", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "quote": "Crawling preferences addressed to the Storebot-Google user agent affect all surfaces of Google Shopping (for example, the Shopping tab in Google Search and Google Shopping).", "status": 200, "pass": true }, { "label": "facebookexternalhit", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "The primary purpose of FacebookExternalHit is to crawl the content of an app or website that was shared on one of Meta’s family of apps, such as Facebook, Instagram, or Messenger.", "status": 200, "pass": true }, { "label": "facebookexternalhit", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "Note that the FacebookExternalHit crawler might bypass robots.txt when performing security or integrity checks, such as checking for malware or malicious content.", "status": 200, "pass": true }, { "label": "meta-externalads", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "The Meta-ExternalAds crawler crawls the web for use cases such as improving advertising and other business-related products and services.", "status": 200, "pass": true }, { "label": "meta-externalads", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.", "status": 200, "pass": true }, { "label": "meta-externalagent", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "The Meta-ExternalAgent crawler crawls the web for use cases such as training foundation AI models or improving products by indexing content directly.", "status": 200, "pass": true }, { "label": "meta-externalagent", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.", "status": 200, "pass": true }, { "label": "meta-externalfetcher", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "The Meta-ExternalFetcher crawler fetches individual links at a user's request and supports product functions such as evaluating and improving agentic AI capabilities", "status": 200, "pass": true }, { "label": "meta-externalfetcher", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "The Meta-ExternalFetcher crawler may bypass robots.txt because it performs fetches that were requested by the user.", "status": 200, "pass": true }, { "label": "meta-webindexer", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users.", "status": 200, "pass": true }, { "label": "meta-webindexer", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "quote": "In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.", "status": 200, "pass": true }, { "label": "adidxbot", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "AdIdxBot is the crawler used by Bing Ads. AdIdxBot crawls ads and follows the websites from those ads for quality control.", "status": 200, "pass": true }, { "label": "adidxbot", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "status": 200, "pass": true }, { "label": "adidxbot", "url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec", "quote": "start the first section of directives applicable to all bots by adding this line", "status": 200, "pass": true }, { "label": "bingbot", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "Bingbot is our standard crawler and handles most of our crawling needs each day.", "status": 200, "pass": true }, { "label": "bingbot", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "status": 200, "pass": true }, { "label": "bingbot", "url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec", "quote": "On finding a specific set of instructions for itself, Bingbot will ignore the directives listed in the generic section", "status": 200, "pass": true }, { "label": "bingbot", "url": "https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26", "quote": "Perform a reverse DNS lookup using the IP address from the logs to verify that it resolves to a name that end with search.msn.com", "status": 200, "pass": true }, { "label": "bingbot", "url": "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a", "quote": "Bing and Copilot search experiences rely on the same core crawling, indexing, and ranking foundation as traditional search.", "status": 200, "pass": true }, { "label": "bingpreview", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "BingPreview generates page snapshots for Bing.", "status": 200, "pass": true }, { "label": "bingpreview", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "status": 200, "pass": true }, { "label": "bingpreview", "url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec", "quote": "start the first section of directives applicable to all bots by adding this line", "status": 200, "pass": true }, { "label": "bingvideopreview", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "BingVideoPreview is a crawler used by Microsoft to provide previews of videos only in Bing.", "status": 200, "pass": true }, { "label": "bingvideopreview", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "status": 200, "pass": true }, { "label": "bingvideopreview", "url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec", "quote": "start the first section of directives applicable to all bots by adding this line", "status": 200, "pass": true }, { "label": "microsoftpreview", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "MicrosoftPreview generates page snapshots for Microsoft products.", "status": 200, "pass": true }, { "label": "microsoftpreview", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "quote": "Robots.txt files can be configured to tell Bing crawlers how to interact with your website.", "status": 200, "pass": true }, { "label": "microsoftpreview", "url": "https://www.bing.com/webmasters/help/how-to-create-a-robots-txt-file-cb7c31ec", "quote": "start the first section of directives applicable to all bots by adding this line", "status": 200, "pass": true }, { "label": "mistralai-index", "url": "https://docs.mistral.ai/robots", "quote": "To help webmasters manage how their sites and content interact with AI, Mistral uses specific robots.txt tags.", "status": 200, "pass": true }, { "label": "mistralai-index", "url": "https://docs.mistral.ai/robots", "quote": "MistralAI-Index is for automated crawling of the web for indexing purposes only.", "status": 200, "pass": true }, { "label": "mistralai-training", "url": "https://docs.mistral.ai/robots", "quote": "MistralAI-Training crawls web content to help build datasets for training Mistral generative AI models. Webmasters can disallow this user agent in their robots.txt file.", "status": 200, "pass": true }, { "label": "mistralai-user", "url": "https://docs.mistral.ai/robots", "quote": "To help webmasters manage how their sites and content interact with AI, Mistral uses specific robots.txt tags.", "status": 200, "pass": true }, { "label": "mistralai-user", "url": "https://docs.mistral.ai/robots", "quote": "MistralAI-User governs which sites these user requests can be made to.", "status": 200, "pass": true }, { "label": "chatgpt-user", "url": "https://developers.openai.com/api/docs/bots", "quote": "Because these actions are initiated by a user, robots.txt rules may not apply.", "status": 200, "pass": true }, { "label": "chatgpt-user", "url": "https://developers.openai.com/api/docs/bots", "quote": "ChatGPT-User is not used for crawling the web in an automatic fashion.", "status": 200, "pass": true }, { "label": "chatgpt-user", "url": "https://developers.openai.com/api/docs/bots", "quote": "OpenAI uses OAI-SearchBot and GPTBot robots.txt tags to enable webmasters to manage how their sites and content work with AI.", "status": 200, "pass": true }, { "label": "gptbot", "url": "https://developers.openai.com/api/docs/bots", "quote": "OpenAI uses OAI-SearchBot and GPTBot robots.txt tags to enable webmasters to manage how their sites and content work with AI. Each setting is independent of the others", "status": 200, "pass": true }, { "label": "gptbot", "url": "https://developers.openai.com/api/docs/bots", "quote": "Disallowing GPTBot indicates a site’s content should not be used in training generative AI foundation models.", "status": 200, "pass": true }, { "label": "oai-adsbot", "url": "https://developers.openai.com/api/docs/bots", "quote": "OAI-AdsBot is used to validate the safety of web pages submitted as ads on ChatGPT.", "status": 200, "pass": true }, { "label": "oai-adsbot", "url": "https://developers.openai.com/api/docs/bots", "quote": "OAI-AdsBot only visits pages submitted as ads, and the data collected by OAI-AdsBot is not used to train generative AI foundation models.", "status": 200, "pass": true }, { "label": "oai-searchbot", "url": "https://developers.openai.com/api/docs/bots", "quote": "OpenAI uses OAI-SearchBot and GPTBot robots.txt tags to enable webmasters to manage how their sites and content work with AI. Each setting is independent of the others", "status": 200, "pass": true }, { "label": "oai-searchbot", "url": "https://developers.openai.com/api/docs/bots", "quote": "Sites that are opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though can still appear as navigational links.", "status": 200, "pass": true }, { "label": "perplexity-user", "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "quote": "When users ask Perplexity a question, it might visit a web page to help provide an accurate answer and include a link to the page in its response.", "status": 200, "pass": true }, { "label": "perplexity-user", "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "quote": "Since a user requested the fetch, this fetcher generally ignores robots.txt rules.", "status": 200, "pass": true }, { "label": "perplexitybot", "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "quote": "PerplexityBot is designed to surface and link websites in search results on Perplexity. It is not used to crawl content for AI foundation models.", "status": 200, "pass": true }, { "label": "perplexitybot", "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "quote": "Webmasters can use the following robots.txt tags to manage how their sites and content interact with Perplexity.", "status": 200, "pass": true }, { "label": "yandexadditionalbot", "url": "https://yandex.com/support/webmaster/en/yandex-ai", "quote": "If you don't want information from your site to be used in Yandex AI answers, add a disallow directive for \"YandexAdditional\" or \"YandexAdditionalBot\"", "status": 200, "pass": true }, { "label": "yandexadditionalbot", "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "quote": "Applies to the pages that have been indexed by the primary crawler. Doesn't make indexing requests.", "status": 200, "pass": true }, { "label": "yandexbot", "url": "https://yandex.com/support/webmaster/en/robot-workings/user-agent", "quote": "User-agent: Yandex # will be used by all Yandex bots", "status": 200, "pass": true }, { "label": "yandexbot", "url": "https://yandex.com/support/webmaster/en/robot-workings/user-agent", "quote": "If the `User-agent: Yandex` string is detected, the `User-agent: *` string is ignored.", "status": 200, "pass": true }, { "label": "yandexbot", "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "quote": "All Yandex robots have names ending in `yandex.ru`, `yandex.net` or `yandex.com`.", "status": 200, "pass": true }, { "label": "yandeximages", "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "quote": "Indexes images to display them in [Yandex Images](http://images.yandex.com).", "status": 200, "pass": true }, { "label": "yandeximages", "url": "https://yandex.com/support/webmaster/en/robot-workings/user-agent", "quote": "User-agent: Yandex # will be used by all Yandex bots", "status": 200, "pass": true }, { "label": "yandeximages", "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "quote": "All Yandex robots have names ending in `yandex.ru`, `yandex.net` or `yandex.com`.", "status": 200, "pass": true }, { "label": "yandexvideo", "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "quote": "Indexes videos for displaying in [Yandex video search](https://video.yandex.com/).", "status": 200, "pass": true }, { "label": "yandexvideo", "url": "https://yandex.com/support/webmaster/en/robot-workings/user-agent", "quote": "User-agent: Yandex # will be used by all Yandex bots", "status": 200, "pass": true }, { "label": "yandexvideo", "url": "https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots", "quote": "All Yandex robots have names ending in `yandex.ru`, `yandex.net` or `yandex.com`.", "status": 200, "pass": true } ] }