{"lang":"en","data":{"results":[{"token":"GPTBot","operator":"OpenAI","purpose":"training","honors_robots_txt":"disputed","vendor_doc_url":"https://developers.openai.com/api/docs/bots","ip_ranges_url":"https://openai.com/gptbot.json","first_documented":"2023-08","status":"active","successor":null,"verified_at":"2026-11-01","notes":"OpenAI states GPTBot honors robots.txt; Cloudflare's August 2025 crawler report and an Atlantic investigation documented GPTBot fetching pages that disallowed it, so the record is disputed.","dataset_version":"1.0.1"},{"token":"OAI-SearchBot","operator":"OpenAI","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://developers.openai.com/api/docs/bots","ip_ranges_url":"https://openai.com/searchbot.json","first_documented":"2024-07","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"ChatGPT-User","operator":"OpenAI","purpose":"user-triggered","honors_robots_txt":"no","vendor_doc_url":"https://developers.openai.com/api/docs/bots","ip_ranges_url":"https://openai.com/chatgpt-user.json","first_documented":"2024","status":"active","successor":null,"verified_at":"2026-11-01","notes":"OpenAI documents this as a user-initiated fetcher, not automatic crawling, and states that robots.txt rules may not apply to it; search opt-outs belong to OAI-SearchBot.","dataset_version":"1.0.1"},{"token":"anthropic-ai","operator":"Anthropic","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ip_ranges_url":null,"first_documented":"2022","status":"renamed","successor":"ClaudeBot","verified_at":"2026-11-01","notes":"Anthropic's original crawler token, replaced by ClaudeBot in its current documentation.","source_note":"Token appears in Anthropic's earlier crawler documentation and in robots.txt files from that period; the current Anthropic crawler page no longer lists it.","dataset_version":"1.0.1"},{"token":"ClaudeBot","operator":"Anthropic","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ip_ranges_url":"https://claude.com/crawling/bots.json","first_documented":"2023-07","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Claude-Web","operator":"Anthropic","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ip_ranges_url":null,"first_documented":"2023","status":"retired","successor":null,"verified_at":"2026-11-01","notes":"Early user-triggered agent for Claude web access, superseded by Claude-User in Anthropic's current documentation.","source_note":"Token appears in Anthropic's earlier crawler documentation and in robots.txt files from that period; the current Anthropic crawler page no longer lists it.","dataset_version":"1.0.1"},{"token":"Claude-User","operator":"Anthropic","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ip_ranges_url":"https://claude.com/crawling/bots.json","first_documented":"2024","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Claude-SearchBot","operator":"Anthropic","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ip_ranges_url":"https://claude.com/crawling/bots.json","first_documented":"2025","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Googlebot","operator":"Google","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"1998","status":"active","successor":null,"verified_at":"2026-11-01","notes":"General search crawler, not an AI agent; blocking it removes the site from Google Search results.","dataset_version":"1.0.1"},{"token":"Googlebot-Image","operator":"Google","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2001","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Googlebot-Video","operator":"Google","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2007","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Googlebot-News","operator":"Google","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2002","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Google-Extended","operator":"Google","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers","ip_ranges_url":"https://www.gstatic.com/ipranges/special-crawlers.json","first_documented":"2023-09","status":"active","successor":null,"verified_at":"2026-11-01","notes":"A robots.txt product token, not a user agent; it controls Gemini training use, and Google said in August 2025 that opted-out content can still appear in Search AI features.","dataset_version":"1.0.1"},{"token":"GoogleOther","operator":"Google","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2023-05","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Crawls for one-off product research and development crawls outside the main Googlebot fleet.","dataset_version":"1.0.1"},{"token":"GoogleOther-Image","operator":"Google","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2023-05","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"GoogleOther-Video","operator":"Google","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2023-05","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Storebot-Google","operator":"Google","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ip_ranges_url":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","first_documented":"2020","status":"active","successor":null,"verified_at":"2026-11-01","dataset_version":"1.0.1"},{"token":"Google-CloudVertexBot","operator":"Google","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers","ip_ranges_url":"https://www.gstatic.com/ipranges/special-crawlers.json","first_documented":"2024","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Fetches pages for Vertex AI customers who ground agents on their own sites; the site owner controls it in robots.txt.","dataset_version":"1.0.1"},{"token":"Google-InspectionTool","operator":"Google","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","ip_ranges_url":"https://www.gstatic.com/ipranges/zero-trust.json","first_documented":"2018","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Fetches a page when a site owner runs the URL Inspection tool in Search Console.","dataset_version":"1.0.1"},{"token":"Google-Read-Aloud","operator":"Google","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","ip_ranges_url":"https://www.gstatic.com/ipranges/zero-trust.json","first_documented":"2020","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Fetches a page to read it aloud when a user asks Google Assistant or Chrome to do so.","dataset_version":"1.0.1"},{"token":"Google-Agent","operator":"Google","purpose":"user-triggered","honors_robots_txt":"no","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","ip_ranges_url":null,"first_documented":"2026","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Agents on Google infrastructure navigating the web on user request; Google's user-triggered fetchers generally ignore robots.txt.","dataset_version":"1.0.1"},{"token":"Google-GeminiNotebook","operator":"Google","purpose":"user-triggered","honors_robots_txt":"no","vendor_doc_url":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","ip_ranges_url":null,"first_documented":"2026","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Fetches individual URLs Gemini Notebook users provide as project sources; the former Google-NotebookLM fetcher was retired in August 2026.","dataset_version":"1.0.1"},{"token":"Applebot","operator":"Apple","purpose":"search-and-training","honors_robots_txt":"yes","vendor_doc_url":"https://support.apple.com/en-us/119829","ip_ranges_url":"https://search.developer.apple.com/applebot.json","first_documented":"2015","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Apple states Applebot data can also be used to train its foundation models; Applebot-Extended is the opt-out token for that training use.","dataset_version":"1.0.1"},{"token":"Applebot-Extended","operator":"Apple","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://support.apple.com/en-us/119829","ip_ranges_url":null,"first_documented":"2024","status":"active","successor":null,"verified_at":"2026-11-01","notes":"A robots.txt token, not a user agent; it opts Applebot-crawled content out of training Apple's foundation models.","dataset_version":"1.0.1"},{"token":"PerplexityBot","operator":"Perplexity","purpose":"search-index","honors_robots_txt":"disputed","vendor_doc_url":"https://docs.perplexity.ai/guides/bots","ip_ranges_url":"https://www.perplexity.com/perplexitybot.json","first_documented":"2023","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Perplexity states the bot honors robots.txt; 2024-2025 reporting by Wired and Cloudflare alleged undisclosed crawlers fetching blocked content, which Perplexity disputes.","dataset_version":"1.0.1"},{"token":"Perplexity-User","operator":"Perplexity","purpose":"user-triggered","honors_robots_txt":"no","vendor_doc_url":"https://docs.perplexity.ai/guides/bots","ip_ranges_url":"https://www.perplexity.com/perplexity-user.json","first_documented":"2025","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Perplexity documents this as a user-directed fetcher that robots.txt does not govern, the same category as a browser visit; that scope statement is separate from the PerplexityBot dispute.","dataset_version":"1.0.1"},{"token":"Meta-ExternalAgent","operator":"Meta","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","ip_ranges_url":null,"first_documented":"2024","status":"active","successor":null,"verified_at":"2026-10-23","dataset_version":"1.0.1"},{"token":"Meta-ExternalFetcher","operator":"Meta","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","ip_ranges_url":null,"first_documented":"2024","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Fetches a specific URL when a Meta AI user shares or asks about it.","dataset_version":"1.0.1"},{"token":"facebookexternalhit","operator":"Meta","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","ip_ranges_url":null,"first_documented":"2010","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Legacy link-preview fetcher that renders shared links on Facebook; it is not an AI training crawler.","dataset_version":"1.0.1"},{"token":"Bytespider","operator":"ByteDance","purpose":"training","honors_robots_txt":"disputed","vendor_doc_url":"https://radar.cloudflare.com/bots","ip_ranges_url":null,"first_documented":"2019","status":"active","successor":null,"verified_at":"2026-10-23","notes":"ByteDance publishes little crawler documentation; whether Bytespider honors robots.txt is disputed, with hosting providers and publishers widely reporting that it ignores the file.","source_note":"No operator documentation found as of 2026-10-23 (ByteDance publishes no crawler page; its /en/crawler URL returns a not-found page). Record rests on Cloudflare Radar's bot directory and independent crawler tracking.","dataset_version":"1.0.1"},{"token":"CCBot","operator":"Common Crawl","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://commoncrawl.org/faq","ip_ranges_url":"https://index.commoncrawl.org/ccbot.json","first_documented":"2011","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Collects the public Common Crawl corpus that many open training datasets are built from.","dataset_version":"1.0.1"},{"token":"bingbot","operator":"Microsoft","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","ip_ranges_url":null,"first_documented":"2010-10","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Bing's search crawler; its index also feeds Copilot answers, so blocking it affects AI answer visibility as well as search.","dataset_version":"1.0.1"},{"token":"MSNBot","operator":"Microsoft","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","ip_ranges_url":null,"first_documented":"2005","status":"renamed","successor":"bingbot","verified_at":"2026-10-23","notes":"Bing's original crawler under MSN Search, retired in favor of bingbot in 2010; old robots.txt files still name it.","dataset_version":"1.0.1"},{"token":"AdIdxBot","operator":"Microsoft","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","ip_ranges_url":null,"first_documented":"2012","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Checks landing pages for Bing Ads quality review.","dataset_version":"1.0.1"},{"token":"Amazonbot","operator":"Amazon","purpose":"search-and-training","honors_robots_txt":"yes","vendor_doc_url":"https://developer.amazon.com/amazonbot","ip_ranges_url":null,"first_documented":"2019","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Amazon's web crawler; content feeds Alexa answers and other Amazon services.","dataset_version":"1.0.1"},{"token":"Amzn-SearchBot","operator":"Amazon","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://developer.amazon.com/amazonbot","ip_ranges_url":"https://developer.amazon.com/amazonbot/searchbot-ip-addresses/","first_documented":"2026","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Improves search experiences in Amazon products; honors robots.txt, falling back to other search bots' directives when unnamed; not used for AI training.","dataset_version":"1.0.1"},{"token":"Amzn-User","operator":"Amazon","purpose":"user-triggered","honors_robots_txt":"no","vendor_doc_url":"https://developer.amazon.com/amazonbot","ip_ranges_url":"https://developer.amazon.com/amazonbot/live-ip-addresses/","first_documented":"2026","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Supports user actions such as Alexa queries needing up-to-date information; may not follow all robots.txt directives; not used for AI training.","dataset_version":"1.0.1"},{"token":"DuckAssistBot","operator":"DuckDuckGo","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/","ip_ranges_url":null,"first_documented":"2023","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Fetches pages when DuckDuckGo's AI chat or search answers need live content.","dataset_version":"1.0.1"},{"token":"AI2Bot","operator":"Allen Institute for AI","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://allenai.org/crawler","ip_ranges_url":null,"first_documented":"2024","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Allen Institute crawler collecting data for open AI research; the operator publishes an opt-out and contact page.","dataset_version":"1.0.1"},{"token":"SemanticScholarBot","operator":"Allen Institute for AI","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://www.semanticscholar.org/","ip_ranges_url":null,"first_documented":"2016","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Fetches scholarly papers and metadata for the Semantic Scholar research index.","dataset_version":"1.0.1"},{"token":"ImagesiftBot","operator":"Hive","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://imagesift.com/","ip_ranges_url":null,"first_documented":"2023","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Collects images for Hive's dataset and moderation products; the operator publishes an opt-out.","dataset_version":"1.0.1"},{"token":"Diffbot","operator":"Diffbot","purpose":"other","honors_robots_txt":"yes","vendor_doc_url":"https://www.diffbot.com/","ip_ranges_url":null,"first_documented":"2011","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Extracts structured data from pages for customers; its corpus work is why it appears on AI-era blocklists.","dataset_version":"1.0.1"},{"token":"Omgilibot","operator":"Webz.io","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://webz.io/","ip_ranges_url":null,"first_documented":"2015","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Webz.io's crawler (formerly Webhose); harvested web data is resold, including for AI training.","dataset_version":"1.0.1"},{"token":"VelenPublicWebCrawler","operator":"Velen","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://velen.io/","ip_ranges_url":null,"first_documented":"2023","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Velen collects public web data for AI training datasets and publishes an opt-out contact.","dataset_version":"1.0.1"},{"token":"FriendlyCrawler","operator":"FriendlyCrawler project","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://radar.cloudflare.com/bots","ip_ranges_url":null,"first_documented":"2023","status":"active","successor":null,"verified_at":"2026-10-23","notes":"An open crawler built for collecting training data politely; it advertises strict robots.txt compliance.","source_note":"Open-source project documentation plus Cloudflare Radar bot listings; no commercial operator documentation exists.","dataset_version":"1.0.1"},{"token":"Timpibot","operator":"Timpi","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://www.timpi.com/","ip_ranges_url":null,"first_documented":"2022","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Crawler for Timpi's decentralized search index.","dataset_version":"1.0.1"},{"token":"MistralAI-User","operator":"Mistral AI","purpose":"user-triggered","honors_robots_txt":"yes","vendor_doc_url":"https://docs.mistral.ai/","ip_ranges_url":null,"first_documented":"2025","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Fetches pages inside Le Chat when a user asks Mistral's assistant to read or use a source.","dataset_version":"1.0.1"},{"token":"xAI-Bot","operator":"xAI","purpose":"search-and-training","honors_robots_txt":"yes","vendor_doc_url":"https://docs.x.ai/","ip_ranges_url":null,"first_documented":"2024","status":"active","successor":null,"verified_at":"2026-10-23","notes":"xAI's crawler, associated with Grok training and answers; operator documentation is thin compared with peers.","source_note":"xAI publishes no dedicated crawler page; record rests on Cloudflare Radar bot reporting and xAI's published crawler references.","dataset_version":"1.0.1"},{"token":"YouBot","operator":"You.com","purpose":"search-index","honors_robots_txt":"yes","vendor_doc_url":"https://you.com/docs/youbot","ip_ranges_url":null,"first_documented":"2022","status":"active","successor":null,"verified_at":"2026-11-01","notes":"Builds the You.com search index used by its AI answer engine.","dataset_version":"1.0.1"},{"token":"cohere-ai","operator":"Cohere","purpose":"training","honors_robots_txt":"yes","vendor_doc_url":"https://radar.cloudflare.com/bots","ip_ranges_url":null,"first_documented":"2023","status":"active","successor":null,"verified_at":"2026-10-23","notes":"Cohere's crawler token, named in robots.txt blocklists since 2023; Cohere publishes no dedicated crawler page.","source_note":"No operator crawler documentation found; record rests on Cloudflare Radar bot reporting and long-standing publisher blocklists naming the token.","dataset_version":"1.0.1"}],"total_count":55,"has_more":true,"next_offset":50,"dataset_version":"1.0.1","dataset_updated":"2026-11-01"},"meta":{"version":"v2026.12.0","corpus_version":"v2026.12.0","total":55,"total_count":55,"has_more":true,"next_offset":50}}