{"lang":"en","data":{"dataset":"ai-crawler-registry","dataset_version":"1.0.1","dataset_updated":"2026-11-01","record_count":55,"operator_count":26,"operators":["Allen Institute for AI","Amazon","Anthropic","Apple","ByteDance","Cohere","Common Crawl","DeepSeek","Diffbot","DuckDuckGo","FriendlyCrawler project","Google","Hive","Internet Archive","Meta","Microsoft","Mistral AI","Moonshot AI","Neeva","OpenAI","Perplexity","Timpi","Velen","Webz.io","You.com","xAI"],"purposes":["training","search-index","user-triggered","search-and-training","other"],"by_purpose":{"training":16,"search-index":14,"user-triggered":13,"other":9,"search-and-training":3},"statuses":["active","retired","renamed"],"last_verified":"2026-11-01","citation":"AI Crawler Registry, movahedi.ca (dataset v1.0.1, updated 2026-11-01). https://movahedi.ca/developers/crawlers/"},"meta":{"version":"v2026.12.0","corpus_version":"v2026.12.0"}}