{
  "schema_version": "1.0",
  "title": "AI Crawler Directory",
  "canonical_url": "https://bozos.si/reference/ai-crawlers",
  "data_url": "https://bozos.si/ai-crawlers.json",
  "updated": "2026-10-20",
  "license": "CC0-1.0",
  "feedback": {
    "contact": "https://bozos.si/contact",
    "email": "mailto:bozos@bozos.si",
    "subject": "[BOZOS agent] correction: ai-crawler-directory-data - <summary> [via:dataset]",
    "corrections_log": "https://bozos.si/corrections.json"
  },
  "record_access": {
    "collections": [
      "crawlers",
      "undocumented_tokens"
    ],
    "permalink": "https://bozos.si/ai-crawlers.json#{id}",
    "resolve": "https://bozos.si/ai-crawlers.json?id={id}",
    "changes": "https://bozos.si/api/v1/changes?dataset=ai-crawler-directory-data&since={YYYY-MM-DD}",
    "review_policy": "Each record is re-checked against its sources every 90 days; review_due is the date that is next due. GET /api/v1/health lists overdue records.",
    "note": "Every record has a permanent id, a permalink, and a cite line: a sentence built only from the record's own fields, with its date and permalink, that can be quoted verbatim. Fetch the resolve URL to get one record; ids are never reused."
  },
  "scope_note": "Crawlers and user agents that AI and search companies document themselves. Every entry in crawlers is taken from the operator's own documentation, read on the verified date. Tokens people often ask about but whose operators publish no documentation are listed separately in undocumented_tokens, with what BOZOS searched for and found; they are not described as fact.",
  "editorial_rule": "Record what the operator states, not what observers report. Where the documentation is silent, say so (robots_txt: not-stated, used_for_ai_training: null) rather than infer.",
  "categories": {
    "training": "Collects content that may be used to train AI models.",
    "search": "Builds a search index that AI answers or search results draw on.",
    "user-fetch": "Fetches a page only when a person asks an assistant something. Often treated as outside robots.txt.",
    "control-token": "Not a crawler. A robots.txt token that controls how another crawler's data is used.",
    "open-dataset": "Builds a public archive that third parties, including AI developers, reuse.",
    "owner-requested": "Crawls only sites whose owners asked for it.",
    "link-preview": "Fetches metadata to render link previews.",
    "other": "Documented purpose that fits none of the above."
  },
  "robots_txt_values": {
    "obeys": "The operator states, or clearly implies, that robots.txt rules for this token are honored.",
    "may-ignore": "The operator states robots.txt may not apply, usually because a user initiated the fetch.",
    "control-token": "The token exists only in robots.txt; there is no separate crawler to block.",
    "not-stated": "The operator's documentation does not say."
  },
  "docs_vs_reports_values": {
    "none-recorded": "BOZOS has not recorded a credible public report that this crawler behaves differently from its documentation. Absence of a report is not verification.",
    "conflict-reported": "A named, primary source has published observations that contradict the operator's documentation. See reported_conflicts, which also records the operator's response."
  },
  "crawlers": [
    {
      "id": "gptbot",
      "permalink": "https://bozos.si/ai-crawlers.json#gptbot",
      "cite": "GPTBot (OpenAI): Crawls content that may be used to train OpenAI's generative AI foundation models. robots.txt: obeys. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#gptbot",
      "review_due": "2027-01-18",
      "token": "GPTBot",
      "operator": "OpenAI",
      "category": "training",
      "purpose": "Crawls content that may be used to train OpenAI's generative AI foundation models.",
      "robots_txt": "obeys",
      "robots_note": "Disallowing GPTBot signals the site's content should not be used for training.",
      "used_for_ai_training": true,
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_ranges": "https://openai.com/gptbot.json",
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "It is used to crawl content that may be used in training our generative AI foundation models.",
          "source": "https://developers.openai.com/api/docs/bots"
        },
        {
          "supports": "robots_txt",
          "quote": "Disallowing GPTBot indicates a site's content should not be used in training generative AI foundation models.",
          "source": "https://developers.openai.com/api/docs/bots"
        }
      ]
    },
    {
      "id": "oai-searchbot",
      "permalink": "https://bozos.si/ai-crawlers.json#oai-searchbot",
      "cite": "OAI-SearchBot (OpenAI): Surfaces sites in ChatGPT search features. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#oai-searchbot",
      "review_due": "2027-01-18",
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "category": "search",
      "purpose": "Surfaces sites in ChatGPT search features.",
      "robots_txt": "obeys",
      "robots_note": "Opted-out sites are not shown in ChatGPT search answers but can still appear as navigational links. Changes take about 24 hours.",
      "used_for_ai_training": null,
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_ranges": "https://openai.com/searchbot.json",
      "user_agent_example": "Mozilla/5.0 ... compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "OAI-SearchBot is used to surface websites in search results in ChatGPT's search features.",
          "source": "https://developers.openai.com/api/docs/bots"
        },
        {
          "supports": "robots_txt",
          "quote": "Sites that are opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though can still appear as navigational links.",
          "source": "https://developers.openai.com/api/docs/bots"
        }
      ]
    },
    {
      "id": "chatgpt-user",
      "permalink": "https://bozos.si/ai-crawlers.json#chatgpt-user",
      "cite": "ChatGPT-User (OpenAI): Visits a page when a ChatGPT or Custom GPT user asks a question, and for GPT Actions. Does not crawl automatically. robots.txt: may-ignore. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#chatgpt-user",
      "review_due": "2027-01-18",
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "category": "user-fetch",
      "purpose": "Visits a page when a ChatGPT or Custom GPT user asks a question, and for GPT Actions. Does not crawl automatically.",
      "robots_txt": "may-ignore",
      "robots_note": "OpenAI states robots.txt rules may not apply because the fetch is user-initiated.",
      "used_for_ai_training": null,
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_ranges": "https://openai.com/chatgpt-user.json",
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "When users ask ChatGPT or a CustomGPT a question, it may visit a web page with a ChatGPT-User agent.",
          "source": "https://developers.openai.com/api/docs/bots"
        },
        {
          "supports": "robots_txt",
          "quote": "Because these actions are initiated by a user, robots.txt rules may not apply.",
          "source": "https://developers.openai.com/api/docs/bots"
        }
      ]
    },
    {
      "id": "oai-adsbot",
      "permalink": "https://bozos.si/ai-crawlers.json#oai-adsbot",
      "cite": "OAI-AdsBot (OpenAI): Checks landing pages submitted as ChatGPT ads for safety and relevance. Visits only submitted ad pages. robots.txt: not-stated. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#oai-adsbot",
      "review_due": "2027-01-18",
      "token": "OAI-AdsBot",
      "operator": "OpenAI",
      "category": "other",
      "purpose": "Checks landing pages submitted as ChatGPT ads for safety and relevance. Visits only submitted ad pages.",
      "robots_txt": "not-stated",
      "robots_note": "OpenAI's page does not say whether robots.txt applies.",
      "used_for_ai_training": false,
      "documentation": "https://developers.openai.com/api/docs/bots",
      "ip_ranges": "https://openai.com/adsbot.json",
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "OAI-AdsBot is used to validate the safety of web pages submitted as ads on ChatGPT.",
          "source": "https://developers.openai.com/api/docs/bots"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "the data collected by OAI-AdsBot is not used to train generative AI foundation models.",
          "source": "https://developers.openai.com/api/docs/bots"
        }
      ]
    },
    {
      "id": "claudebot",
      "permalink": "https://bozos.si/ai-crawlers.json#claudebot",
      "cite": "ClaudeBot (Anthropic): Collects web content that could contribute to training Anthropic's models. robots.txt: obeys. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#claudebot",
      "review_due": "2027-01-18",
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "category": "training",
      "purpose": "Collects web content that could contribute to training Anthropic's models.",
      "robots_txt": "obeys",
      "robots_note": "Also supports the non-standard Crawl-delay extension. Anthropic asks site owners to set rules on every subdomain.",
      "used_for_ai_training": true,
      "documentation": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ip_ranges": "https://claude.com/crawling/bots.json",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "ClaudeBot helps enhance the utility and safety of our generative AI models by collecting web content that could potentially contribute to their training.",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        },
        {
          "supports": "robots_txt",
          "quote": "Anthropic's Bots respect \"do not crawl\" signals by honoring industry standard directives in robots.txt.",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "When a site restricts ClaudeBot access, it signals that the site's future materials should be excluded from our AI model training datasets.",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        }
      ]
    },
    {
      "id": "claude-searchbot",
      "permalink": "https://bozos.si/ai-crawlers.json#claude-searchbot",
      "cite": "Claude-SearchBot (Anthropic): Navigates the web to improve search result quality for Claude users. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#claude-searchbot",
      "review_due": "2027-01-18",
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "category": "search",
      "purpose": "Navigates the web to improve search result quality for Claude users.",
      "robots_txt": "obeys",
      "robots_note": "Covered by Anthropic's general statement that its bots honor robots.txt. Blocking may reduce visibility in Claude search results.",
      "used_for_ai_training": null,
      "documentation": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ip_ranges": "https://claude.com/crawling/bots.json",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "Claude-SearchBot navigates the web to improve search result quality for users.",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        },
        {
          "supports": "robots_txt",
          "quote": "Anthropic's Bots respect \"do not crawl\" signals by honoring industry standard directives in robots.txt.",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        }
      ]
    },
    {
      "id": "claude-user",
      "permalink": "https://bozos.si/ai-crawlers.json#claude-user",
      "cite": "Claude-User (Anthropic): Retrieves pages when Claude users ask questions. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#claude-user",
      "review_due": "2027-01-18",
      "token": "Claude-User",
      "operator": "Anthropic",
      "category": "user-fetch",
      "purpose": "Retrieves pages when Claude users ask questions.",
      "robots_txt": "obeys",
      "robots_note": "Covered by Anthropic's general statement that its bots honor robots.txt. Blocking prevents retrieval in response to user queries.",
      "used_for_ai_training": null,
      "documentation": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "ip_ranges": "https://claude.com/crawling/bots.json",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "When individuals ask questions to Claude, it may access websites using a Claude-User agent.",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        },
        {
          "supports": "robots_txt",
          "quote": "Disabling Claude-User on your site prevents our system from retrieving your content in response to a user query",
          "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
        }
      ]
    },
    {
      "id": "googlebot",
      "permalink": "https://bozos.si/ai-crawlers.json#googlebot",
      "cite": "Googlebot (Google): Crawls for Google Search, including Discover, Images, Video, and News surfaces. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#googlebot",
      "review_due": "2027-01-18",
      "token": "Googlebot",
      "operator": "Google",
      "category": "search",
      "purpose": "Crawls for Google Search, including Discover, Images, Video, and News surfaces.",
      "robots_txt": "obeys",
      "robots_note": "Standard robots.txt rules apply.",
      "used_for_ai_training": null,
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_ranges": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "Crawling preferences addressed to the Googlebot user agent affect Google Search (including Discover and all Google Search features)",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        },
        {
          "supports": "robots_txt",
          "quote": "They always obey robots.txt rules when crawling automatically.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        }
      ]
    },
    {
      "id": "google-extended",
      "permalink": "https://bozos.si/ai-crawlers.json#google-extended",
      "cite": "Google-Extended (Google): Not a crawler. Controls whether content Google crawls may be used to train future Gemini models and for grounding in Gemini Apps and the Vertex AI API. robots.txt: control-token. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#google-extended",
      "review_due": "2027-01-18",
      "token": "Google-Extended",
      "operator": "Google",
      "category": "control-token",
      "purpose": "Not a crawler. Controls whether content Google crawls may be used to train future Gemini models and for grounding in Gemini Apps and the Vertex AI API.",
      "robots_txt": "control-token",
      "robots_note": "Set rules for Google-Extended in robots.txt. It does not affect inclusion in Google Search.",
      "used_for_ai_training": true,
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_ranges": null,
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "category",
          "quote": "the robots.txt user-agent token is used in a control capacity.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "Google-Extended is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        },
        {
          "supports": "purpose",
          "quote": "Google-Extended does not impact a site's inclusion in Google Search nor is it used as a ranking signal in Google Search.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        }
      ]
    },
    {
      "id": "googleother",
      "permalink": "https://bozos.si/ai-crawlers.json#googleother",
      "cite": "GoogleOther (Google): Generic crawler used by Google product teams, for example for internal research. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#googleother",
      "review_due": "2027-01-18",
      "token": "GoogleOther",
      "operator": "Google",
      "category": "other",
      "purpose": "Generic crawler used by Google product teams, for example for internal research.",
      "robots_txt": "obeys",
      "robots_note": "Standard robots.txt rules apply. Affects no specific product.",
      "used_for_ai_training": null,
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_ranges": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "GoogleOther is the generic crawler that may be used by various product teams for fetching publicly accessible content from sites.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        },
        {
          "supports": "robots_txt",
          "quote": "They always obey robots.txt rules when crawling automatically.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        }
      ]
    },
    {
      "id": "google-cloudvertexbot",
      "permalink": "https://bozos.si/ai-crawlers.json#google-cloudvertexbot",
      "cite": "Google-CloudVertexBot (Google): Crawls sites at the site owner's request to build Vertex AI Agents. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#google-cloudvertexbot",
      "review_due": "2027-01-18",
      "token": "Google-CloudVertexBot",
      "operator": "Google",
      "category": "owner-requested",
      "purpose": "Crawls sites at the site owner's request to build Vertex AI Agents.",
      "robots_txt": "obeys",
      "robots_note": "No effect on Google Search.",
      "used_for_ai_training": null,
      "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "ip_ranges": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "Crawling preferences addressed to the Google-CloudVertexBot user agent affect crawls requested by the site owners' for building Vertex AI Agents.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        },
        {
          "supports": "robots_txt",
          "quote": "They always obey robots.txt rules when crawling automatically.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
        }
      ]
    },
    {
      "id": "bingbot",
      "permalink": "https://bozos.si/ai-crawlers.json#bingbot",
      "cite": "bingbot (Microsoft): Crawler for the Bing search index. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-19, medium confidence. https://bozos.si/ai-crawlers.json#bingbot",
      "review_due": "2027-01-17",
      "token": "bingbot",
      "operator": "Microsoft",
      "category": "search",
      "purpose": "Crawler for the Bing search index.",
      "robots_txt": "obeys",
      "robots_note": "Standard robots.txt rules apply. Microsoft's help page renders client-side; the IP list was verified directly.",
      "used_for_ai_training": null,
      "documentation": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "ip_ranges": "https://www.bing.com/toolbox/bingbot.json",
      "notes": "Medium confidence on details beyond the token and IP list: the documentation page could not be read as text.",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-19",
      "confidence": "medium",
      "evidence_status": "not-machine-verifiable: the documentation page renders in the browser, so its text cannot be fetched and checked automatically. Details rest on the token name and the verified IP list."
    },
    {
      "id": "applebot",
      "permalink": "https://bozos.si/ai-crawlers.json#applebot",
      "cite": "Applebot (Apple): Powers search in Spotlight, Siri, and Safari. Apple says crawled data may also be used to train its foundation models. robots.txt: obeys. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#applebot",
      "review_due": "2027-01-18",
      "token": "Applebot",
      "operator": "Apple",
      "category": "search",
      "purpose": "Powers search in Spotlight, Siri, and Safari. Apple says crawled data may also be used to train its foundation models.",
      "robots_txt": "obeys",
      "robots_note": "Apple says Applebot respects standard robots.txt directives in general search crawls targeted at Applebot, and falls back to Googlebot rules when no Applebot rules exist. Does not follow Crawl-delay.",
      "used_for_ai_training": true,
      "documentation": "https://support.apple.com/en-us/119829",
      "ip_ranges": "https://search.developer.apple.com/applebot.json",
      "user_agent_example": "... Applebot/0.1; +http://www.apple.com/go/applebot",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "The data crawled by Applebot is used to power various features, such as the search technology integrated into many user experiences in Apple's ecosystem including Spotlight, Siri, and Safari.",
          "source": "https://support.apple.com/en-us/119829"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "The data crawled by Applebot may also be used to help train Apple foundation models powering generative AI features across Apple products",
          "source": "https://support.apple.com/en-us/119829"
        },
        {
          "supports": "robots_txt",
          "quote": "Applebot respects standard robots.txt directives in general search crawls that are targeted at Applebot.",
          "source": "https://support.apple.com/en-us/119829"
        },
        {
          "supports": "robots_txt",
          "quote": "If robots instructions don't mention Applebot but mention Googlebot, the Apple robot will follow Googlebot instructions.",
          "source": "https://support.apple.com/en-us/119829"
        }
      ]
    },
    {
      "id": "applebot-extended",
      "permalink": "https://bozos.si/ai-crawlers.json#applebot-extended",
      "cite": "Applebot-Extended (Apple): Not a crawler. Controls whether data Applebot already crawled may be used to train Apple's general-purpose foundation models. robots.txt: control-token. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#applebot-extended",
      "review_due": "2027-01-18",
      "token": "Applebot-Extended",
      "operator": "Apple",
      "category": "control-token",
      "purpose": "Not a crawler. Controls whether data Applebot already crawled may be used to train Apple's general-purpose foundation models.",
      "robots_txt": "control-token",
      "robots_note": "Blocking it does not remove pages from Apple search results.",
      "used_for_ai_training": true,
      "documentation": "https://support.apple.com/en-us/119829",
      "ip_ranges": null,
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "used_for_ai_training",
          "quote": "Web publishers can opt-out from having their content used to train generative foundation models by disallowing Applebot-Extended in the robots.txt file.",
          "source": "https://support.apple.com/en-us/119829"
        }
      ]
    },
    {
      "id": "perplexitybot",
      "permalink": "https://bozos.si/ai-crawlers.json#perplexitybot",
      "cite": "PerplexityBot (Perplexity): Surfaces and links websites in Perplexity search results. Perplexity says it is not used to crawl content for AI foundation models. robots.txt: obeys. Used for AI training: no. Cloudflare reported behavior contradicting this documentation (2025-08-04); Perplexity disputes it. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#perplexitybot",
      "review_due": "2027-01-18",
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "category": "search",
      "purpose": "Surfaces and links websites in Perplexity search results. Perplexity says it is not used to crawl content for AI foundation models.",
      "robots_txt": "obeys",
      "robots_note": "Perplexity recommends allowing it in robots.txt; its page does not state compliance in so many words.",
      "used_for_ai_training": false,
      "documentation": "https://docs.perplexity.ai/guides/bots",
      "ip_ranges": "https://www.perplexity.com/perplexitybot.json",
      "docs_vs_reports": "conflict-reported",
      "reported_conflicts": [
        {
          "date": "2025-08-04",
          "reporter": "Cloudflare",
          "title": "Perplexity is using stealth, undeclared crawlers to evade website no-crawl directives",
          "url": "https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/",
          "finding": "Cloudflare reported that when Perplexity's declared crawler was blocked, it used \"a generic browser intended to impersonate Google Chrome on macOS\" from IPs outside its published list, \"ignoring — or sometimes failing to even fetch — robots.txt files\". Cloudflare \"de-listed them as a verified bot\".",
          "operator_response": {
            "date": "2025-08-04",
            "title": "Agents or Bots? Making Sense of AI on the Open Web",
            "url": "https://www.perplexity.ai/hub/blog/agents-or-bots-making-sense-of-ai-on-the-open-web",
            "quote": "Cloudflare fundamentally misattributed 3-6M daily requests from BrowserBase's automated browser service to Perplexity"
          },
          "read": "2026-10-20",
          "quotes": [
            "a generic browser intended to impersonate Google Chrome on macOS",
            "ignoring — or sometimes failing to even fetch — robots.txt files",
            "de-listed them as a verified bot"
          ]
        }
      ],
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "PerplexityBot is designed to surface and link websites in search results on Perplexity.",
          "source": "https://docs.perplexity.ai/guides/bots"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "It is not used to crawl content for AI foundation models.",
          "source": "https://docs.perplexity.ai/guides/bots"
        }
      ]
    },
    {
      "id": "perplexity-user",
      "permalink": "https://bozos.si/ai-crawlers.json#perplexity-user",
      "cite": "Perplexity-User (Perplexity): Visits a page when a Perplexity user asks a question, and links it in the answer. Not used for crawling or training. robots.txt: may-ignore. Used for AI training: no. Cloudflare reported behavior contradicting this documentation (2025-08-04); Perplexity disputes it. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#perplexity-user",
      "review_due": "2027-01-18",
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "category": "user-fetch",
      "purpose": "Visits a page when a Perplexity user asks a question, and links it in the answer. Not used for crawling or training.",
      "robots_txt": "may-ignore",
      "robots_note": "Perplexity's page says both that Perplexity-User \"controls which sites these user requests can access\" and that, since a user requested the fetch, the fetcher \"generally ignores robots.txt rules\".",
      "used_for_ai_training": false,
      "documentation": "https://docs.perplexity.ai/guides/bots",
      "ip_ranges": "https://www.perplexity.com/perplexity-user.json",
      "docs_vs_reports": "conflict-reported",
      "reported_conflicts": [
        {
          "date": "2025-08-04",
          "reporter": "Cloudflare",
          "title": "Perplexity is using stealth, undeclared crawlers to evade website no-crawl directives",
          "url": "https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/",
          "finding": "Cloudflare reported that when Perplexity's declared crawler was blocked, it used \"a generic browser intended to impersonate Google Chrome on macOS\" from IPs outside its published list, \"ignoring — or sometimes failing to even fetch — robots.txt files\". Cloudflare \"de-listed them as a verified bot\".",
          "operator_response": {
            "date": "2025-08-04",
            "title": "Agents or Bots? Making Sense of AI on the Open Web",
            "url": "https://www.perplexity.ai/hub/blog/agents-or-bots-making-sense-of-ai-on-the-open-web",
            "quote": "Cloudflare fundamentally misattributed 3-6M daily requests from BrowserBase's automated browser service to Perplexity"
          },
          "read": "2026-10-20",
          "quotes": [
            "a generic browser intended to impersonate Google Chrome on macOS",
            "ignoring — or sometimes failing to even fetch — robots.txt files",
            "de-listed them as a verified bot"
          ]
        }
      ],
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "When users ask Perplexity a question, it might visit a web page to help provide an accurate answer and include a link to the page in its response.",
          "source": "https://docs.perplexity.ai/guides/bots"
        },
        {
          "supports": "robots_txt",
          "quote": "Since a user requested the fetch, this fetcher generally ignores robots.txt rules.",
          "source": "https://docs.perplexity.ai/guides/bots"
        },
        {
          "supports": "robots_txt",
          "quote": "Perplexity-User controls which sites these user requests can access.",
          "source": "https://docs.perplexity.ai/guides/bots"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "It is not used for web crawling or to collect content for training AI foundation models.",
          "source": "https://docs.perplexity.ai/guides/bots"
        }
      ]
    },
    {
      "id": "meta-externalagent",
      "permalink": "https://bozos.si/ai-crawlers.json#meta-externalagent",
      "cite": "meta-externalagent (Meta): Crawls for uses such as training foundation AI models or improving products by indexing content directly. robots.txt: obeys. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-19, high confidence. https://bozos.si/ai-crawlers.json#meta-externalagent",
      "review_due": "2027-01-17",
      "token": "meta-externalagent",
      "operator": "Meta",
      "category": "training",
      "purpose": "Crawls for uses such as training foundation AI models or improving products by indexing content directly.",
      "robots_txt": "obeys",
      "robots_note": "Block with a disallow rule. Changes can take up to 24 hours.",
      "used_for_ai_training": true,
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ip_ranges": null,
      "user_agent_example": "meta-externalagent/1.1",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-19",
      "confidence": "high",
      "evidence_status": "not-machine-verifiable: the documentation page refuses automated requests (HTTP 400), so quotes cannot be checked automatically. Last read in a browser on the verified date."
    },
    {
      "id": "meta-webindexer",
      "permalink": "https://bozos.si/ai-crawlers.json#meta-webindexer",
      "cite": "meta-webindexer (Meta): Improves Meta AI search results so Meta AI can cite and link content. robots.txt: not-stated. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-19, high confidence. https://bozos.si/ai-crawlers.json#meta-webindexer",
      "review_due": "2027-01-17",
      "token": "meta-webindexer",
      "operator": "Meta",
      "category": "search",
      "purpose": "Improves Meta AI search results so Meta AI can cite and link content.",
      "robots_txt": "not-stated",
      "robots_note": "Meta's page does not describe its robots.txt behavior.",
      "used_for_ai_training": null,
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ip_ranges": null,
      "user_agent_example": "meta-webindexer/1.1",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-19",
      "confidence": "high",
      "evidence_status": "not-machine-verifiable: the documentation page refuses automated requests (HTTP 400), so quotes cannot be checked automatically. Last read in a browser on the verified date."
    },
    {
      "id": "meta-externalfetcher",
      "permalink": "https://bozos.si/ai-crawlers.json#meta-externalfetcher",
      "cite": "meta-externalfetcher (Meta): Fetches individual links at a user's request for agentic AI features. robots.txt: may-ignore. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-19, high confidence. https://bozos.si/ai-crawlers.json#meta-externalfetcher",
      "review_due": "2027-01-17",
      "token": "meta-externalfetcher",
      "operator": "Meta",
      "category": "user-fetch",
      "purpose": "Fetches individual links at a user's request for agentic AI features.",
      "robots_txt": "may-ignore",
      "robots_note": "Meta states this crawler may bypass robots.txt rules.",
      "used_for_ai_training": null,
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ip_ranges": null,
      "user_agent_example": "meta-externalfetcher/1.1",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-19",
      "confidence": "high",
      "evidence_status": "not-machine-verifiable: the documentation page refuses automated requests (HTTP 400), so quotes cannot be checked automatically. Last read in a browser on the verified date."
    },
    {
      "id": "facebookexternalhit",
      "permalink": "https://bozos.si/ai-crawlers.json#facebookexternalhit",
      "cite": "facebookexternalhit (Meta): Fetches title, description, and thumbnail for links shared in Meta apps. robots.txt: may-ignore. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-19, high confidence. https://bozos.si/ai-crawlers.json#facebookexternalhit",
      "review_due": "2027-01-17",
      "token": "facebookexternalhit",
      "operator": "Meta",
      "category": "link-preview",
      "purpose": "Fetches title, description, and thumbnail for links shared in Meta apps.",
      "robots_txt": "may-ignore",
      "robots_note": "Meta states it might bypass robots.txt for security or integrity checks.",
      "used_for_ai_training": null,
      "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "ip_ranges": null,
      "user_agent_example": "facebookexternalhit/1.1",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-19",
      "confidence": "high",
      "evidence_status": "not-machine-verifiable: the documentation page refuses automated requests (HTTP 400), so quotes cannot be checked automatically. Last read in a browser on the verified date."
    },
    {
      "id": "ccbot",
      "permalink": "https://bozos.si/ai-crawlers.json#ccbot",
      "cite": "CCBot (Common Crawl): Builds Common Crawl's open web archive, which anyone may analyze and which is widely used in AI training datasets. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#ccbot",
      "review_due": "2027-01-18",
      "token": "CCBot",
      "operator": "Common Crawl",
      "category": "open-dataset",
      "purpose": "Builds Common Crawl's open web archive, which anyone may analyze and which is widely used in AI training datasets.",
      "robots_txt": "obeys",
      "robots_note": "Common Crawl documents blocking with User-agent: CCBot / Disallow: /. Spoofed CCBot user agents exist; verify by reverse DNS.",
      "used_for_ai_training": null,
      "documentation": "https://commoncrawl.org/ccbot",
      "ip_ranges": "https://index.commoncrawl.org/ccbot.json",
      "user_agent_example": "CCBot/2.0 (https://commoncrawl.org/faq/)",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "producing and maintaining an open repository of web crawl data that is universally accessible and analyzable by anyone.",
          "source": "https://commoncrawl.org/ccbot"
        },
        {
          "supports": "robots_txt",
          "quote": "To prevent Common Crawl from crawling your website, include the following in your robots.txt",
          "source": "https://commoncrawl.org/ccbot"
        }
      ]
    },
    {
      "id": "amazonbot",
      "permalink": "https://bozos.si/ai-crawlers.json#amazonbot",
      "cite": "Amazonbot (Amazon): Improves Amazon products and services. Amazon says content may be used to train Amazon AI models. robots.txt: obeys. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#amazonbot",
      "review_due": "2027-01-18",
      "token": "Amazonbot",
      "operator": "Amazon",
      "category": "training",
      "purpose": "Improves Amazon products and services. Amazon says content may be used to train Amazon AI models.",
      "robots_txt": "obeys",
      "robots_note": "Does not support Crawl-delay. May use a cached robots.txt up to 30 days old.",
      "used_for_ai_training": true,
      "documentation": "https://developer.amazon.com/amazonbot",
      "ip_ranges": "https://developer.amazon.com/amazonbot/ip-addresses/",
      "user_agent_example": "... Amazonbot/0.1 ...",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "Amazonbot is used to improve our products and services.",
          "source": "https://developer.amazon.com/amazonbot"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "may be used to train Amazon AI models.",
          "source": "https://developer.amazon.com/amazonbot"
        },
        {
          "supports": "robots_txt",
          "quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol",
          "source": "https://developer.amazon.com/amazonbot"
        }
      ]
    },
    {
      "id": "amzn-searchbot",
      "permalink": "https://bozos.si/ai-crawlers.json#amzn-searchbot",
      "cite": "Amzn-SearchBot (Amazon): Improves search in Amazon products such as Alexa. Not used for generative AI training. robots.txt: obeys. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#amzn-searchbot",
      "review_due": "2027-01-18",
      "token": "Amzn-SearchBot",
      "operator": "Amazon",
      "category": "search",
      "purpose": "Improves search in Amazon products such as Alexa. Not used for generative AI training.",
      "robots_txt": "obeys",
      "robots_note": "If not named, follows the rules given to other search bots.",
      "used_for_ai_training": false,
      "documentation": "https://developer.amazon.com/amazonbot",
      "ip_ranges": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "Amzn-SearchBot is used to improve search experiences in Amazon products and services.",
          "source": "https://developer.amazon.com/amazonbot"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "Amzn-SearchBot does not crawl content for generative AI model training.",
          "source": "https://developer.amazon.com/amazonbot"
        },
        {
          "supports": "robots_txt",
          "quote": "Automated crawling from these listed user agents respects the Robots Exclusion Protocol",
          "source": "https://developer.amazon.com/amazonbot"
        }
      ]
    },
    {
      "id": "amzn-user",
      "permalink": "https://bozos.si/ai-crawlers.json#amzn-user",
      "cite": "Amzn-User (Amazon): Handles user-initiated requests, for example fetching live data for Alexa. Not used for generative AI training. robots.txt: may-ignore. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#amzn-user",
      "review_due": "2027-01-18",
      "token": "Amzn-User",
      "operator": "Amazon",
      "category": "user-fetch",
      "purpose": "Handles user-initiated requests, for example fetching live data for Alexa. Not used for generative AI training.",
      "robots_txt": "may-ignore",
      "robots_note": "Amazon states it may not always follow robots.txt because it acts on a user's behalf.",
      "used_for_ai_training": false,
      "documentation": "https://developer.amazon.com/amazonbot",
      "ip_ranges": "https://developer.amazon.com/amazonbot/live-ip-addresses/",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "Amzn-User supports user actions, such as responding to Alexa queries that require up-to-date information.",
          "source": "https://developer.amazon.com/amazonbot"
        },
        {
          "supports": "robots_txt",
          "quote": "Because actions taken by Amzn-User can be initiated by a user, it may not follow all robots.txt directives.",
          "source": "https://developer.amazon.com/amazonbot"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "Amzn-User does not crawl content for generative AI model training.",
          "source": "https://developer.amazon.com/amazonbot"
        }
      ]
    },
    {
      "id": "mistralai-training",
      "permalink": "https://bozos.si/ai-crawlers.json#mistralai-training",
      "cite": "MistralAI-Training (Mistral AI): Collects web content to build training datasets for Mistral's generative models. robots.txt: obeys. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#mistralai-training",
      "review_due": "2027-01-18",
      "token": "MistralAI-Training",
      "operator": "Mistral AI",
      "category": "training",
      "purpose": "Collects web content to build training datasets for Mistral's generative models.",
      "robots_txt": "obeys",
      "robots_note": "Mistral says webmasters can disallow it in robots.txt.",
      "used_for_ai_training": true,
      "documentation": "https://docs.mistral.ai/robots",
      "ip_ranges": null,
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "MistralAI-Training crawls web content to help build datasets for training Mistral generative AI models.",
          "source": "https://docs.mistral.ai/robots"
        },
        {
          "supports": "robots_txt",
          "quote": "Webmasters can disallow this user agent in their robots.txt file.",
          "source": "https://docs.mistral.ai/robots"
        }
      ]
    },
    {
      "id": "mistralai-index",
      "permalink": "https://bozos.si/ai-crawlers.json#mistralai-index",
      "cite": "MistralAI-Index (Mistral AI): Crawls for indexing only, feeding Mistral search. Not used for generative AI training. robots.txt: not-stated. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#mistralai-index",
      "review_due": "2027-01-18",
      "token": "MistralAI-Index",
      "operator": "Mistral AI",
      "category": "search",
      "purpose": "Crawls for indexing only, feeding Mistral search. Not used for generative AI training.",
      "robots_txt": "not-stated",
      "robots_note": "Mistral's page does not give bot-specific robots.txt guidance.",
      "used_for_ai_training": false,
      "documentation": "https://docs.mistral.ai/robots",
      "ip_ranges": "https://mistral.ai/mistralai-index-ips.json",
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "MistralAI-Index is for automated crawling of the web for indexing purposes only.",
          "source": "https://docs.mistral.ai/robots"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "Content crawled by MistralAI-Index is not used for generative AI training of any kind.",
          "source": "https://docs.mistral.ai/robots"
        }
      ]
    },
    {
      "id": "mistralai-user",
      "permalink": "https://bozos.si/ai-crawlers.json#mistralai-user",
      "cite": "MistralAI-User (Mistral AI): Visits pages when a user asks Mistral's assistant a question. Not used for automatic crawling. robots.txt: obeys. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#mistralai-user",
      "review_due": "2027-01-18",
      "token": "MistralAI-User",
      "operator": "Mistral AI",
      "category": "user-fetch",
      "purpose": "Visits pages when a user asks Mistral's assistant a question. Not used for automatic crawling.",
      "robots_txt": "obeys",
      "robots_note": "Mistral says the MistralAI-User robots.txt tag governs which sites these user requests can be made to.",
      "used_for_ai_training": false,
      "documentation": "https://docs.mistral.ai/robots",
      "ip_ranges": "https://mistral.ai/mistralai-user-ips.json",
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "When users ask Vibe a question, it may visit a web page to help answer and include a link to the source in its response.",
          "source": "https://docs.mistral.ai/robots"
        },
        {
          "supports": "robots_txt",
          "quote": "MistralAI-User governs which sites these user requests can be made to.",
          "source": "https://docs.mistral.ai/robots"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "It is not used for crawling the web in any automatic fashion, nor to crawl content for generative AI training.",
          "source": "https://docs.mistral.ai/robots"
        }
      ]
    },
    {
      "id": "duckassistbot",
      "permalink": "https://bozos.si/ai-crawlers.json#duckassistbot",
      "cite": "DuckAssistBot (DuckDuckGo): Fetches pages in real time for DuckDuckGo's AI-assisted answers, which cite sources. Not used to train AI models. robots.txt: obeys. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#duckassistbot",
      "review_due": "2027-01-18",
      "token": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "category": "user-fetch",
      "purpose": "Fetches pages in real time for DuckDuckGo's AI-assisted answers, which cite sources. Not used to train AI models.",
      "robots_txt": "obeys",
      "robots_note": "Opt-outs take effect after 72 hours and do not affect organic search rankings.",
      "used_for_ai_training": false,
      "documentation": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "ip_ranges": "https://duckduckgo.com/duckassistbot.json",
      "user_agent_example": "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "DuckAssistBot is a web crawler for DuckDuckGo Search that crawls pages in real-time for our AI-assisted answers, which prominently cite their sources.",
          "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "This data is not used in any way to train AI models.",
          "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
        },
        {
          "supports": "robots_txt",
          "quote": "Once you've updated the file to disallow the DuckAssistBot user agent, the change will take effect after 72 hours and DuckAssistBot will stop crawling your site.",
          "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
        }
      ]
    },
    {
      "id": "diffbot",
      "permalink": "https://bozos.si/ai-crawlers.json#diffbot",
      "cite": "Diffbot (Diffbot): General, proactive web crawling for Diffbot's Knowledge Graph and web search, so sites can be discovered and cited there. Diffbot says this user agent is not used for AI training. robots.txt: obeys. Used for AI training: no. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#diffbot",
      "review_due": "2027-01-18",
      "token": "Diffbot",
      "operator": "Diffbot",
      "category": "search",
      "purpose": "General, proactive web crawling for Diffbot's Knowledge Graph and web search, so sites can be discovered and cited there. Diffbot says this user agent is not used for AI training.",
      "robots_txt": "obeys",
      "robots_note": "Honors disallow and crawl-delay by default. Diffbot says robots.txt can be overridden in specific cases, usually under a partnership or agreement with the site.",
      "used_for_ai_training": false,
      "documentation": "https://www.diffbot.com/docs/crawl/faq/robots-txt",
      "ip_ranges": null,
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "General, proactive web crawling for building a general search engine.",
          "source": "https://www.diffbot.com/docs/crawl/faq/robots-txt"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "It is not used for AI training.",
          "source": "https://www.diffbot.com/docs/crawl/faq/robots-txt"
        },
        {
          "supports": "robots_txt",
          "quote": "By default Diffbot's web crawls adhere to a site's robots.txt instructions, including the disallow and crawl-delay directives.",
          "source": "https://www.diffbot.com/docs/crawl/faq/robots-txt"
        },
        {
          "supports": "robots_txt",
          "quote": "the robots.txt instruction can be ignored/overridden.",
          "source": "https://www.diffbot.com/docs/crawl/faq/robots-txt"
        }
      ]
    },
    {
      "id": "diffbot-user",
      "permalink": "https://bozos.si/ai-crawlers.json#diffbot-user",
      "cite": "Diffbot-User (Diffbot): Requests made on behalf of a person browsing a URL with Diffbot software. robots.txt: not-stated. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#diffbot-user",
      "review_due": "2027-01-18",
      "token": "Diffbot-User",
      "operator": "Diffbot",
      "category": "user-fetch",
      "purpose": "Requests made on behalf of a person browsing a URL with Diffbot software.",
      "robots_txt": "not-stated",
      "robots_note": "Diffbot's robots.txt statement covers its crawls; the page does not say how robots.txt applies to user-initiated Diffbot-User requests.",
      "used_for_ai_training": null,
      "documentation": "https://www.diffbot.com/docs/crawl/faq/robots-txt",
      "ip_ranges": null,
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "This is used by requests originating on behalf of a human user browsing a URL using Diffbot software, in response to their input.",
          "source": "https://www.diffbot.com/docs/crawl/faq/robots-txt"
        }
      ]
    },
    {
      "id": "youbot",
      "permalink": "https://bozos.si/ai-crawlers.json#youbot",
      "cite": "YouBot (You.com): The web crawler that powers the You.com search engine. robots.txt: obeys. Used for AI training: not stated. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#youbot",
      "review_due": "2027-01-18",
      "token": "YouBot",
      "operator": "You.com",
      "category": "search",
      "purpose": "The web crawler that powers the You.com search engine.",
      "robots_txt": "obeys",
      "robots_note": "You.com says YouBot respects robots.txt, including user-agent-specific rules and crawl-delay, and caches robots.txt for 30 minutes.",
      "used_for_ai_training": null,
      "documentation": "https://you.com/docs/youbot",
      "ip_ranges": null,
      "user_agent_example": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; YouBot/1.0; +https://docs.you.com/youbot; env:prod) Chrome/X.X.X.X Safari/537.36",
      "notes": "Verification: You.com recommends checking Web Bot Auth request signatures (keys at https://you.com/.well-known/http-message-signatures-directory); it also documents reverse DNS (youbot-<octets>.search.you.com) and the 68.67.112.0/24 range. No IP list file is published.",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "YouBot is the web crawler that powers the You.com search engine.",
          "source": "https://you.com/docs/youbot"
        },
        {
          "supports": "robots_txt",
          "quote": "YouBot fully respects robots.txt directives, including user-agent specific rules and crawl-delay settings.",
          "source": "https://you.com/docs/youbot"
        },
        {
          "supports": "robots_txt",
          "quote": "YouBot caches robots.txt for 30 minutes.",
          "source": "https://you.com/docs/youbot"
        },
        {
          "supports": "ip_ranges",
          "quote": "You.com supports three ways to verify that requests are genuinely from YouBot: cryptographic signatures (recommended), reverse DNS checks, and IP range validation.",
          "source": "https://you.com/docs/youbot"
        }
      ]
    },
    {
      "id": "ai2bot",
      "permalink": "https://bozos.si/ai-crawlers.json#ai2bot",
      "cite": "AI2Bot (Allen Institute for AI (Ai2)): Explores certain domains to find web content that Ai2 says is used to train open language models. robots.txt: not-stated. Used for AI training: yes. Source: BOZOS, AI Crawler Directory, verified 2026-10-20, high confidence. https://bozos.si/ai-crawlers.json#ai2bot",
      "review_due": "2027-01-18",
      "token": "AI2Bot",
      "operator": "Allen Institute for AI (Ai2)",
      "category": "training",
      "purpose": "Explores certain domains to find web content that Ai2 says is used to train open language models.",
      "robots_txt": "not-stated",
      "robots_note": "The crawler page does not mention robots.txt. It says the user-agent string can be used to filter or reject the crawler's traffic.",
      "used_for_ai_training": true,
      "documentation": "https://allenai.org/crawler",
      "ip_ranges": null,
      "user_agent_example": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)",
      "docs_vs_reports": "none-recorded",
      "verified": "2026-10-20",
      "confidence": "high",
      "evidence": [
        {
          "supports": "purpose",
          "quote": "The AI2 Bot explores certain domains to find web content.",
          "source": "https://allenai.org/crawler"
        },
        {
          "supports": "used_for_ai_training",
          "quote": "This web content is used to train open language models.",
          "source": "https://allenai.org/crawler"
        },
        {
          "supports": "robots_txt",
          "quote": "This user agent string can be used to filter or reject traffic from our crawler if desired.",
          "source": "https://allenai.org/crawler"
        }
      ]
    }
  ],
  "field_notes": {
    "used_for_ai_training": "true: the operator says content may be used to train AI models (directly, or via the token's control). false: the operator says it is not. null: the documentation does not say.",
    "confidence": "high: read directly from the operator's documentation. medium: some details could not be read from the documentation and rest on the token name and a verified IP list.",
    "ip_ranges": "URL of the operator's published IP list, checked to resolve on the verified date. null when none is published.",
    "docs_vs_reports": "Whether a named primary source has reported behavior that contradicts the documentation. See docs_vs_reports_values.",
    "evidence": "Exact quotes from the operator's documentation page, each tagged with the field it supports. Every quote is checked by script against the page's current text; source_snapshots gives a Wayback Machine copy of each page.",
    "evidence_status": "Present only when quotes cannot be checked automatically, with the reason."
  },
  "undocumented_tokens": [
    {
      "id": "bytespider",
      "permalink": "https://bozos.si/ai-crawlers.json#bytespider",
      "cite": "Bytespider: BOZOS found no ByteDance-published page documenting Bytespider's purpose, robots.txt behavior, or IP ranges. Third-party claims about it disagree, so they are not recorded here as fact. Source: BOZOS, AI Crawler Directory, searched 2026-10-20. https://bozos.si/ai-crawlers.json#bytespider",
      "review_due": "2027-01-18",
      "token": "Bytespider",
      "attributed_operator": "ByteDance (as attributed by third-party bot directories)",
      "searched": "2026-10-20",
      "finding": "BOZOS found no ByteDance-published page documenting Bytespider's purpose, robots.txt behavior, or IP ranges. Third-party claims about it disagree, so they are not recorded here as fact.",
      "robots_txt_advice": "You can still name it in robots.txt (User-agent: Bytespider). Compliance cannot be confirmed from operator documentation, so enforce a block at your CDN or firewall if it matters."
    },
    {
      "id": "cohere-ai",
      "permalink": "https://bozos.si/ai-crawlers.json#cohere-ai",
      "cite": "cohere-ai: BOZOS found no Cohere-published crawler documentation for cohere-ai (Cohere's own pages use the name for its SDK packages). Third-party descriptions of its purpose conflict. Source: BOZOS, AI Crawler Directory, searched 2026-10-20. https://bozos.si/ai-crawlers.json#cohere-ai",
      "review_due": "2027-01-18",
      "token": "cohere-ai",
      "attributed_operator": "Cohere (as attributed by third-party bot directories)",
      "searched": "2026-10-20",
      "finding": "BOZOS found no Cohere-published crawler documentation for cohere-ai (Cohere's own pages use the name for its SDK packages). Third-party descriptions of its purpose conflict.",
      "robots_txt_advice": "You can name it in robots.txt (User-agent: cohere-ai). Third parties also list a separate cohere-training-data-crawler token; neither is documented by Cohere."
    }
  ],
  "source_snapshots": [
    {
      "url": "https://allenai.org/crawler",
      "archived": "https://web.archive.org/web/20261002130200/https://allenai.org/crawler",
      "captured": "2026-10-02",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/",
      "archived": "https://web.archive.org/web/20261015105431/https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/",
      "captured": "2026-10-15",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://commoncrawl.org/ccbot",
      "archived": "https://web.archive.org/web/20261005215406/https://commoncrawl.org/ccbot",
      "captured": "2026-10-05",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developer.amazon.com/amazonbot",
      "archived": "https://web.archive.org/web/20261007220743/https://developer.amazon.com/amazonbot",
      "captured": "2026-10-07",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "archived": "https://web.archive.org/web/20261020091238/https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "captured": "2026-10-20",
      "checked": "2026-10-20",
      "quotes_in_snapshot": null
    },
    {
      "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "archived": "https://web.archive.org/web/20261020085128/https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "captured": "2026-10-20",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developers.openai.com/api/docs/bots",
      "archived": "https://web.archive.org/web/20261016233530/https://developers.openai.com/api/docs/bots",
      "captured": "2026-10-16",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://docs.mistral.ai/robots",
      "archived": "https://web.archive.org/web/20261010094213/https://docs.mistral.ai/robots",
      "captured": "2026-10-10",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://docs.perplexity.ai/guides/bots",
      "archived": "https://web.archive.org/web/20261020085457/https://docs.perplexity.ai/guides/bots",
      "captured": "2026-10-20",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "archived": "https://web.archive.org/web/20261013204641/https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "captured": "2026-10-13",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://support.apple.com/en-us/119829",
      "archived": "https://web.archive.org/web/20261016065936/https://support.apple.com/en-us/119829",
      "captured": "2026-10-16",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "archived": "https://web.archive.org/web/20261016233536/https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "captured": "2026-10-16",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "archived": "https://web.archive.org/web/20261019101117/https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "captured": "2026-10-19",
      "checked": "2026-10-20",
      "quotes_in_snapshot": null
    },
    {
      "url": "https://www.diffbot.com/docs/crawl/faq/robots-txt",
      "archived": "https://web.archive.org/web/20261020090915/https://www.diffbot.com/docs/crawl/faq/robots-txt",
      "captured": "2026-10-20",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://www.perplexity.ai/hub/blog/agents-or-bots-making-sense-of-ai-on-the-open-web",
      "archived": "https://web.archive.org/web/20261020085606/https://www.perplexity.ai/hub/blog/agents-or-bots-making-sense-of-ai-on-the-open-web",
      "captured": "2026-10-20",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://you.com/docs/youbot",
      "archived": "https://web.archive.org/web/20261020091006/https://you.com/docs/youbot",
      "captured": "2026-10-20",
      "checked": "2026-10-20",
      "quotes_in_snapshot": true
    }
  ]
}