{
  "schema_version": "1.0",
  "title": "Bot Verification Methods: How to Tell the Real Crawler",
  "canonical_url": "https://bozos.si/reference/bot-verification",
  "data_url": "https://bozos.si/bot-verification.json",
  "updated": "2026-10-21",
  "license": "CC0-1.0",
  "feedback": {
    "contact": "https://bozos.si/contact",
    "email": "mailto:bozos@bozos.si",
    "subject": "[BOZOS agent] correction: bot-verification-data - <summary> [via:dataset]",
    "corrections_log": "https://bozos.si/corrections.json"
  },
  "record_access": {
    "collections": [
      "methods",
      "specifications",
      "operators"
    ],
    "permalink": "https://bozos.si/bot-verification.json#{id}",
    "resolve": "https://bozos.si/bot-verification.json?id={id}",
    "changes": "https://bozos.si/api/v1/changes?dataset=bot-verification-data&since={YYYY-MM-DD}",
    "review_policy": "Each record is re-checked against its sources every 90 days; review_due is the date that is next due. GET /api/v1/health lists overdue records.",
    "note": "Every record has a permanent id, a permalink, and a cite line: a sentence built only from the record's own fields, with its date and permalink, that can be quoted verbatim. Fetch the resolve URL to get one record; ids are never reused."
  },
  "scope_note": "For each AI and search operator in the BOZOS crawler directory, which ways of confirming that a request really came from its bot the operator's own documentation describes: published IP ranges, forward-confirmed reverse DNS, and Web Bot Auth signatures. documented cells quote the operator and give the specifics a verifier needs. not-stated means the pages listed in searched do not mention the method; it is not evidence that the operator does not support it.",
  "editorial_rule": "Record what the operator states, scoped to the pages checked. A specification is quoted for what it is, never as proof that anyone implements it. observed_lists is an observation of a fetch on the date given, not operator documentation.",
  "baseline": {
    "name": "User-Agent string",
    "note": "Every operator in the crawler directory names a user-agent token, but a header is self-reported and any client can send it. Operators warn about this themselves.",
    "evidence": [
      {
        "quote": "Since user agent strings can be spoofed, You.com supports three ways to verify that requests are genuinely from YouBot",
        "source": "https://you.com/docs/youbot"
      },
      {
        "quote": "we are aware of crawlers falsely identifying themselves as CCBot",
        "source": "https://commoncrawl.org/ccbot"
      }
    ]
  },
  "manual_sources": [
    {
      "url": "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting",
      "checked": "2026-10-21",
      "reason": "help.openai.com answers the evidence script's user agent with HTTP 403 but served the article to a browser user agent on this date, so its quotes were checked at fetch time rather than by scripts/evidence.mjs verify."
    }
  ],
  "stance_values": {
    "documented": "The operator's documentation describes the method, for the scope given.",
    "not-stated": "The pages checked do not mention the method. The search terms and pages are listed so the absence can be re-checked.",
    "unknown": "The operator's documentation cannot be read by a script, so neither a statement nor its absence could be verified."
  },
  "specifications": [
    {
      "id": "rfc-9421",
      "permalink": "https://bozos.si/bot-verification.json#rfc-9421",
      "cite": "HTTP Message Signatures (RFC 9421 (Proposed Standard)): The signature mechanism Web Bot Auth builds on. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, updated 2026-10-21. https://bozos.si/bot-verification.json#rfc-9421",
      "review_due": "2027-01-19",
      "name": "HTTP Message Signatures",
      "url": "https://www.rfc-editor.org/rfc/rfc9421.html",
      "status": "RFC 9421 (Proposed Standard)",
      "note": "The signature mechanism Web Bot Auth builds on."
    },
    {
      "id": "webbotauth-architecture",
      "permalink": "https://bozos.si/bot-verification.json#webbotauth-architecture",
      "cite": "HTTP Message Signatures for automated traffic Architecture (Internet-Draft (individual submission at last fetch)): How crawlers attach identity to requests with RFC 9421. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, updated 2026-10-21. https://bozos.si/bot-verification.json#webbotauth-architecture",
      "review_due": "2027-01-19",
      "name": "HTTP Message Signatures for automated traffic Architecture",
      "url": "https://datatracker.ietf.org/doc/draft-meunier-web-bot-auth-architecture/",
      "status": "Internet-Draft (individual submission at last fetch)",
      "note": "How crawlers attach identity to requests with RFC 9421.",
      "evidence": [
        {
          "quote": "allowing HTTP servers to verify their identity with confidence",
          "source": "https://datatracker.ietf.org/doc/draft-meunier-web-bot-auth-architecture/"
        }
      ]
    },
    {
      "id": "http-message-signatures-directory",
      "permalink": "https://bozos.si/bot-verification.json#http-message-signatures-directory",
      "cite": "HTTP Message Signatures Directory (Internet-Draft): The /.well-known/http-message-signatures-directory key list. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, updated 2026-10-21. https://bozos.si/bot-verification.json#http-message-signatures-directory",
      "review_due": "2027-01-19",
      "name": "HTTP Message Signatures Directory",
      "url": "https://datatracker.ietf.org/doc/draft-meunier-http-message-signatures-directory/",
      "status": "Internet-Draft",
      "note": "The /.well-known/http-message-signatures-directory key list."
    }
  ],
  "methods": [
    {
      "id": "published-ip-ranges",
      "permalink": "https://bozos.si/bot-verification.json#published-ip-ranges",
      "cite": "Published IP ranges (operator-practice, Each operator, individually): Fetch the operator's list (usually JSON with a prefixes array of ipv4Prefix and ipv6Prefix entries) and test whether the connecting address falls inside a listed CIDR range. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, updated 2026-10-21. https://bozos.si/bot-verification.json#published-ip-ranges",
      "review_due": "2027-01-19",
      "name": "Published IP ranges",
      "kind": "operator-practice",
      "publisher": "Each operator, individually",
      "how_to_check": "Fetch the operator's list (usually JSON with a prefixes array of ipv4Prefix and ipv6Prefix entries) and test whether the connecting address falls inside a listed CIDR range.",
      "limits": "Only as current as the list: re-fetch on a schedule. An address proves the network, not which of the operator's bots sent the request, unless the operator publishes one list per bot.",
      "spec": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
      "evidence": [
        {
          "quote": "Alternatively, you can identify Googlebot by IP address by matching the crawler's IP address to the lists of Google crawlers' and fetchers' IP ranges:",
          "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
        }
      ],
      "terms": [
        "ip range",
        "ip address",
        "ips\\b",
        "\\.json",
        "cidr"
      ]
    },
    {
      "id": "reverse-forward-dns",
      "permalink": "https://bozos.si/bot-verification.json#reverse-forward-dns",
      "cite": "Reverse then forward DNS (forward-confirmed reverse DNS) (operator-practice, Each operator, individually): Resolve the connecting address to a hostname (PTR record), check the hostname is under the operator's domain, resolve that hostname back to an address, and require it to equal the original address. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, updated 2026-10-21. https://bozos.si/bot-verification.json#reverse-forward-dns",
      "review_due": "2027-01-19",
      "name": "Reverse then forward DNS (forward-confirmed reverse DNS)",
      "kind": "operator-practice",
      "publisher": "Each operator, individually",
      "how_to_check": "Resolve the connecting address to a hostname (PTR record), check the hostname is under the operator's domain, resolve that hostname back to an address, and require it to equal the original address.",
      "limits": "Needs two DNS lookups per new address (cache results). Operators say it is not available everywhere: Common Crawl states reverse DNS is not yet supported over IPv6.",
      "spec": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
      "evidence": [
        {
          "quote": "Run a reverse DNS lookup on the accessing IP address from your logs, using the host command.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
        },
        {
          "quote": "Run a forward DNS lookup on the domain name retrieved in step 1 using the host command on the retrieved domain name.",
          "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
        }
      ],
      "terms": [
        "reverse dns",
        "forward dns",
        "rdns",
        "in-addr",
        "\\bptr\\b",
        "dig -x",
        "host command"
      ]
    },
    {
      "id": "web-bot-auth",
      "permalink": "https://bozos.si/bot-verification.json#web-bot-auth",
      "cite": "Web Bot Auth (HTTP Message Signatures) (ietf-draft-on-rfc, IETF webbotauth working group; signatures per RFC 9421): Read the Signature-Agent header, fetch the key directory it names (/.well-known/http-message-signatures-directory), and verify the Signature and Signature-Input headers over the request per RFC 9421. Operators say only some requests are signed, so fall back to IP or DNS checks. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, updated 2026-10-21. https://bozos.si/bot-verification.json#web-bot-auth",
      "review_due": "2027-01-19",
      "name": "Web Bot Auth (HTTP Message Signatures)",
      "kind": "ietf-draft-on-rfc",
      "publisher": "IETF webbotauth working group; signatures per RFC 9421",
      "how_to_check": "Read the Signature-Agent header, fetch the key directory it names (/.well-known/http-message-signatures-directory), and verify the Signature and Signature-Input headers over the request per RFC 9421. Operators say only some requests are signed, so fall back to IP or DNS checks.",
      "limits": "The profile is an Internet-Draft and may change. Operators sign a subset of traffic. Signature headers can be dropped by intermediaries. A valid signature proves control of the key, not good behavior.",
      "spec": "https://datatracker.ietf.org/wg/webbotauth/about/",
      "evidence": [
        {
          "quote": "The Web Bot Authentication (webbotauth) Working Group will standardize methods for cryptographically authenticating automated clients and providing additional information about their operators to Web sites.",
          "source": "https://datatracker.ietf.org/wg/webbotauth/about/"
        },
        {
          "quote": "This document describes a mechanism for creating, encoding, and verifying digital signatures or message authentication codes over components of an HTTP message.",
          "source": "https://www.rfc-editor.org/rfc/rfc9421.html"
        }
      ],
      "terms": [
        "web bot auth",
        "message signature",
        "signature-agent",
        "signature-input",
        "rfc 9421",
        "http-message-signatures"
      ]
    }
  ],
  "operators": [
    {
      "id": "openai",
      "permalink": "https://bozos.si/bot-verification.json#openai",
      "cite": "OpenAI: published-ip-ranges documented (One list per bot: gptbot.json, searchbot.json, chatgpt-user.json and adsbot.json); reverse-forward-dns not-stated; web-bot-auth documented (ChatGPT's Cloud browser (the article's current name for ChatGPT agent traffic) only. OpenAI's crawler documentation does not mention signatures). Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#openai",
      "review_due": "2027-01-19",
      "operator": "OpenAI",
      "crawler_tokens": [
        "GPTBot",
        "OAI-SearchBot",
        "ChatGPT-User",
        "OAI-AdsBot"
      ],
      "sources": [
        "https://developers.openai.com/api/docs/bots",
        "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "One list per bot: gptbot.json, searchbot.json, chatgpt-user.json and adsbot.json.",
          "lists": [
            "https://openai.com/gptbot.json",
            "https://openai.com/searchbot.json",
            "https://openai.com/chatgpt-user.json",
            "https://openai.com/adsbot.json"
          ],
          "quotes": [
            {
              "quote": "Published IP addresses: https://openai.com/gptbot.json",
              "source": "https://developers.openai.com/api/docs/bots"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://developers.openai.com/api/docs/bots",
              "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "documented",
          "scope": "ChatGPT's Cloud browser (the article's current name for ChatGPT agent traffic) only. OpenAI's crawler documentation does not mention signatures.",
          "signature_agent": "\"https://chatgpt.com\"",
          "directory": "https://chatgpt.com/.well-known/http-message-signatures-directory",
          "quotes": [
            {
              "quote": "Cloud browser uses Web Bot Auth to sign outbound HTTP requests, allowing website operators to verify that requests genuinely originate from ChatGPT.",
              "source": "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting"
            }
          ]
        }
      }
    },
    {
      "id": "anthropic",
      "permalink": "https://bozos.si/bot-verification.json#anthropic",
      "cite": "Anthropic: published-ip-ranges documented (A single list for Anthropic's bots. Anthropic also says blocking by IP address may not reliably opt a site out, because it stops the crawler reading robots.txt; use the list to identify traffic, robots.txt to opt out); reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#anthropic",
      "review_due": "2027-01-19",
      "operator": "Anthropic",
      "crawler_tokens": [
        "ClaudeBot",
        "Claude-SearchBot",
        "Claude-User"
      ],
      "sources": [
        "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "A single list for Anthropic's bots. Anthropic also says blocking by IP address may not reliably opt a site out, because it stops the crawler reading robots.txt; use the list to identify traffic, robots.txt to opt out.",
          "lists": [
            "https://claude.com/crawling/bots.json"
          ],
          "quotes": [
            {
              "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic.",
              "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "google",
      "permalink": "https://bozos.si/bot-verification.json#google",
      "cite": "Google: published-ip-ranges documented (Separate lists for common crawlers, special-case crawlers, and user-triggered fetchers); reverse-forward-dns documented (Hostnames end in googlebot.com, google.com or googleusercontent.com, depending on the crawler category); web-bot-auth documented (Experimental. A subset of requests from the Google-Agent user agent are signed; Google says it does not sign every request and recommends keeping IP and reverse-DNS checks). Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#google",
      "review_due": "2027-01-19",
      "operator": "Google",
      "crawler_tokens": [
        "Googlebot",
        "Google-Extended",
        "GoogleOther",
        "Google-CloudVertexBot"
      ],
      "sources": [
        "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
        "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
        "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "Separate lists for common crawlers, special-case crawlers, and user-triggered fetchers.",
          "lists": [
            "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
            "https://developers.google.com/static/search/apis/ipranges/googlebot.json"
          ],
          "quotes": [
            {
              "quote": "Alternatively, you can identify Googlebot by IP address by matching the crawler's IP address to the lists of Google crawlers' and fetchers' IP ranges:",
              "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "documented",
          "scope": "Hostnames end in googlebot.com, google.com or googleusercontent.com, depending on the crawler category.",
          "hostnames": [
            "crawl-***-***-***-***.googlebot.com",
            "geo-crawl-***-***-***-***.geo.googlebot.com",
            "rate-limited-proxy-***-***-***-***.google.com",
            "***-***-***-***.gae.googleusercontent.com",
            "google-proxy-***-***-***-***.google.com"
          ],
          "quotes": [
            {
              "quote": "Verify that the domain name is either googlebot.com , google.com , or googleusercontent.com .",
              "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
            },
            {
              "quote": "Run a forward DNS lookup on the domain name retrieved in step 1 using the host command on the retrieved domain name.",
              "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
            }
          ]
        },
        "web-bot-auth": {
          "stance": "documented",
          "scope": "Experimental. A subset of requests from the Google-Agent user agent are signed; Google says it does not sign every request and recommends keeping IP and reverse-DNS checks.",
          "signature_agent": "g=\"https://agent.bot.goog\"",
          "directory": "https://agent.bot.goog/.well-known/http-message-signatures-directory",
          "quotes": [
            {
              "quote": "We don't sign every request of a particular agent.",
              "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth"
            },
            {
              "quote": "Not all Google user agents are using Web Bot Auth.",
              "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth"
            }
          ]
        }
      }
    },
    {
      "id": "microsoft",
      "permalink": "https://bozos.si/bot-verification.json#microsoft",
      "cite": "Microsoft: published-ip-ranges unknown; reverse-forward-dns unknown; web-bot-auth unknown. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#microsoft",
      "review_due": "2027-01-19",
      "operator": "Microsoft",
      "crawler_tokens": [
        "bingbot"
      ],
      "sources": [
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
      ],
      "unreadable_sources": [
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "unknown",
          "scope": "The operator's documentation page cannot be read by a script, so absence of a mention cannot be checked.",
          "quotes": []
        },
        "reverse-forward-dns": {
          "stance": "unknown",
          "scope": "The operator's documentation page cannot be read by a script, so absence of a mention cannot be checked.",
          "quotes": []
        },
        "web-bot-auth": {
          "stance": "unknown",
          "scope": "The operator's documentation page cannot be read by a script, so absence of a mention cannot be checked.",
          "quotes": []
        }
      }
    },
    {
      "id": "apple",
      "permalink": "https://bozos.si/bot-verification.json#apple",
      "cite": "Apple: published-ip-ranges documented (One CIDR list for Applebot); reverse-forward-dns documented (Hostnames are under applebot.apple.com); web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#apple",
      "review_due": "2027-01-19",
      "operator": "Apple",
      "crawler_tokens": [
        "Applebot",
        "Applebot-Extended"
      ],
      "sources": [
        "https://support.apple.com/en-us/119829"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "One CIDR list for Applebot.",
          "lists": [
            "https://search.developer.apple.com/applebot.json"
          ],
          "quotes": [
            {
              "quote": "Another way is to match the IP address with a CIDR prefix contained in the following JSON file",
              "source": "https://support.apple.com/en-us/119829"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "documented",
          "scope": "Hostnames are under applebot.apple.com.",
          "hostnames": [
            "*.applebot.apple.com"
          ],
          "quotes": [
            {
              "quote": "Traffic coming from Applebot is generally identified by using reverse DNS",
              "source": "https://support.apple.com/en-us/119829"
            },
            {
              "quote": "The host command can also be used to verify that the DNS points to the same IP address",
              "source": "https://support.apple.com/en-us/119829"
            }
          ]
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://support.apple.com/en-us/119829"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "perplexity",
      "permalink": "https://bozos.si/bot-verification.json#perplexity",
      "cite": "Perplexity: published-ip-ranges documented (One list for PerplexityBot and one for Perplexity-User. The documentation recommends combining User-Agent matching with IP checks); reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#perplexity",
      "review_due": "2027-01-19",
      "operator": "Perplexity",
      "crawler_tokens": [
        "PerplexityBot",
        "Perplexity-User"
      ],
      "sources": [
        "https://docs.perplexity.ai/guides/bots"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "One list for PerplexityBot and one for Perplexity-User. The documentation recommends combining User-Agent matching with IP checks.",
          "lists": [
            "https://www.perplexity.com/perplexitybot.json",
            "https://www.perplexity.com/perplexity-user.json"
          ],
          "quotes": [
            {
              "quote": "combine both User-Agent string matching and IP address verification for enhanced security",
              "source": "https://docs.perplexity.ai/guides/bots"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://docs.perplexity.ai/guides/bots"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://docs.perplexity.ai/guides/bots"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "meta",
      "permalink": "https://bozos.si/bot-verification.json#meta",
      "cite": "Meta: published-ip-ranges unknown; reverse-forward-dns unknown; web-bot-auth unknown. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#meta",
      "review_due": "2027-01-19",
      "operator": "Meta",
      "crawler_tokens": [
        "meta-externalagent",
        "meta-webindexer",
        "meta-externalfetcher",
        "facebookexternalhit"
      ],
      "sources": [
        "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
      ],
      "unreadable_sources": [
        "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "unknown",
          "scope": "The operator's documentation page cannot be read by a script, so absence of a mention cannot be checked.",
          "quotes": []
        },
        "reverse-forward-dns": {
          "stance": "unknown",
          "scope": "The operator's documentation page cannot be read by a script, so absence of a mention cannot be checked.",
          "quotes": []
        },
        "web-bot-auth": {
          "stance": "unknown",
          "scope": "The operator's documentation page cannot be read by a script, so absence of a mention cannot be checked.",
          "quotes": []
        }
      }
    },
    {
      "id": "common-crawl",
      "permalink": "https://bozos.si/bot-verification.json#common-crawl",
      "cite": "Common Crawl: published-ip-ranges documented (One JSON list covering IPv4 and IPv6); reverse-forward-dns documented (Hostnames under crawl.commoncrawl.org. Reverse DNS is not available over IPv6); web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#common-crawl",
      "review_due": "2027-01-19",
      "operator": "Common Crawl",
      "crawler_tokens": [
        "CCBot"
      ],
      "sources": [
        "https://commoncrawl.org/ccbot"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "One JSON list covering IPv4 and IPv6.",
          "lists": [
            "https://index.commoncrawl.org/ccbot.json"
          ],
          "quotes": [
            {
              "quote": "These IP ranges (v4 and v6) are also provided as JSON at https://index.commoncrawl.org/ccbot.json.",
              "source": "https://commoncrawl.org/ccbot"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "documented",
          "scope": "Hostnames under crawl.commoncrawl.org. Reverse DNS is not available over IPv6.",
          "hostnames": [
            "*-*-*-*.crawl.commoncrawl.org"
          ],
          "quotes": [
            {
              "quote": "CCBot is now run on dedicated IP address ranges with reverse DNS (except over IPv6 where reverse DNS is not yet supported.)",
              "source": "https://commoncrawl.org/ccbot"
            }
          ]
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://commoncrawl.org/ccbot"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "amazon",
      "permalink": "https://bozos.si/bot-verification.json#amazon",
      "cite": "Amazon: published-ip-ranges documented (Three HTML pages (Amazonbot, search bot, live/user fetch). They are web pages, not JSON, so they need scraping); reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#amazon",
      "review_due": "2027-01-19",
      "operator": "Amazon",
      "crawler_tokens": [
        "Amazonbot",
        "Amzn-SearchBot",
        "Amzn-User"
      ],
      "sources": [
        "https://developer.amazon.com/amazonbot"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "Three HTML pages (Amazonbot, search bot, live/user fetch). They are web pages, not JSON, so they need scraping.",
          "lists": [
            "https://developer.amazon.com/amazonbot/ip-addresses/",
            "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
            "https://developer.amazon.com/amazonbot/live-ip-addresses/"
          ],
          "quotes": [
            {
              "quote": "Published IP Addresses: https://developer.amazon.com/amazonbot/ip-addresses/",
              "source": "https://developer.amazon.com/amazonbot"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://developer.amazon.com/amazonbot"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://developer.amazon.com/amazonbot"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "mistral-ai",
      "permalink": "https://bozos.si/bot-verification.json#mistral-ai",
      "cite": "Mistral AI: published-ip-ranges documented (Lists for MistralAI-User and MistralAI-Index. The page shows no list under MistralAI-Training); reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#mistral-ai",
      "review_due": "2027-01-19",
      "operator": "Mistral AI",
      "crawler_tokens": [
        "MistralAI-Training",
        "MistralAI-Index",
        "MistralAI-User"
      ],
      "sources": [
        "https://docs.mistral.ai/robots"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "Lists for MistralAI-User and MistralAI-Index. The page shows no list under MistralAI-Training.",
          "lists": [
            "https://mistral.ai/mistralai-user-ips.json",
            "https://mistral.ai/mistralai-index-ips.json"
          ],
          "quotes": [
            {
              "quote": "All published IP addresses:",
              "source": "https://docs.mistral.ai/robots"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://docs.mistral.ai/robots"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://docs.mistral.ai/robots"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "duckduckgo",
      "permalink": "https://bozos.si/bot-verification.json#duckduckgo",
      "cite": "DuckDuckGo: published-ip-ranges documented (One list for DuckAssistBot); reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#duckduckgo",
      "review_due": "2027-01-19",
      "operator": "DuckDuckGo",
      "crawler_tokens": [
        "DuckAssistBot"
      ],
      "sources": [
        "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "One list for DuckAssistBot.",
          "lists": [
            "https://duckduckgo.com/duckassistbot.json"
          ],
          "quotes": [
            {
              "quote": "originates from the following IP addresses (also available in JSON at https://duckduckgo.com/duckassistbot.json )",
              "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "diffbot",
      "permalink": "https://bozos.si/bot-verification.json#diffbot",
      "cite": "Diffbot: published-ip-ranges not-stated; reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#diffbot",
      "review_due": "2027-01-19",
      "operator": "Diffbot",
      "crawler_tokens": [
        "Diffbot",
        "Diffbot-User"
      ],
      "sources": [
        "https://www.diffbot.com/docs/crawl/faq/robots-txt"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://www.diffbot.com/docs/crawl/faq/robots-txt"
            ],
            "terms": [
              "ip range",
              "ip address",
              "ips\\b",
              "\\.json",
              "cidr"
            ],
            "checked": "2026-10-21"
          }
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://www.diffbot.com/docs/crawl/faq/robots-txt"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://www.diffbot.com/docs/crawl/faq/robots-txt"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    },
    {
      "id": "you-com",
      "permalink": "https://bozos.si/bot-verification.json#you-com",
      "cite": "You.com: published-ip-ranges documented (A single /24 range stated in prose, not a machine-readable list); reverse-forward-dns documented (Described generically; the hostname pattern is not stated, and the page's worked example uses a Google address); web-bot-auth documented (YouBot, which the page calls the recommended method. Keys are Ed25519 JWKs). Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#you-com",
      "review_due": "2027-01-19",
      "operator": "You.com",
      "crawler_tokens": [
        "YouBot"
      ],
      "sources": [
        "https://you.com/docs/youbot"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "documented",
          "scope": "A single /24 range stated in prose, not a machine-readable list.",
          "lists": [],
          "quotes": [
            {
              "quote": "Legitimate YouBot requests originate from 68.67.112.0/24.",
              "source": "https://you.com/docs/youbot"
            }
          ]
        },
        "reverse-forward-dns": {
          "stance": "documented",
          "scope": "Described generically; the hostname pattern is not stated, and the page's worked example uses a Google address.",
          "hostnames": [],
          "quotes": [
            {
              "quote": "a forward lookup on that hostname should resolve back to the same IP",
              "source": "https://you.com/docs/youbot"
            }
          ]
        },
        "web-bot-auth": {
          "stance": "documented",
          "scope": "YouBot, which the page calls the recommended method. Keys are Ed25519 JWKs.",
          "signature_agent": null,
          "directory": "https://you.com/.well-known/http-message-signatures-directory",
          "quotes": [
            {
              "quote": "YouBot uses Cloudflare's Web Bot Auth standard for authentication.",
              "source": "https://you.com/docs/youbot"
            }
          ]
        }
      }
    },
    {
      "id": "allen-institute-for-ai",
      "permalink": "https://bozos.si/bot-verification.json#allen-institute-for-ai",
      "cite": "Allen Institute for AI (Ai2): published-ip-ranges not-stated; reverse-forward-dns not-stated; web-bot-auth not-stated. Source: BOZOS, Bot Verification: How to Tell the Real Crawler, verified 2026-10-21. https://bozos.si/bot-verification.json#allen-institute-for-ai",
      "review_due": "2027-01-19",
      "operator": "Allen Institute for AI (Ai2)",
      "crawler_tokens": [
        "AI2Bot"
      ],
      "sources": [
        "https://allenai.org/crawler"
      ],
      "verified": "2026-10-21",
      "cells": {
        "published-ip-ranges": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://allenai.org/crawler"
            ],
            "terms": [
              "ip range",
              "ip address",
              "ips\\b",
              "\\.json",
              "cidr"
            ],
            "checked": "2026-10-21"
          }
        },
        "reverse-forward-dns": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://allenai.org/crawler"
            ],
            "terms": [
              "reverse dns",
              "forward dns",
              "rdns",
              "in-addr",
              "\\bptr\\b",
              "dig -x",
              "host command"
            ],
            "checked": "2026-10-21"
          }
        },
        "web-bot-auth": {
          "stance": "not-stated",
          "scope": "No mention in the operator's pages checked.",
          "quotes": [],
          "searched": {
            "sources": [
              "https://allenai.org/crawler"
            ],
            "terms": [
              "web bot auth",
              "message signature",
              "signature-agent",
              "signature-input",
              "rfc 9421",
              "http-message-signatures"
            ],
            "checked": "2026-10-21"
          }
        }
      }
    }
  ],
  "observed_lists": [
    {
      "url": "https://openai.com/gptbot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-09-22T02:00:07.000000",
      "ipv4_prefixes": 18,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://openai.com/searchbot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-01-02T11:00:00.000000",
      "ipv4_prefixes": 39,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://openai.com/chatgpt-user.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-10-20T17:07:39.673498",
      "ipv4_prefixes": 234,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://openai.com/adsbot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-05-12T03:00:00.000000",
      "ipv4_prefixes": 2,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://claude.com/crawling/bots.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-10-20T00:00:00Z",
      "ipv4_prefixes": 14,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-10-20T14:46:18.000000",
      "ipv4_prefixes": 170,
      "ipv6_prefixes": 147
    },
    {
      "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-10-20T14:46:18.000000",
      "ipv4_prefixes": 170,
      "ipv6_prefixes": 147
    },
    {
      "url": "https://search.developer.apple.com/applebot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-09-15T10:00:00.000000",
      "ipv4_prefixes": 24,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://www.perplexity.com/perplexitybot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2025-02-07T16:56:00.000000",
      "ipv4_prefixes": 8,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://www.perplexity.com/perplexity-user.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2025-10-17T10:17:00.000000",
      "ipv4_prefixes": 4,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://index.commoncrawl.org/ccbot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-08-11T13:40:00Z",
      "ipv4_prefixes": 4,
      "ipv6_prefixes": 1
    },
    {
      "url": "https://developer.amazon.com/amazonbot/ip-addresses/",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "html-or-other"
    },
    {
      "url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "html-or-other"
    },
    {
      "url": "https://developer.amazon.com/amazonbot/live-ip-addresses/",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "html-or-other"
    },
    {
      "url": "https://mistral.ai/mistralai-user-ips.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2025-02-19T12:00:00.000000",
      "ipv4_prefixes": 4,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://mistral.ai/mistralai-index-ips.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-04-19T12:00:00.000000",
      "ipv4_prefixes": 2,
      "ipv6_prefixes": 0
    },
    {
      "url": "https://duckduckgo.com/duckassistbot.json",
      "observed": "2026-10-21",
      "http_status": 200,
      "format": "json",
      "creation_time": "2026-09-01T12:44:58.000000",
      "ipv4_prefixes": 486,
      "ipv6_prefixes": 0
    }
  ],
  "source_snapshots": [
    {
      "url": "https://commoncrawl.org/ccbot",
      "archived": "https://web.archive.org/web/20261005215406/https://commoncrawl.org/ccbot",
      "captured": "2026-10-05",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://datatracker.ietf.org/doc/draft-meunier-web-bot-auth-architecture/",
      "archived": "https://web.archive.org/web/20261005001906/https://datatracker.ietf.org/doc/draft-meunier-web-bot-auth-architecture/",
      "captured": "2026-10-05",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://datatracker.ietf.org/wg/webbotauth/about/",
      "archived": "https://web.archive.org/web/20261015201325/https://datatracker.ietf.org/wg/webbotauth/about/",
      "captured": "2026-10-15",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developer.amazon.com/amazonbot",
      "archived": "https://web.archive.org/web/20261021131042/https://developer.amazon.com/amazonbot",
      "captured": "2026-10-21",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth",
      "archived": "https://web.archive.org/web/20261015115007/https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth",
      "captured": "2026-10-15",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
      "archived": "https://web.archive.org/web/20261021124633/https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
      "captured": "2026-10-21",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://developers.openai.com/api/docs/bots",
      "archived": "https://web.archive.org/web/20261016233530/https://developers.openai.com/api/docs/bots",
      "captured": "2026-10-16",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://docs.mistral.ai/robots",
      "archived": "https://web.archive.org/web/20261010094213/https://docs.mistral.ai/robots",
      "captured": "2026-10-10",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://docs.perplexity.ai/guides/bots",
      "archived": "https://web.archive.org/web/20261021125425/https://docs.perplexity.ai/guides/bots",
      "captured": "2026-10-21",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "archived": "https://web.archive.org/web/20261013204641/https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "captured": "2026-10-13",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting",
      "archived": "https://web.archive.org/web/20261021125120/https://help.openai.com/en/articles/11845367-chatgpt-agent-allowlisting",
      "captured": "2026-10-21",
      "checked": "2026-10-21",
      "quotes_in_snapshot": false
    },
    {
      "url": "https://support.apple.com/en-us/119829",
      "archived": "https://web.archive.org/web/20261016065936/https://support.apple.com/en-us/119829",
      "captured": "2026-10-16",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "archived": "https://web.archive.org/web/20261016233536/https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "captured": "2026-10-16",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://www.rfc-editor.org/rfc/rfc9421.html",
      "archived": "https://web.archive.org/web/20261017150035/https://www.rfc-editor.org/rfc/rfc9421.html",
      "captured": "2026-10-17",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    },
    {
      "url": "https://you.com/docs/youbot",
      "archived": "https://web.archive.org/web/20261020091006/https://you.com/docs/youbot",
      "captured": "2026-10-20",
      "checked": "2026-10-21",
      "quotes_in_snapshot": true
    }
  ]
}