{
  "answer": "For /blog: 1 of 32 directory crawlers are disallowed (GPTBot); the rest are allowed.",
  "path": "/blog",
  "standard": "RFC 9309",
  "results": [
    {
      "token": "GPTBot",
      "operator": "OpenAI",
      "category": "training",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#gptbot",
      "allowed": false,
      "matched_group": [
        "GPTBot"
      ],
      "used_fallback": false,
      "rule": {
        "directive": "disallow",
        "pattern": "/",
        "line": 2
      },
      "reason": "Longest matching rule: Disallow: / (line 2)"
    },
    {
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#oai-searchbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "category": "user-fetch",
      "robots_txt_behavior": "may-ignore",
      "permalink": "https://bozos.si/ai-crawlers.json#chatgpt-user",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)",
      "caution": "OpenAI says ChatGPT-User may not follow robots.txt for user-initiated fetches."
    },
    {
      "token": "OAI-AdsBot",
      "operator": "OpenAI",
      "category": "other",
      "robots_txt_behavior": "not-stated",
      "permalink": "https://bozos.si/ai-crawlers.json#oai-adsbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "category": "training",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#claudebot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#claude-searchbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Claude-User",
      "operator": "Anthropic",
      "category": "user-fetch",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#claude-user",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Googlebot",
      "operator": "Google",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#googlebot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Google-Extended",
      "operator": "Google",
      "category": "control-token",
      "robots_txt_behavior": "control-token",
      "permalink": "https://bozos.si/ai-crawlers.json#google-extended",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "GoogleOther",
      "operator": "Google",
      "category": "other",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#googleother",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Google-CloudVertexBot",
      "operator": "Google",
      "category": "owner-requested",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#google-cloudvertexbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "bingbot",
      "operator": "Microsoft",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#bingbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Applebot",
      "operator": "Apple",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#applebot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Applebot-Extended",
      "operator": "Apple",
      "category": "control-token",
      "robots_txt_behavior": "control-token",
      "permalink": "https://bozos.si/ai-crawlers.json#applebot-extended",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#perplexitybot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "category": "user-fetch",
      "robots_txt_behavior": "may-ignore",
      "permalink": "https://bozos.si/ai-crawlers.json#perplexity-user",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)",
      "caution": "Perplexity says Perplexity-User may not follow robots.txt for user-initiated fetches."
    },
    {
      "token": "meta-externalagent",
      "operator": "Meta",
      "category": "training",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#meta-externalagent",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "meta-webindexer",
      "operator": "Meta",
      "category": "search",
      "robots_txt_behavior": "not-stated",
      "permalink": "https://bozos.si/ai-crawlers.json#meta-webindexer",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "meta-externalfetcher",
      "operator": "Meta",
      "category": "user-fetch",
      "robots_txt_behavior": "may-ignore",
      "permalink": "https://bozos.si/ai-crawlers.json#meta-externalfetcher",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)",
      "caution": "Meta says meta-externalfetcher may not follow robots.txt for user-initiated fetches."
    },
    {
      "token": "facebookexternalhit",
      "operator": "Meta",
      "category": "link-preview",
      "robots_txt_behavior": "may-ignore",
      "permalink": "https://bozos.si/ai-crawlers.json#facebookexternalhit",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)",
      "caution": "Meta says facebookexternalhit may not follow robots.txt for user-initiated fetches."
    },
    {
      "token": "CCBot",
      "operator": "Common Crawl",
      "category": "open-dataset",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#ccbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Amazonbot",
      "operator": "Amazon",
      "category": "training",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#amazonbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Amzn-SearchBot",
      "operator": "Amazon",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#amzn-searchbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Amzn-User",
      "operator": "Amazon",
      "category": "user-fetch",
      "robots_txt_behavior": "may-ignore",
      "permalink": "https://bozos.si/ai-crawlers.json#amzn-user",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)",
      "caution": "Amazon says Amzn-User may not follow robots.txt for user-initiated fetches."
    },
    {
      "token": "MistralAI-Training",
      "operator": "Mistral AI",
      "category": "training",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#mistralai-training",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "MistralAI-Index",
      "operator": "Mistral AI",
      "category": "search",
      "robots_txt_behavior": "not-stated",
      "permalink": "https://bozos.si/ai-crawlers.json#mistralai-index",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "MistralAI-User",
      "operator": "Mistral AI",
      "category": "user-fetch",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#mistralai-user",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "category": "user-fetch",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#duckassistbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Diffbot",
      "operator": "Diffbot",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#diffbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "Diffbot-User",
      "operator": "Diffbot",
      "category": "user-fetch",
      "robots_txt_behavior": "not-stated",
      "permalink": "https://bozos.si/ai-crawlers.json#diffbot-user",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "YouBot",
      "operator": "You.com",
      "category": "search",
      "robots_txt_behavior": "obeys",
      "permalink": "https://bozos.si/ai-crawlers.json#youbot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    },
    {
      "token": "AI2Bot",
      "operator": "Allen Institute for AI (Ai2)",
      "category": "training",
      "robots_txt_behavior": "not-stated",
      "permalink": "https://bozos.si/ai-crawlers.json#ai2bot",
      "allowed": true,
      "matched_group": [
        "*"
      ],
      "used_fallback": true,
      "rule": {
        "directive": "allow",
        "pattern": "/",
        "line": 5
      },
      "reason": "Longest matching rule: Allow: / (line 5)"
    }
  ],
  "groups": [
    {
      "user_agents": [
        "GPTBot"
      ],
      "rules": 1,
      "starts_at_line": 1
    },
    {
      "user_agents": [
        "*"
      ],
      "rules": 1,
      "starts_at_line": 4
    }
  ],
  "sitemaps": [],
  "content_signals": [],
  "crawl_delays": [],
  "warnings": [],
  "note": "robots.txt is a request, not access control. Crawl-delay and Content-Signal are not part of RFC 9309, and support varies by crawler.",
  "deterministic": true,
  "docs": "https://bozos.si/tools/fetch#robots-check"
}