# robots.txt generated by https://bozos.si/tools/robots-builder
# Crawler data: https://bozos.si/ai-crawlers.json (verified 2026-10-20)
# Policy: training=block search=allow user_fetch=allow datasets=block path=/
# robots.txt is a request, not access control. Some user-initiated fetchers state they may ignore it.

# OpenAI GPTBot: training. https://developers.openai.com/api/docs/bots
User-agent: GPTBot
Disallow: /

# OpenAI OAI-SearchBot: search. https://developers.openai.com/api/docs/bots
User-agent: OAI-SearchBot
Allow: /

# OpenAI ChatGPT-User: user-fetch. Operator says robots.txt may not apply. https://developers.openai.com/api/docs/bots
User-agent: ChatGPT-User
Allow: /

# Anthropic ClaudeBot: training. https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: ClaudeBot
Disallow: /

# Anthropic Claude-SearchBot: search. https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: Claude-SearchBot
Allow: /

# Anthropic Claude-User: user-fetch. https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: Claude-User
Allow: /

# Google Googlebot: search. https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
User-agent: Googlebot
Allow: /

# Google Google-Extended: control-token. Control token; no separate crawler. https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
User-agent: Google-Extended
Disallow: /

# Microsoft bingbot: search. https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
User-agent: bingbot
Allow: /

# Apple Applebot: search. https://support.apple.com/en-us/119829
User-agent: Applebot
Allow: /

# Apple Applebot-Extended: control-token. Control token; no separate crawler. https://support.apple.com/en-us/119829
User-agent: Applebot-Extended
Disallow: /

# Perplexity PerplexityBot: search. https://docs.perplexity.ai/guides/bots
User-agent: PerplexityBot
Allow: /

# Perplexity Perplexity-User: user-fetch. Operator says robots.txt may not apply. https://docs.perplexity.ai/guides/bots
User-agent: Perplexity-User
Allow: /

# Meta meta-externalagent: training. https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
User-agent: meta-externalagent
Disallow: /

# Meta meta-webindexer: search. https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
User-agent: meta-webindexer
Allow: /

# Meta meta-externalfetcher: user-fetch. Operator says robots.txt may not apply. https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
User-agent: meta-externalfetcher
Allow: /

# Common Crawl CCBot: open-dataset. https://commoncrawl.org/ccbot
User-agent: CCBot
Disallow: /

# Amazon Amazonbot: training. https://developer.amazon.com/amazonbot
User-agent: Amazonbot
Disallow: /

# Amazon Amzn-SearchBot: search. https://developer.amazon.com/amazonbot
User-agent: Amzn-SearchBot
Allow: /

# Amazon Amzn-User: user-fetch. Operator says robots.txt may not apply. https://developer.amazon.com/amazonbot
User-agent: Amzn-User
Allow: /

# Mistral AI MistralAI-Training: training. https://docs.mistral.ai/robots
User-agent: MistralAI-Training
Disallow: /

# Mistral AI MistralAI-Index: search. https://docs.mistral.ai/robots
User-agent: MistralAI-Index
Allow: /

# Mistral AI MistralAI-User: user-fetch. https://docs.mistral.ai/robots
User-agent: MistralAI-User
Allow: /

# DuckDuckGo DuckAssistBot: user-fetch. https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot
User-agent: DuckAssistBot
Allow: /

# Diffbot Diffbot: search. https://www.diffbot.com/docs/crawl/faq/robots-txt
User-agent: Diffbot
Allow: /

# Diffbot Diffbot-User: user-fetch. https://www.diffbot.com/docs/crawl/faq/robots-txt
User-agent: Diffbot-User
Allow: /

# You.com YouBot: search. https://you.com/docs/youbot
User-agent: YouBot
Allow: /

# Allen Institute for AI (Ai2) AI2Bot: training. https://allenai.org/crawler
User-agent: AI2Bot
Disallow: /

User-agent: *
Allow: /