# OffMeta - Natural Language Magic Card Search
# https://offmeta.app
#
# Grouping rule: robots.txt parsers pick the single most-specific User-agent
# group per crawler and many do NOT merge duplicate groups. So every rule for
# a given UA must live in ONE block below (previously GPTBot / Google-Extended
# / PerplexityBot had a second "Allow: /ai/" block that most crawlers ignored,
# leaving the site fully blocked for them).

# Block AI/LLM training crawlers, but keep /ai/ open for AEO on the ones that
# also power answer engines.
User-agent: GPTBot
Allow: /ai/
Disallow: /

User-agent: Google-Extended
Allow: /ai/
Disallow: /

# Answer-engine search indexers and on-demand fetchers. These power citations
# and user-initiated lookups, not model training, so they stay allowed.
# Vendor documentation confirming the split:
#   OpenAI      GPTBot (training) / OAI-SearchBot (search) / ChatGPT-User (user)
#   Anthropic   ClaudeBot (training) / Claude-SearchBot (search) / Claude-User (user)
#               https://support.claude.com/en/articles/8896518
#   Perplexity  PerplexityBot (search index, explicitly not training) /
#               Perplexity-User (user) https://docs.perplexity.ai/docs/resources/perplexity-crawlers
User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/

User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/

User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/

# Training crawlers, blocked outright.
User-agent: ClaudeBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

# Default policy for every other crawler (Googlebot, Bingbot, DuckDuckBot,
# ChatGPT-User, OAI-SearchBot, etc.)
User-agent: *
Allow: /
Disallow: /api
Disallow: /api/
# Block legacy parameterised search URLs — redirected to /search/:slug
Disallow: /?q=
Disallow: /?*q=
# Block private/admin surface. Paths are prefixes and the app routes have no
# trailing slash (/profile, /reset-password), so the slashless form is required.
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /user/
Disallow: /.lovable/

# Sitemap location
Sitemap: https://offmeta.app/sitemap.xml