# Warhammer 40K Damage Calculator crawler rules
# Public pages are crawlable; technical and non-indexable paths are excluded.

User-agent: *
Allow: /
Allow: /comparison
Disallow: /api/
Disallow: /_vite/
Disallow: /node_modules/
Disallow: /*.json
Disallow: /?q=
Disallow: /*?partner=
Disallow: /*&partner=
Disallow: /s/
Disallow: /brief/
Crawl-delay: 1

# AI search crawlers. Specific groups repeat the technical exclusions because
# they do not inherit the wildcard group's rules.

User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: MistralAI-Index
User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /s/
Disallow: /brief/

# User-requested fetchers may open deliberately shared results, but not APIs.

User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/

# Model-training crawlers retain the owner's existing allow decision while
# excluding technical and non-indexable paths.

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /s/
Disallow: /brief/

# Crawlers excluded entirely.

User-agent: DeepSeekBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Classic search engines.

User-agent: Googlebot
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /s/
Disallow: /brief/

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /s/
Disallow: /brief/
Crawl-delay: 1

Sitemap: https://tactical-cogitator.com/sitemap.xml

# AI-readable product documentation
# https://tactical-cogitator.com/llms.txt
# https://tactical-cogitator.com/llms-full.txt