# BlueCap Strategies robots.txt — see docs/seo-aeo-optimization.md §5
# Policy: allow classic search + AI answer/search crawlers for maximum discoverability and
# answer-engine citation. Model-TRAINING crawlers (GPTBot, ClaudeBot, Google-Extended, …) are BLOCKED (owner decision, 2026-10-23).

# --- Classic search engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# --- AI answer / search crawlers (these can cite us and send referral traffic) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

# --- AI model-training crawlers — blocked (owner decision, 2026-10-23) ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Bytespider
Disallow: /

# Google-Extended is Google's opt-out for Gemini training/grounding only. Google Search and AI
# Overviews are governed by Googlebot (allowed above), so blocking this does not affect search.
User-agent: Google-Extended
Disallow: /

# --- Everything else ---
User-agent: *
Allow: /

# Note: non-compliant scrapers (e.g. Bytespider, stealth crawlers) ignore robots.txt; block those
# at the Cloudflare edge, not here (see docs/gitops-deployment-strategy.md §4.2).

Sitemap: https://www.bluecapstrategies.com/sitemap-index.xml