# BlueCap Strategies robots.txt — see docs/seo-aeo-optimization.md §5
# Policy: allow classic search + AI answer/search crawlers for maximum discoverability and
# answer-engine citation. Training-crawler policy (GPTBot, ClaudeBot, Google-Extended) is an
# owner decision — currently ALLOWED for maximum reach. To opt out of model training, uncomment
# the "Disallow: /" lines in the training-crawler block below.

# --- Classic search engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# --- AI answer / search crawlers (these can cite us and send referral traffic) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

# Google-Extended controls Gemini + Google AI Overviews (this is also a training signal for
# Google — treated as answer-visibility here; move to the training block to opt out).
User-agent: Google-Extended
Allow: /

# --- AI model-training crawlers (owner decision — allowed by default) ---
User-agent: GPTBot
Allow: /
# Disallow: /

User-agent: ClaudeBot
Allow: /
# Disallow: /

# --- Everything else ---
User-agent: *
Allow: /

# Note: non-compliant scrapers (e.g. Bytespider, stealth crawlers) ignore robots.txt; block those
# at the Cloudflare edge, not here (see docs/gitops-deployment-strategy.md §4.2).

Sitemap: https://www.bluecapstrategies.com/sitemap-index.xml