# robots.txt for baileyengineers.com (production)
# Reviewed: 2026-04-20
#
# Purpose: explicitly allow traditional search crawlers AND
# AI/LLM crawlers, to maximize discoverability in both
# traditional search and AI-driven search (ChatGPT, Claude,
# Perplexity, Gemini).

# Default: allow all traditional crawlers (Google, Bing, etc.)
User-agent: *
Allow: /

# --- AI crawlers: explicit allow ---

# OpenAI's GPTBot - crawls for ChatGPT training data
User-agent: GPTBot
Allow: /

# Anthropic's ClaudeBot - crawls for Claude training data
User-agent: ClaudeBot
Allow: /

# Anthropic's Claude-Web (real-time browsing)
User-agent: Claude-Web
Allow: /

# Perplexity AI - real-time web retrieval for answers
User-agent: PerplexityBot
Allow: /

# Google-Extended - Google's AI training opt-in crawler
# (controls whether content is used for Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /

# OpenAI's real-time search crawler
User-agent: OAI-SearchBot
Allow: /

# ChatGPT's browsing-on-behalf-of-users bot
User-agent: ChatGPT-User
Allow: /

# Apple's AI training crawler
User-agent: Applebot-Extended
Allow: /

# --- AI crawlers: explicit disallow ---
# Use this section to OPT OUT of specific crawlers if desired.
# Currently empty - Bailey is opting in to all major AI crawlers.

# --- Content-Signal (isitagentready.com / emerging convention) ---
# Declares per-purpose consent for AI/agent use of this site:
#   ai-train  = yes  : permit use of public content in AI training corpora
#   search    = yes  : permit indexing for traditional + AI search results
#   ai-input  = yes  : permit live retrieval/summarization by AI assistants
# Aligned with Bailey's AI-visibility SEO strategy; do NOT flip ai-train to
# "no" without revisiting that strategy.
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# --- Sitemap location ---
Sitemap: https://baileyengineers.com/sitemap-index.xml