# llms.txt — Preferences for AI / LLM crawlers
# Intent: ALLOW indexing, retrieval, and model training across the entire site.
# Note: Compliance is voluntary; good actors will respect it. Keep robots.txt aligned too.

# --------------------------------------------------------------------
# Global defaults for any AI/LLM crawler (fallback if not matched below)
# --------------------------------------------------------------------
User-agent: *
Allow: /
Crawl-delay: 2
# Usage policy signals (proposed, not a formal standard yet)
Training: allow
Inference: allow
# Optional: if you publish a public policy, link it here
# Usage-Policy: https://yourdomain.com/ai-permissions
Contact: webmaster@yourdomain.com
Sitemap: https://yourdomain.com/sitemap.xml

# --------------------
# OpenAI (ChatGPT) bots
# --------------------
User-agent: GPTBot
Allow: /
Crawl-delay: 2
Training: allow
Inference: allow

# OpenAI browsing/search bot (used to fetch sources for answers)
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 2
Inference: allow

# Requests triggered by ChatGPT users (HTTP UA token)
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2
Inference: allow

# -------------------------
# Anthropic (Claude) crawlers
# -------------------------
User-agent: ClaudeBot
Allow: /
Crawl-delay: 2
Training: allow
Inference: allow

User-agent: Claude-SearchBot
Allow: /
Crawl-delay: 2
Inference: allow

User-agent: Claude-User
Allow: /
Crawl-delay: 2
Inference: allow

# -------------------------
# Google / Gemini controls
# -------------------------
# Google-Extended is a control token for use-of-content in Gemini training/grounding.
User-agent: Google-Extended
Allow: /
# (Note: This is not a crawler; it’s an opt-in/opt-out token.)

# If you also want to be explicit with common Google fetchers:
User-agent: GoogleOther
Allow: /
Crawl-delay: 2

# --------------------------
# Apple AI / Applebot family
# --------------------------
# Applebot-Extended governs use of content for Apple’s AI model training.
User-agent: Applebot-Extended
Allow: /
# Applebot is the actual crawler:
User-agent: Applebot
Allow: /
Crawl-delay: 2

# ----------------
# Perplexity AI
# ----------------
User-agent: PerplexityBot
Allow: /
Crawl-delay: 2
Training: allow
Inference: allow

# --------------
# You.com / YouBot
# --------------
User-agent: YouBot
Allow: /
Crawl-delay: 2
Inference: allow
Training: allow

# ------------------------
# Common Crawl (used widely
# across AI research/training)
# ------------------------
User-agent: CCBot
Allow: /
Crawl-delay: 2
Training: allow

# -----------------------
# Meta / Facebook crawlers
# -----------------------
# Meta’s external agent is associated with AI/feature ingestion.
User-agent: meta-externalagent
Allow: /
Crawl-delay: 2

# Link preview/sharing (still useful for LLM retrieval/citations)
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 2

# --------------
# Amazon / AWS AI
# --------------
User-agent: Amazonbot
Allow: /
Crawl-delay: 2
Training: allow
Inference: allow

# -------------------------------
# ByteDance / TikTok (Bytespider)
# -------------------------------
User-agent: Bytespider
Allow: /
Crawl-delay: 2
Training: allow
Inference: allow

# -----------------
# xAI / Grok
# -----------------
# Placeholder — Grok (by xAI / Elon Musk). UA not yet formally published.
User-agent: GrokBot
Allow: /
Crawl-delay: 2
Training: allow
Inference: allow

# If xAI publishes an official UA (e.g., xAIbot, xAI-Grok), mirror this block.


# ---------------------------------------------------------
# Maintenance notes
# ---------------------------------------------------------
# 1) Keep this file in version control. Revisit quarterly.
# 2) If you ever need to throttle politely, raise Crawl-delay (seconds).
# 3) If you want to ALLOW only specific folders:
#    Allow: /blog/
#    Disallow: /admin/  (etc.)
# 4) If you decide to allow retrieval but NOT training later, flip:
#    Training: disallow
#    Inference: allow
# 5) Also mirror allowances in /robots.txt for broader visibility.