# ============================================================
# robots.txt — grito.in
# GEO-optimised: granular AI crawler policy.
#
# Strategy:
#   - AI search/retrieval bots: full public access (ChatGPT live search,
#     Perplexity, Claude search, Bing/Copilot).
#   - AI training bots: curated path-only access (/llms.txt, /course,
#     /about, /blog, /learn). This allows baseline model knowledge of
#     Grito while protecting checkout, dashboard, and API routes.
#   - Common Crawl (CCBot): blocked — non-attributable bulk scraper.
#
# Last updated: 2026-06-15
# ============================================================

# ── Standard web crawlers ──────────────────────────────────
User-agent: *
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# ── OpenAI ────────────────────────────────────────────────
# GPTBot: model training — allow curated paths for baseline model knowledge
User-agent: GPTBot
Allow: /llms.txt
Allow: /course
Allow: /about
Allow: /blog/
Allow: /learn/
Allow: /resources/
Disallow: /

# OAI-SearchBot: ChatGPT live search — full public access
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# ChatGPT-User: ChatGPT on-demand page fetch — full public access
User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# ── Anthropic / Claude ─────────────────────────────────────
# ClaudeBot: model training — allow curated paths only
User-agent: ClaudeBot
Allow: /llms.txt
Allow: /course
Allow: /about
Allow: /blog/
Allow: /learn/
Allow: /resources/
Disallow: /

# Claude-SearchBot: Claude live search — full public access
User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# Claude-User: Claude on-demand fetch — full public access
User-agent: Claude-User
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# ── Google ─────────────────────────────────────────────────
# Googlebot: standard Google Search — full public access (core SEO)
User-agent: Googlebot
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# Google-Extended: Gemini/AI training — allow curated paths only
User-agent: Google-Extended
Allow: /llms.txt
Allow: /course
Allow: /about
Allow: /blog/
Allow: /learn/
Allow: /resources/
Disallow: /

# ── Perplexity ─────────────────────────────────────────────
# PerplexityBot: AI answer engine — full public access
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# ── Common Crawl ───────────────────────────────────────────
# CCBot: non-attributable bulk scraper — blocked entirely
User-agent: CCBot
Disallow: /

# ── Meta / Facebook ────────────────────────────────────────
User-agent: FacebookBot
Allow: /
Disallow: /dashboard/
Disallow: /api/

# ── Apple ──────────────────────────────────────────────────
User-agent: Applebot
Allow: /

# ── Bing / Microsoft Copilot ───────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /api/
Disallow: /internal/
Disallow: /ads/

# ============================================================
Sitemap: https://grito.in/sitemap.xml