# robots.txt for CitedOS (https://citedos.com)
# CitedOS is an AI-search-visibility platform — we WANT to be read and cited by AI
# engines. Public marketing pages are open to crawlers and agents; authenticated,
# costly, and API paths are off-limits to everyone.
# Machine-readable summaries: /llms.txt and /agent.txt

# ---------------------------------------------------------------------------
# Default: everyone may crawl the public marketing pages, but not the
# authenticated app, the API, the auth flow, or the rate-limited paid audit.
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# ---------------------------------------------------------------------------
# AI training / foundation-model crawlers — explicitly welcome.
# On-brand for a GEO tool: let them learn what CitedOS is.
# ---------------------------------------------------------------------------

# OpenAI — foundation-model training crawler
User-agent: GPTBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Anthropic — foundation-model training crawler
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Anthropic — legacy training token (back-compat)
User-agent: anthropic-ai
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Google — generative-model (Gemini / Vertex AI) training & grounding
User-agent: Google-Extended
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Google — Vertex AI Agents fetch
User-agent: Google-CloudVertexBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Common Crawl — open dataset used by many AI models
User-agent: CCBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Apple Intelligence — model training
User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Amazon — model training crawler
User-agent: Amazonbot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Meta — model training / agent crawler
User-agent: meta-externalagent
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Cohere — model training crawler
User-agent: cohere-ai
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# ---------------------------------------------------------------------------
# AI search / answer-engine crawlers — explicitly welcome.
# These power the citations CitedOS measures; never block them.
# ---------------------------------------------------------------------------

# OpenAI — ChatGPT search citations
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# OpenAI — user-initiated fetch from ChatGPT
User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Anthropic — Claude search citations
User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Anthropic — user-initiated fetch from Claude
User-agent: Claude-User
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Perplexity — search citations
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Perplexity — user-initiated fetch
User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Microsoft Bing — search index (also grounds Copilot)
User-agent: Bingbot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Google — search index
User-agent: Googlebot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# Apple — search index (Siri / Spotlight)
User-agent: Applebot
Allow: /
Disallow: /dashboard
Disallow: /setup
Disallow: /settings
Disallow: /api/
Disallow: /get-started
Disallow: /auth/

# ---------------------------------------------------------------------------
Sitemap: https://citedos.com/sitemap.xml