# robots.txt for x02.me
# Last updated: 2026-09-10

# Allow all major search engines and AI search engines
User-agent: *
Allow: /

# Disallow private user dashboards, authentication routes, and internal endpoints
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /anonymous
Disallow: /oversight
Disallow: /quarantine-management
Disallow: /quarantine/
Disallow: /auth/reset-password
Disallow: /auth/verify-email

# Googlebot, Bingbot and Applebot deliberately have no group of their own: a
# crawler that matches a named group ignores every rule under "*", so a
# "User-agent: Googlebot / Allow: /" group silently un-blocks everything above.

# AI Search & Retrieval Crawlers (ChatGPT Search, Perplexity, Claude, Gemini)
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

# ClaudeBot is the current Anthropic crawler; Claude-Web above is the legacy
# name and is kept so an older crawler still matches a group of its own.
User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

# OAI-SearchBot indexes for ChatGPT Search and is separate from GPTBot, which
# crawls for training. Both are allowed; they are listed apart because blocking
# one without the other is a decision worth making deliberately.
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard

# Social Media Link Preview Bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Discordbot
Allow: /

# Disallow abusive scrapers
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

# Sitemap location
Sitemap: https://x02.me/sitemap.xml

# Article feed and agent-oriented site summary
# https://x02.me/rss.xml
# https://x02.me/llms.txt
# https://x02.me/llms-full.txt
# MCP server card: https://x02.me/.well-known/mcp.json