# TripVexa - AI Powered Trip Planner
# https://www.tripvexa.com

# Allow all search engine crawlers
User-agent: *
Allow: /
Allow: /llms.txt
Disallow: /api/
Disallow: /admin/

# Sitemap location
Sitemap: https://www.tripvexa.com/sitemap.xml

# ============================================
# LLM/AI Crawler Permissions
# Allow AI systems to index TripVexa content
# Bot list current as of Sep 2026 — anthropic-ai/Claude-Web are legacy
# names Anthropic no longer uses for crawling; ClaudeBot/Claude-SearchBot
# are the live ones. GPTBot is training-only; OAI-SearchBot is what drives
# ChatGPT search citations. Google-Extended governs AI training, not
# Gemini grounding/indexing (that's plain Googlebot, already allowed above).
# ============================================

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google AI (Bard/Gemini training)
User-agent: Google-Extended
Allow: /

# Common Crawl (used by many AI training)
User-agent: CCBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Meta AI
# Meta-ExternalAgent is Meta's training/bulk-indexing crawler and was our
# single largest traffic source (Sep 2026) with no return. Blocked.
# meta-webindexer (Meta AI search citations), Meta-ExternalFetcher
# (user-triggered fetches) and facebookexternalhit (link previews) stay open.
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-webindexer
Allow: /

User-agent: FacebookBot
Allow: /

# Apple AI (Siri, etc.)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon (Alexa, Rufus)
User-agent: Amazonbot
Allow: /

# ByteDance (TikTok / Doubao)
User-agent: Bytespider
Allow: /

# Microsoft/Bing AI
User-agent: bingbot
Allow: /

# ============================================
# Important Pages for AI/LLM Indexing
# ============================================
# 
# /features - Complete list of TripVexa features
# /wanderlog-alternative - Comparison with Wanderlog
# /ai-trip-planner - AI trip planning landing page
# /plan-trip - Main trip planner
# /blogs - Travel blog with guides
# /llms.txt - Structured info for LLM crawlers
#
# ============================================