# ==========================================
# FREECV.ORG - ULTIMATE AI OPTIMIZED
# Verified for 2026 Standards
# ==========================================

# ------------------------------------------
# GROUP 1: THE "VIP" AI ALLOW LIST
# We explicitly list every major AI bot here to ensure
# they treat your site as "Opted-In" for traffic & training.
# ------------------------------------------

# OpenAI (ChatGPT / SearchGPT)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai

# Google (Gemini / Vertex AI)
# Note: Google-Extended is the specific token for AI training permission
User-agent: Google-Extended
User-agent: GoogleOther

# Perplexity (AI Search Engine)
User-agent: PerplexityBot
User-agent: Perplexity-User

# Apple (Apple Intelligence / Siri)
User-agent: Applebot
User-agent: Applebot-Extended

# Meta (Llama / Facebook)
User-agent: FacebookBot
User-agent: Meta-ExternalAgent

# Microsoft (Copilot / Bing)
User-agent: bingbot
User-agent: Microsoft-Bing

# Yandex (Yandex Search / Alice)
User-agent: YandexBot

# You.com
User-agent: YouBot

# Data Common Layers (Used by Open Source Models)
User-agent: CCBot
User-agent: cohere-ai
User-agent: Amazonbot
User-agent: Diffbot

# --- RULES FOR AI GROUP ---
# 1. ALLOW everything by default (Resume Builder, Templates, Blogs)
Allow: /

# 2. SECURITY & BUDGET BLOCKS
# Even AI bots should not crawl your admin or backend API.
# We explicitly list these here so the "Allow: /" doesn't override them.
Disallow: /admin/
Disallow: /private/
Disallow: /cv/api/
Disallow: /api/
Disallow: /auth/
Disallow: /tmp/

# 3. EXPLICIT ALLOWS (To prevent any confusion on critical files)
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /llms.txt

# ------------------------------------------
# GROUP 2: GENERAL CRAWLERS (Everyone Else)
# ------------------------------------------
User-agent: *
Allow: /

# Block backend scripts & sensitive areas
Disallow: /cv/api/
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /auth/
Disallow: /tmp/

# Block noisy development files (Safe to block for everyone)
Disallow: /*.lock$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.tar.gz$

# ------------------------------------------
# SITEMAP
# ------------------------------------------
Sitemap: https://freecv.org/sitemap.xml