# robots.txt — Willian Souza SEO
# https://williansouza.dev.br/
#
# Política de /propostas/:
#   Propostas comerciais são documentos privados de cliente.
#   - Google e Bing PODEM rastrear a pasta de propósito: só assim leem o
#     X-Robots-Tag "noindex" (definido no vercel.json) e removem a URL do índice.
#     Bloquear aqui impediria a leitura do noindex e a URL poderia ficar
#     indexada "às cegas" (sem snippet) via links externos.
#   - Todos os demais bots (IA, SEO, genéricos) ficam bloqueados na origem.

# === Bots tradicionais (crawl liberado; noindex resolve /propostas/) ===
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Disallow: /propostas/

User-agent: Slurp
Disallow: /propostas/

User-agent: Yandex
Disallow: /propostas/

User-agent: Baiduspider
Disallow: /propostas/

# === Bots de redes sociais (preview de link) ===
User-agent: facebookexternalhit
Disallow: /propostas/

User-agent: Twitterbot
Disallow: /propostas/

User-agent: LinkedInBot
Disallow: /propostas/

# === Bots de IA Generativa (GEO) ===
# Liberar o site para que a marca seja citada em respostas de LLMs,
# exceto propostas comerciais de clientes.
User-agent: GPTBot
Disallow: /propostas/

User-agent: ChatGPT-User
Disallow: /propostas/

User-agent: OAI-SearchBot
Disallow: /propostas/

User-agent: ClaudeBot
Disallow: /propostas/

User-agent: Claude-Web
Disallow: /propostas/

User-agent: anthropic-ai
Disallow: /propostas/

User-agent: PerplexityBot
Disallow: /propostas/

User-agent: Perplexity-User
Disallow: /propostas/

User-agent: Google-Extended
Disallow: /propostas/

User-agent: Applebot
Disallow: /propostas/

User-agent: Applebot-Extended
Disallow: /propostas/

User-agent: Bytespider
Disallow: /propostas/

User-agent: meta-externalagent
Disallow: /propostas/

User-agent: cohere-ai
Disallow: /propostas/

User-agent: YouBot
Disallow: /propostas/

User-agent: DiffBot
Disallow: /propostas/

User-agent: Amazonbot
Disallow: /propostas/

User-agent: MistralAI-User
Disallow: /propostas/

User-agent: Meta-ExternalFetcher
Disallow: /propostas/

# === Crawlers de SEO (auditoria) ===
User-agent: AhrefsBot
Disallow: /propostas/
Crawl-delay: 5

User-agent: SemrushBot
Disallow: /propostas/
Crawl-delay: 5

User-agent: MJ12bot
Disallow: /propostas/
Crawl-delay: 5

User-agent: rogerbot
Disallow: /propostas/
Crawl-delay: 5

# === Regra padrão para qualquer outro bot ===
User-agent: *
Disallow: /propostas/
Disallow: /.claude/
Disallow: /.vercel/

# === Sitemap ===
Sitemap: https://williansouza.dev.br/sitemap.xml