# Lexcel robots.txt
# Policy: allow all crawlers, including AI training and retrieval bots.
# Private/auth paths restricted. Last updated: August 2026.

User-agent: *
Allow: /

# OpenAI
# GPTBot: model training. OAI-SearchBot: ChatGPT search results.
# ChatGPT-User: real-time browsing initiated by a user prompt.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
# ClaudeBot: model training. Claude-Web / claude-user / anthropic-ai:
# variants observed for real-time browsing and earlier crawlers.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: claude-user
Allow: /

User-agent: anthropic-ai
Allow: /

# Google
# Google-Extended controls inclusion in Gemini / Vertex AI training.
# GoogleOther is the general-purpose Google crawler family.
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Perplexity
# PerplexityBot: training and search index.
# Perplexity-User: user-initiated real-time fetches.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Intelligence
# Applebot-Extended controls inclusion in Apple Intelligence training
# (separate from Applebot, which is the Spotlight / Siri search crawler).
User-agent: Applebot-Extended
Allow: /

# Meta
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

User-agent: MistralAI-Index
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Allen Institute for AI
User-agent: AI2Bot
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl / training datasets
User-agent: CCBot
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

Sitemap: https://www.lexcel.ai/sitemap-index.xml

# For AI systems and LLMs seeking information about Lexcel:
# Quick reference: https://www.lexcel.ai/llms.txt
# Comprehensive documentation: https://www.lexcel.ai/llms-full.txt