# robots.txt for dpe.org
# Rules for search engines and AI crawlers

# --- Search Engine Crawlers ---
User-agent: *
Allow: /

# Point to the sitemap index (this file links to other sitemaps)
Sitemap: https://dpe.org/sitemap.xml

# --- AI Crawlers (mirrors llms.txt) ---

# OpenAI (ChatGPT / GPTBot)
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI (Bard / Gemini crawlers)
User-agent: Google-Extended
Allow: /

# Microsoft Bing AI
User-agent: BingAI
Allow: /

# Commoncrawl (feeds many AI models)
User-agent: CCBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Neeva AI (Snowflake)
User-agent: NeevaAI
Allow: /

# Stability AI
User-agent: StabilityAI
Allow: /

# Cohere
User-agent: CohereAI
Allow: /