# robots.txt for Alpha Compute Corp. (alphacompute.ai)
# Optimized for AI Discovery, AEO/GEO, and Institutional & Financial Research
# Last updated: April 2026

# =============================================================
# Default: Allow all crawlers, block sensitive paths
# =============================================================
User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /_vercel/

# Ensure all crawlers can find our LLM source-of-truth file
Allow: /llms.txt

# =============================================================
# Traditional Search Engine Crawlers
# =============================================================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: MojeekBot
Allow: /

# =============================================================
# AI Answer Engine and Retrieval Crawlers
# (Real-time retrieval for AI answers — critical for AEO/GEO
#  and increasingly used by institutional analysts for
#  company research and due diligence)
# =============================================================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Operator
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: Google-NotebookLM
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Microsoft / Bing Copilot is covered by Bingbot above

# Mistral AI
User-agent: MistralAI-User
Allow: /

User-agent: MistralAI-Index
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# xAI
User-agent: Grok
Allow: /

User-agent: Grokbot
Allow: /

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Phind
User-agent: PhindBot
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# Allen Institute for AI
User-agent: AI2Bot
Allow: /

User-agent: AI2Bot-Dolma
Allow: /

# Tavily (AI research)
User-agent: TavilyBot
Allow: /

# Liner
User-agent: LinerBot
Allow: /

# =============================================================
# AI Training Crawlers
# (Maximize inclusion in foundation model training data)
# =============================================================
User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: Omgili
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: PetalBot
Allow: /

# =============================================================
# Financial, Institutional & Analyst Research Tools
# (SEO and research crawlers commonly used by sell-side analysts,
#  data-room providers, and institutional intelligence platforms
#  for competitive analysis and company research)
# =============================================================
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-OCOB
Allow: /

User-agent: SemrushBot-SWA
Allow: /

User-agent: DataForSeoBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: Sogou web spider
Allow: /

# Professional network preview and reference crawlers
User-agent: LinkedInBot
Allow: /

User-agent: TwitterBot
Allow: /

# =============================================================
# Archival & Historical Reference
# (Preserves a public-record trail of corporate communications,
#  useful for investor due diligence and compliance review)
# =============================================================
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

# =============================================================
# Sitemap and LLM Source Files
# =============================================================
Sitemap: https://alphacompute.ai/sitemap.xml

# AI source-of-truth file:
# https://alphacompute.ai/llms.txt