# ============================================================
# BILARNA.COM – robots.txt
# API is hosted on api.bilarna.com (separate domain, not here)
# Last updated: 2026-06-12
# ============================================================


# ------------------------------------------------------------
# 1. DEFAULT – All crawlers
# ------------------------------------------------------------
User-agent: *
Allow: /

# Explicit allow for AI/agent machine-readable files
Allow: /llms.txt
Allow: /agent-manifest.json
Allow: /.well-known/mcp.json
Allow: /provider/*/agent.json
Allow: /provider/*/llms.txt
Allow: /provider/*/openapi.yaml

# API is served from a separate domain (api.bilarna.com) — never crawl the
# same-origin proxy path here, it wastes crawl budget and backend capacity.
Disallow: /api/

# Throttle default crawl rate — backend runs on a small 2 vCPU host and cannot
# sustain unthrottled concurrent crawling of category/provider pages.
Crawl-delay: 5


# ------------------------------------------------------------
# 2. SEARCH ENGINES – Standard crawlers
# ------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /


# ------------------------------------------------------------
# 3. AI SEARCH & ANSWER ENGINES
#    (indexing + recommendations, NOT training)
# ------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Grok
Allow: /

User-agent: xAI-ContentAgent
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: MistralBot
Allow: /

User-agent: iaskspider
Allow: /

User-agent: YouBot
Allow: /

# ByteDance AI – search/indexing access granted, throttled (known for aggressive crawl rates)
User-agent: Bytespider
Allow: /
Crawl-delay: 20

# Common Crawl – access granted (open web index)
User-agent: CCBot
Allow: /


# ------------------------------------------------------------
# 4. AI TRAINING CRAWLERS – Block (content protection)
# ------------------------------------------------------------
User-agent: Applebot-Extended
Disallow: /

User-agent: AI2Bot-Dolma
Disallow: /

User-agent: Ai2Bot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: *
Disallow: /cdn-cgi/

# ------------------------------------------------------------
# 5. SEO TOOLS – Allow (monitoring & analytics), throttled
#    These are non-search-engine bots crawling aggressively for
#    backlink/keyword data — throttle hard to protect backend capacity.
# ------------------------------------------------------------
User-agent: SemrushBot
Allow: /
Crawl-delay: 20

User-agent: AhrefsBot
Allow: /
Crawl-delay: 20

User-agent: DataForSeoBot
Allow: /
Crawl-delay: 20

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /


# ------------------------------------------------------------
# 6. DATA SCRAPERS – Block (no value, crawl budget waste)
# ------------------------------------------------------------
User-agent: omgili
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: Nicecrawler
Disallow: /


# ------------------------------------------------------------
# 7. SOCIAL / OTHER – Case by case
# ------------------------------------------------------------
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

# The content signals
User-Agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

# ------------------------------------------------------------
# 8. SITEMAPS
# ------------------------------------------------------------
Sitemap: https://bilarna.com/sitemap_index.xml