# robots.txt for https://btlabs.dev
# Auto-generated. Edit via Admin → Einstellungen → KI-Agenten.
# Spec: RFC 9309 (Robots Exclusion Protocol).

Sitemap: https://btlabs.dev/sitemap_index.xml

# ── Default rule — applies to every bot not listed below ──
# This is the policy: content AND the public agent surface are open
# (/api/agents read API, /api/mcp, /api/media for image search,
# /api/favicon, /api/manifest.webmanifest; longest-match wins over
# the /api disallow). AI assistants, search engines and link-preview
# bots are deliberately NOT enumerated — unknown agents are welcome.
# Blocked below: only bots that get LESS than this default.
User-Agent: *
Allow: /
Allow: /api/agents
Allow: /api/mcp
Allow: /api/media
Allow: /api/favicon
Allow: /api/manifest.webmanifest
Disallow: /admin
Disallow: /api
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── Aggressive SEO scrapers (bandwidth-only, no SEO value for us) ──
User-Agent: AhrefsBot
User-Agent: SemrushBot
User-Agent: MJ12bot
User-Agent: DotBot
User-Agent: Barkrowler
User-Agent: Megaindex
User-Agent: BLEXBot
User-Agent: serpstatbot
User-Agent: PetalBot
User-Agent: rogerbot
User-Agent: SiteAuditBot
User-Agent: YisouSpider
User-Agent: Bytespider
Disallow: /

# ── Data-mining / aggregation bots (22 bots) ──
# Setting "Allow data mining" = OFF → these bots are blocked.
User-Agent: Awario
User-Agent: Datenbank
User-Agent: Diffbot
User-Agent: Echobot
User-Agent: ImagesiftBot
User-Agent: Kangaroo
User-Agent: Lightpanda
User-Agent: MyCentralAIScraperBot
User-Agent: NagetBot
User-Agent: PanguBot
User-Agent: TurnitinBot
User-Agent: WARDBot
User-Agent: Webzio-Extended
User-Agent: cohere-training-data-crawler
User-Agent: imageSpider
User-Agent: img2dataset
User-Agent: laion-huggingface-processor
User-Agent: netEstate
User-Agent: newsai
User-Agent: omgili
User-Agent: omgilibot
User-Agent: webzio
Disallow: /

# ── Bots that ignore robots.txt (5 bots — always blocked) ──
# These bots are documented to disregard robots.txt; we still
# signal explicit rejection here.
User-Agent: ISSCyberRiskCrawler
User-Agent: LAIONDownloader
User-Agent: Linguee
User-Agent: Thinkbot
User-Agent: iaskspider
Disallow: /

# ── AI Discovery Resources ──
# llms.txt: https://btlabs.dev/llms.txt
# llms-full.txt: https://btlabs.dev/llms-full.txt
# llms.html: https://btlabs.dev/llms.html
# ai.txt (Policy): https://btlabs.dev/ai.txt
# brand.txt: https://btlabs.dev/brand.txt
# brand.json: https://btlabs.dev/brand.json
# identity.json: https://btlabs.dev/identity.json
# humans.txt: https://btlabs.dev/humans.txt
# faq-ai.txt: https://btlabs.dev/faq-ai.txt
# API-Catalog (RFC 9727): https://btlabs.dev/.well-known/api-catalog
# OpenAPI YAML: https://btlabs.dev/.well-known/openapi.yaml
# MCP-Discovery (mcp.json): https://btlabs.dev/.well-known/mcp.json
# security.txt: https://btlabs.dev/.well-known/security.txt
# TDM Reservation (tdmrep.json): https://btlabs.dev/.well-known/tdmrep.json
# AI-Plugin-Manifest (ai-plugin.json): https://btlabs.dev/.well-known/ai-plugin.json
# AI-Catalog (ARD): https://btlabs.dev/.well-known/ai-catalog.json