# Robots.txt for PodLexicon
# AI-Powered Podcast Search

User-agent: *
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /*?search=
Disallow: /*?episode_id=
Disallow: /*?page=
Disallow: /*?from=
# Internal search results: an unbounded URL space Google's guidelines say not to
# index. /search is the authenticated app route; ?q= also covers /browse/search.
Disallow: /search
Disallow: /*?q=

# The episode sitemap index points at `sitemap-episodes.xml?page=1` and `?page=2`,
# which `Disallow: /*?page=` above matches. This Allow is 21 chars against its 8,
# so it wins under longest-match and keeps sitemap discovery alive. Page 2 is
# reachable ONLY via ?page=2 -- the unpaged URL returns just the first 45,000 of
# the 76,988 episode URLs, so blocking ?page= would have hidden 31,988 of them
# (verified by <loc> count on each file, 2026-09-06). Added 2026-09-06, when
# removing the shadow Googlebot group made `/*?page=` bind on Google and Bing for
# the first time.
Allow: /sitemap-episodes.xml

# NOTE: there is deliberately no `Allow: /browse/` here. `Allow: /` above already
# permits every browse page, and Google resolves Allow/Disallow conflicts by
# LONGEST MATCHING PATTERN -- so `Allow: /browse/` (8 chars) outranked
# `Disallow: /*?q=` and `Disallow: /*?from=` and made every query-string rule
# above a no-op on precisely the /browse/ URLs they exist to block.

# Search engines
# Deliberately NO `User-agent: Googlebot` / `User-agent: Bingbot` groups. Per
# RFC 9309 s2.2.1 a crawler obeys ONLY its most specific matching group, so a
# `Googlebot` group containing just `Allow: /` made Google ignore the entire
# `User-agent: *` block above -- every Disallow in it (/admin, /api/, /search,
# ?q=, ?page= ...) silently did not apply to the two crawlers that matter.
# Found 2026-09-06. Leaving them out is what makes the `*` rules bind.

# AI Crawlers - allow browse content, slower crawl rate
User-agent: GPTBot
Allow: /browse/
Allow: /llms.txt
Allow: /agents
Allow: /developers
Disallow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /browse/
Allow: /llms.txt
Allow: /agents
Allow: /developers
Disallow: /
Crawl-delay: 2

User-agent: ClaudeBot
Allow: /browse/
Allow: /llms.txt
Allow: /agents
Allow: /developers
Disallow: /
Crawl-delay: 2

User-agent: Claude-User
Allow: /browse/
Allow: /llms.txt
Allow: /agents
Allow: /developers
Disallow: /
Crawl-delay: 2

User-agent: Claude-SearchBot
Allow: /browse/
Allow: /llms.txt
Allow: /agents
Allow: /developers
Disallow: /
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /browse/
Allow: /llms.txt
Allow: /agents
Allow: /developers
Disallow: /
Crawl-delay: 2

User-agent: Google-Extended
Allow: /

# Sitemap location
Sitemap: https://podlexicon.com/sitemap.xml

# Crawl delay (be nice to servers)
Crawl-delay: 1