# AI search/retrieval crawlers. These make public pages eligible to be found
# or cited in an answer; they are separate from training crawlers.
#
# Each group repeats the same three Disallow lines the default group carries
# further down (`/api/`, `/ws/`, `/superadmin/`) — a named group does not
# inherit anything from `User-agent: *` (WEB-277 #1865, DEC-1064): until this
# ticket a crawler matching one of these groups by name read only `Allow: /`
# and never saw the three disallowed prefixes at all.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

# Live-query agents: they fetch one page on demand while answering a
# question, rather than crawling ahead of time to build an index. Both are
# distinct user agents from their SearchBot/Bot siblings above and below —
# naming only the crawler and not the live fetcher left the fetcher to fall
# through to `User-agent: *`, which still works but was never a deliberate
# decision until this ticket.
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

# Model-training crawlers. The public marketing site remains explicitly
# allowed under the decision made in WEB-213; changing this group is a data
# policy decision, not an SEO switch.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /superadmin/

# Aggressive scraping, not a citation-driving crawler — blocked.
User-agent: Bytespider
Disallow: /

User-agent: *
Allow: /
Allow: /static/

# Public Pages
Allow: /about
Allow: /contact
Allow: /security
Allow: /privacy
Allow: /terms
Allow: /cookies
Allow: /compliance
Allow: /register

# Legacy auth URLs intentionally remain crawlable so crawlers can read the
# X-Robots-Tag on their redirects. Their /app destinations are noindex.

# Application Routes (Private)
#
# Only routes `cmd/server/main.go` still registers belong here — a Disallow
# for a path nothing serves is dead weight and not a signal (WEB-277 #1865,
# DEC-1064). `/calendar`, `/appointments`, `/case-requests`, `/cases`,
# `/templates` and `/settings` left with the Templ views WEB-233 #1329
# retired and are not registered any more; `/dashboard`, `/users` and
# `/historical-cases` are still `protected.GET` redirects into `/app`, so they
# stay.
Disallow: /dashboard
Disallow: /users
Disallow: /historical-cases
Disallow: /superadmin/

# API & Dynamic
Disallow: /api/
Disallow: /appointment/
Disallow: /ws/

# Prevent indexing of search and query parameters
Disallow: /*?sort=
Disallow: /*?page=
Disallow: /*?filter=

# Crawl-delay (seconds between requests)
Crawl-delay: 1

Sitemap: https://custodio.legal/sitemap.xml