# Sitemap points at the live custom domain — update it (and sitemap.xml, index.html's
# canonical/og:url, and llms.txt) if you ever move to a different domain.
#
# Private/authenticated pages (login, dashboard, account, billing, diagnostic,
# forgot/reset-password) are NOT disallowed here on purpose. Each of those pages already
# carries its own <meta name="robots" content="noindex, nofollow"> tag. Blocking them in
# robots.txt as well would stop Google from ever crawling them to see that noindex tag —
# Google's own guidance is explicit that this backfires: a page linked from elsewhere (e.g.
# the homepage's "Sign in" link) still gets its URL indexed, just with no content, which is
# what caused Search Console's "Duplicate without user-selected canonical" warning (2026-09-17).
# Letting crawlers fetch these pages lets the noindex tag do its job properly instead.
User-agent: *
Allow: /
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /api/

User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /api/

Sitemap: https://zuvoai.co.za/sitemap.xml