Infrastructure

# Path rules are repeated in BOTH groups on purpose.
#
# Under RFC 9309 a crawler obeys exactly ONE group — the most specific one
# matching its token — and ignores every other group, including "*". So a
# named group is not an addition to the "*" rules, it REPLACES them.
#
# This file previously gave each AI crawler its own "User-agent: X /
# Allow: /" group meaning "yes, you may index us". Because those groups
# carried no Disallow lines, they granted GPTBot, OAI-SearchBot,
# ClaudeBot, anthropic-ai and PerplexityBot MORE access than an ordinary
# crawler: every path listed below was open to them and closed to
# everyone else.
#
# Anything added to one group below must be added to the other.
#
# Not every private path belongs in this file. A Disallow line is
# world-readable and therefore a published index of what someone thought
# worth hiding, while being advisory and ignored by the clients that
# matter. Paths whose protection depends on not being known are kept out
# entirely and marked unindexable with an X-Robots-Tag response header
# instead.

User-agent: *
Allow: /

# Auth and user-private pages
Disallow: /login
Disallow: /register
Disallow: /logout
Disallow: /my-account
Disallow: /change-password-account
Disallow: /delete-account
Disallow: /forgot/
Disallow: /verify/
Disallow: /expired

# Payment flow and internal endpoints (pricing is indexable — it's a marketing page)
Disallow: /payment
Disallow: /api/

# AI search, citation and corpus crawlers — explicitly permitted, on the
# same terms as everyone else. One group, many tokens: this is the form
# that keeps the path rules attached to the permission.
#
# CCBot is included deliberately. It is not only a training crawler — its
# corpus is a retrieval source for several AI search systems, so blocking
# it cost citation reach to buy a training-use refusal that a robots
# directive cannot enforce anyway. llms.txt states corpus inclusion as
# permitted, so the two agree.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
Allow: /

Disallow: /login
Disallow: /register
Disallow: /logout
Disallow: /my-account
Disallow: /change-password-account
Disallow: /delete-account
Disallow: /forgot/
Disallow: /verify/
Disallow: /expired
Disallow: /payment
Disallow: /api/

# Not permitted.
User-agent: cohere-ai
Disallow: /

Sitemap: https://appscyborg.com/sitemap.xml