User-agent: *
Allow: /

# ---------------------------------------------------------------------------
# AI assistants and answer engines.
#
# The wildcard above already permits all of these. They are named explicitly so
# the intent is on the record, and so a future wildcard Disallow cannot make us
# invisible to an answer engine by accident. Split into two groups because the
# two groups buy different things.
#
# Verified against vendor docs 2026-09-18:
#   developers.openai.com/api/docs/bots
#   support.claude.com/en/articles/8896518
# ---------------------------------------------------------------------------

# Answer-engine retrieval: these fetch a page in order to answer a user and
# cite it. This is the group that sends traffic.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google-Extended is a control token, not a crawler: Googlebot fetches, and this
# token decides whether Gemini may train on and ground its answers in the result.
User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

# Training corpora: these do not cite and send no traffic. Allowed so the brand
# and the Four Guarantees are present in what the next model generation knows.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ---------------------------------------------------------------------------
# Sitemaps. Canonical host is www: the apex 301s to it, so the apex URLs that
# used to be here cost every crawler a redirect hop.
# ---------------------------------------------------------------------------
Sitemap: https://www.aimrank.io/sitemap-index.xml
Sitemap: https://www.aimrank.io/sitemap-0.xml