User-agent: *
Allow: /

Sitemap: https://ganttpert.com/sitemap.xml

# --- AI crawlers ------------------------------------------------------------
#
# Two different kinds of robot get two different answers, because they do two
# different things to this site.
#
# BLOCKED: crawlers that harvest pages to train or ground a model. They take the
# work and return nothing — no visit, no citation, no link.
#
# ALLOWED (further down, and by the `User-agent: *` group above): agents that
# fetch a page because a person asked a question and then show that person a
# link back here. That is a referral, and refusing it would be refusing traffic.
#
# robots.txt is a request, not a fence. The fence is Cloudflare's bot blocking —
# see docs/OPERATIONS.md.
#
# For the allowed kind: /llms.txt says what this site can do in one fetch, and
# /for-agents documents handing it a task list in a URL.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Asked for by a person, and they get shown the link. Named explicitly so the
# choice is on the record rather than inherited from the wildcard.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /