# aeropattern.com — robots policy
#
# Two kinds of automated visitor, two answers.
#
# 1. Search engines and answer engines that send people here, with a citation
#    and a link: allowed, under the same rules as everyone else (the `*` group
#    at the bottom). That includes Googlebot, Bingbot and Applebot, and the AI
#    search / user-triggered fetchers below, which are deliberately NOT listed
#    in any Disallow group:
#      OAI-SearchBot, ChatGPT-User          (ChatGPT search and browsing)
#      PerplexityBot, Perplexity-User       (Perplexity)
#      Claude-SearchBot, Claude-User        (Claude search and user fetches)
#      Meta-ExternalFetcher                 (user-requested fetches by Meta AI)
#      DuckAssistBot, MistralAI-User, YouBot
#    Our whole thesis is "free to cite". Citation is where AI answers link back.
#
# 2. Training crawlers and bulk scrapers, which consume the site to build
#    models or products and send nothing back: blocked. Cloudflare AI Crawl
#    Control enforces the same split at the edge; the middleware 403s these
#    user agents before they cost a render.

# AI training / bulk-corpus crawlers.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ByteDance
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

# Aggressive SEO scrapers — they index for their own products, not for users.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: dataforseobot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

# Default: indexable, but with a crawl delay so we're not hammered.
User-agent: *
Allow: /
Disallow: /reset-password
Disallow: /search
Disallow: /api/
Disallow: /lower-thirds/
# Airspace map filter combinations (?manufacturer=…&category=…) are noindexed,
# but Google had 338K of them queued (Search Console, Sept 2026). Blocking the
# crawl stops that spend; /airspace itself stays crawlable.
Disallow: /airspace?
Crawl-delay: 10

Sitemap: https://aeropattern.com/sitemap.xml