# Moorlnd, robots policy
# Last reviewed: 2026-10-06. Re-walked quarterly. Rationale: DOCTRINE.md § Anti-scraping.
#
# 6 Oct 2026 change (owner decision 5-6 Oct 2026): being found and cited by AI answer
# engines (Generative Engine Optimisation) now outranks the training opt-out, for the
# PUBLIC MARKETING pages only. So the named model crawlers below may read those pages
# (Allow lines, matched exactly with $) and nothing else (Disallow: / catches the rest
# under longest-match, RFC 9309). Course modules, /content/, the portal, templates, paid
# and audit paths stay closed to them, and to the answer-engine fetchers as well.
# The retired "Claude-Web" token is replaced by Anthropic's current Claude-SearchBot and
# Claude-User (support.claude.com, checked 6 Oct 2026).

# ─── Model crawlers: public marketing pages only ─────────────────────
# Each also feeds a vendor's training or grounding (Google-Extended governs Gemini
# grounding as well as training). Allowed on the pages we want quoted; closed elsewhere.

User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
User-agent: Amazonbot
Allow: /$
Allow: /index.html$
Allow: /governance$
Allow: /governance.html$
Allow: /course$
Allow: /course.html$
Allow: /pricing$
Allow: /pricing.html$
Allow: /faq$
Allow: /faq.html$
Allow: /about$
Allow: /about.html$
Allow: /msp$
Allow: /msp.html$
Allow: /who-we-help$
Allow: /who-we-help.html$
Allow: /solutions$
Allow: /solutions.html$
Allow: /standards-map$
Allow: /standards-map.html$
Allow: /glossary$
Allow: /glossary.html$
Allow: /citations$
Allow: /citations.html$
Allow: /changelog$
Allow: /changelog.html$
Allow: /module-1$
Allow: /module-1.html$
Allow: /llms.txt$
Allow: /sitemap.xml$
Allow: /tdm-policy.json$
Allow: /assets/logo.png$
Disallow: /

# ─── Training-only and unwanted crawlers: BLOCK SITE-WIDE ────────────
# No citation or traffic comes back from these.

User-agent: Bytespider
User-agent: cohere-training-data-crawler
User-agent: MistralAI-Training
User-agent: omgili
User-agent: omgilibot
User-agent: ImagesiftBot
User-agent: YouBot
User-agent: Diffbot
User-agent: magpie-crawler
User-agent: AI2Bot
User-agent: Timpibot
User-agent: PetalBot
User-agent: Webzio-Extended
Disallow: /

# ─── Answer-engine and on-demand fetchers: ALLOW public, BLOCK paid ──
# They cite with a link. A named group replaces the * group entirely, so the paid
# paths are repeated here (before 6 Oct 2026 these bots could reach /module-2 etc.).
# Perplexity-User and MistralAI-User say they may ignore robots.txt on user request;
# listed so the intent is on record. Bingbot, Googlebot, Applebot and DuckDuckBot
# (which feed Copilot, AI Overviews, Siri and DuckDuckGo) fall under *.

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: MistralAI-Index
User-agent: Meta-ExternalFetcher
User-agent: meta-webindexer
User-agent: Amzn-SearchBot
User-agent: Amzn-User
Disallow: /paid/
Disallow: /account/
Disallow: /customer/
Disallow: /api/
Disallow: /admin/
Disallow: /.audit/
Disallow: /portal/
Disallow: /templates/
Disallow: /module-2
Disallow: /module-3
Disallow: /module-4
Disallow: /module-5
Disallow: /module-6
Disallow: /module-7
Disallow: /module-8
Disallow: /module-9
Disallow: /module-10
Disallow: /module-11
Disallow: /module-12
Disallow: /module-copilot
Disallow: /module-manager
Disallow: /module-msp-admin
Disallow: /module-procurement
Disallow: /module-shadow-ai
Disallow: /sector-
Disallow: /content/
Disallow: /.netlify/
Allow: /

# ─── Default (search engines, social link previews, everything else) ─
# Block paid + audit; allow everything else.

User-agent: *
Disallow: /paid/
Disallow: /account/
Disallow: /customer/
Disallow: /api/
Disallow: /admin/
Disallow: /.audit/
# Paid modules: the body is served only to signed-in readers; nothing to index here.
Disallow: /module-2
Disallow: /module-3
Disallow: /module-4
Disallow: /module-5
Disallow: /module-6
Disallow: /module-7
Disallow: /module-8
Disallow: /module-9
Disallow: /module-10
Disallow: /module-11
Disallow: /module-12
Disallow: /module-copilot
Disallow: /module-manager
Disallow: /module-msp-admin
Disallow: /module-procurement
Disallow: /module-shadow-ai
Disallow: /content/
Disallow: /.netlify/
Allow: /

Sitemap: https://moorlnd.ai/sitemap.xml