# needtoknowcomms.com.au
#
# NTK crawler policy, set 2026-09-24 by the operator for ALL THREE stacks (cross-stack #333; basis:
# ntkit reports/T288_crawler_policy_decision.md). Adopted on NTKComms 2026-10-15, structure
# and wording copied from NTKIT's reference file; only the Disallow set is ours.
#
# The policy is ALLOW, deliberately, not by omission. The explicit groups below are
# REDUNDANT against "User-agent: *". They are here so a future reader can tell a considered
# decision from an oversight, and does not "tidy up" silence into a block.
#
# ORDER MATTERS: Disallow lines come BEFORE `Allow: /` in every group. Google/Bing use
# longest-match-wins, but first-match-wins parsers (Python's urllib.robotparser among them)
# would read /cms/ as crawlable with `Allow: /` first.
#
# SERVED BY NGINX, NOT DJANGO: `location = /robots.txt` -> /opt/ntkcomms/static/robots.txt.
# This repo file is the source; scp it there. templates/robots.txt is NOT served.
# Cloudflare caches this for 4h: verify at the origin, expect the edge to lag.

User-agent: *
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

# --- AI retrieval crawlers: these surface and LINK us. Allowing them is the point. ---
# Perplexity's own docs: PerplexityBot "is designed to surface and link websites in search
# results... It is not used to crawl content for AI foundation models."
User-agent: OAI-SearchBot
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

User-agent: Claude-SearchBot
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

User-agent: PerplexityBot
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

# --- User-initiated fetches: a real person asked for the page. ---
# Note OpenAI states that because these are user-initiated, robots.txt may not apply to
# them at all; these groups record intent rather than exert control.
User-agent: ChatGPT-User
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

User-agent: Claude-User
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

User-agent: Perplexity-User
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

# --- Training crawlers: allowed on the operator's explicit call 2026-09-24. ---
# Blocking these would have cost no measured traffic; they were allowed because the site's
# thesis is being the source AI systems reach for, and consistency with that is worth more
# than a symbolic block.
User-agent: GPTBot
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

User-agent: ClaudeBot
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

# Google-Extended is NOT a pure training token. Google's own documentation: it governs
# whether content may be used for training Gemini AND "for grounding (providing content
# from the Google Search index to the model at prompt time) in Gemini Apps and Grounding
# with Google Search on Vertex AI". Blocking it would switch off a live retrieval surface.
# Google also states it does not affect inclusion or ranking in Google Search.
User-agent: Google-Extended
Disallow: /cms/
Disallow: /admin/
Disallow: /django-admin/
Disallow: /search/
Allow: /

# Deliberately NOT listed: CCBot, Bytespider, Amazonbot, meta-externalagent,
# Applebot-Extended. Their vendor documentation has not been read, and this file does not
# assert a position on a crawler nobody here has checked. They fall under "User-agent: *".

Sitemap: https://needtoknowcomms.com.au/sitemap.xml