# diamondedit.co.za, robots.txt v2 (2026-05-27)
#
# Editorial position: allow all reputable crawlers. The Diamond Edit is an
# editorial publication. We want representation in BOTH paths:
#   1. Real-time AI citation crawlers (OAI-SearchBot, PerplexityBot,
#      Claude-User, Claude-SearchBot, Perplexity-User, ChatGPT-User,
#      Bingbot AI, Google AI Overviews) so we appear in generative answers.
#   2. Training corpora crawlers (GPTBot, ClaudeBot, Google-Extended,
#      anthropic-ai, FacebookBot, Applebot-Extended) so future model
#      generations recognise the publication's entity + facts.
#
# A new editorial publication that blocks training crawlers can be cited
# at real time but will not be remembered. Both lanes matter.
#
# Public AI/copyright policy: https://diamondedit.co.za/ai-policy/
# Contact: editor@diamondedit.co.za
# Last reviewed: 2026-05-27

# === Default: allow everything ===
User-agent: *
Allow: /

# === Real-time AI citation crawlers (explicit allow, no crawl-delay) ===
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: DuckAssistBot
Allow: /

# === Disallow only thin auto-generated paths ===
# (none currently; this list is reserved for future programmatic noise)

# === Sitemaps ===
Sitemap: https://diamondedit.co.za/sitemap-index.xml