User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /readme.html
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Disallow: /*?*
Disallow: /*page/*
Disallow: /*/0/*
Allow: /*?*utm_*
Allow: /*?*itm_*
Allow: /*?ver*

# Block training crawlers
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: cohere-ai
Disallow: /

# Allow live fetchers for search/summaries
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: MistralAI-User
Allow: /

# Social preview
User-agent: facebookexternalhit
Allow: /
User-agent: Facebot
Allow: /
User-agent: LinkedInBot
Allow: /
User-agent: Slackbot
Allow: /
User-agent: TelegramBot
Allow: /
User-agent: WhatsApp
Allow: /

# Sitemaps
Sitemap: https://inc42.com/sitemap_index.xml
Sitemap: https://inc42.com/news-sitemap.xml
Sitemap: https://inc42.com/inc42_datalabs.xml