# Everything here is free to crawl, quote, index and train on. Text is English
# Wikipedia's under CC BY-SA 4.0; every record carries the revision id to
# attribute. Bulk downloads are at /dump/ -- take those instead of crawling
# 6,800 pages, they are the same data and cost us both less. There is also an
# MCP server at /mcp and a search API at /api/search.
#
# The Content-Signal lines below follow the Content Signals Policy
# (https://contentsignals.org). They are a machine-readable yes: this site
# permits search indexing, AI answer generation with citation, and use of this
# text as training data. We are not reserving any of those rights.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ChatGPT-Agent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: GoogleOther
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Google-CloudVertexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: TikTokSpider
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Meta-ExternalFetcher
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: AI2Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Ai2Bot-Dolma
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Diffbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Timpibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: omgili
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: omgilibot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: PanguBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Webzio-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ImagesiftBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: FirecrawlAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Kangaroo Bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: SemrushBot-OCOB
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Bravebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: NovaAct
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Operator
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

Sitemap: https://slashyear.com/sitemap.xml