# Note: Collection of TradeGrub data by automated means is
# prohibited unless you have written permission from TradeGrub
# and may only be carried out for the limited purpose.
# See: https://tradegrub.com/terms
#
# Agent-readable site summary: https://tradegrub.com/llms.txt
#
# HISTORY, 2026-08-08: Cloudflare used to prepend a "Cloudflare Managed content"
# block above everything below, setting Disallow: / for ClaudeBot, GPTBot,
# Google-Extended, CCBot, Bytespider, Amazonbot, Applebot-Extended and
# meta-externalagent, plus a Content-Signal of ai-train=no. Nothing in this file
# could override it, because an earlier group wins for the same user-agent — so
# the named Allow groups below were being silently cancelled for those agents.
#
# The "Managed robots.txt" toggle in Cloudflare AI Crawl Control was switched off,
# so this file is now served as written and verified so: zero Disallow: / entries
# and no Content-Signal on this host, charts and ai.
#
# If Disallow: / for training crawlers ever reappears here, nothing in this file
# caused it. That toggle has been turned back on, and hard per-crawler blocking is
# a separate control in the same place. Neither is editable from the repo.

User-agent: *
Allow: /
Sitemap: https://tradegrub.com/sitemap.xml
Sitemap: https://tradegrub.com/imagesitemap.xml

# Retrieval and citation agents: these fetch a page to answer a question and
# cite the source. They are not training crawlers, which is consistent with the
# Content-Signal above (search=yes, ai-train=no, use=reference). Listed
# explicitly so the intent is recorded even where Cloudflare's block does not
# name them.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# Any changes made to robots.txt need to be approved by TradeGrub LLC.