User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/
# Browse: category, inCollection, rarity and page are indexable landing pages (in the sitemap);
# sort/dir/type/wear/q/inCase combinations are an endless, noindexed URL space.
Disallow: /market-hub/browse?*sort=
Disallow: /market-hub/browse?*dir=
Disallow: /market-hub/browse?*type=
Disallow: /market-hub/browse?*wear=
Disallow: /market-hub/browse?*q=
Disallow: /market-hub/browse?*inCase=

# AI training crawlers: welcome on docs and marketing pages, but not on the price data
# (Market Hub, /apps tools) - that data is the paid API product. middleware.ts enforces
# the same list; user-triggered fetchers (ChatGPT-User, Perplexity-User, ...) are allowed.
# Google-Extended and Applebot-Extended are robots.txt-only tokens for AI training (Gemini, Apple
# Intelligence); they don't crawl themselves and don't affect Google Search or AI Overviews.
User-agent: GPTBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Reflectionbot
Disallow: /market-hub/
Disallow: /apps/
Disallow: /api/
# A named group replaces the * group for these bots, so the private paths are repeated here.
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# SEO scrapers that hammer /apps with nested back= URLs.
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: PetalBot
User-agent: MJ12bot
User-agent: DataForSeoBot
Disallow: /apps/
Disallow: /market-hub/
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

Sitemap: https://cspriceapi.com/sitemap.xml