# ─── Search engines ─────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /services/stats/
# /products?search= and /products?sort= are intentionally crawlable.
# The page emits <meta name="robots" content="noindex, follow"> for any
# query-string variant, which lets Google drop them from the index and
# resolves the "Indexed, though blocked by robots.txt" report.

Disallow: /admin/
Disallow: /staff/
Disallow: /pos/
Disallow: /checkout
Disallow: /account
# Legacy WooCommerce .html URLs are intentionally crawlable: the app answers
# them with a 301 to /product/{slug}. Blocking them stopped Google from ever
# seeing the redirect, so the 404s never cleared.
Disallow: /index.rss
Disallow: /*.rss$
# /auth is intentionally crawlable so Googlebot can read its noindex + canonical
# and collapse the /auth?redirect=... duplicates that internal "Sign in" links create.

User-agent: Bingbot
Allow: /
Disallow: /services/stats/
Disallow: /admin/
Disallow: /staff/
Disallow: /pos/
Disallow: /checkout
Disallow: /account
# .html crawlable on purpose — the app 301s them to /product/{slug}
Disallow: /index.rss
Disallow: /*.rss$
# /auth crawlable on purpose (noindex + canonical handle duplicates)

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ─── Social previewers ──────────────────────────────────────────
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# ─── AI / LLM crawlers (explicitly welcomed) ────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Bytespider
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Mistral-Bot
Allow: /

# ─── Default: everyone else ─────────────────────────────────────
User-agent: *
Allow: /
Disallow: /services/stats/
Disallow: /admin/
Disallow: /staff/
Disallow: /pos/
Disallow: /checkout
Disallow: /account
# /auth crawlable on purpose (noindex + canonical handle duplicates)

# ─── Legacy WordPress / WooCommerce noise (404s in logs) ────────
# Stops bots from probing these and polluting our error reports.
Disallow: /wp-admin/
Disallow: /wp-login
Disallow: /wp-json/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /cart/add/
Disallow: /compare/
Disallow: /account/wishlistadd/
Disallow: /cookielaw/
Disallow: /cookie-law/
Disallow: /feed/
Disallow: /tag/
Disallow: /product-tag/
Disallow: /portfolio/
Disallow: /category/
Disallow: /services/
Disallow: /apple-app-site-association
Disallow: /*?attachment_id=
Disallow: /*?add-to-cart=
# Legacy WooCommerce product/artist pages ended in .html at the site root.
# These now resolve to a 301 towards /product/{slug}, so they must stay
# crawlable for Google to process the redirect and drop the 404s.
# Old WordPress RSS endpoints (/index.rss, /*/feed/…)
Disallow: /index.rss
Disallow: /*.rss$

# ─── Sitemaps & LLM manifests ───────────────────────────────────
# /sitemap.xml is a static sitemapindex; every child sitemap is served
# from this same domain (public/sitemaps/*.xml.gz, refreshed at build time).
Sitemap: https://www.craterecords.be/sitemap.xml

# LLM-friendly site summary (https://llmstxt.org/)
# Available at /llms.txt (concise) and /llms-full.txt (full catalogue)