# --- AI / LLM crawlers: only lightweight discovery files allowed ---
# ChatGPT / OpenAI
User-agent: GPTBot
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

User-agent: OAI-SearchBot
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

# Anthropic / Claude
User-agent: Anthropic-AI
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

User-agent: Claude-Web
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

User-agent: ClaudeBot
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

User-agent: Claude-SearchBot
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

# Google Gemini / AI Overviews training & grounding
User-agent: Google-Extended
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

User-agent: GoogleOther
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

# Common Crawl (often used by LLM training)
User-agent: CCBot
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

# Perplexity
User-agent: PerplexityBot
Allow: /llms.txt
Allow: /sitemap-brands.xml
Disallow: /

# --- SEO-tool crawlers: heavy load, no benefit to us ---
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# --- Amazon product crawler: no benefit to us ---
User-agent: Amzn-SearchBot
Disallow: /

# --- Meta crawlers: needed for ads/feeds, but slowed down and
#     kept out of infinite pagination/filter URL permutations ---
User-agent: meta-webindexer
Crawl-delay: 10
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*manufacturer_id=

User-agent: meta-external-agent
Crawl-delay: 10

User-agent: meta-externalagent
Crawl-delay: 10

User-agent: facebookexternalhit
Crawl-delay: 5

# --- Everyone else (Google, Bing, ...): allowed, but not into
#     paginated/filtered catalog permutations (cars are in the sitemap) ---
User-agent: *
Allow: /
Crawl-delay: 10
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*manufacturer_id=
Disallow: /*?*generation_id=
Clean-param: gad_source

# Popular brand landing pages only (autonuma.com). No filter/query URLs.
# Cross-host Sitemap lines are ignored by Google for other domains sharing this file.
Sitemap: https://autonuma.com/sitemap-brands.xml