# robots.txt for usdiesel.com
# Public company-site crawl policy with pointers to US Diesel retail catalog resources.

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: *
Disallow: /admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /search/
Disallow: /store/item-search/
Disallow: /*?q=
Disallow: /*?query=
Disallow: /*?search=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*?page=

Allow: /
Allow: /llms.txt
Allow: /data.xml

# Canonical sitemap resources
Sitemap: https://usdiesel.com/sitemap-index.xml
Sitemap: https://retail.usdiesel.com/sitemap.xml

# Non-standard discovery hints for AI/semantic crawlers.
# These are ignored by traditional robots parsers but useful as explicit machine-readable breadcrumbs.

LLMS: https://usdiesel.com/llms.txt
Data-Index: https://usdiesel.com/data.xml

Retail-Catalog: https://retail.usdiesel.com/
Retail-Sitemap: https://retail.usdiesel.com/sitemap.xml

Product-Feed-Page: https://retail.usdiesel.com/product-feed/
Retail-Data-Page: https://retail.usdiesel.com/product-feed/

Product-Feed: https://usd.imagesforcatalog.com/product_feed.json

Retail-LLMS: https://retail.usdiesel.com/llms.txt
Retail-Data-Index: https://retail.usdiesel.com/data.xml