# robots.txt — YourHealthier (2026-06)
# Strategy: Allow retrieval/citation crawlers → maximize AI citations
#           Allow major AI training crawlers → build training-data presence (GEO)
#           Block only abusive / low-value scrapers
#           Updated: 2026-06-15

User-agent: *
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /pages/llms-txt
Allow: /pages/llms-full-txt
Disallow: /collections/*/tagged
  
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /72155594903/checkouts
Disallow: /72155594903/orders
Disallow: /carts
Disallow: /account
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: */collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products
Disallow: /services/login_with_shop
Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Sitemap: https://yourhealthier.com/sitemap.xml

User-agent: adsbot-google
  
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /72155594903/checkouts
Disallow: /72155594903/orders
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js
Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /services/login_with_shop

User-agent: Nutch
  
Disallow: /

User-agent: AhrefsBot
  
Crawl-delay: 10
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /72155594903/checkouts
Disallow: /72155594903/orders
Disallow: /carts
Disallow: /account
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: */collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /services/login_with_shop
Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Sitemap: https://yourhealthier.com/sitemap.xml

User-agent: AhrefsSiteAudit
  
Crawl-delay: 10
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /72155594903/checkouts
Disallow: /72155594903/orders
Disallow: /carts
Disallow: /account
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: */collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /services/login_with_shop
Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Sitemap: https://yourhealthier.com/sitemap.xml

User-agent: MJ12bot
  
Crawl-delay: 10

User-agent: Pinterest
  
Crawl-delay: 1


# ══════════════════════════════════════════════
# ALLOW: AI Search & Retrieval Crawlers
# These drive citations in ChatGPT, Perplexity,
# Google AI Overviews, Bing Copilot, Claude
# ══════════════════════════════════════════════

# OpenAI search/retrieval (ChatGPT Search citations)
User-agent: OAI-SearchBot
Allow: /

# OpenAI user-triggered real-time fetch
User-agent: ChatGPT-User
Allow: /

# Perplexity search (drives ~46.7% Reddit-style citations)
User-agent: PerplexityBot
Allow: /

# Perplexity user-triggered fetch
User-agent: Perplexity-User
Allow: /

# Anthropic search/retrieval (Claude search citations)
User-agent: Claude-SearchBot
Allow: /

# Anthropic user-triggered fetch
User-agent: Claude-User
Allow: /

# Apple Siri / Apple Intelligence retrieval
User-agent: Applebot
Allow: /

# Bing (powers Copilot citations — critical)
User-agent: Bingbot
Allow: /

# Google (powers AI Overviews / AI Mode)
User-agent: Googlebot
Allow: /

# ══════════════════════════════════════════════
# ALLOW: AI Training Crawlers
# Being present in model training data is a top GEO
# citation signal — let the major model-makers learn
# the brand natively. Freshness is covered by the
# retrieval bots allowed above.
# ══════════════════════════════════════════════

# OpenAI training crawler
User-agent: GPTBot
Allow: /

# Anthropic training crawler
User-agent: ClaudeBot
Allow: /

# Google AI training (feeds Gemini)
User-agent: Google-Extended
Allow: /

# Apple AI training
User-agent: Applebot-Extended
Allow: /

# Common Crawl (open dataset — broadest model distribution)
User-agent: CCBot
Allow: /

# Meta AI training crawler (feeds Llama / Meta AI)
User-agent: Meta-ExternalAgent
Allow: /

# Meta general crawler
User-agent: FacebookBot
Allow: /

# Cohere AI training
User-agent: cohere-ai
Allow: /

# Amazon AI training (Rufus / Alexa)
User-agent: Amazonbot
Allow: /

# Diffbot (knowledge-graph extraction — aids entity recognition)
User-agent: Diffbot
Allow: /

# ══════════════════════════════════════════════
# BLOCK: Abusive / Low-Value Scrapers
# ══════════════════════════════════════════════

# ByteDance crawler (aggressive, negligible citation value)
User-agent: Bytespider
Disallow: /

# Omgili / Webz.io data reseller
User-agent: omgili
Disallow: /

# ══════════════════════════════════════════════
# BLOCK/THROTTLE: Aggressive/Low-Value Crawlers
# ══════════════════════════════════════════════

User-agent: SemrushBot
Crawl-delay: 2

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Crawl-delay: 5

# ══════════════════════════════════════════════
# AI / GEO Discovery
# ══════════════════════════════════════════════
# LLM content index: https://yourhealthier.com/llms.txt
# Full reference doc: https://yourhealthier.com/llms-full.txt

Sitemap: https://yourhealthier.com/sitemap.xml
Sitemap: https://yourhealthier.com/sitemap_agentic_discovery.xml