# =================================================================
# MiriCanvas robots.txt
# Last updated: 2026-03-17
# Strategy: GEO-First - Maximize AI search & training visibility
#
# STRUCTURE:
#   1. Default rules (traditional search + unknown bots)
#   2. AI Search & User-Initiated bots (full allow, private blocked)
#   3. AI Training bots (selective allow, public pages only)
#   4. Social preview bots (full allow, private blocked)
#   5. Google Ads & Inspection (full allow, private blocked)
#   6. Aggressive / Low-value crawlers (blocked)
#   7. Sitemaps
#
# PRIVATE PATHS (blocked for ALL bots):
#   user editor - /design, /v2/design, /v2/*/design, /design2, /v2/design2, /v2/*/design2
#   user dashboard - /workspace/
# =================================================================


# =================================================================
# 1. DEFAULT - Traditional search engines & unknown crawlers
#    Googlebot, Yandex, etc. match here.
#    Conservative allow-list: only public marketing pages.
# =================================================================
User-agent: *
Allow: /$
Allow: /page
Allow: /miricle
Allow: /illust
Allow: /edu
Allow: /b2b
Allow: /templates/
Allow: /template/
Allow: /pricing
Allow: /features/
Allow: /match-your-vibe
Disallow: /design
Disallow: /v2/design
Disallow: /v2/*/design
Disallow: /design2
Disallow: /v2/design2
Disallow: /v2/*/design2
Disallow: /workspace/
Disallow: /*?utm_source
Disallow: /*?utm_medium
Disallow: /*?utm_campaign
Disallow: /*?utm_term
Disallow: /*?utm_content
Disallow: /*?gclid


# =================================================================
# 2. AI SEARCH & USER-INITIATED BOTS - Full allow (private blocked)
#
#    These power AI search results, citations, and inline links.
#    Full site access maximizes GEO visibility.
#    Private paths still blocked for user data protection.
# =================================================================
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: bingbot
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: GrokBot
Allow: /
Disallow: /design
Disallow: /v2/design
Disallow: /v2/*/design
Disallow: /design2
Disallow: /v2/design2
Disallow: /v2/*/design2
Disallow: /workspace/


# =================================================================
# 3. AI TRAINING BOTS - Selective allow (public pages only)
#
#    Training data -> base knowledge in AI models.
#    Users asking "what is MiriCanvas?" get answers from this data.
#    Restrict to public marketing/content pages only.
# =================================================================
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: anthropic-ai
User-agent: claude-web
Allow: /$
Allow: /page
Allow: /miricle
Allow: /illust
Allow: /edu
Allow: /b2b
Allow: /templates/
Allow: /template/
Allow: /pricing
Allow: /features/
Allow: /match-your-vibe
Disallow: /design
Disallow: /v2/design
Disallow: /v2/*/design
Disallow: /design2
Disallow: /v2/design2
Disallow: /v2/*/design2
Disallow: /workspace/
Disallow: /*?utm_source
Disallow: /*?utm_medium
Disallow: /*?utm_campaign
Disallow: /*?utm_term
Disallow: /*?utm_content
Disallow: /*?gclid


# =================================================================
# 4. SOCIAL PREVIEW - Open Graph / Twitter Cards
#    Full allow for link previews. Private paths blocked.
# =================================================================
User-agent: facebookexternalhit
User-agent: Twitterbot
Allow: /
Disallow: /design
Disallow: /v2/design
Disallow: /v2/*/design
Disallow: /design2
Disallow: /v2/design2
Disallow: /v2/*/design2
Disallow: /workspace/


# =================================================================
# 5. GOOGLE ADS & INSPECTION - Full allow, private blocked
# =================================================================
User-agent: AdsBot-Google
User-agent: AdsIdxBot
User-agent: Google-InspectionTool
Allow: /
Disallow: /design
Disallow: /v2/design
Disallow: /v2/*/design
Disallow: /design2
Disallow: /v2/design2
Disallow: /v2/*/design2
Disallow: /workspace/


# =================================================================
# 6. AGGRESSIVE / LOW-VALUE CRAWLERS - Blocked
#
#    No search visibility benefit. Reasons to block:
#    - Crawl budget protection: reduce server load so important
#      bots (Googlebot, GPTBot) crawl more efficiently
#    - Data protection: prevent opaque data collection
#    - Server performance: fewer requests -> better response times
# =================================================================
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Ai2Bot
User-agent: Ai2Bot-Dolma
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: Timpibot
User-agent: Omgilibot
User-agent: omgili
User-agent: Webzio-Extended
User-agent: VelenPublicWebCrawler
User-agent: FriendlyCrawler
User-agent: ICC-Crawler
User-agent: PanguBot
User-agent: Kangaroo Bot
Disallow: /


# =================================================================
# 7. SITEMAPS
# =================================================================
Sitemap: https://www.miricanvas.com/sitemap.xml
Sitemap: https://www.miricanvas.com/sitemaproot.xml
Sitemap: https://www.miricanvas.com/sitemap-search-illust.xml