# robots.txt, https://www.acmez.com
#
# The whole site is open to crawlers. Only URLs that produce duplicate or
# non-canonical views are excluded, so that crawl budget goes to real pages.

User-agent: *
Allow: /

# Filtered listing views duplicate the main index and carry noindex already.
Disallow: /insights/?type=
Disallow: /insights/?topic=
Disallow: /search/

# Server-side directories that hold no public content.
Disallow: /includes/
Disallow: /storage/
Disallow: /design/

# Assets must stay crawlable: blocking CSS, JS or images prevents search engines
# from rendering the pages correctly and hurts both indexing and Core Web Vitals
# assessment.
Allow: /assets/

# Answer engines and AI crawlers.
#
# Acmez publishes this site to be read, quoted and cited, including by AI search
# products. These agents are explicitly permitted rather than left to guess.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

# Aggressive SEO crawlers that provide no benefit to us and consume bandwidth.
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

Sitemap: https://www.acmez.com/sitemap.xml

# Curated summary for answer engines (llmstxt.org convention).
# Company facts, service map and common questions in one plain-text file.
# Llms-Txt: https://www.acmez.com/llms.txt