# monit.rs — robots policy
# Last reviewed: 2026-06-13

User-agent: *
Allow: /

# Don't waste bot budget on internal surfaces (all of these require auth
# and would 401/302 to the login page — wasted requests for both sides).
Disallow: /dashboard/
Disallow: /auth/
Disallow: /api/

# Public regression-report share links are tokenized one-off URLs — they
# should never be indexed (users explicitly opt-in to sharing one, not
# the entire population).
Disallow: /report/

# ── LLM crawlers ──────────────────────────────────────────────────────
# Reversed in 2026-06-13 from the original "block all AI" posture. For a
# developer tool that depends on being recommended by ChatGPT/Claude/
# Perplexity/AI Overviews when users ask "how do I catch API regressions",
# blocking these UAs gates off our highest-leverage discovery channel.
# The cost-benefit shifted: real-time user-query bots send qualified
# referrals; training crawlers claim permanent shelf-space in future
# model recommendations. Both are now worth more than the bandwidth.
#
# See also /llms.txt for the structured product description these
# crawlers read in one fetch.

# Real-time user-query crawlers — fired when a user asks an LLM a
# question and the LLM does a live web fetch to ground its answer.
# Each hit can become an attributed referral.
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

# Training crawlers — claim shelf-space in future model recommendations.
# Acceptable trade for a content-rich SEO surface where the content is
# what we want recommended.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# AI scrapers that don't drive meaningful inbound and don't show good
# attribution. Update this list as the landscape changes.
User-agent: Bytespider
Disallow: /

User-agent: omgilibot
Disallow: /

Sitemap: https://monit.rs/sitemap.xml