# What may be indexed (CAS-1493).
#
# A TEMPLATE since CAS-1609: scripts/write-seo.mjs fills https://runscout.dev
# from the VITE_CANONICAL_ORIGIN build setting and writes dist/robots.txt. On a
# build with VITE_NOINDEX=1 (dev) it writes a disallow-all file instead, and this
# allowlist is not used.
#
# An ALLOWLIST. The obvious shape — "everything except /:slug/*" — fails open:
# a route added tomorrow is crawlable by default, and the one that matters is
# the one nobody thought about. This is the same reasoning already written into
# src/analytics/analyticsPaths.ts, and robots.test.ts holds the two files
# together so a new route cannot land in only one of them.
#
# Until this file existed there was no robots.txt at all, so every route was
# crawlable by default — including all 19 workspace pages, which render Entry
# content, member identities and a live workspace API key.
#
# `$` and `*` are standard: RFC 9309 section 2.2.2. Every marketing path is
# anchored with `$` on purpose. Unanchored, `Allow: /security` would also
# permit /security/dashboard — the workspace pages of a team whose slug happens
# to be "security".

User-agent: *
Disallow: /

# The marketing pages, each twice (CAS-1870). `$` anchors the end of the
# path AND query, so `Allow: /why$` alone refuses `/why?utm_source=x` — and a
# share crawler (Twitterbot, facebookexternalhit, LinkedInBot, Slackbot) that
# is refused draws no card at all for a UTM-tagged link. The `?` twin opens
# the same page with a query string and nothing else: `?` is a literal here,
# so `/why?` does not match /whyfoo or /why/anything. The canonical tag on
# each page dedupes the query variants for search.
Allow: /$
Allow: /?
Allow: /what-it-does$
Allow: /what-it-does?
Allow: /why$
Allow: /why?
Allow: /security$
Allow: /security?
Allow: /price$
Allow: /price?
Allow: /startups$
Allow: /startups?
Allow: /demo$
Allow: /demo?
Allow: /manifesto$
Allow: /manifesto?
Allow: /as-seen-on$
Allow: /as-seen-on?
Allow: /status$
Allow: /status?
Allow: /support$
Allow: /support?
Allow: /terms$
Allow: /terms?
Allow: /privacy$
Allow: /privacy?
Allow: /subprocessors$
Allow: /subprocessors?

# The conference pages (CAS-2405). Unlisted and not in the sitemap, and each
# page carries `noindex`. Allowed so a share crawler can draw the card for a
# texted or posted link, and so a search crawler can read that noindex.
Allow: /hello$
Allow: /hello?
Allow: /hello/qr$
Allow: /hello/qr?

# Citation file (CAS-1853). Anchored so it does not open /llms.txt/anything.
Allow: /llms.txt$

# IndexNow ownership key. The search engine fetches it to verify our URL
# submissions; refused here, it cannot. Anchored like the line above.
Allow: /9959ea6dc1c70b2571942e1fd370a7c0.txt$

# Docs: the index, and the seven topic pages under it.
Allow: /docs$
Allow: /docs?
Allow: /docs/

# Articles (CAS-2571). The index, and every published slug under it.
# A slug with no prerendered file 404s. The prefix is what lets a crawler
# fetch the ones the build wrote. Same shape as /docs/.
Allow: /articles$
Allow: /articles?
Allow: /articles/

# Googlebot and Bingbot render JavaScript, and cannot render a page whose
# script and stylesheet are blocked. Blocking these would make the crawlers
# that CAN read this site see the same blank shell as the ones that cannot.
Allow: /assets/
Allow: /favicon.svg
Allow: /icons.svg
Allow: /brand/

# Share card (CAS-1870). Pages name the content-hashed copy under /assets/,
# allowed above; this is the original fixed URL, kept for cards already
# posted. Anchored so it does not open /og.jpg/anything.
Allow: /og.jpg$

# Deliberately NOT allowed, and each for its own reason:
#
#   /login /signin /signup /register
#       analyticsPaths.ts: they "render an email input and nothing else".
#       There is no content to index, and the four are two pages under two
#       spellings each, so indexing them would file duplicates as well.
#
#   /start
#       Trial-link signup (CAS-2568). The code is in the query string.
#
#   /loud/product/*.html
#       Internal design mocks. Seven of them, served as real HTML, and until
#       this file they were the most crawlable HTML on the production host.
#
#   /:slug/*, /admin, /account, /onboarding, /invite/:token, /oauth/resume
#       Workspace and session pages. ANALYTICS_EXCLUDED in analyticsPaths.ts
#       says what each one renders and why it is kept out.

Sitemap: https://runscout.dev/sitemap.xml