# robots.txt for https://www.alixco.com
# Last updated: 2026-09-09
# Source: Alixco AI-Readiness Audit (§8), May 2026
#         Crawl-budget review, July 2026 (AI traffic statistics)
#         Crawl-budget review, Sept 2026: crawls down ~80% since July.
#         Remaining problem is NOT a missing rule - it is that "/*:"
#         relies on the "*" wildcard, which not every crawler supports.
#         YouBot was still fetching "…layout.footer:setlocale/xx" on
#         09.09.2026, eight weeks after the rule went live. A rule that
#         needs a wildcard cannot be relied on; only plain prefixes
#         ("Disallow: /") and a server-side block are universal.
# ------------------------------------------------------------------
# Policy
#   - Allow the major AI search bots so Alixco can be cited in
#     ChatGPT Search, Perplexity, Claude, Gemini, Copilot, etc.
#   - Block private / interactive areas (/main, /account, /js).
#   - Block auth and form-handler URLs: they carry no citable content,
#     yet the July 2026 stats showed /login as the single most-crawled
#     URL on the whole site (~10% of all hits). Every hit spent there
#     is a hit not spent on a campaign page.
#   - Drop the deprecated "Noindex:" directive (replaced by
#     <meta name="robots" content="noindex"> where needed).
#
# IMPORTANT — how robots.txt group matching works:
#   A crawler obeys ONLY the single most specific User-agent group that
#   matches its token; all other groups, including "*", are ignored
#   entirely. So a bare "User-agent: GPTBot / Allow: /" group does NOT
#   inherit the Disallow rules from the "*" group — it would let GPTBot
#   crawl /login, /main and /account. Every rule set below therefore
#   repeats the full Disallow list. If you add a Disallow, add it to
#   EVERY group, not just to "*".
# ------------------------------------------------------------------

# ---------- Default rules for all crawlers ----------
User-agent: *
Disallow: /main
Disallow: /account
Disallow: /js
Disallow: /login
Disallow: /register
Disallow: /verifyemail
Disallow: /basicinformation
Disallow: /*.form$
# Tapestry component-event URLs ("page.component:eventname/context").
# The footer language switcher alone yields one such URL per page per
# locale — all of them only redirect back to a page the crawler already
# has. A colon never occurs in a real Alixco content URL.
Disallow: /*:
Disallow: *?*passwordreminder*
Disallow: *?*jsessionid*
# jsessionid is a path parameter (";jsessionid="), not a query string —
# fallback rule in case URL rewriting is ever re-enabled (F-30):
Disallow: *;jsessionid*
# JWKS is an API artefact for partner token validation, not content.
# Plain prefix rule (no wildcard), so every parser honours it; the
# machine-to-machine clients that actually need it ignore robots.txt
# anyway. Removes ~160 pointless crawler hits.
Disallow: /.well-known/jwks.json
Allow: /

# ---------- AI search engines (allowed: we WANT to be cited) ----------
# Same rule set as "*" — repeated because these groups override it.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: meta-externalagent
User-agent: DuckAssistBot
User-agent: MistralAI-User
Disallow: /main
Disallow: /account
Disallow: /js
Disallow: /login
Disallow: /register
Disallow: /verifyemail
Disallow: /basicinformation
Disallow: /*.form$
# Tapestry component-event URLs ("page.component:eventname/context").
# The footer language switcher alone yields one such URL per page per
# locale — all of them only redirect back to a page the crawler already
# has. A colon never occurs in a real Alixco content URL.
Disallow: /*:
Disallow: *?*passwordreminder*
Disallow: *?*jsessionid*
Disallow: *;jsessionid*
# JWKS is an API artefact for partner token validation, not content.
# Plain prefix rule (no wildcard), so every parser honours it; the
# machine-to-machine clients that actually need it ignore robots.txt
# anyway. Removes ~160 pointless crawler hits.
Disallow: /.well-known/jwks.json
Allow: /

# ---------- Specific bots we do NOT want crawling at all ----------
# (Originally Amazonbot was fully disallowed - kept intentionally.)
User-agent: Amazonbot
Disallow: /

# Bytespider (ByteDance) crawls aggressively and gives nothing back.
User-agent: Bytespider
Disallow: /

# ---------- Sitemap ----------
# /sitemap.xml requires a server-side rewrite to /sitemapnew.xml
# (Apache mod_rewrite or Cloudflare Page Rules). See docs/AI-READINESS.md.
Sitemap: https://www.alixco.com/sitemap.xml
Sitemap: https://www.alixco.com/sitemapnew.xml