# Apier.no crawl policy (advisory only; real access control is enforced in the auth layer).
# Public API + agent-discovery surfaces are crawlable; auth-gated /api/ paths stay blocked.
# Allow lines precede Disallow so first-match crawlers permit the public subpaths before
# the broad /api/ block; Google's longest-match resolves the same either way.
User-agent: *
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
# Discovery reads that llms.txt advertises: agents and AI crawlers may fetch
# them. /api/v1/sandbox/fixtures, /api/v1/errors and /api/mcp answer with
# X-Robots-Tag: noindex, so they are read, not indexed. /api/knowledge (the
# public Knowledge Catalog JSON) sends no noindex and may be indexed, like the
# public API routes above. "Allow: /api/mcp$" matches only the exact path,
# i.e. the keyless GET discovery document. The agent-readiness probe and the
# payment options read are advertised by llms.txt and the agent-payments guide.
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

# Named AI-crawler groups below intentionally restate the SAME Allow/Disallow
# policy as User-agent: *. A crawler obeys only its single most-specific
# matching group and does NOT inherit the * group, so the /api/ disallow
# MUST be repeated in every named block — otherwise these bots would
# be unconstrained on the API + MCP surfaces. Each block's Allow set is
# identical to the * group above.
User-agent: GPTBot
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

User-agent: ClaudeBot
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

User-agent: Google-Extended
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

User-agent: PerplexityBot
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

User-agent: Bingbot
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

User-agent: Applebot-Extended
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

User-agent: DuckAssistBot
Allow: /api/v1/public/
Allow: /api/v1/sandbox/public/
Allow: /api/v1/tools/
Allow: /api/v1/capabilities
Allow: /api/v1/pricing
Allow: /api/v1/comparison/
Allow: /.well-known/
Allow: /api/v1/sandbox/fixtures
Allow: /api/v1/errors
Allow: /api/knowledge
Allow: /api/mcp$
Allow: /api/v1/health/agent-readiness
Allow: /api/v1/payments/options
Disallow: /api/

# LLM / AI-agent guidance (comment only — there is no standard robots.txt
# directive for llms.txt, and unknown directives can confuse strict parsers):
# a machine-readable site summary is served at https://www.apier.no/llms.txt
# (full version: https://www.apier.no/llms-full.txt).

Sitemap: https://www.apier.no/sitemap.xml