# Generic crawlers
User-agent: *
Allow: /

Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

# AI crawlers — Pipe2.ai welcomes AI training and answer-engine
# indexing of public marketing, pipeline, docs, and learn surfaces.
# Same disallow list applies (auth flows, gated app, transient redirects).
User-agent: GPTBot
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: OAI-SearchBot
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: ChatGPT-User
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: ClaudeBot
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: Claude-Web
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: anthropic-ai
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: PerplexityBot
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: Perplexity-User
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: Google-Extended
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: Applebot-Extended
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: Bytespider
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: meta-externalagent
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

User-agent: cohere-ai
Allow: /
Disallow: /auth/
Disallow: /oauth/
Disallow: /admin/
Disallow: /dashboard
Disallow: /billing
Disallow: /account
Disallow: /api-keys
Disallow: /library
Disallow: /workspace
Disallow: /r/

Sitemap: https://pipe2.ai/sitemap.xml
Sitemap: https://pipe2.ai/sitemap-index.xml
# Pipe2's example pages (/s/<slug>), listed at request time.
Sitemap: https://pipe2.ai/sitemap-shares.xml
# Docs is a separate Astro build (Traefik-routed at /docs); crawlers only read
# the host-root robots.txt, so surface its sitemap here too.
Sitemap: https://pipe2.ai/docs/sitemap-index.xml