# Datox robots policy: every public page may be crawled, including by AI search
# and answer crawlers (listed explicitly below so the policy is clear).
# /_serverFn/ is TanStack Start's internal RPC endpoint, not content.
# A crawler that matches a named group ignores "*", so each group repeats the rules.

# Search engines
User-agent: Googlebot
Allow: /
Disallow: /_serverFn/

User-agent: Bingbot
Allow: /
Disallow: /_serverFn/

# Link previews
User-agent: Twitterbot
Allow: /
Disallow: /_serverFn/

User-agent: facebookexternalhit
Allow: /
Disallow: /_serverFn/

# OpenAI
User-agent: OAI-SearchBot
Allow: /
Disallow: /_serverFn/

User-agent: ChatGPT-User
Allow: /
Disallow: /_serverFn/

User-agent: GPTBot
Allow: /
Disallow: /_serverFn/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /_serverFn/

User-agent: Claude-SearchBot
Allow: /
Disallow: /_serverFn/

User-agent: Claude-User
Allow: /
Disallow: /_serverFn/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /_serverFn/

User-agent: Perplexity-User
Allow: /
Disallow: /_serverFn/

# Google and Apple AI use controls
User-agent: Google-Extended
Allow: /
Disallow: /_serverFn/

User-agent: Applebot-Extended
Allow: /
Disallow: /_serverFn/

# Everyone else
User-agent: *
Allow: /
Disallow: /_serverFn/

Sitemap: https://datox.ai/sitemap.xml