# https://www.robotstxt.org/robotstxt.html
# Ivorycom CRM — public marketing pages are crawlable; the
# authenticated app surface (under /dashboard, /leads, etc.) is a SPA
# and isn't useful to crawl, but blocking it here would also block
# preview crawlers like Slackbot/Twitterbot from rendering OG tags on
# shared links. Allow everything by default; rely on noindex meta tags
# inside the SPA for any specific page that shouldn't surface.
User-agent: *
Allow: /
Disallow: /api/
# ─── AI / LLM crawlers — explicit allows ───────────────────────────────
# These bots train + power the live-search features in ChatGPT, Claude,
# Perplexity, Gemini, Copilot. Many SaaS sites block them by default
# (or get them blocked by Cloudflare's bot-fight mode) and then wonder
# why they're invisible in AI answers. Explicit allow = explicit intent.
User-agent: GPTBot
Allow: /
Disallow: /api/
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
User-agent: ClaudeBot
Allow: /
Disallow: /api/
User-agent: Claude-Web
Allow: /
Disallow: /api/
User-agent: anthropic-ai
Allow: /
Disallow: /api/
User-agent: PerplexityBot
Allow: /
Disallow: /api/
User-agent: Perplexity-User
Allow: /
Disallow: /api/
User-agent: Google-Extended
Allow: /
Disallow: /api/
User-agent: Googlebot
Allow: /
Disallow: /api/
User-agent: Bingbot
Allow: /
Disallow: /api/
User-agent: CCBot
Allow: /
Disallow: /api/
User-agent: Applebot
Allow: /
Disallow: /api/
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
User-agent: cohere-ai
Allow: /
Disallow: /api/
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
User-agent: YouBot
Allow: /
Disallow: /api/
# Sitemap pointer so search engines + LLM crawlers can find every
# public page in one fetch instead of crawling links blindly.
Sitemap: https://ivorycomcrm.com/sitemap.xml
# llms.txt is the emerging soft standard for telling LLM crawlers what
# the site is, who it's for, and where the canonical sources live.
# https://llmstxt.org/