# ============================================================ # NomadTaxCalc — robots.txt # Updated: June 2026 # Strategy: block AI TRAINING crawlers, allow AI SEARCH & # RETRIEVAL agents that drive citation traffic. # ============================================================ # ── Default: allow all well-behaved crawlers ───────────────── User-agent: * Allow: / Allow: /authors/ # Block private/auth routes Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ # Allow auth entry points Allow: /login Allow: /register # Block internal API & session-only routes (not public pages) Disallow: /tax-calculator/session Disallow: /tax-calculator/step-1 Disallow: /tax-calculator/step-2 Disallow: /tax-calculator/save Disallow: /tax-calculator/email-results Disallow: /tax-calculator/generate-link Disallow: /tax-calculator/visa-optimizer Disallow: /tax-calculator/se-comparison Disallow: /tax-calculator/report/view Disallow: /currencies # Block Laravel internals & Cloudflare endpoints Disallow: /_ignition Disallow: /sanctum Disallow: /horizon Disallow: /telescope Disallow: /cdn-cgi/ # Block personal token routes Disallow: /newsletter/unsubscribe/ # ── WordPress blog — allow public content, block admin ─────── Allow: /blog/ Allow: /blog/sitemap_index.xml Allow: /blog/post-sitemap.xml Disallow: /blog/wp-admin/ Disallow: /blog/wp-login.php Disallow: /blog/wp-json/ Disallow: /blog/xmlrpc.php Disallow: /blog/?p= Disallow: /blog/?s= Disallow: /blog/tag/ # ── Google ──────────────────────────────────────────────────── # Googlebot: NEVER block — powers Google Search + AI Overviews # Google-Extended: opt-out from Gemini/Vertex AI TRAINING only. # Blocking Google-Extended has zero effect on search rankings # or AI Overview citations. Safe to block for IP protection. User-agent: Google-Extended Disallow: / # ── Bing / Copilot ──────────────────────────────────────────── # Bingbot: standard crawler → allow (powers Copilot answers) User-agent: Bingbot Allow: / # adidxbot: Bing Ads quality bot → allow User-agent: adidxbot Allow: / # msnbot-media: image crawler only → block (no citation value) User-agent: msnbot-media Disallow: / User-agent: msnbot Disallow: / # ── OpenAI ──────────────────────────────────────────────────── # GPTBot: training crawler → BLOCK User-agent: GPTBot Disallow: / # OAI-SearchBot: ChatGPT live web search / citation → ALLOW User-agent: OAI-SearchBot Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ Disallow: /blog/wp-admin/ Disallow: /blog/wp-json/ # ChatGPT-User: user-triggered live retrieval → ALLOW User-agent: ChatGPT-User Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ # ── Anthropic ───────────────────────────────────────────────── # ClaudeBot: training scraper → BLOCK User-agent: ClaudeBot Disallow: / # anthropic-ai: legacy training name → BLOCK User-agent: anthropic-ai Disallow: / # Claude-Web: live retrieval / citation → ALLOW User-agent: Claude-Web Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ Disallow: /blog/wp-admin/ Disallow: /blog/wp-json/ # Claude-SearchBot: search-mode citation → ALLOW User-agent: Claude-SearchBot Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ # ── Perplexity ──────────────────────────────────────────────── # PerplexityBot: indexes pages for Sonar citation answers → ALLOW User-agent: PerplexityBot Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ Disallow: /blog/wp-admin/ Disallow: /blog/wp-json/ # Perplexity-User: live retrieval → ALLOW User-agent: Perplexity-User Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ # ── xAI / Grok ─────────────────────────────────────────────── # xAI-SearchBot: Grok search index → ALLOW User-agent: xAI-SearchBot Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ # ── You.com ────────────────────────────────────────────────── User-agent: YouBot Allow: / Disallow: /dashboard Disallow: /profile Disallow: /my-calculations Disallow: /auth/ # ── Meta ───────────────────────────────────────────────────── # All Meta agents confirmed training scrapers as of 2026 → BLOCK User-agent: Meta-ExternalAgent Disallow: / User-agent: Meta-ExternalFetcher Disallow: / User-agent: FacebookBot Disallow: / # ── Amazon ─────────────────────────────────────────────────── User-agent: Amazonbot Disallow: / # ── Common Crawl ───────────────────────────────────────────── # Powers many LLM training pipelines → BLOCK User-agent: CCBot Disallow: / # ── Bytedance / TikTok ─────────────────────────────────────── User-agent: Bytespider Disallow: / # ── Diffbot ────────────────────────────────────────────────── User-agent: Diffbot Disallow: / # ── Apple ──────────────────────────────────────────────────── # Applebot: standard Siri search → ALLOW (sends traffic) User-agent: Applebot Allow: / # Applebot-Extended: Apple Intelligence training → BLOCK User-agent: Applebot-Extended Disallow: / # ── Cohere ─────────────────────────────────────────────────── User-agent: cohere-ai Disallow: / # ── AI2 / Allen Institute ──────────────────────────────────── User-agent: AI2Bot Disallow: / # ── DataForSEO ─────────────────────────────────────────────── User-agent: DataForSeoBot Disallow: / # ── Omgili / Webz.io ───────────────────────────────────────── User-agent: omgili Disallow: / User-agent: omgilibot Disallow: / # ── Timpibot ───────────────────────────────────────────────── User-agent: Timpibot Disallow: / Sitemap: https://nomadtaxcalc.com/sitemap.xml Sitemap: https://nomadtaxcalc.com/blog/post-sitemap.xml