# BuiltABot robots.txt
# All pages are allowed by default; only private/internal routes are blocked

User-agent: *
Allow: /

# =============================================================================
# BLOCKED ROUTES (Private & Internal)
# =============================================================================
# User dashboard (requires authentication)
Disallow: /dashboard
Disallow: /dashboard/*

# API endpoints (not for crawling)
Disallow: /api/

# Auth callback routes (internal OAuth/auth flows)
Disallow: /auth/callback
Disallow: /auth/update-password

# =============================================================================
# CRAWLABLE PUBLIC PAGES (covered by Allow: / above)
# =============================================================================
# Homepage, About, Pricing, Contact
# Blog: /blog, /blog/*
# Auth: /auth/signin, /auth/signup
#
# Help & Documentation:
#   /tutorials
#   /documentation
#   /help/troubleshooting
#
# PSEO Pages (61+ pages):
#   /solutions/, /solutions/[industry], /solutions/[industry]/[city]
#   /use-cases/, /use-cases/[use-case]
#   /locations/, /locations/[city]
#   /compare/, /compare/[competitor]
#   /features/, /features/[feature]
#
# Free Tools (21 tools):
#   /tools (landing page)
#   Chatbot content generators:
#   /tools/chatbot-personality-generator
#   /tools/chatbot-name-generator
#   /tools/customer-service-script-generator
#   /tools/welcome-message-generator
#   /tools/faq-generator
#   /tools/quick-reply-generator
#   /tools/support-response-generator
#   /tools/roi-calculator
#   SEO cluster:
#   /tools/sitemap-url-extractor
#   /tools/sitemap-checker
#   /tools/sitemap-generator
#   Markdown / content prep cluster:
#   /tools/convert-webpage-to-markdown
#   /tools/convert-paste-to-markdown
#   /tools/convert-pdf-to-markdown
#   /tools/convert-docx-to-markdown
#   /tools/convert-json-to-markdown
#   /tools/convert-xml-to-markdown
#   /tools/pdf-faq-generator
#   AI / LLM utility cluster:
#   /tools/ai-answer-generator
#   /tools/ai-prompt-optimizer
#   /tools/openai-api-pricing-calculator

# =============================================================================
# CRAWLER SETTINGS
# =============================================================================
# Crawl delay (helps with server load)
Crawl-delay: 1

# Sitemap location (crawlers will discover all public pages here)
Sitemap: https://builtabot.com/sitemap.xml

# =============================================================================
# AI / LLM INGEST FILES (llmstxt.org convention)
# =============================================================================
# Lightweight LLM index (curated): https://builtabot.com/llms.txt
# Full machine-generated catalog (all blogs + tools): https://builtabot.com/llms-full.txt
# Both files are also linked from sitemap-discoverable pages. AI agents and
# crawlers should prefer these over scraping individual pages.