# robots.txt for brandscanner.org
# AI-friendly configuration with llms.txt integration
# Last updated: 2025-01-15

# AI Crawlers and LLM Training Data Collection
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: BingBot
Allow: /

User-agent: BingPreview
Allow: /

# Search Engine Crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# Social Media Crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# All other crawlers
User-agent: *
Allow: /

# Crawl-delay for polite crawling (1 second)
Crawl-delay: 1

# Important resources and standards
# LLMs.txt file for AI usage permissions and guidelines
# See: https://llmstxt.org/ for specification details
Sitemap: https://brandscanner.org/sitemap.xml

# AI Usage Guidelines
# For detailed AI usage permissions, training guidelines, and attribution requirements,
# please refer to our LLMs.txt file at: https://brandscanner.org/llms.txt
# 
# This file contains:
# - Usage permissions for AI inference, training, and commercial use
# - Attribution requirements and rate limiting guidelines  
# - Canonical URL references and contact information
# - Technical standards and API documentation links