# Robots.txt for Agentic Automation
# Optimized for AI agent discovery and crawling

User-agent: *
Allow: /

# AI Agents - Full Access
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: Google-Extended
User-agent: PerplexityBot
Allow: /

# Sitemaps
Sitemap: https://agentic-automation.co.uk/sitemap.xml

# MCP and NL Web Discovery
Allow: /.well-known/
Allow: /ask
Allow: /agent
Allow: /feed.xml
Allow: /feed.json
Allow: /rss.xml
Allow: /nl-manifest.json
Allow: /api/services
Allow: /api/tools/

# Rate limits apply to /api/tools/* endpoints
# Authentication required for transactional endpoints