# robots.txt — robinsonlandscape.com
# robots.txt controls CRAWLING, not indexing.

# ----------------------------------------------------------
# Default — all standard crawlers (Googlebot, Bingbot, etc.)
# ----------------------------------------------------------
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# ----------------------------------------------------------
# AI SEARCH / RETRIEVAL CRAWLERS — ALLOW (drive AI-search visibility)
# ----------------------------------------------------------
# OpenAI (ChatGPT)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini / AI Overviews — separate from Googlebot)
User-agent: Google-Extended
Allow: /

# Apple (Siri / Apple Intelligence)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon (Alexa / shopping AI)
User-agent: Amazonbot
Allow: /

# Meta (Meta AI / Llama)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Common Crawl (upstream dataset for many open models)
User-agent: CCBot
Allow: /

# ----------------------------------------------------------
# BLOCKED — aggressive scrapers / low US AI-search value
# (flip to Allow: / if you decide the upside is worth it)
# ----------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: cohere-ai
Disallow: /

# ----------------------------------------------------------
Sitemap: https://www.robinsonlandscape.com/sitemap_index.xml