# ============================================================
# robots.txt — NursingHomeNews.org
# Optimized for MAXIMUM AI Search Visibility
# Last updated: 2026-03-23
# ============================================================

# --- Default ---
User-agent: *
Allow: /
Crawl-delay: 1

# Allow sitemaps and AI documentation
Allow: /sitemaps/
Allow: /llms.txt
Allow: /llms-full.txt

# Block development and internal directories
Disallow: /dev-site/
Disallow: /data/
Disallow: /includes/
Disallow: /scripts/
Disallow: /templates/
Disallow: /video_test/

# --- GOOGLE (Search + AdSense) ---
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Crawl-delay: 0

User-agent: Googlebot-News
Allow: /
Crawl-delay: 0

User-agent: AdsBot-Google
Allow: /
Crawl-delay: 0

User-agent: AdsBot-Google-Mobile
Allow: /
Crawl-delay: 0

User-agent: Mediapartners-Google
Allow: /
Crawl-delay: 0

User-agent: Google-AdSense
Allow: /
Crawl-delay: 0

# --- GOOGLE AI (Gemini training) ---
User-agent: Google-Extended
Allow: /

# --- OPENAI ---
# GPTBot: Model training data
User-agent: GPTBot
Allow: /
Disallow: /includes/
Disallow: /data/

# OAI-SearchBot: CRITICAL — powers ChatGPT search results
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User: User-initiated retrieval
User-agent: ChatGPT-User
Allow: /

# --- ANTHROPIC (Claude) ---
User-agent: ClaudeBot
Allow: /
Disallow: /includes/

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# --- PERPLEXITY ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- APPLE ---
User-agent: Applebot-Extended
Allow: /

# --- META ---
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# --- OTHER AI BOTS ---
User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: CCBot
Allow: /

# --- BING ---
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# --- OTHER SEARCH ENGINES ---
User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# --- SOCIAL MEDIA ---
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 1

User-agent: Twitterbot
Allow: /
Crawl-delay: 1

User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

# Sitemaps
Sitemap: https://nursinghomenews.org/sitemap.xml