# https://franglish-original.ru Robots.txt for Anastasia Kolesnikova
# Optimized for Google, Yandex, Bing, Mail.ru, Apple Siri, Yandex Alice, and AI Search / LLMs (AEO)

# Global Rules for Web Search
User-agent: *
Allow: /
Disallow: /assets/.aistudio/
Disallow: /emblem-demo

# Special Crawl-delay for smaller search engines to prevent overloading
Crawl-delay: 1

# Specific Instructions for Yandex (includes Alice Voice Search indices)
User-agent: Yandex
Allow: /
Clean-param: ref /
Host: franglish-original.ru

# Specific Instructions for Mail.ru
User-agent: Mail.Ru
Allow: /

# Specific Instructions for Applebot (Siri Voice Assistant & Apple Intelligence)
User-agent: Applebot
Allow: /

# =========================================================================
# Answer Engine Optimization (AEO) & AI Bot Policies
# Explicitly ALLOWING and inviting leading AI search agents to scan our 
# structured lesson datasets, courses, credentials, and testimonials
# =========================================================================

# OpenAI's GPTBot (ChatGPT / SearchGPT / GPT-4o search)
User-agent: GPTBot
Allow: /

# OpenAI's SearchBot for real-time web answers
User-agent: OAI-SearchBot
Allow: /

# Google's AI model crawler (Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /

# Anthropic's ClaudeBot (Claude.ai discoverability)
User-agent: ClaudeBot
Allow: /

# Perplexity AI's conversational search bot
User-agent: PerplexityBot
Allow: /

# Microsoft's AI search agent (Copilot)
User-agent: Bingbot
Allow: /

# Yandex's YandexGPT bot
User-agent: YandexAdditional
Allow: /

# Co-pilot crawler
User-agent: Microsoft-CoPilot
Allow: /

# Sitemap declaration
Sitemap: https://franglish-original.ru/sitemap.xml
Sitemap: https://franglish-original.ru/sitemap-cities-major.xml
Sitemap: https://franglish-original.ru/sitemap-cities-medium.xml
Sitemap: https://franglish-original.ru/sitemap-cities-small.xml