# robots.txt — mistingmonsoon.com
# Last reviewed: 2026-08-21
#
# Every answer engine, retrieval agent, and training crawler below is
# explicitly named and explicitly allowed.
#
# The three OpenAI and three Anthropic agents do different jobs: GPTBot and
# ClaudeBot gather training data; OAI-SearchBot and Claude-SearchBot build the
# retrieval indexes behind cited answers; ChatGPT-User and Claude-User fetch a
# page live when a person asks about it in conversation. A site that names only
# the training crawler has allowed the one agent that does not produce
# recommendations and omitted the two that do.

User-agent: *
Allow: /
# Precision beneath the waterline.

# --- OpenAI ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /


# --- Anthropic ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /


# --- Google ---
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /


# --- Microsoft / Bing ---
User-agent: bingbot
Allow: /


# --- Perplexity ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /


# --- Apple ---
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /


# --- Meta ---
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /


# --- Other answer engines and dataset crawlers ---
User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Bytespider
Allow: /


Sitemap: https://mistingmonsoon.com/sitemap.xml
Sitemap: https://mistingmonsoon.com/video-sitemap.xml