User-agent: *
Allow: /

# Block staging, API, and internal utility paths
Disallow: /api/*
Disallow: /v2/
Disallow: /upload-csv
Disallow: /upload-csv-2
Disallow: /upload-clarity
Disallow: /upload-clarity-2
Disallow: /funnel/
Disallow: /pe/

# =====================================================
# AI, LLM, and Search Crawlers, Allow ALL
# GOSO wants maximum visibility across every AI search
# engine, assistant, and citation system. Every crawler
# that finds us is a potential customer finding us.
#
# Source: github.com/ai-robots-txt/ai.robots.txt
# Last updated: 2026-03-21
# =====================================================

# --- OpenAI (ChatGPT, SearchGPT, Operator) ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ChatGPT Agent
Allow: /

User-agent: OpenAI
Allow: /

User-agent: Operator
Allow: /

# --- Anthropic (Claude) ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# --- Google (Gemini, AI Overviews, NotebookLM) ---
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

User-agent: GoogleOther-Video
Allow: /

User-agent: GoogleAgent-Mariner
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: CloudVertexBot
Allow: /

User-agent: Google-Firebase
Allow: /

User-agent: Google-NotebookLM
Allow: /

User-agent: NotebookLM
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

# --- Microsoft (Bing, Copilot) ---
User-agent: Bingbot
Allow: /

User-agent: AzureAI-SearchBot
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- xAI (Grok) ---
User-agent: GrokBot
Allow: /

# --- Apple (Siri, Apple Intelligence) ---
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# --- Meta (Meta AI, Facebook) ---
User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: meta-webindexer
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: facebookexternalhit
Allow: /

# --- Amazon (Alexa, Bedrock, Kendra) ---
User-agent: Amazonbot
Allow: /

User-agent: AmazonBuyForMe
Allow: /

User-agent: Amzn-SearchBot
Allow: /

User-agent: Amzn-User
Allow: /

User-agent: amazon-kendra
Allow: /

User-agent: bedrockbot
Allow: /

# --- Mistral AI ---
User-agent: MistralAI-User
Allow: /

User-agent: MistralAI-User/1.0
Allow: /

# --- DeepSeek ---
User-agent: DeepSeekBot
Allow: /

# --- Brave Search ---
User-agent: Bravebot
Allow: /

# --- DuckDuckGo ---
User-agent: DuckAssistBot
Allow: /

# --- You.com ---
User-agent: YouBot
Allow: /

# --- Phind (developer AI search) ---
User-agent: PhindBot
Allow: /

# --- iAsk AI ---
User-agent: iaskspider
Allow: /

User-agent: iAskBot
Allow: /

# --- Kagi Search ---
User-agent: kagi-fetcher
Allow: /

# --- Cohere AI ---
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# --- LinkedIn ---
User-agent: LinkedInBot
Allow: /

# --- ByteDance (TikTok) ---
User-agent: Bytespider
Allow: /

User-agent: TikTokSpider
Allow: /

# --- Common Crawl (used by many AI systems) ---
User-agent: CCBot
Allow: /

# --- Yandex ---
User-agent: YandexAdditional
Allow: /

User-agent: YandexAdditionalBot
Allow: /

# --- Other AI search and citation crawlers ---
User-agent: Andibot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ExaBot
Allow: /

User-agent: LinerBot
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: TavilyBot
Allow: /

User-agent: LinkupBot
Allow: /

User-agent: Manus-User
Allow: /

User-agent: WRTNBot
Allow: /

User-agent: PanguBot
Allow: /

User-agent: SBIntuitionsBot
Allow: /

User-agent: ChatGLM-Spider
Allow: /

User-agent: Cloudflare-AutoRAG
Allow: /

User-agent: FirecrawlAgent
Allow: /

User-agent: Crawl4AI
Allow: /

User-agent: Poggio-Citations
Allow: /

User-agent: AddSearchBot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: Brightbot 1.0
Allow: /

User-agent: atlassian-bot
Allow: /

User-agent: NovaAct
Allow: /

User-agent: Devin
Allow: /

User-agent: bigsur.ai
Allow: /

Sitemap: https://goso.io/sitemap-index.xml
Sitemap: https://goso.io/sitemap-0.xml