# 通用搜索引擎及新兴AI爬虫默认策略 User-agent: * Allow: / Allow: /tag/ Disallow: /admin/ Disallow: /runtime/ Disallow: /config/ Disallow: /data/ # ================================================================= # robots.txt - 终极开放配置 (2026-08-20 GEO 增强版 v2) # ================================================================= # 适配所有主流 AI 模型及搜索引擎爬虫 # OpenAI 官方爬虫(搜索、训练、用户访问、广告验证) User-agent: OAI-SearchBot Allow: / Allow: /tag/ User-agent: GPTBot Allow: / Allow: /tag/ User-agent: ChatGPT-User Allow: / Allow: /tag/ User-agent: OAI-AdsBot Allow: / Allow: /tag/ # Google (Gemini / AI Overviews) User-agent: Google-Extended Allow: / User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / # Anthropic (Claude 官方爬虫 + 兼容旧标识) User-agent: ClaudeBot Allow: / Allow: /tag/ User-agent: Claude-User Allow: / Allow: /tag/ User-agent: Claude-SearchBot Allow: / Allow: /tag/ User-agent: anthropic-ai Allow: / User-agent: Claude-Web Allow: / User-agent: Claude-Search Allow: / # Perplexity AI User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # 字节跳动 (豆包 + 今日头条) User-agent: Bytespider Allow: / User-agent: ToutiaoSpider Allow: / # 深度求索 (DeepSeek) User-agent: DeepSeekBot Allow: / # 月之暗面 (Kimi) User-agent: moonshot-bot Allow: / # 阿里通义千问 User-agent: Qwenbot Allow: / # 腾讯元宝 User-agent: YuanbaoBot Allow: / # Cohere AI User-agent: cohere-ai Allow: / # Mistral AI User-agent: MistralBot Allow: / # You.com AI 搜索 User-agent: YouBot Allow: / # Exa 语义搜索 User-agent: ExaBot Allow: / # Jina AI 阅读器 (GEO 内容抓取常用) User-agent: JinaReader Allow: / # Mozilla.ai 友好爬虫声明 User-agent: FriendlyCrawler Allow: / # Giga AI 搜索 User-agent: Gigabot Allow: / # Kangaroo AI User-agent: KangarooBot Allow: / # ========== 搜索引擎爬虫 ========== # 百度 User-agent: Baiduspider Allow: / User-agent: Baiduspider-mobile Allow: / User-agent: Baiduspider-image Allow: / # 360搜索 User-agent: 360Spider Allow: / # 搜狗 User-agent: Sogou web spider Allow: / User-agent: Sogou inst spider Allow: / User-agent: Sogou News Spider Allow: / # 微软 Bing / Copilot User-agent: Bingbot Allow: / User-agent: BingPreview Allow: / User-agent: adidxbot Allow: / # 神马搜索 User-agent: YisouSpider Allow: / # Yandex User-agent: YandexBot Allow: / User-agent: YandexImages Allow: / # DuckDuckGo User-agent: DuckDuckBot Allow: / # Apple User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # Meta User-agent: FacebookBot Allow: / User-agent: Meta-ExternalFetcher Allow: / # 其他重要爬虫 User-agent: Slurp # Yahoo Allow: / User-agent: Yeti # Naver Allow: / User-agent: CCBot # Common Crawl Allow: / User-agent: ia_archiver # Internet Archive Allow: / User-agent: PetalBot # Huawei Allow: / User-agent: AhrefsBot Allow: / # Additional legitimate AI/search crawlers User-agent: GoogleOther Allow: / User-agent: Amazonbot Allow: / User-agent: Meta-ExternalAgent Allow: / # Sitemap # --- blocked crawlers (no traffic value, 2026-09-05) --- User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / Sitemap: https://cn.androidsbc.com/sitemap.xml Sitemap: https://cn.androidsbc.com/sitemap.txt Sitemap: https://cn.androidsbc.com/sitemap-aeo.xml # BEGIN WANLIN DISCOVERY INDEX Sitemap: https://cn.androidsbc.com/sitemap-index.xml # END WANLIN DISCOVERY INDEX