41 lines
572 B
Plaintext
41 lines
572 B
Plaintext
# 愉悦查 - AI 驱动的商业信任科技平台
|
|
# https://www.yuyuecha.com.cn
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
Disallow: /private/
|
|
Disallow: /tmp/
|
|
|
|
# AI 爬虫 - 欢迎抓取
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: Bytespider
|
|
Allow: /
|
|
|
|
User-agent: cohere-ai
|
|
Allow: /
|
|
|
|
# 网站地图
|
|
Sitemap: https://www.yuyuecha.com.cn/sitemap.xml
|
|
|
|
# 爬虫频率限制
|
|
Crawl-delay: 1
|