# 屏蔽不必要爬虫
User-agent: MJ12Bot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: SERankingBacklinksBot
Disallow: /
User-agent: Serpstatbot
Disallow: /
User-agent: IbouBot
Disallow: /
User-agent: IBOUBOT
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: LeakIX
Disallow: /
User-agent: l9scan
Disallow: /

# ========= 威胁不大的爬虫，限制频率 =========
User-agent: Amzn-SearchBot
Crawl-delay: 60
Disallow: /article_page/  # 屏蔽目录 article_page
Disallow: /category_collection/  # 屏蔽目录 category_collection 

# ========= 屏蔽AI爬虫 =========
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-SearchBot
Disallow: /
User-agent: Claude-Web
Disallow: /

# ========= 屏蔽恶意/高风险爬虫 =========
User-agent: Barkrowler
Disallow: /

# 允许主流搜索引擎和必需服务
# 允许Google爬虫
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Mobile
Allow: /

# 允许Facebook爬虫
User-agent: FacebookBot
Allow: /
User-agent: facebookexternalhit
Allow: /

User-agent: *
Disallow: