# 1. GOOGLE系列
User-agent: Googlebot
Disallow: /api/*
Disallow: /search*
Disallow: /preview*
Crawl-delay: 1

User-agent: Googlebot-Image
Crawl-delay: 2

# 2. 微软BING系列
User-agent: bingbot
Disallow: /api/*
Disallow: /search*
Disallow: /preview*
Crawl-delay: 1

User-agent: msnbot
Disallow: /api/*
Disallow: /search*
Disallow: /preview*
Crawl-delay: 2

# 3. AI/LLM爬虫
User-agent: GPTBot
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 3

User-agent: ChatGPT-User
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 2

User-agent: CCBot
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 3

# 4. 其他搜索引擎
User-agent: Applebot
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 3

User-agent: DuckDuckBot
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 2

# 5. 社交媒体
User-agent: facebookexternalhit
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 10

User-agent: Twitterbot
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 10

User-agent: LinkedInBot
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 10

# 6. SEO工具 - 严格禁止
User-agent: AhrefsBot
Disallow: /
Crawl-delay: 30

User-agent: SemrushBot
Disallow: /
Crawl-delay: 30

User-agent: MJ12bot
Disallow: /
Crawl-delay: 30

User-agent: DotBot
Disallow: /
Crawl-delay: 30

User-agent: rogerbot
Disallow: /
Crawl-delay: 30

User-agent: Amazonbot
Disallow: /
Crawl-delay: 30

User-agent: Dataprovider.com
Disallow: /
Crawl-delay: 30


# 7. 通用规则（必须放在最后！）
User-agent: *
Disallow: /api/*
Disallow: /preview*
Crawl-delay: 5

# Sitemaps
Sitemap: https://www.ocleannutra.com/sitemap.xml