# =========================
# Crawl-delay settings for specific bots
# =========================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: bingbot
Crawl-delay: 5

User-agent: msnbot
Crawl-delay: 5

User-agent: msnbot-media
Crawl-delay: 5

# =========================
# Block aggressive bots completely
# =========================

User-agent: MJ12bot
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: dotbot
Disallow: /

# =========================
# Default rules for all other bots
# =========================

User-agent: *
Disallow: /admin/
Disallow: /system/
Disallow: /phone/
Disallow: /sales/
Disallow: /wholesale/
Disallow: /purchasing/
Disallow: /search # Block search/faceted URLs
Disallow: /*?page= # Block pagination URLs
Allow: /content/
Crawl-delay: 10

# =========================
# SEMrush and related bots
# =========================

User-agent: SiteAuditBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: SemrushBot-COUB
Disallow: /