User-agent: *
Crawl-delay: 20
Disallow: /accounting/
Disallow: /ads.txt
Disallow: /app-ads.txt
Disallow: /arc/
Disallow: /cgi/
Disallow: /feed
Disallow: /feeds
Disallow: /internal/
Disallow: /reports/
Disallow: /usr/

User-Agent: ahc
Disallow: /

User-Agent: Adsbot
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-Agent: Baiduspider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: exensa.com
Disallow: /

User-agent: facebookexternalhit
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: opensiteexplorer
Disallow: /

User-agent: petalbot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: Siteimprove
Disallow: /

User-agent: siteimprove-lookup
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: Webmeup
Disallow: /

User-Agent: XTbot
Disallow: /

User-Agent: yandex
Disallow: /