# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
User-agent: *
Crawl-delay: 5
Disallow: /*?*page=*
Disallow: /editions/*
Disallow: /users/*
Disallow: /feed
Disallow: /feeds
Disallow: /rss
Disallow: /*?*q=*
User-agent: SemrushBot
Disallow: /
User-agent: PetalBot
Disallow: /