User-agent: Scrapy
Allow: /

User-agent: Googlebot
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Yahoo-slurp
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Slurp
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Msnbot
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Bingbot
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Mediapartners-Google*
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Googlebot-Image
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: Yahoo-MMCrawler
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: TwitterBot
Allow: /
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /assets
Disallow: /services/replyconfirm/

User-agent: *
Disallow: /