User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Googlebot-News
Disallow: /$
Disallow: /biography/
Disallow: /tag/
Disallow: /collections/
Disallow: /collections-list/
Disallow: /commentary/$
Disallow: /issue/$
Disallow: /multimedia/$
Disallow: /what-were-reading/
Disallow: /voices-of-tomorrow/
Disallow: /uncategorized/
Disallow: /special-topics/
Disallow: /science-denial/
Disallow: /press-releases/
Disallow: /presentation/
Disallow: /personal-essay/
Disallow: /opinion/
Disallow: /nuclear-roundup/
Disallow: /nuclear-risk/
Disallow: /multimedia/
Disallow: /missile-defense/
Disallow: /interviews/
Disallow: /explain-this/
Disallow: /cat-events/
Disallow: /cat-doomsday-clock/
Disallow: /disruptive-technologies/
Disallow: /cat-columnists/
Disallow: /collections-list/
Disallow: /climate-change/
Disallow: /chemical-weapons/
Disallow: /book-review/
Disallow: /cat-arts-initiative/
Disallow: /annual-report/
Disallow: /announcement/
Disallow: /analysis/
Disallow: /2019-doomsday-clock-mentions/
Disallow: /announcement/
Disallow: /analysis/