Infrastructure

# Our Content is made available for your personal, non-commercial use subject to our Terms of Service here: https://www.medianewsgroup.com/terms-of-use/
# Any other uses are prohibited, including but not limited to:
# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in the Digital Single Market;
# (2) use of any Content or information available on the Site for purposes of retrieval augmented generation, grounding, training, or development of machine learning models, algorithms, or artificial intelligence (AI) systems, or to generate substitute content or develop any products, services, or technology;
# (3) caching or archiving the Content; and/or
# (4) any commercial purposes.
# Contact us at https://www.bostonherald.com/contact-us/ for assistance.

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /trackback/
Disallow: /comments/

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Archive.org_bot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Brightbot 1.0
Disallow: /

User-Agent: Bytespider
Disallow: /

User-agent: carbon-umbrella-bot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Searchbot
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Cohere-training-data-crawler
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: Dalvik
Disallow: /

User-agent: Datenbank Crawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: DocSpotBot
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: GnowitNewsbot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: grok-crawler
Disallow: /

User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: imageSpider
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: laion-huggingface-processor
Disallow: /

User-agent: LCC
Disallow: /

User-agent: Mantisbot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: netEstate Imprint Crawler
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

User-agent: SirdataBot
Disallow: /

User-agent: Spider
Disallow: /

User-agent: Thinkbot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: Vivaldi
Disallow: /

User-agent: Webzio-extended
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: AmazonAdBot
Disallow:

User-agent: facebookexternalhit
Disallow:

User-agent: grapeshot
Disallow:

User-agent: Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)
Allow: /
User-agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Booldog/1.0)
Allow: /
User-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36(KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 (compatible; mrfCompass-Booldog/1.0)
Allow: /
User-agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; mrfCompass-Marshall/1.0)
Allow: /
User-agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA51N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Mobile Safari/537.36 (compatible; mrfCompass-Jukebox/1.0)
Allow: /
# Sitemap archive
Sitemap: https://www.bostonherald.com/sitemap.xml