Infrastructure

# All content on this website, including text, graphics, images, web design, film, sound 
# and other content, is protected by the Swedish Copyright Act (1960:729) (hereinafter
# "the Copyright Act"), both in terms of copyright and related rights. Bonnier News AB and
# its group companies (“Bonnier News”/”us”) reserve all rights to the content, including the
# right to use the content and the press publication for text and data extraction in
# accordance with Section 15a of the Copyright Act and Article 4 of the Copyright Directive.
#
# You may not use, nor may you facilitate or authorize the use of, the websites and/or any
# content for any purpose without prior written permission by Bonnier News, including, but
# not limited to, scraping the content or reproducing, modifying, collecting or extracting
# the content for text and data aggregation, analysis or extraction or commercial use.
# However, permission for website scraping and indexing may be granted for inclusion in a
# search engine provided that such authorization has been made explicitly by us through our
# technical instructions or similar specifications. Any form of scraping and indexing not
# expressly authorized by our technical instructions or specifications is strictly
# prohibited.

# Common Crawl robot, the resulting dataset is the primary training corpus in every LLM.
User-agent: CCBot
Disallow: /

# ChatGPT robot, used to improve the ChatGPT LLM. 
User-agent: ChatGPT-User
Disallow: /

# ChatGPT robot, may be used to improve the ChatGPT LLM. 
User-agent: GPTBot
Disallow: /

# Robot used to improve Bard and Vertex AI LLMs.
User-agent: Google-Extended
Disallow: /

# Associated with Google Vertex AI agents
User-agent: Google-CloudVertexBot
Disallow: /

# webz.io robot, the resulting dataset can and is purchased to train LLMs.
User-agent: Omgilibot
Disallow: /

# webz.io robot, the resulting dataset can and is purchased to train LLMs.
User-agent: Omgili
Disallow: /

# FacebookBot crawls public web pages to improve LLMs for Facebook's speech recognition technology. 
User-agent: FacebookBot
Disallow: /

# Another agent used by Anthropic that is more specifically related to Claude
User-agent: ClaudeBot
Disallow: /

# AI search indexing for Claude search capabilities
User-agent: Claude-SearchBot
Disallow: /

# AI agent for real-time web access when Claude users browse
User-agent: Claude-User
Disallow: /

# Diffbot crawls the web in or others to train their LLMs.
User-agent: Diffbot
Disallow: /

# Uses scraped data on-the-fly to create answers for DuckAssist.
User-agent: DuckAssistBot
Disallow: /

# Used by perplexity.ai. Generates text based on scraped material.
User-agent: PerplexityBot
Disallow: /

# AI agent for real-time browsing when Perplexity users request information
User-agent: Perplexity-User
Disallow: /

#  Cohere’s chatbot.
User-agent: cohere-ai
Disallow: /

#  Cohere’s chatbot.
User-agent: cohere-training-data-crawler
Disallow: /

# Use cases such as training AI models or improving products by indexing content directly.
User-agent: Meta-ExternalAgent
Disallow: /

# Crawler performs user-initiated fetches of individual links in support of some AI tools.
User-agent: Meta-ExternalFetcher
Disallow: /

# Used to improve Meta AI search.
User-agent: Meta-WebIndexer
Disallow: /

# Used by Timpi to scrape data for training their Large Language Models.
User-agent: Timpibot
Disallow: /

# Used by Webz.io to indicate that your site should not be included those using it to train AI models.
User-agent: Webzio-Extended
Disallow: /

# Crawler behind You.com’s AI search and browser assistant, indexing content for real-time answers.
User-agent: YouBot
Disallow: /

# Amazonbot is used to train Amazon services such as Alexa.
User-agent: Amazonbot
Disallow: /

# Bytespider is ByteDance's bot and may not respect robots.txt.
User-agent: Bytespider
Disallow: /

# Robot used to improve Anthropic AI LLMs.
User-agent: anthropic-ai
Disallow: /

# OpenAI search bot 
User-agent: OAI-SearchBot
Disallow: /

# Velen.io/Hunter.io "build business datasets and machine learning models to better understand the web" - seems to focus on collecting email adresses for spam though. 
User-agent: VelenPublicWebCrawler
Disallow: /

# AI research agent for Google Gemini’s Deep Research feature
User-agent: Gemini-Deep-Research
Disallow: /

# AI and machine learning data collection
User-agent: ICC-Crawler
Disallow: /

# Mistral’s real-time citation fetcher for “Le Chat” assistant
User-agent: MistralAI-User
Disallow: /


User-agent: *
Disallow: /account/new
Disallow: /auth/facebook
Disallow: /video-player/api/
Disallow: /sok/?q=
Disallow: /*?bullet=
Disallow: /*?channel=
Disallow: /*&channel=
Disallow: /*login?*
Disallow: /*login?return_uri=
Disallow: /*register?return_uri=
Disallow: /read-more-recommendations

Sitemap: https://www.expressen.se/googlenews
Sitemap: https://www.expressen.se/sitemap.xml