Infrastructure

# Automated access to this website by web crawlers is allowed only for the use in public search engines (e.g. Google or Bing).
# Automated data collection, data mining or mirroring (scraping) of any content on this website for any other purpose than public search engines without explicit authorization is strictly prohibited.
# Irresponsible or illegal access may be blocked at any time, and we reserve the right to take legal action against such activities.
#
User-agent: *
User-Agent: AhrefsBot
#
# Legacy URLs
Disallow: /ads/
Disallow: /xml/
Disallow: /berichte/
Disallow: /blog/
Disallow: /data/
Disallow: /download/
Disallow: /hinweis/datenleck/
Disallow: /important/dataleak/
Disallow: /important/fuite-donnees/
Disallow: /importante/fuga-de-datos/
Disallow: /favorit-hinzufuegen/
Disallow: /add-favorite/
Disallow: /ajouter-favoris/
Disallow: /anadir-busqueda/
Disallow: /favorit-loeschen/
Disallow: /remove-favorite/
Disallow: /supprimer-favoris/
Disallow: /eliminar-busqueda/
Disallow: /eliminar-favoritos/
#
# Forms
Disallow: /statistik/report-content/
Disallow: /statistics/report-content/
Disallow: /statistiques/report-content/
Disallow: /estadisticas/report-content/
#
# Products (06/24 Update)
# 03/25 companies discontinued & removed
Disallow: /perspectiva/
Disallow: /perspective/
Disallow: /global-consumer-survey/tool/
#
# Pagination (04/22)
Disallow: /statistics/popular/p/
Disallow: /statistik/top/p/
Disallow: /statistiques/populaires/p/
Disallow: /estadisticas/populares/p/
Disallow: /statistik/aktuell/p/
Disallow: /statistics/recent/p/
Disallow: /statistiques/recentes/p/
Disallow: /estadisticas/recientes/p/
#Update 03/25
Disallow: /statistics/recommendations/?p=
Disallow: /estadisticas/recomendaciones/?p=
Disallow: /statistiques/recommandees/?p=
Disallow: /statistik/empfehlungen/?p=
Disallow: /statistics/forecast/p/
Disallow: /estadisticas/prevision/p/
Disallow: /statistiques/prevision/p/
Disallow: /statistik/prognose/p/
Disallow: /statistics/updated/?p=
Disallow: /estadisticas/actualizadas/?p=
Disallow: /statistiques/mis-a-jour/?p=
Disallow: /statistik/aktualisiert/?p=
Disallow: /statistics/new/p/
Disallow: /estadisticas/nuevas/p/
Disallow: /statistiques/nouveau/p/
Disallow: /statistik/neu/p/
#
# Editors (08/23 opened for crawler)
#
# Study search (04/22)
Disallow: /studies-and-reports/*?q=
Disallow: /studies-and-reports/*&q=
Disallow: /studien-und-reports/*?q=
Disallow: /studien-und-reports/*&q=
#
# Personal information
Disallow: /profile/
Disallow: /profil/
Disallow: /perfil/
Disallow: /profil/
Disallow: /login/password/
Disallow: /login/passwort_vergessen/
Disallow: /acceso/contrasena/
Disallow: /login/resend-confirmation-link
Disallow: /login/bestaetigungslink-erneut-senden
Disallow: /login/reenviar-enlace-de-confirmacion
Disallow: /login/renvoyer-le-lien-de-confirmation
Disallow: /direct/purchase/
Disallow: /request/custom-solution/
#
# Internal search (04/22)
Disallow: /search/
Disallow: /statistik/suche/
Disallow: /buscar/
Disallow: /recherche/
Disallow: /serp
#
# Research AI (07/24)
Disallow: /research-ai/
Allow: /research-ai/$
#
# Sitemaps all languages
Sitemap: https://www.statista.com/sitemap/
Sitemap: https://de.statista.com/sitemap/
Sitemap: https://es.statista.com/sitemap/
Sitemap: https://fr.statista.com/sitemap/
#
#Ahrefs Slowdown
User-Agent: AhrefsBot
Crawl-Delay: 2
#
# Scraping protection
User-agent: sitecheck.internetseer.com
User-agent: Zealbot
User-agent: MSIECrawler
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebZIP
User-agent: linko
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: Xenu
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Download Ninja
User-agent: Brightbot 1.0
Disallow: /
#
# Disallow LLMs and related bots/crawlers 03/25
#
# Amazon
User-agent: Amazonbot # https://developer.amazon.com/amazonbot
#
# Anthropic
User-agent: anthropic-ai # https://darkvisitors.com/agents/anthropic-ai
User-agent: claude-web
User-agent: ClaudeBot # https://privacy.anthropic.com/en/articles/10023637-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
#
# OpenAI
User-agent: Operator # https://openai.com/index/introducing-operator/
User-agent: cohere-training-data-crawler # https://openai.com/index/introducing-operator/
User-agent: cohere-ai
User-agent: ChatGPT-User # https://platform.openai.com/docs/bots
User-agent: GPTBot # https://platform.openai.com/docs/bots
User-agent: OAI-SearchBot # https://platform.openai.com/docs/bots
#
# Facebook/Meta
User-agent: meta-externalfetcher # https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
User-agent: facebookbot # https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
User-agent: meta-externalagent # https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
#
# DuckDuckGo
User-agent: DuckAssistBot # https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/
#
# Other
User-agent: AI2Bot # https://allenai.org/crawler
User-agent: Diffbot # https://darkvisitors.com/agents/diffbot
User-agent: Timpibot # https://timpi.io/
User-agent: Kangaroo LLM # https://2.de-2.isolation.mimecastprotect.com/?sessionID=804d42c408cb5df1124d
User-agent: YouBot # you.com
User-agent: ImagesiftBot
User-agent: FriendlyCrawler
User-agent: img2dataset
User-agent: Peer39_crawler
User-agent: Magpie-crawler
#
# ByteDance/Tiktok
User-agent: Bytespider
#
# Apple
User-agent: Applebot-Extended # https://support.apple.com/en-us/119829
User-agent: Applebot # https://support.apple.com/en-us/119829
#
# CommonCrawl and other archives
User-agent: CCBot # https://commoncrawl.org/faq
User-agent: archive.org_bot # https://archive.org/details/archive.org_bot
User-agent: Arquivo-web-crawler # https://sobre.arquivo.pt/en/help/crawling-and-archiving-web-content/
User-agent: ia_archiver
User-agent: ia_archiver-web.archive.org
#
# Huawei
User-agent: PanguBot
#
# Perplexity
User-agent: PerplexityBot # https://docs.perplexity.ai/guides/bots#perplexity-crawlers
User-agent: Perplexity-User #https://url.de.m.mimecastprotect.com/s/ONS8C57m38IMQALAhLt0UkiR8l?domain=docs.perplexity.ai
#
# Liner AI
User-agent: LinerBot # https://darkvisitors.com/agents/linerbot
#
# Google
User-agent: Google-Extended # https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-extended
User-agent: GoogleOther # https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#googleother
User-agent: GoogleOther-Video # https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#googleother-video
User-agent: GoogleOther-Image # https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#googleother-image
User-agent: Google-CloudVertexBot # https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cloudvertexbot
#
# Webz.io
User-agent: omgili
User-agent: omgilibot
User-agent: Webzio-Extended
#
User-agent: Claude-Web
#
Disallow: /
Allow: /$
Allow: /statistik/info$
Allow: /kontakt/
Allow: /contact/
Allow: /contacto/
Allow: /accounts/
Allow: /solutions/
Allow: /getting-started/
Allow: /infografik/
Allow: /chartoftheday/
Allow: /chart/
Allow: /graphique-du-jour/
Allow: /infographie/
Allow: /grafico-del-dia/
Allow: /grafico/
#
Allow: /karriere/
Allow: /working-at-statista/
Allow: /business/
Allow: /impressum/
Allow: /imprint/
Allow: /lexikonListe/
Allow: /lexikon/
Allow: /statistics-glossary/
Allow: /presse/
Allow: /press/
Allow: /prensa/
Allow: /research-ai/$
Allow: /webinar$
Allow: /webinar/
Allow: /site/