Infrastructure

# Ranker content is made available for your personal, non-commercial
# use subject to our Terms of Service here: https://www.ranker.com/terms
# Use of any device, tool, or process designed to data mine or scrape the
# content using automated means is prohibited without prior written permission.
# from Ranker, Inc.
# Prohibited uses include but are not limited to:
# (1) text and data mining activities under Art. 4 of the EU Directive on
# Copyright in the Digital Single Market;
# (2) the development of any software, machine learning, artificial
# intelligence (AI), and/or large language models (LLMs);
# (3) creating or providing archived or cached data sets containing our
# content to others; and/or
# (4) any commercial purposes.
# Contact https://business.ranker.com/contact for assistance.


# (A) ALLOW: SOCIAL PREVIEWS & SEARCH AI (RAG / Citations)
# We allow these for real-time user-requested citations and link previews.
User-agent: Applebot
User-agent: BingBot
User-agent: BingPreview
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckDuckBot
User-agent: facebookexternalhit
User-agent: Grok-DeepSearch
User-agent: Googlebot
User-agent: GoogleOther
User-agent: LinkedInBot
User-agent: meta-externalfetcher
User-agent: meta-webindexer
User-agent: MistralAI-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Slackbot
User-agent: Twitterbot
User-agent: xAI-Grok
Disallow: /app
Disallow: /commentary
Disallow: /listcommentary
Disallow: /extlink
Disallow: /node
Disallow: /suggestions
Disallow: /widget
Disallow: /pics
Disallow: /review
Disallow: /amp
Disallow: /*US_CENSUS_NAME*
Allow: /


# (B) BLOCK: AI TRAINING & DATA AGGREGATORS
# These bots are known to scrape for LLM training or commercial data reselling.
User-agent: AI2Bot
User-agent: AI2Bot-Dolma
User-agent: AliyunSecBot
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: bigsur.ai
User-agent: Bytespider
User-agent: CCBot
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: DeepseekBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: GPTBot
User-agent: GrokBot
User-agent: iAskBot
User-agent: ImagesiftBot
User-agent: meta-externalagent
User-agent: omgili
User-agent: omgilibot
User-agent: Petalbot
User-agent: Timpibot
User-agent: webzio
User-agent: webzio-extended
Disallow: /


# (C) MEDIA CRAWLERS (Images & Video)
# Note: /pics is intentionally allowed here for media indexing.
User-agent: Googlebot-Image 
User-agent: Googlebot-Video 
Disallow: /app 
Disallow: /commentary 
Disallow: /listcommentary 
Disallow: /extlink 
Disallow: /node 
Disallow: /suggestions 
Disallow: /widget 
Disallow: /review 
Disallow: /amp 
Disallow: /*US_CENSUS_NAME*
Allow: /  


# (D) GENERAL SEARCH ENGINES & SEO TOOLS
User-agent: *
Disallow: /app
Disallow: /commentary
Disallow: /listcommentary
Disallow: /extlink
Disallow: /node
Disallow: /suggestions
Disallow: /widget
Disallow: /pics
Disallow: /review
Disallow: /amp
Disallow: /*US_CENSUS_NAME*


# Sitemaps
Sitemap: https://www.ranker.com/sitemaps/list/sitemap-list-index.xml
Sitemap: https://www.ranker.com/sitemaps/user/sitemap-user-index.xml
Sitemap: https://www.ranker.com/sitemaps/google-news/sitemap-google-news-index.xml
Sitemap: https://www.ranker.com/sitemaps/collection/sitemap-collection-index.xml