Infrastructure

# Allow all search engines to crawl
User-agent: *
Allow: /

# AI Crawlers
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Google-Extended
Allow: /

# Disallow Parameters
# GA traffic source parameters
User-agent: *
Disallow: */?*utm_source=*
Disallow: */?*utm_campaign=*
Disallow: */?*utm_medium=*
Disallow: /*?source=*
Disallow: */?*utm_content =*
Disallow: */?*utm_newsbreak=*

# Allow crawling ads
User-agent: *
allow: /ads.txt

# Embedded widget parameters
User-agent: *
Disallow: *?embed*

# Influencers/Affiliate Links parameters
User-agent: *
Disallow: */*a_aid=*
Disallow: /*?partner=*

# Experiments testing team
User-agent: *
Disallow: */*?mm-experiments=*

# Search box param
User-agent: *
Disallow: /*?*s=*

# Apps param
User-agent: *
Disallow: *?app=*

# FB campaigns
User-agent: *
Disallow: *?fbclid=*


# GEO targeting:
User-agent: *
Disallow: /*?setLocale=*
Disallow: *?georedirect=*

# Unknown Parameters
User-agent: *
Disallow: /*?term=*
Disallow: *?ref=*
Disallow: /*?view_source=*
Disallow: /*?view_medium=*
Disallow: /*?initialLeagueId=*



# GA parameters - Generated from Google caching
User-agent: *
Disallow: *_ga_*
Disallow: */?_gl=*

# Generated from Voltax API url
User-agent: *
Disallow: */api/*

# Voltax HTML Unknown crawled urls
User-agent: *
Disallow: */videos/undefinedc_fill,w_360,ar_16:9,f_auto,q_auto,g_auto/undefined
Disallow: */teams/mainNavigationChevron_icon.svg?*
Disallow: */leagues/mainNavigationChevron_icon.svg?*
Disallow: */undefinedc_fill,w_360,ar_16:9,f_auto,q_auto,g_auto/undefined

# Generated from 90min and TBL Monolith
User-agent: *
Disallow: */xposts/
Disallow: */reader/
Disallow: */embed/*
Disallow: */ads/*
Disallow: */editor/*
Disallow: */posts/*/edit$
Disallow: */posts/*/publish$
Disallow: */singlepage/pipe/*
Disallow: */singlepage/uncached_pipe/*
Disallow: */singlepage/epipe/*
Disallow: */reads/*/read$
Disallow: */embed_code
Disallow: */matches/*
Disallow: */es/partidos/*
Disallow: */it/partite/*
Disallow: */de/spiele/*
Disallow: */zh-CN/*
Disallow: */ping
Disallow: */sessions/*
Disallow: */admin/*
Disallow: */management/*
Disallow: */castr
Disallow: */unfeature$
Disallow: */unfeature/*
Disallow: */videos/*
Disallow: */channels/videos/*

# Generated from Mentalfloss WP and Drupal
User-agent: *
Disallow: /_partial/*
Disallow: */amazingfactgenerator/*
Disallow: /_modules/*
Disallow: /subscribe/v3/*
Disallow: /search/*
Disallow: */file/*
Disallow: */node/*
Disallow: /sites/*
Disallow: /longform/*
Disallow: /blogs/*
Disallow: */store/*
Disallow: */us/*
Disallow: */store/*
Disallow: */us/*
Disallow: */HDYK/*
Disallow: /dist/*
Disallow: */shopping/*
Disallow: /trivia/*
Disallow: /worksheets/*
Disallow: /music/*
Disallow: /magazine/*
Disallow: /puzzle/*

# Generated from Fansided WP
User-agent: *
Disallow: */files/*
Disallow: */wp-admin/*
Disallow: */?*utm_newsbreak=*
Disallow: */?*utm_content =*
Disallow: */wp-content/*
Disallow: */wp-includes/*
Disallow: */app/*
Disallow: */embed_code*
Disallow: */{{url/*
Disallow: */v2/*

#Social Media Robots
User-agent: Twitterbot
Allow:*
User-agent: FacebookBot
Allow:*

# Sitemap XML
User-agent: *
Sitemap: https://www.mentalfloss.com/feed-sitemap.xml
Sitemap: https://www.mentalfloss.com/news-sitemap.xml
Sitemap: https://www.mentalfloss.com/sitemap.xml