Infrastructure

# NOTICE: Collection of content and other data on https://www.marketwatch.com/ through
# automated means is prohibited unless you have express written
# permission from Dow Jones & Company, Inc. and may only be conducted for the
# limited purpose contained in said permission.
#
# Dow Jones & Company, Inc. Terms of Use may be found at
# https://www.dowjones.com/terms-of-use/
#
# If you would like to apply for permission to license the
# intellectual property and/or other materials of Dow Jones & Company, Inc.’s
# brands, please contact us via email at copyright@dowjones.com.


User-agent: *
Disallow: /

User-agent: googlebot
User-agent: googlebot-image
User-agent: GoogleOther
User-agent: Googlebot-Video
User-agent: Google-InspectionTool
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
User-agent: Storebot-Google
User-agent: google-read-aloud
User-agent: mediapartners-google
User-agent: bingbot
User-agent: msnbot
User-agent: bingpreview
User-agent: slurp
User-agent: yahoo
User-agent: yandex
User-agent: YandexImages
User-agent: baiduspider
User-agent: Pinterestbot
User-agent: Yeti
User-agent: MojeekBot
User-agent: 360Spider
User-agent: google-cloudvertexbot
User-agent: duckduckbot
User-agent: petalbot
User-agent: Applebot
User-agent: AppleNewsBot
User-agent: flipboard
User-agent: qwantbot
User-agent: SeznamBot
User-agent: yisouspider


User-agent: proximic
User-agent: admantx
User-agent: thetradedesk
User-agent: outbrain
User-agent: ias_crawler
User-agent: AmazonAdBot
User-agent: pubmatic
User-agent: smartologybot
User-agent: parselybot


User-agent: Screaming Frog SEO Spider
User-agent: DeepCrawl
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: SimilarWebBot
User-agent: SISTRIX
User-agent: OnCrawl
User-agent: botify
User-agent: Chrome-Lighthouse


User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: OAI-SearchBot


User-agent: facebookexternalhit
User-agent: facebot
User-agent: twitterbot
User-agent: linkedinbot
User-agent: telegrambot
User-agent: snapchat
User-agent: newrelicpinger
User-agent: siteimprovebot
User-agent: uptimerobot
User-agent: pingdom.com_bot_version_1.4
User-agent: sentry
User-agent: schema-validator
User-agent: googleimageproxy
User-agent: Iframely
User-agent: Vocabtracker
User-agent: EpvzCrawl6194680250
User-agent: Citoid
User-agent: ZoteroTranslationServer


Allow: /

Disallow: /2/
Disallow: /3com/
Disallow: /admin/
Disallow: /bgpoxy/
Disallow: /bin/
Disallow: /cDN_contENt/
Disallow: /cgi-bin/
Disallow: /client/
Disallow: /data/
Disallow: /dbcfiles/
Disallow: /dhtml/
Disallow: /dhtmlmenu/
Disallow: /doubleclick/
Disallow: /dynamiclogic/
Disallow: /email/
Disallow: /feeds/
Disallow: /free/
Disallow: /http2_data/
Disallow: /game/*
Disallow: /games/*
Disallow: /*?tickers=
Disallow: /*?submit=
Disallow: /*?exchange=
Disallow: /login/
Disallow: /MarketPerception/
Disallow: /marketperception/
Disallow: /MyPortfolio/
Disallow: /myportfolio/
Disallow: /news/current/
Disallow: /portfolio/
Disallow: /print/
Disallow: /RealMedia/
Disallow: /realmedia/
Disallow: /registration/
Disallow: /search/
Disallow: /search
Disallow: /story/twittershare
Disallow: /tools/alerts/
Disallow: /tpx/
Disallow: /investing/secfile/
Disallow: /tools/quotes/secfilings.asp
Disallow: /tools/quotes/form10k.asp
Disallow: /tools/quotes/form10q.asp
Disallow: /tools/quotes/form8k.asp
Disallow: /tools/quotes/registrationstate.asp
Disallow: /tools/quotes/sched13d.asp
Disallow: /tools/quotes/sched13f.asp
Disallow: /tools/quotes/sched13g.asp
Disallow: /tools/quotes/secarticle.asp
Disallow: /wsj/
Disallow: /xhtml/v/
Disallow: /user/login/profile
Disallow: /auth/

User-agent: googlebot
Disallow: /client

Sitemap: https://www.marketwatch.com/sitemaps/web/mw/en/sitemap_mw_en_index.xml
Sitemap: https://www.marketwatch.com/sitemaps/web/mw-video/en/sitemap_mw-video_en_index.xml
Sitemap: https://www.marketwatch.com/mw_news_sitemap.xml
Sitemap: https://www.marketwatch.com/mw_live_news_sitemap.xml
Sitemap: https://www.marketwatch.com/api-video/sitemaps/google/sitemap-google-video-marketwatch.asp
Sitemap: https://www.marketwatch.com/mw/mw_quote_index_sitemap.xml
Sitemap: https://www.marketwatch.com/sitemaps/web/mw-pr/en/sitemap_mw-pr_en_index.xml