Infrastructure

# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449
#

# Ziff Davis content is made available for your non-commercial use subject to our
# Terms of Use here: https://www.ziffdavis.com/terms-of-use
# Use of any robot, crawler, or other tool to scrape, harvest, extract, or retrieve any content on
# this website using automated means is prohibited without written permission from Ziff Davis.
# Prohibited uses include but are not limited to:
# (1) text and data mining under Art. 4 of the EU Directive on Copyright in the Digital Single
# Market;
# (2) development or operation of artificial intelligence or machine learning software or
# databases, including by training, fine-tuning, embedding, and retrieval-augmented generation;
# (3) creating data sets containing our content or sharing it with others; and
# (4) any commercial purposes.
# Contact licensing@ziffdavis.com for assistance.

User-agent: *

Disallow: */*max-price=*
Disallow: */*min-price=*
Disallow: */Xhr*
Disallow: */account-information/*
Disallow: */ad/ad-cookie/*
Disallow: */ajax/
Disallow: */alert/*
Disallow: */cnet_global_app_redirect.xml
Disallow: */component/*
Disallow: */edit/
Disallow: */edit/change-email/*
Disallow: */edit/change-password/*
Disallow: */email/editor/*
Disallow: */email/member/*
Disallow: */email/newsletter/*
Disallow: */error/*
Disallow: */ext/make/ingest/*
Disallow: */feature/*
Disallow: */finder/*/*/?*filter=*_*_*_*_*
Disallow: */finder/*/?*filter=*_*_*_*_*
Disallow: */forums/*
Disallow: */g00/*
Disallow: */informacion-de-mi-cuenta/*
Disallow: */members/*
Disallow: */products/*?*filter=*
Disallow: */roadshow/finder/*
Disallow: */search/*
Disallow: */user/*
Disallow: */videos/manifest/*
Disallow: */videos/secure/*
Disallow: */videos/selector/*
Disallow: */wrap-function*
Disallow: */wrap-xhr*
Disallow: */xhr*
Disallow: *?sort=popularity*
Disallow: *?sort=price*
Disallow: *?sort=user-rating*
Disallow: *hub-preview-version=*
Disallow: /.well-known/*
Disallow: /api/editionurlgenerator/
Disallow: /apple-news/*
Disallow: /au/*
Disallow: /es/*
Disallow: /html/story/*
Disallow: /index.php/*
Disallow: /products/*
Disallow: /uk/*

# Ziff Davis bot list
user-agent: AddSearchBot
user-agent: AI2Bot
user-agent: Ai2Bot-DeepResearchEval
user-agent: Ai2Bot-Dolma
user-agent: Amazonbot
user-agent: AmazonBuyForMe
user-agent: Anchor Browser
user-agent: anthropic-ai
user-agent: Applebot
user-agent: Applebot-extended
user-agent: atlassian-bot
user-agent: AutoRAG
user-agent: AwarioSmartBot
user-agent: Big Sur AI
user-agent: bigsur.ai
user-agent: Brandwatch
user-agent: Bravebot
user-agent: Bytespider
user-agent: CCBot
user-agent: Channel3Bot
user-agent: ChatGLM-Spider
user-agent: ChatGPT-User
user-agent: Claude-SearchBot
user-agent: Claude-User
user-agent: Claude-Web
user-agent: ClaudeBot
user-agent: cohere-ai
user-agent: cohere-training-data-crawler
user-agent: Cotoyogi
user-agent: Crawl4AI
user-agent: Datenbank Crawler
user-agent: DeepSeekBot
user-agent: Devin
user-agent: Diffbot
user-agent: Direqt Anomura
user-agent: DuckAssistBot
user-agent: Echobot Bot
user-agent: FacebookBot
user-agent: Factset_spyderbot
user-agent: GPTBot
user-agent: iAsk
user-agent: iaskspider
user-agent: ICC Crawler
user-agent: ICC-Crawler
user-agent: ImageSiftBot
user-agent: imageSpider
user-agent: Kangaroo Bot
user-agent: KlaviyoAIBot
user-agent: Kunato
user-agent: laion-huggingface-processor
user-agent: LCC
user-agent: LINER Bot
user-agent: LinerBot
user-agent: LinkupBot
user-agent: Manus-User
user-agent: meta-externalagent
user-agent: meta-externalfetcher
user-agent: meta-webindexer
user-agent: mistral.ai
user-agent: MistralAI-User
user-agent: netEstate Imprint Crawler
user-agent: NovaAct
user-agent: Novellum AI Crawl
user-agent: OAI-SearchBot
user-agent: omgili
user-agent: PanguBot
user-agent: PeopleInc-DCipher-Scraper/1.1.0
user-agent: Perplexity-User
user-agent: PerplexityBot
user-agent: PetalBot
user-agent: Poggio-Citations
user-agent: QualifiedBot
user-agent: SBIntuitionsBot
user-agent: SeekrBot
user-agent: SemrushBot-OCOB
user-agent: SemrushBotSwa
user-agent: ShapBot
user-agent: Spider
user-agent: TavilyBot
user-agent: Timpibot
user-agent: TwinAgent
user-agent: Velen Crawler
user-agent: VelenPublicWebCrawler
user-agent: Webzio-Extended
user-agent: Wrtn
user-agent: YouBot
user-agent: ZanistaBot
Allow: /home/internet/xfinity-vs-att-which-internet-provider-is-best/
Allow: /home/internet/xfinity-vs-verizon-which-internet-provider-is-best/
Allow: /news/ai-automation-changing-how-we-file-taxes/
Allow: /paid-content/news/enjoy-unlimited-wi-fi-and-predictable-pricing-with-on-your-home-internet/
Allow: /paid-content/news/from-tacos-to-tvs-doordash-delivers-your-game-day-essentials/
Allow: /paid-content/news/get-an-unlimited-5g-phone-plan-for-25-per-month-with-metro-by-t-mobile/
Allow: /paid-content/news/h-r-blocks-local-tax-pros-help-you-navigate-complex-returns/
Allow: /paid-content/news/how-galaxy-ai-camera-tools-help-you-take-better-photos-automatically-on-the-s25/
Allow: /paid-content/news/how-h-r-block-helps-you-maximize-freelance-and-gig-worker-write-offs/
Allow: /paid-content/news/how-h-r-block-simplifies-taxes-for-first-time-filers/
Allow: /paid-content/news/how-h-r-blocks-experts-can-help-you-avoid-common-filing-mistakes/
Allow: /paid-content/news/how-samsungs-galaxy-ai-boosts-gaming-performance/
Allow: /paid-content/news/how-sling-is-disrupting-the-old-live-tv-streaming-model/
Allow: /paid-content/news/how-t-mobile-can-help-you-save-thousands-on-your-family-wireless-plan/
Allow: /paid-content/news/how-to-watch-live-sports-and-avoid-wheres-the-game-chaos-with-xfinity/
Allow: /paid-content/news/i-tried-internxt-an-encrypted-secure-cloud-storage-built-for-the-post-quantum-era/
Allow: /paid-content/news/is-diy-tax-filing-right-for-you-heres-what-h-r-blocks-tools-make-possible/
Allow: /paid-content/news/is-wi-fi-7-worth-the-upgrade-heres-how-msis-roamii-mesh-systems-fit-in/
Allow: /paid-content/news/life-changes-and-new-laws-can-affect-your-tax-refund-how-to-estimate/
Allow: /paid-content/news/maximize-your-refund-with-h-r-blocks-smart-tax-tools/
Allow: /paid-content/news/must-know-tax-season-tips-for-small-businesses-and-solopreneurs/
Allow: /paid-content/news/selling-stuff-online-what-you-could-be-on-the-hook-for-tax-wise/
Allow: /paid-content/news/t-mobile-5g-home-internet-offers-a-solution-to-contracts-cords-and-hidden-fees/
Allow: /paid-content/news/tempos-fresh-chef-crafted-meals-make-your-meal-prep-smarter-and-tastier/
Allow: /paid-content/news/what-americans-pay-for-internet-in-2026-and-how-xfinity-helps-you-save/
Allow: /paid-content/news/why-a-reliable-fiber-powered-network-from-xfinity-is-the-ultimate-solution-for-home-wi-fi/
Allow: /paid-content/news/why-tax-filers-are-choosing-h-r-block-this-season/
Allow: /tech/mobile/samsung-galaxy-s26-and-s26-plus-hands-on-first-take/
Allow: /tech/mobile/samsung-galaxy-s26-ultra-vs-galaxy-s26-and-s26-plus-flagship-specs-comparison/
Allow: /tech/services-and-software/a-guide-to-data-removal-services-should-you-pay-for-privacy/
Allow: /tech/services-and-software/complex-income-tax-filing-rules-2026/
Allow: /tech/services-and-software/crypto-new-tax-form-rules/
Allow: /tech/services-and-software/data-brokers-how-your-personal-data-becomes-business/
Allow: /tech/services-and-software/end-year-tax-checklist-maximize-tax-refund/
Allow: /tech/services-and-software/end-year-tax-checklist-maximize-tax-refund/
Allow: /tech/services-and-software/free-tax-software-cost-upgrades/
Allow: /tech/services-and-software/how-h-r-block-helps-freelancers-and-gig-workers-stay-ahead-of-tax-season/
Allow: /tech/services-and-software/how-to-prevent-irs-penalties-and-delays-this-tax-season/
Allow: /tech/services-and-software/married-single-or-self-employed-how-your-filing-status-changes-everything/
Allow: /tech/services-and-software/qualified-tips-overtime-tax-deduction-rules/
Allow: /tech/services-and-software/self-employment-tax-guide-1099-contractor/
Allow: /tech/services-and-software/w2-vs-1099-difference/
Allow: /tech/services-and-software/what-is-vibe-coding-how-anyone-can-make-apps-with-the-help-of-ai/
Allow: /tech/services-and-software/why-would-a-business-need-to-use-a-proxy-server/
Allow: /videos/2026-tax-season-checklist-what-you-need-before-you-file/
Allow: /videos/5-finance-and-tax-tips-for-freelancers/
Allow: /videos/navigating-your-tax-filing-options-simple-vs-complex-situations/
Allow: /videos/tax-breaks-you-might-miss-in-2026-and-how-to-find-them/
Disallow: /

#CNET restricted bots
User-agent: "008"
User-agent: AwarioRssBot
User-agent: DDM-DCipher/1.0.7
User-agent: HTTrack
User-agent: Magpie-crawler
User-agent: Nutch
User-agent: Offline Explorer
User-agent: Omgilibot
User-agent: Peer39_crawler/1.0
User-agent: Scrapy
Disallow: /

Sitemap: https://www.cnet.com/sitemaps/article/index.xml
Sitemap: https://www.cnet.com/sitemaps/best.xml
Sitemap: https://www.cnet.com/sitemaps/gallery/index.xml
Sitemap: https://www.cnet.com/sitemaps/how-to/index.xml
Sitemap: https://www.cnet.com/sitemaps/news.xml
Sitemap: https://www.cnet.com/sitemaps/review/index.xml
Sitemap: https://www.cnet.com/sitemaps/static.xml
Sitemap: https://www.cnet.com/sitemaps/updating_story/index.xml
Sitemap: https://www.cnet.com/sitemaps/video/index.xml