Infrastructure

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# Crawl Delay added to slow down some well behaving bots.
Crawl-delay: 10
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg

# Directories
Disallow: /core/
Disallow: /profiles/

# Files
# Although deprecated, leaving Noindex for non-html files. This can/should be moved to nginx config.
Noindex: /README.txt
Noindex: /web.config

Disallow: /README.txt
Disallow: /web.config

# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/

# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips/
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/

################################################################################

# Core
Disallow: /book/export/

# MSK: Content-types
Disallow: /block/
# Disallow: /podcasts/
# Disallow: /slideshows/
# Disallow: /teaser/
Disallow: /term/
# Disallow: /media-coverage/

# MSK: Modules and entities
# Disallow: /dialog
Disallow: /msk_markup/
# Disallow: /taxonomy/
Disallow: /templates
# Disallow: /user/

# MSK tools
# Disallow: /print
# Disallow: /pdf
# Disallow: /mail
# Disallow: */print/
# Disallow: */pdf/

# MSK: Redirects
Disallow: */redirect$
Disallow: */redirect-01$
Disallow: */redirect-02$
Disallow: */redirect-03$
Disallow: */redirect-04$
Disallow: */redirect-05$
Disallow: */redirect-06$
Disallow: */redirect-07$
Disallow: */redirect-08$
Disallow: */redirect-09$
Disallow: */redirect-10$

################################################################################

# MSK: Old website

Disallow: /mskcc/print/


# MSK: Misc.

Disallow: /calgbtraining

# Disallow: /cancer-care/patient-education/herbs/

Disallow: /drupal-guide

Disallow: /msk-prepare

Disallow: /experience/become-patient/first-appointment/begin-registration-msk-your-first-visit

Disallow: /archived

Disallow: /news/available-download

Disallow: /appointments/request-appointment-01

Disallow: /ario2016/presentations

Disallow: /monmouthopening

Disallow: /hcp-education-training/medical-students/elective/apply/canadian-medical-students/students-international-medical-schools

Disallow: /research-advantage/core-facilities/comparative-medicine-pathology-0/new-animal-user-onboarding

Disallow: /videos/msk-kimmel

Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/meditation-101

Disallow: /departments/surgery/endometrial-cancer-research-day

Disallow: /community-affairs/programs-services/high-school-outreach/registration-major-trends

Disallow: /research-programs/cell-engineering/eleventh-annual-center-cell-engineering-retreat

Disallow: /experience/become-patient/remotesecondopinion

Disallow: /sites/default/files/node/168388/documents/sample-remote-2nd-opinion-report-2.pdf

Disallow: /sites/default/files/node/168388/documents/sample-remote-2nd-opinion-report-2_0.pdf

Disallow: /videos/msk-impact-informational-video

Disallow: /hcp-education-training/cne/msk-2-day-cne-discounted-fee

Disallow: /hcp-education-training/cne/msk-1-day-cne-discounted-fee

Disallow: /departments/division-subspecialty-medicine/integrative-medicine/fundamentals-oncology-acupuncture-online-course

Disallow: /lw

Disallow: /profile/kathleen-foley

Disallow: /videos/anusha

Disallow: /videos/how-can-msk-direct-care-advisor-help-me-and-my-family

Disallow: /videos/msk-direct-getting-right-diagnosis

Disallow: /elc

Disallow: /lung-screening/not-eligible

Disallow: /lung-screening/eligible

Disallow: /videos/innovations-breast-screening

Disallow: /direct/*

Disallow: /profile/jerome-posner

Disallow: /research-programs/stem-cell-biology/2020-gforce-meeting

Disallow: /patient-screening-form

Disallow: /remote-access-resources-msk-employees

Disallow: /research-programs/cell-engineering/special-events/twelfth-annual-center-cell-engineering-retreat

Disallow: /hics-medical-staff-redeployment-registration-form

Disallow: /mayor-de-blasio-press-conference

Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/therapies/how-we-can-help-you-stay-calm-and-connected-through-covid-19

Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/therapies/how-we-can-help-you-stay-calm-and-connected-through-covid-19/mind-body-and-spirit-classes

Disallow: /msk-strategic-partnerships-resources-and-information-hub

Disallow: /videos/msk-cancer-alliance-final

Disallow: /videos/msk-cancer-alliance-quality-lecture-series-skin-toxicities-patients-receiving-targeted-therapy

Disallow: /videos/msk-survivorship-lecture-series-food-choice-during-and-after

Disallow: /videos/msk-survivorship-lecture-series-cancer-immunotherapy-101-overview-non-oncologist

Disallow: /videos/how-treat-cns-lymphoma-mskcc-bmt-lecture-series

Disallow: /videos/quality-and-outcomes-lecture-series-radiation-related-mucositis

Disallow: /departments/radiology/radiology-image-request

Disallow: /msk-why

Disallow: /drupal-training-registration-form

Disallow: /index.php/*

Disallow: /msk-news/fall-2020/nursing-excellence-honored-robbins-family-awards

Disallow: /msk-news/spring-2020/infectious-disease-specialist-mini-kamboj-discusses-covid-19

Disallow: /msk-news/spring-2020/msk-enters-new-era-opening-david-koch-center-cancer-care-msk-cancer

Disallow: /resource-sheet

Disallow: /videos/multidisciplinary-management-early-stage-triple-negative-and-her-2-breast-cancers-neoadjuvant-therapy

Disallow: /videos/personalized-treatment-soft-tissue-sarcoma-primary-management

Disallow: /event/optimal-living-program-exercise-and-breast

Disallow: /optimal-living-program-exercise-and-breast-cancer-series

Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/membership/renewal-form

Disallow: /experience/become-patient/becoming-patient

Disallow: /videos/united-face-crisis-msk-s-response-covid-19

Disallow: /lung-cancer-awareness-month-day-1-events-1-00-pm-5-00-pm

Disallow: /lung-cancer-awareness-month-day-2-events-10-00-am-2-00-pm

Disallow: /videos/personalized-treatment-recurrent-and-metastatic-soft-tissue-sarcoma

Disallow: /nursing-request-redeployment-staff

Disallow: /nursing-offer-redeploy-staff

Disallow: /ert-c*

Disallow: /research-advantage/marie-josee-kravis-women-science-endeavor-wise/kravis-wise-symposium

Disallow: /coronavirus/update-msk-community-covid-19-vaccines

Disallow: /coronavirus/una-actualizacion-para-nuestra-comunidad-msk-sobre-las-vacunas-contra-la-covid-19

Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/membership/integrative-medicine-home-member-resources

Disallow: /videos/morning-meditation

Disallow: /videos/chair-yoga

Disallow: /videos/beginner-mat-yoga

Disallow: /videos/30-minute-fitness-everyone

Disallow: /videos/60-minute-fitness-everyone

Disallow: /videos/barre-basics

Disallow: /videos/barre-express

Disallow: /videos/dance-cardio

Disallow: /videos/core-strength

Disallow: /videos/fitness-everyone

Disallow: /videos/guided-imagery-meditation-mountain-walk

Disallow: /videos/guided-imagery-meditation-floating-ocean

Disallow: /videos/tai-chi

Disallow: /videos/intermediate-mat-yoga

Disallow: /cancer-care/diagnosis-treatment/symptom-management/integrative-medicine/membership/integrative-medicine-home-member-resources/videos

Disallow: /employee-town-hall-question

Disallow: /hcp-education-training/cne/aphon-chemotherapy-biotherapy-course

Disallow: /msk-search-feedback

Disallow: /join-us-gsk-cocktail-reception

Disallow: /sites/default/files/node/2739/documents/2021-dos-research-booklet.pdf

Disallow: /mskdirect/msk-kids-testimonial

Disallow: /videos/september-2022-surgical-oncology-talk

Disallow: /surgical-oncology-teaching-conference

Disallow: /videos/causal-inference-methods-observational-research

Disallow: /teaser/msk-direct-summit-24-flyers.pdf

Disallow: /teaser/patient-report.pdf

Disallow: /appointments/request-appointment-rapid-diagnosis

Disallow: /teaser/asco-quality-care-symposium-poster.pdf

Disallow: /teaser/2025-msk-direct-marketing-toolkit.pdf

Disallow: /teaser/firefighter-cancer-support-network.pdf

Disallow: /teaser/msk-direct-clinical-leadership-roundtable.pdf


# MSK: Sitemap

Sitemap: https://www.mskcc.org/sitemap.xml


# MSK: News Sitemap

Sitemap: https://www.mskcc.org/news.xml


User-agent: Twitterbot

Allow: /news*

Allow: /news-releases*

Allow: /cancer-care/types*

Allow: /cancer-care/patient-education/*

Allow: /cancer-care/integrative-medicine/herbs*