Infrastructure

#

# robots.txt

#

# This file is to prevent the crawling and indexing of certain parts

# of your site by web crawlers and spiders run by sites like Yahoo!

# and Google. By telling these "robots" where not to go on your site,

# you save bandwidth and server resources.

#

# This file will be ignored unless it is at the root of your host:

# Used:    http://example.com/robots.txt

# Ignored: http://example.com/site/robots.txt

#

# For more information about the robots.txt standard, see:

# http://www.robotstxt.org/robotstxt.html


User-agent: *

# CSS, JS, Images

Allow: /core/*.css$

Allow: /core/*.css?

Allow: /core/*.js$

Allow: /core/*.js?

Allow: /core/*.gif

Allow: /core/*.jpg

Allow: /core/*.jpeg

Allow: /core/*.png

Allow: /core/*.svg

Allow: /profiles/*.css$

Allow: /profiles/*.css?

Allow: /profiles/*.js$

Allow: /profiles/*.js?

Allow: /profiles/*.gif

Allow: /profiles/*.jpg

Allow: /profiles/*.jpeg

Allow: /profiles/*.png

Allow: /profiles/*.svg

# Directories

Disallow: /core/

Disallow: /profiles/

Disallow: /node/

Disallow: /media/

# Files

Disallow: /README.txt

Disallow: /web.config

# Paths with query strings

Allow: /sitemap.xml?

Allow: /medicare-coverage-database/*?

Disallow: /*?

# Paths (clean URLs)

Disallow: /admin/

Disallow: /comment/reply/

Disallow: /filter/tips

Disallow: /node/add/

Disallow: /user/

Disallow: /es/

# Paths (no clean URLs)

Disallow: /index.php/

# Sitemap

Sitemap: https://www.cms.gov/sitemap.xml