Infrastructure

#

# robots.txt

#

# This file is to prevent the crawling and indexing of certain parts

# of your site by web crawlers and spiders run by sites like Yahoo!

# and Google. By telling these "robots" where not to go on your site,

# you save bandwidth and server resources.

#

# This file will be ignored unless it is at the root of your host:

# Used:    http://example.com/robots.txt

# Ignored: http://example.com/site/robots.txt

#

# For more information about the robots.txt standard, see:

# http://www.robotstxt.org/robotstxt.html


User-agent: *

Crawl-delay: 1
# CSS, JS, Images

Allow: /core/*.css$

Allow: /core/*.css?

Allow: /core/*.js$

Allow: /core/*.js?

Allow: /core/*.gif

Allow: /core/*.jpg

Allow: /core/*.jpeg

Allow: /core/*.png

Allow: /core/*.svg

Allow: /profiles/*.css$

Allow: /profiles/*.css?

Allow: /profiles/*.js$

Allow: /profiles/*.js?

Allow: /profiles/*.gif

Allow: /profiles/*.jpg

Allow: /profiles/*.jpeg

Allow: /profiles/*.png

Allow: /profiles/*.svg

# Directories

Disallow: /core/

Disallow: /profiles/

# Files

Disallow: /README.txt

Disallow: /web.config

# Paths (clean URLs)

Disallow: /admin/

Disallow: /comment/reply/

Disallow: /filter/tips

Disallow: /node/add/

Disallow: /search/

Disallow: /user/register/

Disallow: /user/password/

Disallow: /user/login/

Disallow: /user/logout/

# Paths (no clean URLs)

Disallow: /index.php/admin/

Disallow: /index.php/comment/reply/

Disallow: /index.php/filter/tips

Disallow: /index.php/node/add/

Disallow: /index.php/search/

Disallow: /index.php/user/password/

Disallow: /index.php/user/register/

Disallow: /index.php/user/login/

Disallow: /index.php/user/logout/



# CUSTOM CONFIG BELOW

Disallow /redirects/

# Filterable view pages

Disallow: /research/findings/evidence-based-reports/search.html?

Disallow: /research/findings/studies/index.html?

Disallow: /news/newsletters/e-newsletter/index.html?

Disallow: /news/events/other/index.html?

Disallow: /news/newsroom/press-releases/index.html?

Disallow: /news/newsroom/noticias/index.html?

Disallow: /news/newsroom/articles-of-interest/index.html?

Disallow: /news/newsroom/case-studies/index.html?

Disallow: /cahps/bibliography/index.html?

Disallow: /cahps/surveys-guidance/item-sets/search.html?

Disallow: /cahps/faq/index.html?

Disallow: /programs/index.html?

Disallow: /tools/index.html?

Disallow: /data/dataresources/index.html?


# PBRN Registry filtered pages - prevent aggressive bot crawling

Disallow: /ncepcr/communities/pbrn/registry/index.html?


# Block aggressive AI/ML bots from PBRN Registry

User-agent: GPTBot

Disallow: /ncepcr/communities/pbrn/


User-agent: ChatGPT-User

Disallow: /ncepcr/communities/pbrn/


User-agent: CCBot

Disallow: /ncepcr/communities/pbrn/


User-agent: anthropic-ai

Disallow: /ncepcr/communities/pbrn/


User-agent: Claude-Web

Disallow: /ncepcr/communities/pbrn/



# Googlebot - Allow all paths

User-agent: Googlebot

Allow: /

Disallow:


# Special use pages

Disallow: /research/findings/ta/technical-assistance-form-thank-you.html

Disallow: /funding/contracts/proposal-instructions.html


# Deprecated pages

Disallow: /sites/default/files/publications/files/organization-chart.pdf

Disallow: /whatsnew.html


# Temporary

Disallow: /news/events/ahrq-research-summit-learning.html

Allow: /professionals/quality-patient-safety/hais/tools/perinatal-care/index.html

Disallow: /professionals/quality-patient-safety/hais/tools/perinatal-care/

Disallow: /nhguide/toolkits/determine-whether-to-treat/antibiotic-tool.html

Disallow: /topics/index-v2-f5L2sheVSpUMT9tL.html

Disallow: /learning-health-systems/

Disallow: /whatsnew.html

Disallow: /health-literacy

Disallow: /rural-post-partum-challenge

Disallow: /questions

Disallow: /challenges

Disallow: /innovations