Infrastructure

#

# robots.txt

#

# This file is to prevent the crawling and indexing of certain parts

# of your site by web crawlers and spiders run by sites like Yahoo!

# and Google. By telling these "robots" where not to go on your site,

# you save bandwidth and server resources.

#

# This file will be ignored unless it is at the root of your host:

# Used:    http://example.com/robots.txt

# Ignored: http://example.com/site/robots.txt

#

# For more information about the robots.txt standard, see:

# http://www.robotstxt.org/robotstxt.html


User-agent: *

Crawl-delay: 5
# CSS, JS, Images

Allow: /core/*.css$

Allow: /core/*.css?

Allow: /core/*.js$

Allow: /core/*.js?

Allow: /core/*.gif

Allow: /core/*.jpg

Allow: /core/*.jpeg

Allow: /core/*.png

Allow: /core/*.svg

Allow: /profiles/*.css$

Allow: /profiles/*.css?

Allow: /profiles/*.js$

Allow: /profiles/*.js?

Allow: /profiles/*.gif

Allow: /profiles/*.jpg

Allow: /profiles/*.jpeg

Allow: /profiles/*.png

Allow: /profiles/*.svg

# Directories

Disallow: /core/

Disallow: /profiles/

# Files

Disallow: /README.txt

Disallow: /web.config

# Paths (clean URLs)

Disallow: /admin/

Disallow: /comment/reply/

Disallow: /filter/tips

Disallow: /node/add/

Disallow: /search/

Disallow: /user/register

Disallow: /user/password

Disallow: /user/login

Disallow: /user/logout

# Paths (no clean URLs)

Disallow: /index.php/admin/

Disallow: /index.php/comment/reply/

Disallow: /index.php/filter/tips

Disallow: /index.php/node/add/

Disallow: /index.php/search/

Disallow: /index.php/user/password

Disallow: /index.php/user/register

Disallow: /index.php/user/login

Disallow: /index.php/user/logout


# For link-checking site crawlers.

Disallow: /es/node/*

Disallow: /*?combine=

Disallow: /*?*combine=*

Disallow: /*?items_per_page=

Disallow: /*?*items_per_page=*

Disallow: /*/comment/*

Disallow: /*/comment/reply/*

Disallow: /news-events/events-calendar/2018/03/ninth-annual-african-consumer-protection-dialogue-conference

Disallow: /news-events/events-calendar/2018/03/ninth-annual-african-consumer-protection-dialogue-conference/RSVP

Disallow: /news-events/events-calendar/2019/05/tenth-annual-african-consumer-protection-dialogue-conference

Disallow: /sites/default/files/documents/public_statements/joint-fda/ftc-warning-letter-concerning-product-labeling-human-chrorionic-gonadotropin-hcg-drugs/ucm282052.htm

Disallow: /sites/default/files/documents/videos/final-ftc-privacy-report-press-conference/privacyframework.shtm

Disallow: /sites/default/files/documents/public_statements/statement-ftc-chairman-jon-leibowitz-senate-judiciary-committees-passage-preserve-access-affordable.369/pfdvote.shtm

Disallow: /sites/default/files/documents/public_statements/statement-ftc-chairman-jon-leibowitz-regarding-announcement-arthur-d.levinson-has-resigned-googles-board/google.shtm

Disallow: /sites/default/files/documents/public_statements/statement-ftc-chairman-jon-leibowitz-passage-h.r.3126-consumer-financial-protection-act-2009-house-energy-and-commerce-committee/cfpa.shtm