Infrastructure

#

# robots.txt

#

# This file is to prevent the crawling and indexing of certain parts

# of your site by web crawlers and spiders run by sites like Yahoo!

# and Google. By telling these "robots" where not to go on your site,

# you save bandwidth and server resources.

#

# This file will be ignored unless it is at the root of your host:

# Used:    http://example.com/robots.txt

# Ignored: http://example.com/site/robots.txt

#

# For more information about the robots.txt standard, see:

# http://www.robotstxt.org/robotstxt.html


User-agent: *

# CSS, JS, Images

Allow: /core/*.css$

Allow: /core/*.css?

Allow: /core/*.js$

Allow: /core/*.js?

Allow: /core/*.gif

Allow: /core/*.jpg

Allow: /core/*.jpeg

Allow: /core/*.png

Allow: /core/*.svg

Allow: /profiles/*.css$

Allow: /profiles/*.css?

Allow: /profiles/*.js$

Allow: /profiles/*.js?

Allow: /profiles/*.gif

Allow: /profiles/*.jpg

Allow: /profiles/*.jpeg

Allow: /profiles/*.png

Allow: /profiles/*.svg

# Directories

Disallow: /core/

Disallow: /profiles/

# Files

Disallow: /README.txt

Disallow: /web.config

# Paths (clean URLs)

Disallow: /admin/

Disallow: /comment/reply/

Disallow: /filter/tips/

Disallow: /node/add/

Disallow: /jobs/search/

Disallow: /user/register/

Disallow: /user/password/

Disallow: /user/login/

Disallow: /user/logout/

# Paths (no clean URLs)

Disallow: /index.php/admin/

Disallow: /index.php/comment/reply/

Disallow: /index.php/filter/tips/

Disallow: /index.php/node/add/

Disallow: /index.php/search/

Disallow: /index.php/user/password/

Disallow: /index.php/user/register/

Disallow: /index.php/user/login/

Disallow: /index.php/user/logout/

Disallow: /node/*

Disallow: /taxonomy/term/*

# Disallow Ajax loaded pages

Disallow: /feedback-form

Disallow: /report-concern

Disallow: /sites/default/files/asset-library/documents/library/documents/annual_reports/2000-management-report-brands-de.pdf

# Disallow brand search page

Disallow: /brands/brandssearchlist
Disallow: /cdn-cgi
Disallow: /node
# Simple sitemap
Sitemap: https://www.nestle.com/sitemap.xml