Infrastructure

# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

# Targets all user agents
User-agent: *
# ================================
# Directories
# ================================
# Prevent indexing of core Drupal directories that contain system files.
Disallow: /core/
Disallow: /profiles/
# ================================
# Files
# ================================
# Prevent indexing of configuration and readme files that are not relevant to search engines.
Disallow: /README.md
Disallow: /composer/
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# ================================
# Paths (clean URLs)
# ================================
# Administrative, user related paths (all languages)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /media/oembed
Disallow: /node/add/
Disallow: /user/
Disallow: /*/admin/
Disallow: /*/comment/reply/
Disallow: /*/filter/tips
Disallow: /*/media/oembed
Disallow: /*/node/add/
Disallow: /*/user/
 
# Specific search result pages (all languages)
Disallow: /boardactions-search
Disallow: /information-documents-search
Disallow: /meetings-search
Disallow: /news-search
Disallow: /official-documents-search
Disallow: /osgstatements-search
Disallow: /podcast-search
Disallow: /pressmaterial-search
Disallow: /project-search
Disallow: /publications-search
Disallow: /search
Disallow: /upcoming-meetings-search
Disallow: /*/boardactions-search
Disallow: /*/information-documents-search
Disallow: /*/meetings-search
Disallow: /*/news-search
Disallow: /*/official-documents-search
Disallow: /*/osgstatements-search
Disallow: /*/podcast-search
Disallow: /*/pressmaterial-search
Disallow: /*/project-search
Disallow: /*/publications-search
Disallow: /*/search
Disallow: /*/upcoming-meetings-search
 
# ================================
# Paths (no clean URLs)
# ================================
# Block duplicate or legacy URLs using index.php.
Disallow: /index.php/
 
# ================================
# Allowed assets
# ================================
# Explicitly allow crawlers to access CSS, JS, and image files 
# so they can render pages correctly and understand visual layout.
Allow: /*.css
Allow: /*.js
Allow: /*.gif
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.svg
 
# ================================
# Sitemap
# ================================
# Provides the sitemap location for efficient crawling.
Sitemap: https://unctad.org/sitemap.xml