Infrastructure

# NOTICE: Collection of content and other data on www.theaustralian.com.au through automated means is prohibited unless you have express written permission from the publisher of the website and may only be conducted for the limited purpose contained in said permission.

# Website Terms of Use may be found at https://www.theaustralian.com.au/termsconditions

#Agent Specific Allowed Section
# ==========================
User-agent: ScreamingFrogSEOSpider
User-agent: proximic
User-agent: Moatbot
User-agent: Adbeat
User-agent: Expo9
User-agent: Outbrain
User-agent: uipbot
User-agent: Weborama-fetcher
User-agent: Yahoo Ad monitoring
User-agent: NetSeer crawler
User-agent: SemrushBot
User-agent: facebookexternalhit
User-agent: ias_crawler
User-agent: ias_wombles
User-agent: IASbot
User-agent: identrics
User-agent: MeltwaterNews
User-agent: links.streem.com.au
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: redditbot
User-agent: Mediapartners-Google
User-agent: Google-InspectionTool
User-agent: Adsbot-Google
User-agent: Storebot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
User-agent: APIs-Google
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: Google-Ads-Creatives-Assistant
User-agent: Google-adwords
User-agent: Google-amphtml
User-agent: Google-cloudvertexbot
User-agent: Google-play-newsstand
User-agent: Google-read-aloud
User-agent: Google-Safety
User-agent: Google-site-verification
User-agent: Google-Structured-Data-Testing-Tool
User-agent: Googleimageproxy
User-agent: Twitterbot
User-agent: Applebot
User-agent: AppleNewsBot
User-agent: AdIdxBot
User-agent: admantx
User-agent: admantx-usbatch/3.1
User-agent: AHC
User-agent: AHC/2.1
User-agent: AmazonAdBot
User-agent: bidswitchbot/1.0
User-agent: botify
User-agent: checkMy
User-agent: Chrome-Lighthouse
User-agent: CriteoBot/0.1
User-agent: datadog-synthetics
User-agent: dlvrit
User-agent: EchoboxBot/1.0
User-agent: flipboard
User-agent: GrapeshotCrawler
User-agent: Verity
User-agent: HyScore/1.0
User-agent: ia_archiver
User-agent: Iframely
User-agent: labne
User-agent: Leikibot
User-agent: newrelicpinger
User-agent: okta
User-agent: parselybot
User-agent: peer39_crawler/1.0
User-agent: photon
User-agent: pingdom
User-agent: Pinterestbot
User-agent: proofpoint
User-agent: proximic
User-agent: pubmatic
User-agent: schema-validator
User-agent: sentry
User-agent: SirdataBot
User-agent: site24x7
User-agent: siteimprovebot
User-agent: Slack-ImgProxy
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: smartologybot
User-agent: Snap URL Preview Service
User-agent: snapchat
User-agent: SocialFlow
User-agent: statuscake
User-agent: Taboolabot
User-agent: telegrambot
User-agent: thetradedesk
User-agent: Uptimebot
User-agent: YahooMailProxy
User-agent: yahoo-linkpreview
Disallow:

# Web Search Engines
==========================
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Bingbot
User-agent: bingpreview
User-agent: MicrosoftPreview
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: yahoo
Disallow:
Disallow: /*?expand_comments=true
Disallow: /*&expand_comments=true
Disallow: /*/comments-*
Disallow: /*SIT*
Disallow: /*.swf
Disallow: /printpage/
Disallow: /archive/
Disallow: /tablet-t3/
Disallow: /enewsletters/
Disallow: /newsnetwork/
Disallow: /news/*/collection/*
Disallow: /myaustralian/
Disallow: /myaus/
Disallow: /app-route/
Disallow: /double-rainbow/
Disallow: /it-test-only/
Disallow: /business/stockhead/news/*/news-story/*
Disallow: /world/the-times/*/news-story/*
Disallow: /akam/
Disallow: /*?gclid=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?arena_mid=
Disallow: /*?matchid=
Disallow: /*?keyevent=
Disallow: /*?from=
Disallow: /*?nk=
Disallow: /*?sv=

User-agent: Google-Extended
Disallow: /

# All remaining visitors
# ==========================
User-agent: *
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /


# ==========================
Sitemap: https://www.theaustralian.com.au/sitemap.xml
Sitemap: https://www.theaustralian.com.au/news-sitemap.xml