# www.yacreader.com
# Installed by tools/upgrade/90-site-setup.sh (source: joomla/robots.txt in the site repo).

User-agent: *
# Joomla system folders (the pages load their CSS/JS/images from /media/, which stays open)
Disallow: /administrator/
Disallow: /api/
Disallow: /bin/
Disallow: /cache/
Disallow: /cli/
Disallow: /components/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /layouts/
Disallow: /libraries/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /tmp/
# Forum search: spam sites link to search URLs with junk queries, and every one is a database
# search (Sep 2026: 14,744 Bingbot requests, a third of all crawler traffic)
Disallow: /forum/search
Disallow: /*option=com_kunena&view=search
# Forum pages with no content for search engines: user profiles, forms, actions, personal lists
Disallow: /forum/profile
Disallow: /forum/newtopic
Disallow: /forum/mylatest
Disallow: /forum/announcement
Disallow: /forum/*/topic/create
Disallow: /forum/*/manage
Disallow: /forum/*task-
Disallow: /forum/*/reply
Disallow: /forum/*/edit/
Disallow: /forum/*/moderate
# User account pages
Disallow: /component/users/
Disallow: /registration

# SEO and marketing crawlers: no benefit for the site, a lot of requests
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: trendictionbot
User-agent: serpstatbot
Disallow: /