# Sitemap: https://www.jre.eu/sitemap.xml

# Crawlers Setup
User-agent: *

# Directories
Disallow: /*_*/$
Disallow: /cgi-bin/
Disallow: /404/
Disallow: /cgi-bin/
Disallow: /search/
Disallow: /forgot_password
Disallow: /assets/cache/

# Paths (no clean URLs)
Disallow: /*.php$
Disallow: /*?SID=
Disallow: /rss*
Disallow: /*PHPSESSID
Disallow: /*?Q=

# Files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /get.php

# Block specific bots
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot-BM
Disallow: /
User-agent: domaincrawler.com
Disallow: /
User-agent: spbot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: BLEXBot
Disallow: /

# Delay specific bots
User-agent: Slurp
Crawl-delay: 10

User-agent: DuckDuckGo Bot
Crawl-delay: 10