# ---------------------------------------------------------------------------
# Served for www.mediamonkey.com and mediamonkey.com via the RewriteRule in
# public_html/.htaccess.  Language subdomains get assets/robots_lang.txt
# (which already Disallows /forum/ entirely); forum./faq. get robots_off.txt.
#
# 2026-08-31: added the phpBB interaction endpoints.  posting.php (73.8k/day)
# and ucp.php (38.8k/day) were 45% of all dynamic forum traffic and carry no
# SEO value at all -- they are login/reply/PM forms.  sid= URLs are phpBB
# session ids leaking into links and create unbounded duplicate crawl space
# (20.8k hits/day).  Verified Applebot and ClaudeBot were being 429'd by the
# in-app throttle purely because they were crawling these paths.
#
# Amazonbot is the dominant wiki crawler (~60% of /wiki hits) -- throttle it
# and keep it off dynamic, uncacheable wiki URLs.  A named UA group is used
# exclusively by that bot, so it must repeat all rules.
# ---------------------------------------------------------------------------

User-agent: Amazonbot
Crawl-delay: 30
Disallow: /wiki/*?
Disallow: /wiki/Special:
Disallow: /wiki/*/4.0
Disallow: /forum/posting.php
Disallow: /forum/ucp.php
Disallow: /forum/search.php
Disallow: /forum/memberlist.php
Disallow: /forum/report.php
Disallow: /forum/mcp.php
Disallow: /forum/app.php/user/
Disallow: /forum/*sid=
Disallow: /addons/download

User-agent: *
Crawl-delay: 10
Disallow: /wiki/*?
Disallow: /wiki/Special:
Disallow: /wiki/*/4.0
Disallow: /forum/posting.php
Disallow: /forum/ucp.php
Disallow: /forum/search.php
Disallow: /forum/memberlist.php
Disallow: /forum/report.php
Disallow: /forum/mcp.php
Disallow: /forum/app.php/user/
Disallow: /forum/*sid=
Disallow: /addons/download