# Static robots.txt served by nginx-proxy (not PHP), shared across all vhosts. Added 2026-06-14.
# Mirrors the proxy's bot policy: junk crawlers disallowed, greedy crawlers asked to self-pace.

# Search engines: full access
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

# Greedy-but-polite crawlers: self-pace at the source (proxy also caps them at 1 r/s)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: Amazonbot
Crawl-delay: 10

User-agent: Applebot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: Bytespider
Crawl-delay: 10

User-agent: DataForSeoBot
Crawl-delay: 10

User-agent: PetalBot
Crawl-delay: 10

User-agent: GPTBot
Crawl-delay: 10

User-agent: ClaudeBot
Crawl-delay: 10

User-agent: CCBot
Crawl-delay: 10

# Junk crawlers: disallowed (also hard-blocked at the proxy)
User-agent: crawler_eb_germany
Disallow: /

User-agent: Timpibot
Disallow: /

# Everyone else
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php