# A gentle robots.txt for a public-domain literary archive
# Purpose: Protect server resources and admin areas, but otherwise welcome all crawlers.

User-agent: *
Allow: /

# Protect system and private areas
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /private/
Disallow: /search/
Disallow: /tmp/
Disallow: /backup/
Disallow: /cgi-bin/
Disallow: /*.php$
Disallow: /*.sql$
Disallow: /*.env$

# Block only the most aggressive SEO scrapers (they consume bandwidth, index nothing, and send zero traffic)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# A polite, gentle crawl delay (2 seconds between requests keeps server happy, but doesn't frustrate Google)
Crawl-delay: 2

# Help bots find everything
Sitemap: http://www.skirret.com/sitemap.xml