User-agent: *
Allow: /

# Disallow sensitive and system paths
Disallow: /logs/
Disallow: /php/
Disallow: /*.log$
Disallow: /*.php$
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /404.html
Disallow: /403.html
Disallow: /500.html

# Allow all major search engine crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Baiduspider
Allow: /

# Block scrapers and aggressive bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

# Sitemap location
Sitemap: https://nexus1onion.shop/sitemap.xml

# Crawl delay for general bots
Crawl-delay: 2