# robots.txt for proefpersoon.nl
# Rate limiting and access control for web crawlers

# Default rules for all user agents
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /*.json$
Disallow: /*?*
Disallow: /search
Disallow: /cart/
Disallow: /checkout/
Crawl-delay: 1
Request-rate: 30/60

User-agent: bingbot
Disallow: /
Crawl-delay: 2

# Rate limit Facebook and social media crawlers
User-agent: facebookexternalhit
Disallow: /admin/
Disallow: /private/
Crawl-delay: 2
Request-rate: 10/60

User-agent: Twitterbot
Disallow: /admin/
Disallow: /private/
Crawl-delay: 1

User-agent: LinkedInBot
Disallow: /admin/
Disallow: /private/
Crawl-delay: 1

User-agent: WhatsApp
Disallow: /admin/
Disallow: /private/

User-agent: Slurp
Disallow: /admin/
Disallow: /private/
Crawl-delay: 1

# Allow Google and Bing with reasonable limits
User-agent: Googlebot
Disallow: /admin/
Disallow: /private/
Crawl-delay: 0.5

User-agent: Bingbot
Disallow: /admin/
Disallow: /private/
Crawl-delay: 1