User-Agent: *
Allow: /
Allow: /api/og/*
Allow: /tools/
Allow: /features/
Allow: /compare/
Allow: /about/
Allow: /pricing/

# Block sensitive areas
Disallow: /pricing/lifetime
Disallow: /app/
Disallow: /dashboard/
Disallow: /login
Disallow: /signup
Disallow: /api/
Disallow: /_next/

# Block image generation endpoints to reduce crawl load
Disallow: /blogs/*/opengraph-image
Disallow: /blogs/*/twitter-image
Disallow: /user/*/opengraph-image
Disallow: /user/*/twitter-image
Disallow: /user/*/collections/*/opengraph-image
Disallow: /user/*/collections/*/twitter-image

# Crawl delay to prevent overloading
Crawl-delay: 1

Sitemap: https://betterstacks.com/sitemap.xml