User-agent: *
Allow: /

# Allow all important pages
Allow: /generator
Allow: /verification
Allow: /blog/
Allow: /about
Allow: /contact
Allow: /privacy
Allow: /terms

# Disallow admin and API endpoints
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /node_modules/

# Allow important files
Allow: /llms.txt
Allow: /sitemap.xml
Allow: /favicon.ico

# Sitemap location
Sitemap: https://llmtext.net/sitemap.xml

# Crawl delay (optional - be respectful)
Crawl-delay: 1