# Explicitly allow Facebook's crawler
User-agent: facebookexternalhit
Allow: /

# Explicitly allow Twitter's crawler
User-agent: Twitterbot
Allow: /

User-agent: *
Allow: /

# Disallow admin and API routes
Disallow: /admin/
Disallow: /api/
Disallow: /auth/

# Allow the read-only endpoints crawlers must fetch to render rate pages.
# Longest-match wins, so private/mutating /api routes above stay blocked.
Allow: /api/rates
Allow: /api/rate-trends
Allow: /api/lenders
Allow: /api/articles

# Allow all public calculators and pages
Allow: /calculators/
Allow: /rates
Allow: /embed-demo

# Sitemap location
Sitemap: https://rates.now/sitemap.xml

# Crawl delay (optional - be respectful to crawlers)
Crawl-delay: 1