# ── General crawlers ──
User-agent: *
Allow: /
Disallow: /admin
Crawl-delay: 5

# ── AI / Generative Engine crawlers ──
User-agent: GPTBot
Allow: /
Crawl-delay: 10

User-agent: ClaudeBot
Allow: /
Crawl-delay: 10

User-agent: BardBot
Allow: /
Crawl-delay: 10

User-agent: PerplexityBot
Allow: /
Crawl-delay: 10

User-agent: Amazonbot
Allow: /
Crawl-delay: 10

User-agent: AI2Bot
Allow: /
Crawl-delay: 10

User-agent: Meta-ExternalFetcher
Allow: /
Crawl-delay: 10

User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 10

# ── Sitemaps ──
Sitemap: https://www.digambaramastrologer.in/sitemap.xml
Sitemap: https://www.digambaramastrologer.in/llms.txt