User-agent: *

Sitemap: https://www.shanthicabs.com/sitemap.xml

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

# Prevent indexing of duplicate or unnecessary pages
Disallow: /search?q=
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/

# Allow all user-agents to crawl public pages
Allow: /

# Block specific bots known for spam
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /