# New Indian Sweets & Restaurant Robots.txt
# Website: https://www.newindiansweetsandrestaurant.ca
# Last Updated: 2025-08-30

# Main directives for all bots
User-agent: *
Allow: /
Disallow: /assets/
Disallow: /components/
Disallow: /logs/
Disallow: /*.js$
Disallow: /*.css$
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Allow search engines to access important resources
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
Allow: /
Allow: /*.css$
Allow: /*.js$
Allow: /assets/images/
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# AI Bot specific directives - Allow access to llms.txt
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Google Extended (Bard)
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# Social Media Bots
User-agent: FacebookBot
User-agent: TwitterBot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
Allow: /

# Google Ads Bot
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
Allow: /

# Image crawlers
User-agent: Googlebot-Image
User-agent: bingbot-image
Allow: /assets/images/
Allow: /assets/gallery/
Allow: /assets/dishes/

# Crawl delay for heavy bots
User-agent: Baiduspider
User-agent: YandexBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
Crawl-delay: 10

# Block bad bots
User-agent: BadBot
User-agent: ScraperBot
User-agent: CopyBot
User-agent: EmailCollector
User-agent: EmailSiphon
User-agent: WebBandit
User-agent: WebCopier
User-agent: WebStripper
Disallow: /

# Sitemap location
Sitemap: https://www.newindiansweetsandrestaurant.ca/sitemap.xml

# LLMs.txt location for AI agents
# AI agents should check this file for structured information
# Path: /llms.txt