# BelleTorus Corporation - Robots.txt
# https://www.robotstxt.org/robotstxt.html

# Allow all crawlers
User-agent: *

# Allow crawling of all content
Allow: /

# Sitemap location
Sitemap: https://belle.ai/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# Block access to admin areas (if any)
Disallow: /admin/
Disallow: /private/
Disallow: /internal/

# Block access to temporary files
Disallow: /temp/
Disallow: /tmp/
Disallow: /*.tmp$
Disallow: /*.temp$

# Block access to development files
Disallow: /dev/
Disallow: /test/
Disallow: /staging/

# Allow access to important directories
Allow: /assets/
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /documents/

# Block access to API endpoints (if they exist)
Disallow: /api/
Disallow: /rest/
Disallow: /graphql/

# Block access to configuration files
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*.txt$
Disallow: /*.log$

# Allow access to public documents
Allow: /documents/*.pdf
Allow: /documents/*.doc
Allow: /documents/*.docx