# robots.txt for https://website.adsmediamix.id
User-agent: *
Allow: /

# Important pages
Allow: /about
Allow: /services
Allow: /portfolio
Allow: /blog
Allow: /pricing
Allow: /contact

# Admin pages - disallow
Disallow: /admin/
Disallow: /login
Disallow: /api/

# Instructions for search engine crawlers
User-agent: Googlebot
Crawl-delay: 2

User-agent: Bingbot
Disallow: /admin/
Disallow: /api/
Crawl-delay: 3

# AI/LLM Crawlers - Allow with rate limiting
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Claude-Web
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: GoogleOther
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Omgilibot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: cohere-ai
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Diffbot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

# Prevent crawling on non-production environments
User-agent: *
Disallow: /staging/
Disallow: /dev/
Disallow: /test/

# Sitemaps
Sitemap: https://website.adsmediamix.id/sitemap-index.xml
Sitemap: https://website.adsmediamix.id/sitemap.xml
Sitemap: https://website.adsmediamix.id/sitemap-blog.xml

# AI-specific information files
AI-Agent-Info: https://website.adsmediamix.id/ai.txt
LLM-Context: https://website.adsmediamix.id/llms.txt
LLM-Context-Full: https://website.adsmediamix.id/llms-full.txt