# robots.txt — anti-plagiat.uz
# Yangilangan: 2026-04-12

# ─── Umumiy qoidalar (barcha botlar) ───────────────────────────────────────
User-agent: *
Allow: /
Allow: /uz/
Allow: /ru/
Allow: /en/
Allow: /uz/plagiat
Allow: /ru/plagiat
Allow: /en/plagiat
Allow: /uz/si
Allow: /ru/si
Allow: /en/si

# API va autentifikatsiya marshrutlarini bloklash
Disallow: /api/
Disallow: /_next/
Disallow: /uz/cabinet/
Disallow: /ru/cabinet/
Disallow: /en/cabinet/

# Crawl tezligi (ortiqcha yuklanmasin)
Crawl-delay: 2

# ─── Google ───────────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /_next/
Crawl-delay: 1

User-agent: Googlebot-Image
Disallow: /

# ─── Bing ─────────────────────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /_next/
Crawl-delay: 2

# ─── AI Crawler agentlari ─────────────────────────────────────────────────
# GPT / OpenAI
User-agent: GPTBot
Allow: /
Allow: /uz/
Allow: /ru/
Allow: /en/
Disallow: /api/
Disallow: /_next/
Disallow: /resources/media/

# Claude (Anthropic)
User-agent: ClaudeBot
Allow: /
Allow: /uz/
Allow: /ru/
Allow: /en/
Disallow: /api/
Disallow: /_next/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /_next/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /_next/

# Gemini / Google AI
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /_next/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /_next/

# Common Crawl (AI training ma'lumotlar to'plami)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /resources/media/

# ─── Noxush botlar ────────────────────────────────────────────────────────
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# ─── Sitemap manzili ──────────────────────────────────────────────────────
Sitemap: https://anti-plagiat.uz/sitemap.xml

# ─── AI ma'lumot fayli ────────────────────────────────────────────────────
# AI modellar uchun: https://anti-plagiat.uz/llms.txt