113 lines
2.9 KiB
Plaintext
113 lines
2.9 KiB
Plaintext
# robots.txt — anti-plagiat.uz
|
|
# Yangilangan: 2026-04-12
|
|
|
|
# ─── Umumiy qoidalar (barcha botlar) ───────────────────────────────────────
|
|
User-agent: *
|
|
Allow: /
|
|
Allow: /uz/
|
|
Allow: /ru/
|
|
Allow: /en/
|
|
Allow: /uz/plagiat
|
|
Allow: /ru/plagiat
|
|
Allow: /en/plagiat
|
|
Allow: /uz/si
|
|
Allow: /ru/si
|
|
Allow: /en/si
|
|
|
|
# API va autentifikatsiya marshrutlarini bloklash
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
Disallow: /uz/cabinet/
|
|
Disallow: /ru/cabinet/
|
|
Disallow: /en/cabinet/
|
|
|
|
# Crawl tezligi (ortiqcha yuklanmasin)
|
|
Crawl-delay: 2
|
|
|
|
# ─── Google ───────────────────────────────────────────────────────────────
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
Crawl-delay: 1
|
|
|
|
User-agent: Googlebot-Image
|
|
Disallow: /
|
|
|
|
# ─── Bing ─────────────────────────────────────────────────────────────────
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
Crawl-delay: 2
|
|
|
|
# ─── AI Crawler agentlari ─────────────────────────────────────────────────
|
|
# GPT / OpenAI
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
Allow: /uz/
|
|
Allow: /ru/
|
|
Allow: /en/
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
Disallow: /resources/media/
|
|
|
|
# Claude (Anthropic)
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
Allow: /uz/
|
|
Allow: /ru/
|
|
Allow: /en/
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
|
|
# Perplexity
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
|
|
# Meta AI
|
|
User-agent: Meta-ExternalAgent
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
|
|
# Gemini / Google AI
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
|
|
# Cohere
|
|
User-agent: cohere-ai
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
|
|
# Common Crawl (AI training ma'lumotlar to'plami)
|
|
User-agent: CCBot
|
|
Allow: /
|
|
Disallow: /api/
|
|
Disallow: /_next/
|
|
Disallow: /resources/media/
|
|
|
|
# ─── Noxush botlar ────────────────────────────────────────────────────────
|
|
User-agent: MJ12bot
|
|
Disallow: /
|
|
|
|
User-agent: DotBot
|
|
Disallow: /
|
|
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
# ─── Sitemap manzili ──────────────────────────────────────────────────────
|
|
Sitemap: https://anti-plagiat.uz/sitemap.xml
|
|
|
|
# ─── AI ma'lumot fayli ────────────────────────────────────────────────────
|
|
# AI modellar uchun: https://anti-plagiat.uz/llms.txt
|