new sitemap.xml file

2026-04-13 18:54:40 +05:00
parent 18724e135b
commit fa9df55856
3 changed files with 385 additions and 60 deletions
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -0,0 +1,112 @@
+# robots.txt — anti-plagiat.uz
+# Yangilangan: 2026-04-12
+
+# ─── Umumiy qoidalar (barcha botlar) ───────────────────────────────────────
+User-agent: *
+Allow: /
+Allow: /uz/
+Allow: /ru/
+Allow: /en/
+Allow: /uz/plagiat
+Allow: /ru/plagiat
+Allow: /en/plagiat
+Allow: /uz/si
+Allow: /ru/si
+Allow: /en/si
+
+# API va autentifikatsiya marshrutlarini bloklash
+Disallow: /api/
+Disallow: /_next/
+Disallow: /uz/cabinet/
+Disallow: /ru/cabinet/
+Disallow: /en/cabinet/
+
+# Crawl tezligi (ortiqcha yuklanmasin)
+Crawl-delay: 2
+
+# ─── Google ───────────────────────────────────────────────────────────────
+User-agent: Googlebot
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+Crawl-delay: 1
+
+User-agent: Googlebot-Image
+Disallow: /
+
+# ─── Bing ─────────────────────────────────────────────────────────────────
+User-agent: Bingbot
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+Crawl-delay: 2
+
+# ─── AI Crawler agentlari ─────────────────────────────────────────────────
+# GPT / OpenAI
+User-agent: GPTBot
+Allow: /
+Allow: /uz/
+Allow: /ru/
+Allow: /en/
+Disallow: /api/
+Disallow: /_next/
+Disallow: /resources/media/
+
+# Claude (Anthropic)
+User-agent: ClaudeBot
+Allow: /
+Allow: /uz/
+Allow: /ru/
+Allow: /en/
+Disallow: /api/
+Disallow: /_next/
+
+# Perplexity
+User-agent: PerplexityBot
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+
+# Meta AI
+User-agent: Meta-ExternalAgent
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+
+# Gemini / Google AI
+User-agent: Google-Extended
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+
+# Cohere
+User-agent: cohere-ai
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+
+# Common Crawl (AI training ma'lumotlar to'plami)
+User-agent: CCBot
+Allow: /
+Disallow: /api/
+Disallow: /_next/
+Disallow: /resources/media/
+
+# ─── Noxush botlar ────────────────────────────────────────────────────────
+User-agent: MJ12bot
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+# ─── Sitemap manzili ──────────────────────────────────────────────────────
+Sitemap: https://anti-plagiat.uz/sitemap.xml
+
+# ─── AI ma'lumot fayli ────────────────────────────────────────────────────
+# AI modellar uchun: https://anti-plagiat.uz/llms.txt