Block bots

src/data/ai_bots.ts

@@ -0,0 +1,38 @@
+export const AI_BOTS = [
+  "AI2Bot",
+  "Ai2Bot-Dolma",
+  "Amazonbot",
+  "Applebot",
+  "Applebot-Extended",
+  "Bytespider",
+  "CCBot",
+  "ChatGPT-User",
+  "Claude-Web",
+  "ClaudeBot",
+  "Diffbot",
+  "FacebookBot",
+  "FriendlyCrawler",
+  "GPTBot",
+  "Google-Extended",
+  "GoogleOther",
+  "GoogleOther-Image",
+  "GoogleOther-Video",
+  "ICC-Crawler",
+  "ImagesiftBot",
+  "Meta-ExternalAgent",
+  "Meta-ExternalFetcher",
+  "OAI-SearchBot",
+  "PerplexityBot",
+  "PetalBot",
+  "Scrapy",
+  "Timpibot",
+  "VelenPublicWebCrawler",
+  "Webzio-Extended",
+  "YouBot",
+  "anthropic-ai",
+  "cohere-ai",
+  "facebookexternalhit",
+  "img2dataset",
+  "omgili",
+  "omgilibot",
+];

src/layouts/PublishedContentLayout.astro

@@ -152,7 +152,9 @@ const thumbnail =
         >
           <IconSun width="1.25rem" height="1.25rem" class="hidden dark:block" />
           <IconMoon width="1.25rem" height="1.25rem" class="block dark:hidden" />
-          <span class="sr-only select-none" id="label-toggle-dark-mode">{t(props.lang, "published_content/toggle_dark_mode")}</span>
+          <span class="sr-only select-none" id="label-toggle-dark-mode"
+            >{t(props.lang, "published_content/toggle_dark_mode")}</span
+          >
         </button>
       </div>
     </div>

src/pages/robots.txt.ts

@@ -0,0 +1,18 @@
+import type { APIRoute } from "astro";
+import { AI_BOTS } from "../data/ai_bots";
+
+export const GET: APIRoute = async () => {
+  const robots = [
+    AI_BOTS.map((bot) => `User-agent: ${bot}`),
+    "Disallow: /",
+    "",
+    "User-agent: *",
+    "Disallow: .htaccess",
+    "Disallow: /stories/drafts/",
+    "Disallow: /games/drafts/",
+  ]
+    .flat()
+    .join("\n");
+
+  return new Response(robots, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
+};