Block bots

src/pages/robots.txt.ts

@@ -0,0 +1,18 @@
+import type { APIRoute } from "astro";
+import { AI_BOTS } from "../data/ai_bots";
+
+export const GET: APIRoute = async () => {
+  const robots = [
+    AI_BOTS.map((bot) => `User-agent: ${bot}`),
+    "Disallow: /",
+    "",
+    "User-agent: *",
+    "Disallow: .htaccess",
+    "Disallow: /stories/drafts/",
+    "Disallow: /games/drafts/",
+  ]
+    .flat()
+    .join("\n");
+
+  return new Response(robots, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
+};