feat: robots.txt file

packages/website/public/robots.txt

@@ -1,7 +0,0 @@
-User-agent: *
-Allow: /
-
-Host: https://blitzkit.app
-
-Sitemap: https://blitzkit.app/sitemap.txt
-Sitemap: https://blitzkit.app/sitemap.xml

packages/website/src/pages/ads.txt.ts

@@ -1,8 +1,6 @@
 import { assertSecret } from '@blitzkit/core';
 import type { APIContext } from 'astro';
 
-export const prerender = false;
-
 export async function GET({}: APIContext) {
   const response = await fetch(
     `https://api.nitropay.com/v1/ads-${assertSecret(

packages/website/src/pages/robots.txt.ts

@@ -0,0 +1,20 @@
+import { assertSecret } from '@blitzkit/core';
+import type { APIContext } from 'astro';
+
+export async function GET({}: APIContext) {
+  const allowRobots = assertSecret(import.meta.env.ALLOW_ROBOTS) === 'true';
+
+  return new Response(`
+User-agent: *
+${allowRobots ? 'Allow' : 'Disallow'}: /
+
+${
+  allowRobots
+    ? `
+Host: https://blitzkit.app
+Sitemap: https://blitzkit.app/sitemap.txt
+`
+    : ''
+}
+`);
+}