From dfc55b745ba63dea3d8856e4dc8bc6cefc5faa42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com>
Date: Mon, 14 Oct 2024 10:47:49 -0500
Subject: [PATCH] feat: robots.txt file

---
 packages/website/public/robots.txt       |  7 -------
 packages/website/src/pages/ads.txt.ts    |  2 --
 packages/website/src/pages/robots.txt.ts | 20 ++++++++++++++++++++
 3 files changed, 20 insertions(+), 9 deletions(-)
 delete mode 100644 packages/website/public/robots.txt
 create mode 100644 packages/website/src/pages/robots.txt.ts

diff --git a/packages/website/public/robots.txt b/packages/website/public/robots.txt
deleted file mode 100644
index 8f953d4fa..000000000
--- a/packages/website/public/robots.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-User-agent: *
-Allow: /
-
-Host: https://blitzkit.app
-
-Sitemap: https://blitzkit.app/sitemap.txt
-Sitemap: https://blitzkit.app/sitemap.xml
\ No newline at end of file
diff --git a/packages/website/src/pages/ads.txt.ts b/packages/website/src/pages/ads.txt.ts
index 629443ab3..d24ab2c9a 100644
--- a/packages/website/src/pages/ads.txt.ts
+++ b/packages/website/src/pages/ads.txt.ts
@@ -1,8 +1,6 @@
 import { assertSecret } from '@blitzkit/core';
 import type { APIContext } from 'astro';
 
-export const prerender = false;
-
 export async function GET({}: APIContext) {
   const response = await fetch(
     `https://api.nitropay.com/v1/ads-${assertSecret(
diff --git a/packages/website/src/pages/robots.txt.ts b/packages/website/src/pages/robots.txt.ts
new file mode 100644
index 000000000..22edfd3f9
--- /dev/null
+++ b/packages/website/src/pages/robots.txt.ts
@@ -0,0 +1,20 @@
+import { assertSecret } from '@blitzkit/core';
+import type { APIContext } from 'astro';
+
+export async function GET({}: APIContext) {
+  const allowRobots = assertSecret(import.meta.env.ALLOW_ROBOTS) === 'true';
+
+  return new Response(`
+User-agent: *
+${allowRobots ? 'Allow' : 'Disallow'}: /
+
+${
+  allowRobots
+    ? `
+Host: https://blitzkit.app
+Sitemap: https://blitzkit.app/sitemap.txt
+`
+    : ''
+}
+`);
+}