From b9895b1c0f8fc7c8c94564a2295cd265f4befdfc Mon Sep 17 00:00:00 2001
From: tresabhi <43380238+tresabhi@users.noreply.github.com>
Date: Sun, 17 Nov 2024 12:29:57 -0600
Subject: [PATCH 1/2] feat: allow robots

---
 .github/workflows/cd-website.yml         |  2 +-
 packages/website/src/pages/robots.txt.ts | 26 +++++++++++++-----------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/cd-website.yml b/.github/workflows/cd-website.yml
index ae03cfb35..d0064a0c2 100644
--- a/.github/workflows/cd-website.yml
+++ b/.github/workflows/cd-website.yml
@@ -22,7 +22,7 @@ env:
   PUBLIC_ASSET_REPO: ${{ secrets.PUBLIC_ASSET_REPO }}
   PUBLIC_ASSET_BRANCH: ${{ secrets.PUBLIC_ASSET_BRANCH }}
   PUBLIC_PROMOTE_OPENTEST: ${{ secrets.PUBLIC_PROMOTE_OPENTEST }}
-  ALLOW_ROBOTS: ${{ secrets.ALLOW_ROBOTS }}
+  ALLOW_ROBOTS: true
   WOTB_GLOSSARY: ${{ secrets.WOTB_GLOSSARY }}
 
 jobs:
diff --git a/packages/website/src/pages/robots.txt.ts b/packages/website/src/pages/robots.txt.ts
index bfb868ae0..3b9cb2c61 100644
--- a/packages/website/src/pages/robots.txt.ts
+++ b/packages/website/src/pages/robots.txt.ts
@@ -5,17 +5,19 @@ export async function GET({}: APIContext) {
   const allowRobots = assertSecret(import.meta.env.ALLOW_ROBOTS) === 'true';
   const site = assertSecret(import.meta.env.SITE);
 
-  return new Response(`
-    User-agent: *
-    ${allowRobots ? 'Allow' : 'Disallow'}: /
+  return new Response(
+    `
+User-agent: *
+${allowRobots ? 'Allow' : 'Disallow'}: /
 
-    ${
-      allowRobots
-        ? `
-          Host: ${site}
-          Sitemap: ${site}/sitemap-index.xml
-          `
-        : ''
-    }
-  `);
+${
+  allowRobots
+    ? `
+Host: ${site}
+Sitemap: ${site}/sitemap-index.xml
+`
+    : ''
+}
+`.trim(),
+  );
 }

From eb6aa5bc5bbb4d87ff96d5aae7103acfb536d740 Mon Sep 17 00:00:00 2001
From: tresabhi <43380238+tresabhi@users.noreply.github.com>
Date: Sun, 17 Nov 2024 12:44:42 -0600
Subject: [PATCH 2/2] allow

---
 packages/website/src/pages/robots.txt.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/website/src/pages/robots.txt.ts b/packages/website/src/pages/robots.txt.ts
index 3b9cb2c61..c54bff4e4 100644
--- a/packages/website/src/pages/robots.txt.ts
+++ b/packages/website/src/pages/robots.txt.ts
@@ -8,7 +8,7 @@ export async function GET({}: APIContext) {
   return new Response(
     `
 User-agent: *
-${allowRobots ? 'Allow' : 'Disallow'}: /
+Allow: /
 
 ${
   allowRobots