Merge branch 'main' into dev

.github/workflows/cd-website.yml

@@ -22,7 +22,7 @@ env:
   PUBLIC_ASSET_REPO: ${{ secrets.PUBLIC_ASSET_REPO }}
   PUBLIC_ASSET_BRANCH: ${{ secrets.PUBLIC_ASSET_BRANCH }}
   PUBLIC_PROMOTE_OPENTEST: ${{ secrets.PUBLIC_PROMOTE_OPENTEST }}
-  ALLOW_ROBOTS: ${{ secrets.ALLOW_ROBOTS }}
+  ALLOW_ROBOTS: true
   WOTB_GLOSSARY: ${{ secrets.WOTB_GLOSSARY }}
 
 jobs:

packages/website/src/pages/robots.txt.ts

@@ -5,17 +5,19 @@ export async function GET({}: APIContext) {
   const allowRobots = assertSecret(import.meta.env.ALLOW_ROBOTS) === 'true';
   const site = assertSecret(import.meta.env.SITE);
 
-  return new Response(`
+  return new Response(
-    User-agent: *
+    `
-    ${allowRobots ? 'Allow' : 'Disallow'}: /
+User-agent: *
+Allow: /
 
-    ${
+${
-      allowRobots
+  allowRobots
-        ? `
+    ? `
-          Host: ${site}
+Host: ${site}
-          Sitemap: ${site}/sitemap-index.xml
+Sitemap: ${site}/sitemap-index.xml
-          `
+`
-        : ''
+    : ''
-    }
+}
-  `);
+`.trim(),
+  );
 }