fix: disallow robots on non-master domains

.github/workflows/website.yml

@@ -19,7 +19,7 @@ concurrency:
 env:
   PUBLIC_WARGAMING_APPLICATION_ID: ${{ secrets.PUBLIC_WARGAMING_APPLICATION_ID }}
   PUBLIC_PROMOTE_OPENTEST: ${{ secrets.PUBLIC_PROMOTE_OPENTEST }}
-  ALLOW_ROBOTS: 'true'
+  ALLOW_ROBOTS: ${{ secrets.ALLOW_ROBOTS }}
   WOTB_GLOSSARY: ${{ secrets.WOTB_GLOSSARY }}
   PUBLIC_GOOGLE_TAG_MANAGER_ID: ${{ secrets.PUBLIC_GOOGLE_TAG_MANAGER_ID }}
   PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID: ${{ secrets.PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID }}

packages/website/src/pages/robots.txt.ts

@@ -1,9 +1,12 @@
 import type { APIContext } from 'astro';
 
+const ALLOW_ROBOTS = import.meta.env.ALLOW_ROBOTS === 'true';
+
 export async function GET({ request }: APIContext) {
   const flags: Record<string, string | null> = {
     'User-agent': '*',
-    Allow: '/',
+    Allow: ALLOW_ROBOTS ? '/' : null,
+    Disallow: ALLOW_ROBOTS ? null : '/',
 
     space: null,
     Sitemap: `${new URL(request.url).origin}/sitemap-index.xml`,