fix: disallow robots on non-master domains

packages/website/src/pages/robots.txt.ts

@@ -1,9 +1,12 @@
 import type { APIContext } from 'astro';
 
+const ALLOW_ROBOTS = import.meta.env.ALLOW_ROBOTS === 'true';
+
 export async function GET({ request }: APIContext) {
   const flags: Record<string, string | null> = {
     'User-agent': '*',
-    Allow: '/',
+    Allow: ALLOW_ROBOTS ? '/' : null,
+    Disallow: ALLOW_ROBOTS ? null : '/',
 
     space: null,
     Sitemap: `${new URL(request.url).origin}/sitemap-index.xml`,