more modern way of handling id discovery

.github/workflows/website.yml

@@ -2,20 +2,37 @@ name: Website
 
 on:
   push:
-  workflow_dispatch:
 
   schedule:
     - cron: "0 4 * * *"
 
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment
+
+        required: true
+        # default: production
+        default: beta
+        type: choice
+
+        options:
+          - development
+          - beta
+          - production
+          - preview
+          - opentest
+
 permissions:
   contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  # cancel-in-progress: ${{ github.ref == 'refs/heads/main' }}
   cancel-in-progress: true
 
 env:
+  PUBLIC_ENVIRONMENT: ${{ inputs.environment || 'beta' }}
+
   # blitzkit
   PUBLIC_BRANCH: main
   PUBLIC_PROMOTE_OPENTEST: false
@@ -47,15 +64,9 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
-        include:
-          - chunk: default
-            node: blitzkit/blitzkit.github.io
-            path: dist/website
-
-          - chunk: api
-            node: blitzkit/api
-            path: dist/website/api
+        chunk: [default, api]
 
     steps:
       - name: Free disk space
@@ -124,18 +135,24 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: chunk-${{ matrix.chunk }}
-          path: ${{ matrix.path }}
-
-      - name: Save artifact ID
-        run: |
-          echo "${{ steps.upload.outputs.artifact-id }}"
+          path: dist/website
 
       - name: Trigger deployment
+
+        if: github.ref_name == 'production' || github.ref_name == 'infrastructure'
+
         env:
           TOKEN: ${{ secrets.GH_NODE_DEPLOY_TOKEN }}
-          ARTIFACT_ID: ${{ steps.upload.outputs.artifact-id }}
-          TARGET_REPO: ${{ matrix.node }}
+          CHUNK: ${{ matrix.chunk }}
+
         run: |
+          ENV_FILE="environments/${PUBLIC_ENVIRONMENT}.json"
+          TARGET_REPO=$(jq -r ".nodes.\"$CHUNK\"" "$ENV_FILE")
+
+          if [ "$TARGET_REPO" == "null" ] || [ -z "$TARGET_REPO" ]; then
+            exit 1
+          fi
+
           curl \
             -X POST \
             -H "Accept: application/vnd.github+json" \
@@ -146,7 +163,7 @@ jobs:
                   \"client_payload\": {
                     \"source_repo\": \"${{ github.repository }}\",
                     \"run_id\": \"${{ github.run_id }}\",
-                    \"artifact_id\": \"$ARTIFACT_ID\",
-                    \"chunk\": \"${{ matrix.chunk }}\"
+                    \"artifact_id\": \"${{ steps.upload.outputs.artifact-id }}\",
+                    \"chunk\": \"$CHUNK\"
                   }
                 }"

package.json

@@ -6,7 +6,8 @@
   "scripts": {
     "dev": "cd packages/website && bun run dev",
     "build": "cd packages/website && bun run build",
-    "build:averages": "cd packages/scripts && bun run build:averages"
+    "build:averages": "cd packages/scripts && bun run build:averages",
+    "build:ids": "cd packages/scripts && bun run build:ids"
   },
   "type": "module",
   "workspaces": [

packages/core/src/blitzkit/api/client.ts

@@ -6,6 +6,7 @@ import {
   EquipmentDefinitions,
   Gallery,
   GameDefinitions,
+  MapDefinitions,
   ModelDefinitions,
   ProvisionDefinitions,
   SkillDefinitions,
@@ -61,4 +62,8 @@ export class ClientBlitzKitAPI extends BlitzKitAPI {
   galleryDefinitions() {
     return fetchPB(alias("api", "/definitions/gallery.pb"), Gallery);
   }
+
+  mapDefinitions() {
+    return fetchPB(alias("api", "/definitions/maps.pb"), MapDefinitions);
+  }
 }

packages/core/src/blitzkit/environment.ts

@@ -23,7 +23,7 @@ for (const key in environments) {
 }
 
 export function environmentConfig() {
-  const environment = assertSecret(import.meta.env.ENVIRONMENT);
+  const environment = assertSecret(import.meta.env.PUBLIC_ENVIRONMENT);
 
   if (!(environment in environments)) {
     throw new Error(`Invalid environment "${environment}"`);

packages/scripts/src/discoverIds.ts

@@ -1,165 +1,87 @@
-import {
-  DidsReadStream,
-  DidsWriteStream,
-  DiscoveredIdsDefinitions,
-  MIN_REGION_IDS,
-  REGIONS,
-  Region,
-  asset,
-  getAccountInfo,
-  idToRegion,
-  retryAbleBlitzFetchEvent,
-} from "@blitzkit/core";
-import { chunk, times, uniq } from "lodash-es";
-import { compress, decompress } from "lz4js";
-import { AssetUploader } from "./core/github/assetUploader";
+const MAX_IDS = 100;
+const API_RATE = 10;
 
-const CHUNK_SIZE = 2 ** 21;
-const RUN_TIME = 1000 * 60 * 60 * 5.5;
-const PROGRESS_UPDATE_FREQUENCY = 1000 * 60;
-const MAX_REQUESTS = 10;
-const ACCOUNTS_PER_CALL = 100;
-const TERMINATION_THRESHOLD = 1000;
-const startTime = Date.now();
-const indexableRegions = [...REGIONS];
-let ids: number[] = [];
-const preDiscoveredManifest = (await fetch(asset("ids/manifest.json")).then(
-  (response) => response.json()
-)) as DiscoveredIdsDefinitions;
+const ACCOMMODATED_ID_COUNT = 10_000_000_000;
+const BYTES_PER_ID = 64 / 8;
+const MAX_BYTES_PER_BATCH = 100 * 1_000_000;
 
-console.log(
-  `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`
+const regions = ["eu", "com", "asia"];
+const seeds = [5e8, 10e8, 20e8].map(BigInt);
+
+export interface QuickInfo {
+  status: string;
+  data: Record<string, {} | null>;
+}
+
+const batches = BigInt(
+  Math.ceil((ACCOMMODATED_ID_COUNT * BYTES_PER_ID) / MAX_BYTES_PER_BATCH),
 );
+const ids: bigint[] = [];
 
-let chunkIndex = 0;
-while (chunkIndex < preDiscoveredManifest.chunks) {
-  const preDiscovered = await fetch(asset(`ids/${chunkIndex}.dids.lz4`)).then(
-    async (response) => {
-      const buffer = await response.arrayBuffer();
-      const decompressed = decompress(new Uint8Array(buffer)).buffer;
-      return new DidsReadStream(decompressed as ArrayBuffer).dids();
+const t0 = performance.now();
+const maxT = 5 * 60 * 60 * 1000;
+const t1 = t0 + maxT;
+
+const unfilled = Array.from({ length: seeds.length });
+const offsets = unfilled.map(() => 0n);
+const queues = unfilled.map(() => [] as bigint[]);
+
+function mountIds() {
+  for (let i = 0; i < regions.length; i++) {
+    const queue = queues[i];
+
+    if (queue.length >= MAX_IDS) continue;
+
+    const availableSpace = MAX_IDS - queue.length;
+
+    const seed = seeds[i];
+    const offset = offsets[i]++;
+
+    for (let j = 0n; j < availableSpace; j++) {
+      const id = seed + offset + j;
+      queue.push(id);
     }
-  );
 
-  // no spread syntax: https://github.com/oven-sh/bun/issues/11734
-  ids = ids.concat(preDiscovered);
-
-  console.log(
-    `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`
-  );
-
-  chunkIndex++;
+    offsets[i] += BigInt(availableSpace);
+  }
 }
 
-const regionalIdIndex: Record<Region, number> = {
-  asia: ids.findLast((id) => idToRegion(id) === "asia") ?? MIN_REGION_IDS.asia,
-  com: ids.findLast((id) => idToRegion(id) === "com") ?? MIN_REGION_IDS.com,
-  eu: ids.findLast((id) => idToRegion(id) === "eu") ?? MIN_REGION_IDS.eu,
-};
-const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
-let regionIndex = 0;
-let outOfTimeFlagged = false;
+let i = 0;
+while (performance.now() < t1) {
+  mountIds();
+  await sleep();
 
-async function verify(region: Region, ids: number[]) {
-  const infos = await getAccountInfo(region, ids, undefined, {
-    // cheeky way of requesting nothing
-    fields: "account_id,-account_id",
-  });
-  const filtered = ids.filter((_, index) => {
-    const info = infos[index];
-    return info !== null;
-  });
+  const region = regions[i];
+  const queue = queues[i];
 
-  return filtered;
+  const accountIds = queue.join(",");
+  const fields = "account_id%2C-account_id";
+  const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
+
+  const url = `https://api.wotblitz.${
+    region
+  }/wotb/account/info/?application_id=${applicationId}&fields=${
+    fields
+  }&account_id=${accountIds}`;
+  const response = await fetch(url);
+  const data = (await response.json()) as QuickInfo;
+
+  if (data.status !== "ok") continue;
+
+  for (const key in data.data) {
+    const value = data.data[key];
+
+    if (value === null) continue;
+
+    const id = BigInt(key);
+
+    ids.push(id);
+  }
+
+  queues[i] = [];
+  i = (i + 1) % regions.length;
 }
 
-let discardAttempts = 0;
-let lastProgressUpdate = 0;
-
-retryAbleBlitzFetchEvent.on(() => discardAttempts++);
-
-const interval = setInterval(async () => {
-  if (discardAttempts > 0) return discardAttempts--;
-
-  const region = indexableRegions[regionIndex];
-  const idsToVerify = times(
-    ACCOUNTS_PER_CALL,
-    (index) => regionalIdIndex[region] + index
-  );
-  const verified = await verify(region, idsToVerify);
-
-  if (Date.now() - lastProgressUpdate > PROGRESS_UPDATE_FREQUENCY) {
-    lastProgressUpdate = Date.now();
-    console.log(`Discovered ${ids.length} ids in ${Date.now() - startTime}ms`);
-  }
-
-  if (verified.length === 0) {
-    zeroStreak[region]++;
-
-    if (zeroStreak[region] >= TERMINATION_THRESHOLD) {
-      const sliceableIndex = indexableRegions.indexOf(region);
-
-      if (sliceableIndex !== -1) {
-        console.log(`Stopped discovery for ${region}`);
-        indexableRegions.splice(sliceableIndex, 1);
-
-        if (indexableRegions.length === 0) {
-          console.log("All players discovered exhaustively!");
-          await post();
-        }
-      }
-    }
-  } else {
-    zeroStreak[region] = 0;
-    ids.push(...verified);
-  }
-
-  if (Date.now() - startTime > RUN_TIME && !outOfTimeFlagged) {
-    outOfTimeFlagged = true;
-    console.log("Out of time");
-    await post();
-  }
-
-  regionIndex = (regionIndex + 1) % indexableRegions.length;
-  regionalIdIndex[region] += ACCOUNTS_PER_CALL;
-}, 1000 / MAX_REQUESTS);
-
-async function post() {
-  clearInterval(interval);
-  ids = uniq(ids).sort((a, b) => a - b);
-  console.log(`Uploading ${ids.length} ids...`);
-
-  using uploader = new AssetUploader("discovered ids");
-  const idsChunked = chunk(ids, CHUNK_SIZE);
-
-  let index = 0;
-  for (const chunk of idsChunked) {
-    const didsWriteStream = new DidsWriteStream().dids(chunk);
-    const buffer = didsWriteStream.uint8Array;
-    const content = compress(buffer);
-
-    console.log(
-      `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`
-    );
-
-    await uploader.add({
-      content,
-      path: `ids/${index}.dids.lz4`,
-    });
-
-    index++;
-  }
-
-  await uploader.add({
-    content: new TextEncoder().encode(
-      JSON.stringify({
-        chunks: idsChunked.length,
-        count: ids.length,
-        time: Date.now(),
-      } satisfies DiscoveredIdsDefinitions)
-    ),
-    path: "ids/manifest.json",
-  });
-
-  await uploader.flush();
+function sleep() {
+  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
 }

packages/website/src/constants/lightingEnvironments.ts

@@ -1,7 +0,0 @@
-export const ENVIRONMENTS = [
-  'lobby',
-  'apartment',
-  'warehouse',
-  'dawn',
-  'sunset',
-] as const;

packages/website/src/pages/[...chunk_api]/definitions/maps.pb.ts

@@ -0,0 +1,11 @@
+import { MapDefinitions } from "@blitzkit/core";
+import { api } from "../../../core/blitzkit/api";
+
+export { getStaticPaths } from "../_index";
+
+export async function GET() {
+  const definitions = await api.mapDefinitions();
+  const bytes = MapDefinitions.encode(definitions).finish();
+
+  return new Response(bytes);
+}

packages/website/src/pages/[...chunk_api]/icons/equipment/[id].webp.ts

@@ -69,8 +69,6 @@ export const getStaticPaths = mixStaticPaths(_getStaticPaths, async () => {
     }
   }
 
-  console.log(paths);
-
   return paths;
 });
 

packages/website/src/pages/[...chunk_default]/[...locale]/tools/tankopedia/[id]/index.astro

@@ -5,6 +5,8 @@ import LegacyRedirect from "../../../../../../layouts/LegacyRedirect.astro";
 import { getStaticPaths as _getStaticPaths } from "../index.astro";
 
 export const getStaticPaths = mixStaticPaths(_getStaticPaths, async () => {
+  const tankDefinitions = await api.tankDefinitions();
+
   return Object.values(tankDefinitions.tanks).map(({ id }) => ({
     params: { id },
   }));