proper concurrency

.github/workflows/website.yaml

@@ -2,7 +2,6 @@ name: Website
 
 on:
   push:
-    branches: production
 
   schedule:
     - cron: "0 4 * * *"
@@ -34,14 +33,40 @@ on:
           - preview
           - opentest
 
+      client_provider:
+        description: Client provider
+
+        type: choice
+        required: true
+        default: steam
+
+        options:
+          - steam
+          - zip
+          - wgpkg
+
+      client_source:
+        description: Client source
+
+        type: string
+        required: true
+        default: 444200/windows
+
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ inputs.environment || github.ref }}
   cancel-in-progress: true
 
 env:
+  # client
+  STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+  STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
+  CLIENT_PROVIDER: ${{ inputs.client_provider || 'steam' }}
+  CLIENT_SOURCE: ${{ inputs.client_source || '444200/windows' }}
+  CLIENT_DIR: ../../temp/game
+
   # blitzkit
   PUBLIC_ENVIRONMENT: ${{ inputs.environment || 'production' }}
   PUBLIC_PROMOTE_OPENTEST: false
@@ -52,7 +77,6 @@ env:
   WOTB_GLOSSARY: ${{ secrets.WOTB_GLOSSARY }}
   WOTB_GLOSSARY_DOMAIN: ${{ secrets.WOTB_GLOSSARY_DOMAIN }}
   WOTB_GLOSSARY_PATH: ${{ secrets.WOTB_GLOSSARY_PATH }}
-  GAME_DIR: ../../temp/game
 
   # google
   PUBLIC_GOOGLE_TAG_MANAGER_ID: ${{ secrets.PUBLIC_GOOGLE_TAG_MANAGER_ID }}
@@ -60,12 +84,6 @@ env:
   # wargaming
   PUBLIC_WARGAMING_APPLICATION_ID: ${{ secrets.PUBLIC_WARGAMING_APPLICATION_ID }}
 
-  # steam
-  STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
-  STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }}
-  STEAM_APP: 444200
-  STEAM_OS: windows
-
 jobs:
   build:
     name: Build website
@@ -73,7 +91,7 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         chunk: [default, api]
 
@@ -96,44 +114,16 @@ jobs:
         with:
           repo-token: ${{ secrets.GH_TOKEN }}
 
-      - name: Configure pages
+      - name: Setup CLIs
-        uses: actions/configure-pages@v6
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y aria2 libarchive-tools
 
       - name: Install dependencies
         run: bun install
 
-      - name: Setup DepotDownloader
+      - name: Download Client
-        run: |
+        run: bun run download-client
-          REPO_USER="SteamRE"
-          REPO_NAME="DepotDownloader"
-          TEMP="temp"
-
-          OUT="$TEMP/$REPO_NAME"
-          ZIP="$TEMP/$REPO_NAME.zip"
-
-          URL=$(
-            curl -s "https://api.github.com/repos/$REPO_USER/$REPO_NAME/releases/latest" \
-              | jq -r '.assets[]
-                | select(.name | contains("linux-x64"))
-                | .browser_download_url' \
-              | head -n 1
-          )
-
-          mkdir -p "$TEMP"
-          curl -L "$URL" -o "$ZIP"
-          unzip -p "$ZIP" "$REPO_NAME" > "$OUT"
-          rm "$ZIP"
-
-          chmod +x "$OUT"
-
-      - name: Download game
-        run: |
-          TEMP="temp"
-
-          DEPOT_DOWNLOADER="$TEMP/DepotDownloader"
-          GAME="$TEMP/game"
-
-          $DEPOT_DOWNLOADER -os $STEAM_OS -app $STEAM_APP -username $STEAM_USERNAME -password $STEAM_PASSWORD -dir $GAME -max-downloads 256
 
       - name: Build
         run: bun run --bun build
@@ -149,6 +139,8 @@ jobs:
 
       - name: Trigger deployment
 
+        if: ${{ github.ref == 'refs/heads/production' }}
+
         env:
           TOKEN: ${{ secrets.GH_NODE_DEPLOY_TOKEN }}
           CHUNK: ${{ matrix.chunk }}

docs/changelogs/1.31.1.md

@@ -0,0 +1,3 @@
+# BlitzKit 1.31.1
+
+This update adds capabilities to release BlitzKit OpenTest and Preview.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "blitzkit",
-  "version": "1.31.0",
+  "version": "1.31.1",
   "license": "BlitzKit License",
   "private": true,
   "scripts": {
@@ -8,7 +8,8 @@
     "build": "cd packages/website && bun run build",
     "discover-ids": "cd packages/scripts && bun run discover-ids",
     "build-protos": "cd packages/scripts && bun run build-protos",
-    "record-performance": "cd packages/scripts && bun run record-performance"
+    "record-performance": "cd packages/scripts && bun run record-performance",
+    "download-client": "cd packages/scripts && bun run download-client"
   },
   "type": "module",
   "workspaces": [

packages/scripts/package.json

@@ -4,7 +4,8 @@
   "scripts": {
     "discover-ids": "bun src/discoverIds/index.ts",
     "build-protos": "bun src/protos.ts",
-    "record-performance": "bun src/recordPerformance/index.ts"
+    "record-performance": "bun src/recordPerformance/index.ts",
+    "download-client": "bun src/downloadClient/index.ts"
   },
   "dependencies": {
     "@blitzkit/closed": "workspace:*",

packages/scripts/src/discoverIds/index.ts

@@ -4,7 +4,7 @@ import { exit } from "node:process";
 import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
 import { IdArray } from "../core/blitzkit/idArray";
 import { CHUNK_COUNT, MAX_DRY_STREAK, MAX_IDS_PER_CALL } from "./constants";
-import { sleep, withinTimeLimit } from "./time";
+import { atMaxRate } from "./time";
 import { IdsManifest, QuickInfo, RegionDescriptor } from "./types";
 
 const storage = await new GitObjectStorage(
@@ -30,18 +30,16 @@ for (let i = 0; i < manifest.chunks; i++) {
 
 console.log(`Found ${manifest.chunks} pre-existing chunks`);
 
-manifest.chunks = CHUNK_COUNT;
-
 let chunks: IdArray[] = [];
 
-if (discoveredChunks.length === manifest.chunks) {
+if (manifest.chunks === CHUNK_COUNT) {
   console.log("Pre-existing chunking is consistent, passing as-is...");
 
   chunks = discoveredChunks as IdArray[];
 } else {
   console.log("Pre-existing chunking is not consistent, re-chunking...");
 
-  chunks = Array.from({ length: manifest.chunks }).map(() => new IdArray());
+  chunks = Array.from({ length: CHUNK_COUNT }).map(() => new IdArray());
 
   for (let i = 0; i < discoveredChunks.length; i++) {
     const ids = discoveredChunks[i];
@@ -49,7 +47,7 @@ if (discoveredChunks.length === manifest.chunks) {
 
     for (let j = 0; j < size; j++) {
       const id = ids!.get(j);
-      const chunk = chunks[id % manifest.chunks];
+      const chunk = chunks[id % CHUNK_COUNT];
 
       chunk.push(id);
     }
@@ -60,6 +58,9 @@ if (discoveredChunks.length === manifest.chunks) {
   console.log("Sorting chunks...");
 
   for (let i = 0; i < chunks.length; i++) chunks[i].sort();
+
+  manifest.chunks = CHUNK_COUNT;
+  manifest.last_verified = manifest.last_verified % CHUNK_COUNT;
 }
 
 const regions: RegionDescriptor[] = [
@@ -116,7 +117,6 @@ async function save() {
   manifest.total_count = totalFound;
 
   await storage.write("manifest.json", JSON.stringify(manifest, null, 2));
-
   await storage.commit();
 
   console.log(
@@ -129,41 +129,44 @@ async function save() {
 process.on("SIGINT", save);
 process.on("SIGTERM", save);
 
-const queue: { region: RegionDescriptor; url: string; size: number }[] = [];
 const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
 const fields = "account_id%2C-account_id";
 let regionI = 0;
 
-function fillQueue() {
+console.log("Starting discovery loop...");
-  if (queue.length >= 1) return;
 
-  while (regions.length > 0) {
+atMaxRate(async ({ _break: break1 }) => {
-    regionI = (regionI + 1) % regions.length;
+  if (regions.length === 0) {
-    const region = regions[regionI];
+    console.log("All regions exhausted, exiting discovery loop...");
+    return break1();
+  }
 
-    if (region.dry_streak >= MAX_DRY_STREAK) {
+  regionI = (regionI + 1) % regions.length;
-      console.log(
+  const region = regions[regionI];
-        `Removing region ${region.domain} removed due to ${region.dry_streak} dry streak...`,
-      );
-      regions.splice(regionI, 1);
-      continue;
-    }
 
-    const id0 = region.seed;
+  if (region.dry_streak >= MAX_DRY_STREAK) {
-    const id1 = Math.min(id0 + MAX_IDS_PER_CALL - 1, region.max);
+    console.log(
-    const idRange = range(id0, id1 + 1);
+      `Removing region ${region.domain} removed due to ${region.dry_streak} dry streak...`,
+    );
+    regions.splice(regionI, 1);
 
-    region.seed += idRange.length;
+    return;
+  }
 
-    const accountIds = idRange.join(",");
+  const id0 = region.seed;
-    const url = `https://api.wotblitz.${
+  const id1 = Math.min(id0 + MAX_IDS_PER_CALL - 1, region.max);
-      region.domain
+  const idRange = range(id0, id1 + 1);
-    }/wotb/account/info/?application_id=${applicationId}&fields=${
-      fields
-    }&account_id=${accountIds}`;
 
-    queue.push({ region, url, size: idRange.length });
+  region.seed += idRange.length;
 
+  const accountIds = idRange.join(",");
+  const url = `https://api.wotblitz.${
+    region.domain
+  }/wotb/account/info/?application_id=${applicationId}&fields=${
+    fields
+  }&account_id=${accountIds}`;
+
+  atMaxRate(async ({ _break: break2 }) => {
     if (idRange.length < MAX_IDS_PER_CALL) {
       console.log(
         `Removing region ${region.domain} removed due to ID range exhaustion...`,
@@ -171,74 +174,39 @@ function fillQueue() {
       regions.splice(regionI, 1);
     }
 
-    return;
+    const response = await fetch(url);
-  }
+    const data = (await response.json()) as QuickInfo;
-}
 
-console.log("Starting discovery loop...");
+    if (data.status !== "ok") {
+      console.warn("Failed API call; trying again...");
+      return;
+    }
 
-while (withinTimeLimit() && regions.length > 0) {
+    let foundIds = 0;
-  await sleep();
 
-  fillQueue();
+    for (const key in data.data) {
+      const value = data.data[key];
 
-  if (queue.length === 0) {
+      if (value === null) continue;
-    console.log("Queue empty, ending discovery loop...");
-    break;
-  }
 
-  const request = queue.shift()!;
+      foundIds++;
-  const response = await fetch(request.url);
-  const data = (await response.json()) as QuickInfo;
 
-  if (data.status !== "ok") {
+      const id = Number(key);
-    console.warn("Failed API call; pushing back into queue...");
+      const chunkIndex = id % manifest.chunks;
-    queue.push(request);
+      const chunk = chunks[chunkIndex];
 
-    continue;
+      chunk.push(id);
-  }
+    }
 
-  let foundIds = 0;
+    if (foundIds === 0) {
+      region.dry_streak += idRange.length;
+    } else {
+      region.dry_streak = 0;
+      // console.log(`Found ${foundIds} ids in ${request.region.domain}`);
+    }
 
-  for (const key in data.data) {
+    break2();
-    const value = data.data[key];
+  });
-
+});
-    if (value === null) continue;
-
-    foundIds++;
-
-    const id = Number(key);
-    const chunkIndex = id % manifest.chunks;
-    const chunk = chunks[chunkIndex];
-
-    chunk.push(id);
-  }
-
-  if (foundIds === 0) {
-    request.region.dry_streak += request.size;
-  } else {
-    request.region.dry_streak = 0;
-    // console.log(`Found ${foundIds} ids in ${request.region.domain}`);
-  }
-}
-
-// console.log("Starting verification loop...");
-
-// while (withinTimeLimit()) {
-//   const verificationIndex = (manifest.last_verified + 1) % manifest.chunks;
-//   manifest.last_verified = verificationIndex;
-//   const chunk = chunks[verificationIndex];
-//   const size = chunk.size();
-
-//   let i0 = 0;
-//   while (withinTimeLimit() && i0 < size) {
-//     await sleep();
-
-//     const i1 = Math.min(i0 + MAX_IDS_PER_CALL - 1, size - 1);
-//     const indices = range(i0, i1 + 1);
-//   }
-
-//   console.log(`Verification complete for chunk ${verificationIndex}`);
-// }
 
 await save();

packages/scripts/src/discoverIds/time.ts

@@ -9,6 +9,37 @@ export function withinTimeLimit() {
   return performance.now() < t1;
 }
 
-export function sleep() {
+export function sleep(t: number) {
-  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
+  return new Promise((resolve) => setTimeout(resolve, t));
+}
+
+interface MethodContext {
+  _break(): void;
+}
+
+let lastT = 0;
+const maxDelta = 1000 / API_RATE;
+const DELTA_BUFFER = 0;
+export async function atMaxRate(method: (context: MethodContext) => void) {
+  let isBroken = false;
+
+  const context: MethodContext = {
+    _break() {
+      isBroken = true;
+    },
+  };
+
+  while (!isBroken) {
+    if (!withinTimeLimit()) break;
+
+    const t = performance.now();
+    const delta = t - lastT;
+
+    if (delta < maxDelta) {
+      await sleep(maxDelta - delta + DELTA_BUFFER);
+    }
+
+    method(context);
+    lastT = t;
+  }
 }

packages/scripts/src/downloadClient/constants.ts

@@ -0,0 +1,9 @@
+// dava engine has a lot of tiny files, this is faster
+export const DEPOT_DOWNLOADER_MAX_DOWNLOADS = 256;
+export const DEPOT_DOWNLOADER_OUTPUT = "../../temp/depotdownloader";
+export const DEPOT_DOWNLOADER_EXECUTABLE = `${DEPOT_DOWNLOADER_OUTPUT}/DepotDownloader`;
+
+export const ARIA2_OUTPUT = "../../temp/aria2";
+export const ARIA2_EXECUTABLE = `${ARIA2_OUTPUT}/aria2c`;
+
+export const WGPKG_DIR = "../../temp/wgpkgs";

packages/scripts/src/downloadClient/index.ts

@@ -0,0 +1,98 @@
+import { assertSecret } from "@blitzkit/core";
+import { $ } from "bun";
+import { mkdir, rm } from "node:fs/promises";
+import {
+  DEPOT_DOWNLOADER_EXECUTABLE,
+  DEPOT_DOWNLOADER_MAX_DOWNLOADS,
+  DEPOT_DOWNLOADER_OUTPUT,
+  WGPKG_DIR,
+} from "./constants";
+import { QuickGitHubRelease } from "./types";
+
+const provider = assertSecret(import.meta.env.CLIENT_PROVIDER);
+const source = assertSecret(import.meta.env.CLIENT_SOURCE);
+const dir = assertSecret(import.meta.env.CLIENT_DIR);
+
+await rm(dir, { recursive: true, force: true });
+await mkdir(dir, { recursive: true });
+
+if (provider === "steam") {
+  const [id, platform] = source.split("/");
+
+  console.log("Downloading Depot Downloader...");
+
+  const releases = await fetch(
+    "https://api.github.com/repos/SteamRE/DepotDownloader/releases/latest",
+  ).then((response) => response.json() as Promise<QuickGitHubRelease>);
+  const asset = releases.assets.find((asset) =>
+    asset.name.includes("linux-x64"),
+  );
+
+  if (!asset) {
+    throw new Error("No linux-x64 asset found for Steam DepotDownloader");
+  }
+
+  await rm(DEPOT_DOWNLOADER_OUTPUT, { recursive: true, force: true });
+
+  await $`curl -L "${asset.browser_download_url}" -o "${DEPOT_DOWNLOADER_OUTPUT}.zip"`;
+  await $`unzip "${DEPOT_DOWNLOADER_OUTPUT}.zip" -d "${DEPOT_DOWNLOADER_OUTPUT}"`;
+  await $`chmod +x "${DEPOT_DOWNLOADER_EXECUTABLE}"`;
+
+  console.log("Downloading client...");
+
+  const username = assertSecret(import.meta.env.STEAM_USERNAME);
+  const password = assertSecret(import.meta.env.STEAM_PASSWORD);
+
+  await $`${DEPOT_DOWNLOADER_EXECUTABLE} -os ${
+    platform
+  } -app ${id} -username ${username} -password ${
+    password
+  } -dir ${dir} -max-downloads ${DEPOT_DOWNLOADER_MAX_DOWNLOADS}`;
+} else if (provider === "zip") {
+  const url = assertSecret(import.meta.env.CLIENT_SOURCE);
+
+  console.log("Downloading and extracting zipped client...");
+
+  await mkdir(dir, { recursive: true });
+  await $`curl -L "${url}" | bsdtar -xf- --strip-components=1 -C ${dir}`;
+} else if (provider === "wgpkg") {
+  let index = 1;
+  const urls: string[] = [];
+
+  console.log("Discovering packages...");
+
+  while (true) {
+    const url = `${source}.${index.toString().padStart(3, "0")}`;
+    const response = await fetch(url);
+
+    if (response.status === 200) {
+      urls.push(url);
+    } else if (response.status === 404) {
+      break;
+    } else {
+      throw new Error(`Wargaming response error status: ${response.status}`);
+    }
+
+    index++;
+  }
+
+  console.log(`Downloading ${urls.length} packages...`);
+
+  await rm(WGPKG_DIR, { recursive: true });
+
+  const inputContent = urls
+    .map((url) => {
+      const name = url.split("/").pop()!;
+      return `${url}\n  out=${name}\n  dir=${WGPKG_DIR}`;
+    })
+    .join("\n");
+
+  await $`echo ${inputContent} | aria2c -j ${urls.length} -x 2 -s 2 -i -`;
+
+  const firstChunk = `${WGPKG_DIR}/${urls[0].split("/").pop()!}`;
+  await $`7z x "${firstChunk}" -o${dir} -y`;
+
+  await rm(WGPKG_DIR, { recursive: true });
+} else {
+  throw new Error(`Unknown provider: ${provider}`);
+}

packages/scripts/src/downloadClient/types.ts

@@ -0,0 +1,8 @@
+export interface QuickGitHubRelease {
+  assets: Asset[];
+}
+
+interface Asset {
+  name: string;
+  browser_download_url: string;
+}

packages/website/src/core/blitzkit/vfs.ts

@@ -1,4 +1,4 @@
 import { assertSecret } from "@blitzkit/core";
 import { LocalVFS } from "@blitzkit/core/src/blitzkit/vfs/local";
 
-export const vfs = new LocalVFS(assertSecret(import.meta.env.GAME_DIR));
+export const vfs = new LocalVFS(assertSecret(import.meta.env.CLIENT_DIR));