ok cool

.github/workflows/ids.yaml

@@ -0,0 +1,42 @@
+name: Discover IDs
+
+on:
+  push:
+    branches: infrastructure
+
+  schedule:
+    - cron: 0 */6 * * *
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  discover:
+    name: Discover IDs
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: recursive
+          token: ${{ secrets.GH_TOKEN }}
+
+      - name: Setup bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Setup Git
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Discover IDs
+        run: bun run discover-ids
+        env:
+          PUBLIC_WARGAMING_APPLICATION_ID: ${{ secrets.PUBLIC_WARGAMING_APPLICATION_ID }}
+          GH_TOKEN: ${{ secrets.IDS_TOKEN }}

package.json

@@ -6,8 +6,7 @@
   "scripts": {
     "dev": "cd packages/website && bun run dev",
     "build": "cd packages/website && bun run build",
-    "build:averages": "cd packages/scripts && bun run build:averages",
-    "build:ids": "cd packages/scripts && bun run build:ids"
+    "discover-ids": "cd packages/scripts && bun run discover-ids"
   },
   "type": "module",
   "workspaces": [

packages/scripts/package.json

@@ -36,18 +36,7 @@
   },
   "private": true,
   "scripts": {
-    "build:assets": "bun src/buildAssets/index.ts",
-    "build:averages": "bun src/averages.ts",
-    "setup:depotdownloader": "bun src/setupDepotDownloader.ts",
-    "setup:patches": "bun src/applyPatches.ts",
-    "build:reviews": "bun src/reviews.ts",
-    "build:ids": "bun src/discoverIds/index.ts",
-    "build:ratingleaderboard": "bun src/buildRatingLeaderboard.ts",
-    "build:proxyclient": "bun src/buildProxyClient.ts",
-    "build:catalogitemaccesssor": "bun src/buildCatalogItemAccessor.ts",
-    "prunestrings": "bun src/pruneStrings.ts",
-    "build:protos": "bun src/protos.ts",
-    "patchnotes": "bun src/patchNotes.ts"
+    "discover-ids": "bun src/discoverIds/index.ts"
   },
   "type": "module"
 }

packages/scripts/src/discoverIds/index.ts

@@ -8,13 +8,15 @@ import { IdArray } from "./idArray";
 interface ManifestV1 {
   version: 1;
   last_verified: number;
+  chunks: number;
+  total_count: number;
 }
 
 interface RegionDescriptor {
   domain: string;
   seed: number;
   max: number;
-  streak: number;
+  dry_streak: number;
 }
 
 export interface QuickInfo {
@@ -34,16 +36,8 @@ const MAX_IDS_PER_CALL = 100;
 const API_RATE = 10; // 10Hz
 const MAX_DRY_STREAK = 10_000;
 
-const chunkCount = Math.ceil(
-  (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
-);
-
-console.log(
-  `${chunkCount.toLocaleString()} will be needed for ${ACCOMMODATED_ID_COUNT.toLocaleString()} ids`,
-);
-
 const t0 = performance.now();
-const maxT = 5 * 60 * 60 * 1000; // 5hr
+const maxT = 5.5 * 60 * 60 * 1000; // 5hr 30min
 const t1 = t0 + maxT;
 
 await mkdir(TEMP, { recursive: true });
@@ -51,19 +45,26 @@ await rm(WORKING_DIR, { recursive: true, force: true });
 
 console.log(`Cloning ${REPO} to ${WORKING_DIR}`);
 
-await $`git clone --depth 1 https://github.com/${REPO} ${WORKING_DIR}`;
+const token = process.env.GH_TOKEN;
+const repoUrl = token
+  ? `https://x-access-token:${token}@github.com/${REPO}.git`
+  : `https://github.com/${REPO}.git`;
+
+await $`git clone --depth 1 ${repoUrl} ${WORKING_DIR}`;
 await mkdir(CHUNKS_DIR, { recursive: true });
 
 console.log(`Discovering pre-existing chunks...`);
 
 const discoveredChunks: (IdArray | null)[] = [];
 
-for (let i = 0; true; i++) {
+const manifest: ManifestV1 = await Bun.file(
+  `${WORKING_DIR}/manifest.json`,
+).json();
+
+for (let i = 0; i < manifest.chunks; i++) {
   const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`;
   const file = Bun.file(path);
 
-  if (!(await file.exists())) break;
-
   const bytes = await file.bytes();
   const uncompressed = decompress(bytes);
 
@@ -72,18 +73,22 @@ for (let i = 0; true; i++) {
   discoveredChunks.push(ids);
 }
 
-console.log(`Found ${discoveredChunks.length} pre-existing chunks`);
+console.log(`Found ${manifest.chunks} pre-existing chunks`);
+
+manifest.chunks = Math.ceil(
+  (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
+);
 
 let chunks: IdArray[] = [];
 
-if (discoveredChunks.length === chunkCount) {
+if (discoveredChunks.length === manifest.chunks) {
   console.log("Pre-existing chunking is consistent, passing as-is...");
 
   chunks = discoveredChunks as IdArray[];
 } else {
   console.log("Pre-existing chunking is not consistent, re-chunking...");
 
-  chunks = Array.from({ length: chunkCount }).map(() => new IdArray());
+  chunks = Array.from({ length: manifest.chunks }).map(() => new IdArray());
 
   for (let i = 0; i < discoveredChunks.length; i++) {
     const ids = discoveredChunks[i];
@@ -91,7 +96,7 @@ if (discoveredChunks.length === chunkCount) {
 
     for (let j = 0; j < size; j++) {
       const id = ids!.get(j);
-      const chunk = chunks[id % chunkCount];
+      const chunk = chunks[id % manifest.chunks];
 
       chunk.push(id);
     }
@@ -105,14 +110,14 @@ if (discoveredChunks.length === chunkCount) {
 }
 
 const regions: RegionDescriptor[] = [
-  { domain: "eu", seed: 5e8, max: 10e8 - 1, streak: 0 },
-  { domain: "com", seed: 10e8, max: 20e8 - 1, streak: 0 },
-  { domain: "asia", seed: 20e8, max: 31e8 - 1, streak: 0 },
+  { domain: "eu", seed: 5e8, max: 10e8 - 1, dry_streak: 0 },
+  { domain: "com", seed: 10e8, max: 20e8 - 1, dry_streak: 0 },
+  { domain: "asia", seed: 20e8, max: 31e8 - 1, dry_streak: 0 },
 ];
 
 console.log("Finding new seed ids...");
 
-for (let i = 0; i < chunkCount; i++) {
+for (let i = 0; i < manifest.chunks; i++) {
   const chunk = chunks[i];
   const size = chunk.size();
 
@@ -156,14 +161,21 @@ async function save() {
     await Bun.write(path, compressed);
   }
 
+  manifest.total_count = totalFound;
+
+  await Bun.write(
+    `${WORKING_DIR}/manifest.json`,
+    JSON.stringify(manifest, null, 2),
+  );
+
   const git = $.cwd(WORKING_DIR);
 
   await git`git add .`.quiet();
-  await git`git commit --amend -m "ids update ${new Date().toISOString()}"`.quiet();
+  await git`git commit --amend --reset-author -m "ids update ${new Date().toISOString()}"`.quiet();
   await git`git push --force-with-lease`.quiet();
 
   console.log(
-    `Committed ${totalFound.toLocaleString()} total ids across ${chunkCount} chunks`,
+    `Committed ${totalFound.toLocaleString()} total ids across ${manifest.chunks} chunks`,
   );
 
   exit(0);
@@ -172,8 +184,6 @@ async function save() {
 process.on("SIGINT", save);
 process.on("SIGTERM", save);
 
-console.log("Starting discovery loop...");
-
 const queue: { region: RegionDescriptor; url: string; size: number }[] = [];
 const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
 const fields = "account_id%2C-account_id";
@@ -182,45 +192,55 @@ let regionI = 0;
 function fillQueue() {
   if (queue.length >= 1) return;
 
-  regionI = (regionI + 1) % regions.length;
-  const region = regions[regionI];
+  while (regions.length > 0) {
+    regionI = (regionI + 1) % regions.length;
+    const region = regions[regionI];
+
+    if (region.dry_streak >= MAX_DRY_STREAK) {
+      console.log(
+        `Removing region ${region.domain} removed due to ${region.dry_streak} dry streak...`,
+      );
+      regions.splice(regionI, 1);
+      continue;
+    }
+
+    const id0 = region.seed;
+    const id1 = Math.min(id0 + MAX_IDS_PER_CALL - 1, region.max);
+    const idRange = range(id0, id1 + 1);
+
+    region.seed += idRange.length;
+
+    const accountIds = idRange.join(",");
+    const url = `https://api.wotblitz.${
+      region.domain
+    }/wotb/account/info/?application_id=${applicationId}&fields=${
+      fields
+    }&account_id=${accountIds}`;
+
+    queue.push({ region, url, size: idRange.length });
+
+    if (idRange.length < MAX_IDS_PER_CALL) {
+      console.log(
+        `Removing region ${region.domain} removed due to ID range exhaustion...`,
+      );
+      regions.splice(regionI, 1);
+    }
 
-  if (region.streak >= MAX_DRY_STREAK) {
-    console.log(
-      `Removing region ${region.domain} removed due to ${region.streak} dry streak...`,
-    );
-    regions.splice(regionI, 1);
     return;
   }
-
-  const id0 = region.seed;
-  const id1 = Math.min(id0 + MAX_IDS_PER_CALL - 1, region.max);
-  const idRange = range(id0, id1 + 1);
-
-  region.seed += idRange.length;
-
-  const accountIds = idRange.join(",");
-  const url = `https://api.wotblitz.${
-    region.domain
-  }/wotb/account/info/?application_id=${applicationId}&fields=${
-    fields
-  }&account_id=${accountIds}`;
-
-  queue.push({ region, url, size: idRange.length });
-
-  if (idRange.length < MAX_IDS_PER_CALL) {
-    console.log(
-      `Removing region ${region.domain} removed due to ID range exhaustion...`,
-    );
-    regions.splice(regionI, 1);
-  }
 }
 
 function sleep() {
   return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
 }
 
-while (performance.now() < t1 && regions.length > 0) {
+function withinTimeLimit() {
+  return performance.now() < t1;
+}
+
+console.log("Starting discovery loop...");
+
+while (withinTimeLimit() && regions.length > 0) {
   await sleep();
 
   fillQueue();
@@ -251,17 +271,37 @@ while (performance.now() < t1 && regions.length > 0) {
     foundIds++;
 
     const id = Number(key);
-    const chunkIndex = id % chunkCount;
+    const chunkIndex = id % manifest.chunks;
     const chunk = chunks[chunkIndex];
 
     chunk.push(id);
   }
 
   if (foundIds === 0) {
-    request.region.streak += request.size;
+    request.region.dry_streak += request.size;
   } else {
+    request.region.dry_streak = 0;
     // console.log(`Found ${foundIds} ids in ${request.region.domain}`);
   }
 }
 
+// console.log("Starting verification loop...");
+
+// while (withinTimeLimit()) {
+//   const verificationIndex = (manifest.last_verified + 1) % manifest.chunks;
+//   manifest.last_verified = verificationIndex;
+//   const chunk = chunks[verificationIndex];
+//   const size = chunk.size();
+
+//   let i0 = 0;
+//   while (withinTimeLimit() && i0 < size) {
+//     await sleep();
+
+//     const i1 = Math.min(i0 + MAX_IDS_PER_CALL - 1, size - 1);
+//     const indices = range(i0, i1 + 1);
+//   }
+
+//   console.log(`Verification complete for chunk ${verificationIndex}`);
+// }
+
 await save();

packages/website/src/pages/[...chunk_api]/icons/shells/[id].webp.ts

@@ -11,6 +11,8 @@ export const getStaticPaths = mixStaticPaths(_getStaticPaths, async () => {
     .dir(`Data/Gfx/Shared/tank-supply/ammunition/big`)
     .then((files) => files.filter((file) => file.endsWith("_l.txt")));
 
+  console.log("Shell file paths", files);
+
   for (const file of files) {
     const id = file.match(/(.+)_l\.txt/)![1];
     const sizes = (

packages/website/src/pages/[...chunk_default]/404.astro

@@ -5,6 +5,8 @@ import { LinkI18n } from "../../components/LinkI18n";
 import { PageWrapper } from "../../components/PageWrapper";
 import BaseLayout from "../../layouts/BaseLayout.astro";
 
+export { getStaticPaths } from "./_index";
+
 const { locale } = Astro.props;
 ---