feat: start on fixing pre work

scripts/averages.ts

@@ -1,10 +1,8 @@
 import { times } from 'lodash';
 import { argv } from 'process';
-import ProgressBar from 'progress';
 import { Region, REGIONS } from '../src/constants/regions';
-import getTankStats from '../src/core/blitz/getTankStats';
+import { blitzFetchQueueAvailableEvent } from '../src/core/blitz/fetchBlitz';
-import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion';
+import { idToRegion } from '../src/core/blitz/idToRegion';
-import { asset } from '../src/core/blitzkit/asset';
 import {
   AverageDefinitions,
   AverageDefinitionsAllStats,
@@ -13,9 +11,8 @@ import {
 } from '../src/core/blitzkit/averageDefinitions';
 import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants';
 import { commitAssets } from '../src/core/blitzkit/commitAssets';
+import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
 import { superCompress } from '../src/core/blitzkit/superCompress';
-import { DidsReadStream } from '../src/core/streams/dids';
-import { IndividualTankStats } from '../src/types/tanksStats';
 
 interface DataPoint {
   x: number;
@@ -27,120 +24,29 @@ const RUN_TIME = 1000 * 60 * 60;
 const MAX_REQUESTS = 10;
 
 const production = argv.includes('--production');
-const startTime = Date.now();
+const preDiscoveredIds = await fetchPreDiscoveredIds(!production);
-const fields = 'all,tank_id,battle_life_time';
+const playerIds: Record<Region, number[]> = {
+  asia: preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'),
+  com: preDiscoveredIds.filter((id) => idToRegion(id) === 'com'),
+  eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'),
+};
+const regionalIndices: Record<Region, number> = {
+  asia: playerIds.asia.length - 1,
+  com: playerIds.com.length - 1,
+  eu: playerIds.eu.length - 1,
+};
+let regionIndex = 0;
+const availableRegions = [...REGIONS];
 
-const players: IndividualTankStats[][] = [];
+times(MAX_REQUESTS, () => {
-const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then(
+  validate();
-  async (response) => {
+  blitzFetchQueueAvailableEvent.on(validate);
-    if (response.status === 200) {
+});
-      const buffer = await response.arrayBuffer();
-      return new DidsReadStream(buffer).dids();
-    }
 
-    return undefined;
+function validate() {
-  },
+  const region = availableRegions[regionIndex];
-);
-const discoveredIds: number[] = [];
-let processes = 0;
 
-if (preDiscovered) {
+  regionIndex = (regionIndex + 1) % availableRegions.length;
-  console.log(
-    `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`,
-  );
-
-  const preDiscoveredBar = new ProgressBar(':bar', {
-    total: preDiscovered.length,
-  });
-  let preDiscoveredIndex = 0;
-
-  async function chainPreDiscovery() {
-    processes++;
-
-    if (preDiscoveredIndex === preDiscovered!.length) {
-      processes--;
-
-      if (processes !== 0) return;
-
-      console.log(
-        `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`,
-      );
-      discover();
-
-      return;
-    }
-
-    const id = preDiscovered![preDiscoveredIndex++];
-    const region = idToRegion(id);
-
-    if (id > MAX_IDS[region]) return;
-
-    const stats = await getTankStats(region, id, { fields });
-
-    if (stats !== null && stats.length > 0) {
-      discoveredIds.push(id);
-      players.push(stats);
-      preDiscoveredBar.tick();
-    }
-
-    processes--;
-    setTimeout(chainPreDiscovery); // circumvent max call stack
-  }
-
-  times(MAX_REQUESTS, chainPreDiscovery);
-} else {
-  console.log('No pre-discovered ids found :(');
-  discover();
-}
-
-function discover() {
-  const index: Record<Region, number> = {
-    asia:
-      preDiscovered?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
-    com: preDiscovered?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com,
-    eu: preDiscovered?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu,
-  };
-
-  let regionIndex = 0;
-  let done = 0;
-
-  async function chainDiscovery() {
-    regionIndex = (regionIndex + 1) % REGIONS.length;
-    const region = REGIONS[regionIndex];
-    const id = index[region]++;
-
-    if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) {
-      if (++done !== MAX_REQUESTS) return;
-
-      console.log(
-        `Total discovery run time: ${
-          Date.now() - startTime
-        }ms\nDiscovered ids: ${discoveredIds.length}`,
-      );
-
-      postWork();
-
-      return;
-    }
-
-    if (id > MAX_IDS[region]) return;
-
-    const stats = await getTankStats(region, id, { fields });
-
-    if (stats !== null && stats.length > 0) {
-      discoveredIds.push(id);
-      players.push(stats);
-
-      console.log(
-        `discovered ${id} in ${region} (time left: ${RUN_TIME - (Date.now() - startTime)}ms)`,
-      );
-    }
-
-    setTimeout(chainDiscovery); // circumvent max call stack
-  }
-
-  console.log(`Spawning ${MAX_REQUESTS} discovery chains...`);
-  times(MAX_REQUESTS, chainDiscovery);
 }
 
 function postWork() {

scripts/discoverIds.ts

@@ -9,7 +9,7 @@ import { asset } from '../src/core/blitzkit/asset';
 import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
 
-interface DiscoverIdsManifest {
+export interface DiscoverIdsManifest {
   time: number;
   chunks: number;
   count: number;
@@ -44,7 +44,7 @@ while (chunkIndex < preDiscoveredManifest.chunks) {
   ).then(async (response) => {
     const buffer = await response.arrayBuffer();
     const decompressed = decompress(new Uint8Array(buffer)).buffer;
-    return new DidsReadStream(decompressed).dids();
+    return new DidsReadStream(decompressed as ArrayBuffer).dids();
   });
 
   // no spread syntax: https://github.com/oven-sh/bun/issues/11734

src/core/blitz/fetchBlitz.ts

@@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch';
 const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE'];
 
 export const retryAbleBlitzFetchEvent = new EventManager();
+export const blitzFetchQueueAvailableEvent = new EventManager();
 
 type BlitzResponse<Data extends object> =
   | {
@@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10;
 const queue: { url: string; resolve: (data: object) => void }[] = [];
 let inProgress = 0;
 
+function decrement() {
+  inProgress--;
+  blitzFetchQueueAvailableEvent.emit(undefined);
+  manageQueue();
+}
+
 async function manageQueue() {
   if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) {
     inProgress++;
 
-    const timeout = setTimeout(() => {
+    const timeout = setTimeout(() => decrement, 1000);
-      inProgress--;
-      manageQueue();
-    }, 1000);
-
     const request = queue.shift()!;
     const data = (await patientFetch(request.url)
       .then((response) => response.json())
@@ -50,13 +53,11 @@ async function manageQueue() {
         clearTimeout(timeout);
         setTimeout(() => {
           queue.push(request);
-          inProgress--;
+          decrement();
-          manageQueue();
         }, 1000 / MAX_CALLS_PER_SECOND);
       } else {
         clearTimeout(timeout);
-        inProgress--;
+        decrement();
-        manageQueue();
 
         throw new Error(`Wargaming response error status: "${data.status}"`, {
           cause: `Message: "${data.error.message}"\nURL: "${request.url}"`,

src/core/blitzkit/fetchPreDiscoveredIds.ts

@@ -0,0 +1,37 @@
+import { times } from 'lodash';
+import { decompress } from 'lz4js';
+import { DiscoverIdsManifest } from '../../../scripts/discoverIds';
+import { DidsReadStream } from '../streams/dids';
+import { asset } from './asset';
+
+export async function fetchPreDiscoveredIds(dev: boolean) {
+  const idChunks: number[][] = [];
+  const preDiscoveredManifest = (await fetch(
+    asset('ids/manifest.json', dev),
+  ).then((response) => response.json())) as DiscoverIdsManifest;
+
+  console.log(
+    `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
+  );
+
+  await Promise.all(
+    times(preDiscoveredManifest.chunks, async (chunkIndex) => {
+      const preDiscovered = await fetch(
+        asset(`ids/${chunkIndex}.dids.lz4`, dev),
+      ).then(async (response) => {
+        const buffer = await response.arrayBuffer();
+        const decompressed = decompress(new Uint8Array(buffer)).buffer;
+        return new DidsReadStream(decompressed as ArrayBuffer).dids();
+      });
+
+      // no spread syntax: https://github.com/oven-sh/bun/issues/11734
+      idChunks.push(preDiscovered);
+
+      console.log(
+        `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
+      );
+    }),
+  );
+
+  return idChunks.sort((a, b) => a[0] - b[0]).flat();
+}