feat: start on new discover ids script

scripts/averages.ts

@@ -1,184 +1,95 @@
-import { times } from 'lodash';
-import { compress } from 'lz4js';
 import { argv } from 'process';
-import ProgressBar from 'progress';
-import { Region, REGIONS } from '../src/constants/regions';
+import { REGIONS, Region } from '../src/constants/regions';
+import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
 import getTankStats from '../src/core/blitz/getTankStats';
-import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion';
-import { asset } from '../src/core/blitzkit/asset';
+import { idToRegion } from '../src/core/blitz/idToRegion';
 import {
   AverageDefinitions,
   AverageDefinitionsAllStats,
 } from '../src/core/blitzkit/averageDefinitions';
 import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants';
-import { commitAssets } from '../src/core/blitzkit/commitAssets';
-import { superCompress } from '../src/core/blitzkit/superCompress';
-import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
-import { IndividualTankStats } from '../src/types/tanksStats';
+import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
 
-const RUN_TIME = 1000 * 60 * 60;
 const MAX_REQUESTS = 10;
+const RUN_TIME = 1000 * 20;
 
-const production = argv.includes('--production');
 const startTime = Date.now();
-const fields = 'all,tank_id,battle_life_time';
+const production = argv.includes('--production');
 
-const players: IndividualTankStats[][] = [];
-const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then(
-  async (response) => {
-    if (response.status === 200) {
-      const buffer = await response.arrayBuffer();
-      return new DidsReadStream(buffer).dids();
-    }
+console.log(`Running in ${production ? 'production' : 'development'} mode`);
 
-    return undefined;
-  },
-);
-const discoveredIds: number[] = [];
-let processes = 0;
+const ids = await fetchPreDiscoveredIds(!production);
+const groups: Record<Region, number[]> = { asia: [], com: [], eu: [] };
 
-if (preDiscovered) {
-  console.log(
-    `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`,
-  );
+ids.forEach((id) => {
+  groups[idToRegion(id)].push(id);
+});
 
-  const preDiscoveredBar = new ProgressBar(':bar', {
-    total: preDiscovered.length,
-  });
-  let preDiscoveredIndex = 0;
+const tankIds: number[] = [];
+const tanks: Record<number, AverageDefinitionsAllStats[]> = {};
+const indices: Record<Region, number> = {
+  asia: groups.asia.length,
+  com: groups.com.length,
+  eu: groups.eu.length,
+};
+let players = 0;
+let regionIndex = 0;
+let discardAttempts = 0;
 
-  async function chainPreDiscovery() {
-    processes++;
+const interval = setInterval(async () => {
+  if (discardAttempts > 0) return discardAttempts--;
+  if (Date.now() - startTime > RUN_TIME) post();
 
-    if (preDiscoveredIndex === preDiscovered!.length) {
-      processes--;
-
-      if (processes !== 0) return;
-
-      console.log(
-        `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`,
-      );
-      discover();
-
-      return;
-    }
-
-    const id = preDiscovered![preDiscoveredIndex++];
-    const region = idToRegion(id);
-
-    if (id > MAX_IDS[region]) return;
-
-    const stats = await getTankStats(region, id, { fields });
-
-    if (stats !== null && stats.length > 0) {
-      discoveredIds.push(id);
-      players.push(stats);
-      preDiscoveredBar.tick();
-    }
-
-    processes--;
-    setTimeout(chainPreDiscovery); // circumvent max call stack
-  }
-
-  times(MAX_REQUESTS, chainPreDiscovery);
-} else {
-  console.log('No pre-discovered ids found :(');
-  discover();
-}
-
-function discover() {
-  const index: Record<Region, number> = {
-    asia:
-      preDiscovered?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
-    com: preDiscovered?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com,
-    eu: preDiscovered?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu,
-  };
-
-  let regionIndex = 0;
-  let done = 0;
-
-  async function chainDiscovery() {
-    regionIndex = (regionIndex + 1) % REGIONS.length;
-    const region = REGIONS[regionIndex];
-    const id = index[region]++;
-
-    if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) {
-      if (++done !== MAX_REQUESTS) return;
-
-      console.log(
-        `Total discovery run time: ${
-          Date.now() - startTime
-        }ms\nDiscovered ids: ${discoveredIds.length}`,
-      );
-
-      postWork();
-
-      return;
-    }
-
-    if (id > MAX_IDS[region]) return;
-
-    const stats = await getTankStats(region, id, { fields });
-
-    if (stats !== null && stats.length > 0) {
-      discoveredIds.push(id);
-      players.push(stats);
-
-      console.log(
-        `discovered ${id} in ${region} (time left: ${RUN_TIME - (Date.now() - startTime)}ms)`,
-      );
-    }
-
-    setTimeout(chainDiscovery); // circumvent max call stack
-  }
-
-  console.log(`Spawning ${MAX_REQUESTS} discovery chains...`);
-  times(MAX_REQUESTS, chainDiscovery);
-}
-
-function postWork() {
-  const tankIds: number[] = [];
-  const sorted: Record<number, IndividualTankStats[]> = {};
-
-  players.forEach((tanks) => {
-    tanks.forEach((tank) => {
-      if (tank.all.battles === 0) return;
-
-      if (!sorted[tank.tank_id]) {
-        tankIds.push(tank.tank_id);
-        sorted[tank.tank_id] = [];
-      }
-      sorted[tank.tank_id].push(tank);
-    });
+  const region = REGIONS[regionIndex];
+  const index = --indices[region];
+  const id = groups[region][index];
+  const stats = await getTankStats(region, id, {
+    fields: 'all,battle_life_time,tank_id',
   });
 
+  if (stats !== null) players++;
+
+  stats?.forEach((stat) => {
+    if (!tankIds.includes(stat.tank_id)) {
+      tanks[stat.tank_id] = [];
+      tankIds.push(stat.tank_id);
+    }
+
+    // goofy ahh optimization instead of using spread syntax
+    const composite = stat.all as AverageDefinitionsAllStats;
+    composite.battle_life_time = stat.battle_life_time;
+
+    tanks[stat.tank_id].push(composite);
+  });
+
+  regionIndex = (regionIndex + 1) % REGIONS.length;
+}, 1000 / MAX_REQUESTS);
+
+retryAbleBlitzFetchEvent.on(() => discardAttempts++);
+
+function post() {
+  clearInterval(interval);
   console.log(
-    `Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`,
+    `Out of time, computing ${tankIds.length} tanks from ${players} players...`,
   );
 
   const averages: AverageDefinitions = {};
 
   tankIds.forEach((id) => {
-    const tanks = sorted[id];
-    const samples = tanks.length;
+    const samples = tanks[id];
+    const n = samples.length;
 
     function sum(slice: (tank: AverageDefinitionsAllStats) => number) {
-      return tanks.reduce((acc, tank) => {
-        return (
-          acc + slice({ battle_life_time: tank.battle_life_time, ...tank.all })
-        );
+      return samples.reduce((acc, tank) => {
+        return acc + slice(tank);
       }, 0);
     }
 
     const denominator = sum((tank) => tank.battles);
 
     function moment(slice: (tank: AverageDefinitionsAllStats) => number) {
-      return tanks.reduce((acc, tank) => {
-        return (
-          acc +
-          tank.all.battles *
-            slice({ battle_life_time: tank.battle_life_time, ...tank.all })
-        );
+      return samples.reduce((acc, tank) => {
+        return acc + tank.battles * slice(tank);
       }, 0);
     }
 
@@ -188,79 +99,46 @@ function postWork() {
       return moment(slice) / denominator;
     }
 
-    const mu = averageDefinitionsAllStatsKeys.reduce<
-      Partial<AverageDefinitionsAllStats>
-    >((acc, key) => {
-      acc[key] = weightedAverage((tank) => tank[key]);
-      return acc;
-    }, {}) as AverageDefinitionsAllStats;
-    const sigma = averageDefinitionsAllStatsKeys.reduce<
-      Partial<AverageDefinitionsAllStats>
-    >((acc, key) => {
-      acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / samples);
-      return acc;
-    }, {}) as AverageDefinitionsAllStats;
-    const r = averageDefinitionsAllStatsKeys.reduce<
-      Partial<AverageDefinitionsAllStats>
-    >((acc, key) => {
-      acc[key] =
-        (samples *
-          sum(
-            (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
-          ) -
-          sum((tank) => tank[key] / tank.battles) *
-            sum((tank) => tank.wins / tank.battles)) /
-        Math.sqrt(
-          (samples * sum((tank) => (tank[key] / tank.battles) ** 2) -
-            sum((tank) => tank[key] / tank.battles) ** 2) *
-            (samples * sum((tank) => (tank.wins / tank.battles) ** 2) -
-              sum((tank) => tank.wins / tank.battles) ** 2),
-        );
-      return acc;
-    }, {}) as AverageDefinitionsAllStats;
-    const m = averageDefinitionsAllStatsKeys.reduce<
-      Partial<AverageDefinitionsAllStats>
-    >((acc, key) => {
-      acc[key] =
-        (samples *
-          sum(
-            (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
-          ) -
-          sum((tank) => tank[key] / tank.battles) *
-            sum((tank) => tank.wins / tank.battles)) /
-        (samples * sum((tank) => (tank[key] / tank.battles) ** 2) -
-          sum((tank) => tank[key] / tank.battles) ** 2);
-      return acc;
-    }, {}) as AverageDefinitionsAllStats;
-    const b = averageDefinitionsAllStatsKeys.reduce<
-      Partial<AverageDefinitionsAllStats>
-    >((acc, key) => {
-      acc[key] =
-        (sum((tank) => tank.wins / tank.battles) -
-          m[key] * sum((tank) => tank[key] / tank.battles)) /
-        samples;
-      return acc;
-    }, {}) as AverageDefinitionsAllStats;
+    const sumBattles = sum((tank) => tank.battles);
 
-    averages[id] = { samples, mu, sigma, r, m, b };
+    function reduce(method: (key: keyof AverageDefinitionsAllStats) => number) {
+      return averageDefinitionsAllStatsKeys.reduce<
+        Partial<AverageDefinitionsAllStats>
+      >((acc, key) => {
+        acc[key] = method(key);
+        return acc;
+      }, {}) as AverageDefinitionsAllStats;
+    }
+
+    const mu = reduce(
+      (key) => sum((tank) => tank.battles * tank[key]) / sumBattles,
+    );
+
+    // const sigma = averageDefinitionsAllStatsKeys.reduce<
+    //   Partial<AverageDefinitionsAllStats>
+    // >((acc, key) => {
+    //   acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / n);
+    //   return acc;
+    // }, {}) as AverageDefinitionsAllStats;
+    // const r = averageDefinitionsAllStatsKeys.reduce<
+    //   Partial<AverageDefinitionsAllStats>
+    // >((acc, key) => {
+    //   acc[key] =
+    //     (n *
+    //       sum(
+    //         (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
+    //       ) -
+    //       sum((tank) => tank[key] / tank.battles) *
+    //         sum((tank) => tank.wins / tank.battles)) /
+    //     Math.sqrt(
+    //       (n * sum((tank) => (tank[key] / tank.battles) ** 2) -
+    //         sum((tank) => tank[key] / tank.battles) ** 2) *
+    //         (n * sum((tank) => (tank.wins / tank.battles) ** 2) -
+    //           sum((tank) => tank.wins / tank.battles) ** 2),
+    //     );
+    //   return acc;
+    // }, {}) as AverageDefinitionsAllStats;
+
+    averages[id] = { n: n, mu, sigma, r };
   });
-
-  commitAssets(
-    'averages',
-    [
-      {
-        path: 'definitions/averages.cdon.lz4',
-        content: superCompress(averages),
-        encoding: 'base64',
-      },
-      {
-        path: 'definitions/discovered.dids.lz4',
-        content: Buffer.from(
-          compress(new DidsWriteStream().dids(discoveredIds).uint8Array),
-        ).toString('base64'),
-        encoding: 'base64',
-      },
-    ],
-    production,
-  );
 }

scripts/discoverIds.ts

@@ -1,15 +1,15 @@
 import { chunk, times, uniq } from 'lodash';
-import { compress, decompress } from 'lz4js';
+import { compress } from 'lz4js';
 import { argv } from 'process';
 import { REGIONS, Region } from '../src/constants/regions';
 import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
 import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
 import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion';
-import { asset } from '../src/core/blitzkit/asset';
 import { commitAssets } from '../src/core/blitzkit/commitAssets';
-import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
+import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
+import { DidsWriteStream } from '../src/core/streams/dids';
 
-interface DiscoverIdsManifest {
+export interface DiscoverIdsManifest {
   time: number;
   chunks: number;
   count: number;
@@ -28,34 +28,7 @@ console.log(`Running in ${production ? 'production' : 'development'} mode`);
 
 const startTime = Date.now();
 const indexableRegions = [...REGIONS];
-let ids: number[] = [];
-const preDiscoveredManifest = (await fetch(
-  asset('ids/manifest.json', !production),
-).then((response) => response.json())) as DiscoverIdsManifest;
-
-console.log(
-  `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
-);
-
-let chunkIndex = 0;
-while (chunkIndex < preDiscoveredManifest.chunks) {
-  const preDiscovered = await fetch(
-    asset(`ids/${chunkIndex}.dids.lz4`, !production),
-  ).then(async (response) => {
-    const buffer = await response.arrayBuffer();
-    const decompressed = decompress(new Uint8Array(buffer)).buffer;
-    return new DidsReadStream(decompressed).dids();
-  });
-
-  // no spread syntax: https://github.com/oven-sh/bun/issues/11734
-  ids = ids.concat(preDiscovered);
-
-  console.log(
-    `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
-  );
-
-  chunkIndex++;
-}
+let ids = await fetchPreDiscoveredIds(!production);
 
 const regionalIdIndex: Record<Region, number> = {
   asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
@@ -82,9 +55,7 @@ async function verify(region: Region, ids: number[]) {
 let discardAttempts = 0;
 let lastProgressUpdate = 0;
 
-retryAbleBlitzFetchEvent.on(() => {
-  discardAttempts++;
-});
+retryAbleBlitzFetchEvent.on(() => discardAttempts++);
 
 const interval = setInterval(async () => {
   if (discardAttempts > 0) return discardAttempts--;