Merge branch 'dev-static-updates' into dev

scripts/averages.ts

@@ -1,144 +1,186 @@
+import { chunk, times } from 'lodash';
 import { argv } from 'process';
-import { REGIONS, Region } from '../src/constants/regions';
-import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
+import { Region, REGIONS } from '../src/constants/regions';
+import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
 import getTankStats from '../src/core/blitz/getTankStats';
 import { idToRegion } from '../src/core/blitz/idToRegion';
 import {
   AverageDefinitions,
   AverageDefinitionsAllStats,
+  AverageDefinitionsEntry,
+  AverageDefinitionsEntrySubPartial,
 } from '../src/core/blitzkit/averageDefinitions';
 import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants';
+import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
+import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64';
 
-const MAX_REQUESTS = 10;
-const RUN_TIME = 1000 * 20;
+interface DataPoint {
+  x: number;
+  y: number;
+  w: number;
+}
+
+const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 120;
+const MIN_BATTLES = 5000;
+const RUN_TIME = 1000 * 60 * 60 * 5.9;
+const THREADS = 10;
+const PLAYER_IDS_PER_CALL = 100;
 
 const startTime = Date.now();
 const production = argv.includes('--production');
+const preDiscoveredIds = await fetchPreDiscoveredIds(!production);
+const playerIds: Record<Region, number[][]> = {
+  asia: chunk(
+    preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'),
+    PLAYER_IDS_PER_CALL,
+  ),
+  com: chunk(
+    preDiscoveredIds.filter((id) => idToRegion(id) === 'com'),
+    PLAYER_IDS_PER_CALL,
+  ),
+  eu: chunk(
+    preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'),
+    PLAYER_IDS_PER_CALL,
+  ),
+};
+let regionIndex = 0;
+let checkedPlayers = 0;
+let includedPlayers = 0;
+const availableRegions = [...REGIONS];
+const tankIds: number[] = [];
+const tanksSorted: Record<number, AverageDefinitionsAllStats[]> = {};
+let postWorkRequested = false;
 
-console.log(`Running in ${production ? 'production' : 'development'} mode`);
+times(THREADS, async () => {
+  while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) {
+    const region = availableRegions[regionIndex];
+    const regionIds = playerIds[region];
+    const [ids] = regionIds.splice(
+      Math.floor(Math.random() * regionIds.length),
+      1,
+    );
+    const accountInfo = await getAccountInfo(region, ids, undefined, {
+      fields: 'last_battle_time,statistics.all.battles',
+    });
+    checkedPlayers += ids.length;
 
-const ids = await fetchPreDiscoveredIds(!production);
-const groups: Record<Region, number[]> = { asia: [], com: [], eu: [] };
+    /**
+     * Pass rates
+     * none: 1000 / 1000
+     * activity: 55 / 1000
+     * battles: 30 / 1000
+     * both: 23 / 1000
+     */
 
-ids.forEach((id) => {
-  groups[idToRegion(id)].push(id);
+    const filteredIds = ids.filter((_, index) => {
+      const info = accountInfo[index];
+      return (
+        info !== null &&
+        Date.now() - info.last_battle_time * 1000 <= MAX_ACTIVITY_TIME &&
+        info.statistics.all.battles > MIN_BATTLES
+      );
+    });
+    includedPlayers += filteredIds.length;
+    const players = await Promise.all(
+      filteredIds.map((id) => getTankStats(region, id)),
+    );
+
+    players.forEach((tanks) => {
+      tanks?.forEach((tank) => {
+        if (tank.all.battles === 0) return;
+
+        if (!tankIds.includes(tank.tank_id)) {
+          tankIds.push(tank.tank_id);
+          tanksSorted[tank.tank_id] = [];
+        }
+
+        tanksSorted[tank.tank_id].push({
+          ...tank.all,
+          battle_life_time: tank.battle_life_time,
+        });
+      });
+    });
+
+    if (regionIds.length === 0 && availableRegions.includes(region)) {
+      availableRegions.splice(availableRegions.indexOf(region), 1);
+    }
+    regionIndex = (regionIndex + 1) % availableRegions.length;
+  }
+
+  if (!postWorkRequested) {
+    postWorkRequested = true;
+    postWork();
+  }
 });
 
-const tankIds: number[] = [];
-const tanks: Record<number, AverageDefinitionsAllStats[]> = {};
-const indices: Record<Region, number> = {
-  asia: groups.asia.length,
-  com: groups.com.length,
-  eu: groups.eu.length,
-};
-let players = 0;
-let regionIndex = 0;
-let discardAttempts = 0;
-
-const interval = setInterval(async () => {
-  if (discardAttempts > 0) return discardAttempts--;
-  if (Date.now() - startTime > RUN_TIME) post();
-
-  const region = REGIONS[regionIndex];
-  const index = --indices[region];
-  const id = groups[region][index];
-  const stats = await getTankStats(region, id, {
-    fields: 'all,battle_life_time,tank_id',
-  });
-
-  if (stats !== null) players++;
-
-  stats?.forEach((stat) => {
-    if (!tankIds.includes(stat.tank_id)) {
-      tanks[stat.tank_id] = [];
-      tankIds.push(stat.tank_id);
-    }
-
-    // goofy ahh optimization instead of using spread syntax
-    const composite = stat.all as AverageDefinitionsAllStats;
-    composite.battle_life_time = stat.battle_life_time;
-
-    tanks[stat.tank_id].push(composite);
-  });
-
-  regionIndex = (regionIndex + 1) % REGIONS.length;
-}, 1000 / MAX_REQUESTS);
-
-retryAbleBlitzFetchEvent.on(() => discardAttempts++);
-
-function post() {
-  clearInterval(interval);
+async function postWork() {
   console.log(
-    `Out of time, computing ${tankIds.length} tanks from ${players} players...`,
+    `Generating statistics based off ${includedPlayers.toLocaleString()} players (${checkedPlayers.toLocaleString()} checked in total) and ${tankIds.length} tanks...`,
   );
 
   const averages: AverageDefinitions = {};
 
   tankIds.forEach((id) => {
-    const samples = tanks[id];
-    const n = samples.length;
+    const tanks = tanksSorted[id];
+    const samples = tanks.length;
+    const entry: AverageDefinitionsEntrySubPartial = {
+      samples,
+      mu: {},
+      sigma: {},
+      r: {},
+    };
 
-    function sum(slice: (tank: AverageDefinitionsAllStats) => number) {
-      return samples.reduce((acc, tank) => {
-        return acc + slice(tank);
-      }, 0);
-    }
+    averageDefinitionsAllStatsKeys.forEach((key) => {
+      const data: DataPoint[] = tanks.map((tank) => ({
+        w: tank.battles,
+        x: tank[key] / tank.battles,
+        y: tank.wins / tank.battles,
+      }));
 
-    const denominator = sum((tank) => tank.battles);
+      if (data.length === 0) return;
 
-    function moment(slice: (tank: AverageDefinitionsAllStats) => number) {
-      return samples.reduce((acc, tank) => {
-        return acc + tank.battles * slice(tank);
-      }, 0);
-    }
+      function sum(slicer: (data: DataPoint) => number) {
+        return data.reduce(
+          (accumulator, data) => accumulator + slicer(data),
+          0,
+        );
+      }
 
-    function weightedAverage(
-      slice: (tank: AverageDefinitionsAllStats) => number,
-    ) {
-      return moment(slice) / denominator;
-    }
+      const sum_w = sum(({ w }) => w);
+      const sum_wx = sum(({ w, x }) => w * x);
+      const sum_wy = sum(({ w, y }) => w * y);
+      const x_bar = sum_wx / sum_w;
+      const y_bar = sum_wy / sum_w;
+      const mu = x_bar;
 
-    const sumBattles = sum((tank) => tank.battles);
+      const sigma_numerator = sum(({ w, x }) => w * (x - x_bar) ** 2);
+      const sigma = Math.sqrt(sigma_numerator / sum_w);
 
-    function reduce(method: (key: keyof AverageDefinitionsAllStats) => number) {
-      return averageDefinitionsAllStatsKeys.reduce<
-        Partial<AverageDefinitionsAllStats>
-      >((acc, key) => {
-        acc[key] = method(key);
-        return acc;
-      }, {}) as AverageDefinitionsAllStats;
-    }
+      const r_numerator = sum(({ w, x, y }) => w * (x - x_bar) * (y - y_bar));
+      const r_denominator_x = sum(({ w, x }) => w * (x - x_bar) ** 2);
+      const r_denominator_y = sum(({ w, y }) => w * (y - y_bar) ** 2);
+      const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y);
 
-    const mu = reduce(
-      (key) => sum((tank) => tank.battles * tank[key]) / sumBattles,
-    );
+      entry.mu[key] = mu;
+      entry.sigma[key] = sigma;
+      entry.r[key] = isNaN(r) ? 0 : r;
+    });
 
-    // const sigma = averageDefinitionsAllStatsKeys.reduce<
-    //   Partial<AverageDefinitionsAllStats>
-    // >((acc, key) => {
-    //   acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / n);
-    //   return acc;
-    // }, {}) as AverageDefinitionsAllStats;
-    // const r = averageDefinitionsAllStatsKeys.reduce<
-    //   Partial<AverageDefinitionsAllStats>
-    // >((acc, key) => {
-    //   acc[key] =
-    //     (n *
-    //       sum(
-    //         (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
-    //       ) -
-    //       sum((tank) => tank[key] / tank.battles) *
-    //         sum((tank) => tank.wins / tank.battles)) /
-    //     Math.sqrt(
-    //       (n * sum((tank) => (tank[key] / tank.battles) ** 2) -
-    //         sum((tank) => tank[key] / tank.battles) ** 2) *
-    //         (n * sum((tank) => (tank.wins / tank.battles) ** 2) -
-    //           sum((tank) => tank.wins / tank.battles) ** 2),
-    //     );
-    //   return acc;
-    // }, {}) as AverageDefinitionsAllStats;
-
-    averages[id] = { n: n, mu, sigma, r };
+    averages[id] = entry as AverageDefinitionsEntry;
   });
+
+  commitAssets(
+    'averages',
+    [
+      {
+        path: 'definitions/averages.pb',
+        content: await encodeToBase64('blitzkit.AverageDefinitions', {
+          averages,
+        }),
+        encoding: 'base64',
+      },
+    ],
+    production,
+  );
 }

scripts/discoverIds.ts

@@ -9,7 +9,7 @@ import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
 import { DidsWriteStream } from '../src/core/streams/dids';
 
-export interface DiscoverIdsManifest {
+export export interface DiscoverIdsManifest {
   time: number;
   chunks: number;
   count: number;
@@ -28,7 +28,34 @@ console.log(`Running in ${production ? 'production' : 'development'} mode`);
 
 const startTime = Date.now();
 const indexableRegions = [...REGIONS];
-let ids = await fetchPreDiscoveredIds(!production);
+let ids: number[] = [];
+const preDiscoveredManifest = (await fetch(
+  asset('ids/manifest.json', !production),
+).then((response) => response.json())) as DiscoverIdsManifest;
+
+console.log(
+  `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
+);
+
+let chunkIndex = 0;
+while (chunkIndex < preDiscoveredManifest.chunks) {
+  const preDiscovered = await fetch(
+    asset(`ids/${chunkIndex}.dids.lz4`, !production),
+  ).then(async (response) => {
+    const buffer = await response.arrayBuffer();
+    const decompressed = decompress(new Uint8Array(buffer)).buffer;
+    return new DidsReadStream(decompressed as ArrayBuffer).dids();
+  });
+
+  // no spread syntax: https://github.com/oven-sh/bun/issues/11734
+  ids = ids.concat(preDiscovered);
+
+  console.log(
+    `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
+  );
+
+  chunkIndex++;
+}
 
 const regionalIdIndex: Record<Region, number> = {
   asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,