feat: complete averages script

docs/technical/bkni.md

@@ -58,7 +58,7 @@ $$
 The weight of an atomic statistic is $w$ where $r$ is the correlation coefficient between the statistic and win rate, $\mu$ is the mean of the statistic across all players, and $m$ is the slope of the least squares regression line.
 
 $$
-w=\mu mr
+w=\left|r\right|\mu m
 $$
 
 The raw metric is the weighted average of all atomic statistics.

package.json

@@ -65,6 +65,7 @@
     "parse-dds": "^1.2.1",
     "pickleparser": "^0.2.1",
     "postprocessing": "^6.34.1",
+    "progress": "^2.0.3",
     "protobufjs": "^7.2.6",
     "react": "^18.2.0",
     "react-device-detect": "^2.2.3",
@@ -95,6 +96,7 @@
     "@types/node": "^20.6.5",
     "@types/parse-dds": "^1.0.3",
     "@types/pngjs": "^6.0.4",
+    "@types/progress": "^2.0.7",
     "@types/react": "^18.2.22",
     "@vanilla-extract/next-plugin": "^2.3.1",
     "compare-versions": "^6.1.0",

scripts/averages.ts

@@ -1,29 +1,31 @@
-import { mkdir, writeFile } from 'fs/promises';
 import { times } from 'lodash';
 import { compress } from 'lz4js';
+import { argv } from 'process';
+import ProgressBar from 'progress';
 import { Region, REGIONS } from '../src/constants/regions';
 import getTankStats from '../src/core/blitz/getTankStats';
 import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion';
 import { asset } from '../src/core/blitzkit/asset';
+import {
+  AverageDefinitions,
+  AverageDefinitionsAllStats,
+} from '../src/core/blitzkit/averageDefinitions';
+import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants';
+import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { superCompress } from '../src/core/blitzkit/superCompress';
 import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
 import { IndividualTankStats } from '../src/types/tanksStats';
 
-const RUN_TIME = 1000 * 60;
+const RUN_TIME = 1000 * 0;
 const MAX_REQUESTS = 10;
-const startTime = Date.now();
 
-const list: IndividualTankStats[][] = [];
+const production = argv.includes('--production');
+const startTime = Date.now();
+const fields = 'all,tank_id,battle_life_time';
+
+const players: IndividualTankStats[][] = [];
 const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then(
   async (response) => {
-    return [
-      1000000048, 500000058, 500000076, 2000000096, 2000000097, 2000000102,
-      2000000103, 2000000109, 1000000116, 500000131, 2000000138, 2000000141,
-      2000000140, 2000000143, 2000000144, 2000000145, 500000147, 2000000150,
-      2000000173, 2000000174, 2000000178, 2000000177, 2000000182, 2000000189,
-      500000191, 2000000191, 2000000192, 2000000196,
-    ];
-
     if (response.status === 200) {
       const buffer = await response.arrayBuffer();
       return new DidsReadStream(buffer).dids();
@@ -40,17 +42,22 @@ if (preDiscovered) {
     `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`,
   );
 
+  const preDiscoveredBar = new ProgressBar(':bar', {
+    total: preDiscovered.length,
+  });
   let preDiscoveredIndex = 0;
 
   async function chainPreDiscovery() {
     processes++;
 
-    if (preDiscoveredIndex >= preDiscovered!.length) {
+    if (preDiscoveredIndex === preDiscovered!.length) {
       processes--;
 
       if (processes !== 0) return;
 
-      console.log(`Rediscovered ids in ${Date.now() - startTime}ms`);
+      console.log(
+        `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`,
+      );
       discover();
 
       return;
@@ -61,13 +68,12 @@ if (preDiscovered) {
 
     if (id > MAX_IDS[region]) return;
 
-    const stats = await getTankStats(region, id, { felids: 'all' });
+    const stats = await getTankStats(region, id, { fields });
 
     if (stats !== null && stats.length > 0) {
       discoveredIds.push(id);
-      list.push(stats);
-
-      console.log(`re-discovered ${id} in ${region}`);
+      players.push(stats);
+      preDiscoveredBar.tick();
     }
 
     processes--;
@@ -89,51 +95,170 @@ function discover() {
   };
 
   let regionIndex = 0;
+  let done = 0;
 
   async function chainDiscovery() {
-    processes++;
-
-    if (Date.now() - startTime > RUN_TIME) {
-      processes--;
-
-      if (processes !== 0) return;
-
-      const discoveredIdsContent = compress(
-        new DidsWriteStream().dids(discoveredIds).uint8Array,
-      );
-
-      await mkdir('temp/averages', { recursive: true });
-      writeFile('temp/averages/discovered.dids.lz4', discoveredIdsContent);
-      writeFile('temp/averages/list.cdon.lz4', superCompress(list));
-
-      console.log(
-        `Total run time: ${
-          Date.now() - startTime
-        }ms\nDiscovered ids: ${discoveredIds.length}`,
-      );
-
-      return;
-    }
-
     regionIndex = (regionIndex + 1) % REGIONS.length;
     const region = REGIONS[regionIndex];
     const id = index[region]++;
 
+    if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) {
+      if (++done !== MAX_REQUESTS) return;
+
+      console.log(
+        `Total discovery run time: ${
+          Date.now() - startTime
+        }ms\nDiscovered ids: ${discoveredIds.length}`,
+      );
+
+      postWork();
+
+      return;
+    }
+
     if (id > MAX_IDS[region]) return;
 
-    const stats = await getTankStats(region, id, { felids: 'all' });
+    const stats = await getTankStats(region, id, { fields });
 
     if (stats !== null && stats.length > 0) {
       discoveredIds.push(id);
-      list.push(stats);
+      players.push(stats);
 
       console.log(`discovered ${id} in ${region}`);
     }
 
-    processes--;
     setTimeout(chainDiscovery); // circumvent max call stack
   }
 
   console.log(`Spawning ${MAX_REQUESTS} discovery chains...`);
   times(MAX_REQUESTS, chainDiscovery);
 }
+
+function postWork() {
+  const tankIds: number[] = [];
+  const sorted: Record<number, IndividualTankStats[]> = {};
+
+  players.forEach((tanks) => {
+    tanks.forEach((tank) => {
+      if (tank.all.battles === 0) return;
+
+      if (!sorted[tank.tank_id]) {
+        tankIds.push(tank.tank_id);
+        sorted[tank.tank_id] = [];
+      }
+      sorted[tank.tank_id].push(tank);
+    });
+  });
+
+  console.log(
+    `Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`,
+  );
+
+  const averages: AverageDefinitions = {};
+
+  tankIds.forEach((id) => {
+    const tanks = sorted[id];
+    const samples = tanks.length;
+
+    function sum(slice: (tank: AverageDefinitionsAllStats) => number) {
+      return tanks.reduce((acc, tank) => {
+        return (
+          acc + slice({ battle_life_time: tank.battle_life_time, ...tank.all })
+        );
+      }, 0);
+    }
+
+    const denominator = sum((tank) => tank.battles);
+
+    function moment(slice: (tank: AverageDefinitionsAllStats) => number) {
+      return tanks.reduce((acc, tank) => {
+        return (
+          acc +
+          tank.all.battles *
+            slice({ battle_life_time: tank.battle_life_time, ...tank.all })
+        );
+      }, 0);
+    }
+
+    function weightedAverage(
+      slice: (tank: AverageDefinitionsAllStats) => number,
+    ) {
+      return moment(slice) / denominator;
+    }
+
+    const mu = averageDefinitionsAllStatsKeys.reduce<
+      Partial<AverageDefinitionsAllStats>
+    >((acc, key) => {
+      acc[key] = weightedAverage((tank) => tank[key]);
+      return acc;
+    }, {}) as AverageDefinitionsAllStats;
+    const sigma = averageDefinitionsAllStatsKeys.reduce<
+      Partial<AverageDefinitionsAllStats>
+    >((acc, key) => {
+      acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / samples);
+      return acc;
+    }, {}) as AverageDefinitionsAllStats;
+    const r = averageDefinitionsAllStatsKeys.reduce<
+      Partial<AverageDefinitionsAllStats>
+    >((acc, key) => {
+      acc[key] =
+        (samples *
+          sum(
+            (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
+          ) -
+          sum((tank) => tank[key] / tank.battles) *
+            sum((tank) => tank.wins / tank.battles)) /
+        Math.sqrt(
+          (samples * sum((tank) => (tank[key] / tank.battles) ** 2) -
+            sum((tank) => tank[key] / tank.battles) ** 2) *
+            (samples * sum((tank) => (tank.wins / tank.battles) ** 2) -
+              sum((tank) => tank.wins / tank.battles) ** 2),
+        );
+      return acc;
+    }, {}) as AverageDefinitionsAllStats;
+    const m = averageDefinitionsAllStatsKeys.reduce<
+      Partial<AverageDefinitionsAllStats>
+    >((acc, key) => {
+      acc[key] =
+        (samples *
+          sum(
+            (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
+          ) -
+          sum((tank) => tank[key] / tank.battles) *
+            sum((tank) => tank.wins / tank.battles)) /
+        (samples * sum((tank) => (tank[key] / tank.battles) ** 2) -
+          sum((tank) => tank[key] / tank.battles) ** 2);
+      return acc;
+    }, {}) as AverageDefinitionsAllStats;
+    const b = averageDefinitionsAllStatsKeys.reduce<
+      Partial<AverageDefinitionsAllStats>
+    >((acc, key) => {
+      acc[key] =
+        (sum((tank) => tank.wins / tank.battles) -
+          m[key] * sum((tank) => tank[key] / tank.battles)) /
+        samples;
+      return acc;
+    }, {}) as AverageDefinitionsAllStats;
+
+    averages[id] = { samples, mu, sigma, r, m, b };
+  });
+
+  commitAssets(
+    'averages',
+    [
+      {
+        path: 'definitions/averages.cdon.lz4',
+        content: superCompress(averages),
+        encoding: 'base64',
+      },
+      {
+        path: 'definitions/discovered.dids.lz4',
+        content: Buffer.from(
+          compress(new DidsWriteStream().dids(discoveredIds).uint8Array),
+        ).toString('base64'),
+        encoding: 'base64',
+      },
+    ],
+    production,
+  );
+}

src/core/blitzkit/averageDefinitions/constants.ts

@@ -0,0 +1,12 @@
+import { AverageDefinitionsAllStats } from '.';
+
+import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants';
+
+export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = {
+  ...emptyAllStats,
+  battle_life_time: 0,
+};
+
+export const averageDefinitionsAllStatsKeys = Object.keys(
+  emptyAverageDefinitionsAllStats,
+) as (keyof AverageDefinitionsAllStats)[];

src/core/blitzkit/averageDefinitions/index.ts

@@ -0,0 +1,28 @@
+import { AllStats } from '../../blitz/getAccountInfo';
+import { asset } from '../asset';
+import { fetchCdonLz4 } from '../fetchCdonLz4';
+
+export interface AverageDefinitionsAllStats extends AllStats {
+  battle_life_time: number;
+}
+
+export interface AverageDefinitions {
+  [id: number]: {
+    samples: number;
+
+    // average
+    mu: AverageDefinitionsAllStats;
+    // standard deviation
+    sigma: AverageDefinitionsAllStats;
+    // correlation coefficient
+    r: AverageDefinitionsAllStats;
+    // least squares regression line slope
+    m: AverageDefinitionsAllStats;
+    // least squares regression line y-intercept
+    b: AverageDefinitionsAllStats;
+  };
+}
+
+export const averageDefinitions = fetchCdonLz4<AverageDefinitions>(
+  asset('definitions/averages.cdon.lz4'),
+);

yarn.lock

@@ -2311,6 +2311,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/progress@^2.0.7":
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/@types/progress/-/progress-2.0.7.tgz#798b309935ef1cf5bef3b3f7bb8da7b0335bc67e"
+  integrity sha512-iadjw02vte8qWx7U0YM++EybBha2CQLPGu9iJ97whVgJUT5Zq9MjAPYUnbfRI2Kpehimf1QjFJYxD0t8nqzu5w==
+  dependencies:
+    "@types/node" "*"
+
 "@types/prop-types@*":
   version "15.7.11"
   resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.11.tgz#2596fb352ee96a1379c657734d4b913a613ad563"
@@ -4944,6 +4951,11 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
+progress@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
+  integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
+
 promise-inflight@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"