Merge branch 'dev-static-updates' into dev

.github/workflows/cd-averages.yml

@@ -2,8 +2,8 @@ name: CD Averages
 
 on:
   workflow_dispatch:
-  # schedule:
-  #   - cron: '0 12 * * 0'
+  schedule:
+    - cron: '0 8 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

package.json

@@ -106,5 +106,8 @@
     "prisma": "^5.4.1",
     "typescript": "^5.4.5"
   },
-  "packageManager": "[email protected]"
+  "packageManager": "[email protected]",
+  "trustedDependencies": [
+    "protobufjs"
+  ]
 }

scripts/averages.ts

@@ -1,144 +1,186 @@
+import { chunk, times } from 'lodash';
 import { argv } from 'process';
-import { REGIONS, Region } from '../src/constants/regions';
-import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
+import { Region, REGIONS } from '../src/constants/regions';
+import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
 import getTankStats from '../src/core/blitz/getTankStats';
 import { idToRegion } from '../src/core/blitz/idToRegion';
 import {
   AverageDefinitions,
   AverageDefinitionsAllStats,
+  AverageDefinitionsEntry,
+  AverageDefinitionsEntrySubPartial,
 } from '../src/core/blitzkit/averageDefinitions';
 import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants';
+import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
+import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64';
 
-const MAX_REQUESTS = 10;
-const RUN_TIME = 1000 * 20;
+interface DataPoint {
+  x: number;
+  y: number;
+  w: number;
+}
+
+const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 120;
+const MIN_BATTLES = 5000;
+const RUN_TIME = 1000 * 60 * 60 * 5.9;
+const THREADS = 10;
+const PLAYER_IDS_PER_CALL = 100;
 
 const startTime = Date.now();
 const production = argv.includes('--production');
+const preDiscoveredIds = await fetchPreDiscoveredIds(!production);
+const playerIds: Record<Region, number[][]> = {
+  asia: chunk(
+    preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'),
+    PLAYER_IDS_PER_CALL,
+  ),
+  com: chunk(
+    preDiscoveredIds.filter((id) => idToRegion(id) === 'com'),
+    PLAYER_IDS_PER_CALL,
+  ),
+  eu: chunk(
+    preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'),
+    PLAYER_IDS_PER_CALL,
+  ),
+};
+let regionIndex = 0;
+let checkedPlayers = 0;
+let includedPlayers = 0;
+const availableRegions = [...REGIONS];
+const tankIds: number[] = [];
+const tanksSorted: Record<number, AverageDefinitionsAllStats[]> = {};
+let postWorkRequested = false;
 
-console.log(`Running in ${production ? 'production' : 'development'} mode`);
+times(THREADS, async () => {
+  while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) {
+    const region = availableRegions[regionIndex];
+    const regionIds = playerIds[region];
+    const [ids] = regionIds.splice(
+      Math.floor(Math.random() * regionIds.length),
+      1,
+    );
+    const accountInfo = await getAccountInfo(region, ids, undefined, {
+      fields: 'last_battle_time,statistics.all.battles',
+    });
+    checkedPlayers += ids.length;
 
-const ids = await fetchPreDiscoveredIds(!production);
-const groups: Record<Region, number[]> = { asia: [], com: [], eu: [] };
+    /**
+     * Pass rates
+     * none: 1000 / 1000
+     * activity: 55 / 1000
+     * battles: 30 / 1000
+     * both: 23 / 1000
+     */
 
-ids.forEach((id) => {
-  groups[idToRegion(id)].push(id);
+    const filteredIds = ids.filter((_, index) => {
+      const info = accountInfo[index];
+      return (
+        info !== null &&
+        Date.now() - info.last_battle_time * 1000 <= MAX_ACTIVITY_TIME &&
+        info.statistics.all.battles > MIN_BATTLES
+      );
+    });
+    includedPlayers += filteredIds.length;
+    const players = await Promise.all(
+      filteredIds.map((id) => getTankStats(region, id)),
+    );
+
+    players.forEach((tanks) => {
+      tanks?.forEach((tank) => {
+        if (tank.all.battles === 0) return;
+
+        if (!tankIds.includes(tank.tank_id)) {
+          tankIds.push(tank.tank_id);
+          tanksSorted[tank.tank_id] = [];
+        }
+
+        tanksSorted[tank.tank_id].push({
+          ...tank.all,
+          battle_life_time: tank.battle_life_time,
+        });
+      });
+    });
+
+    if (regionIds.length === 0 && availableRegions.includes(region)) {
+      availableRegions.splice(availableRegions.indexOf(region), 1);
+    }
+    regionIndex = (regionIndex + 1) % availableRegions.length;
+  }
+
+  if (!postWorkRequested) {
+    postWorkRequested = true;
+    postWork();
+  }
 });
 
-const tankIds: number[] = [];
-const tanks: Record<number, AverageDefinitionsAllStats[]> = {};
-const indices: Record<Region, number> = {
-  asia: groups.asia.length,
-  com: groups.com.length,
-  eu: groups.eu.length,
-};
-let players = 0;
-let regionIndex = 0;
-let discardAttempts = 0;
-
-const interval = setInterval(async () => {
-  if (discardAttempts > 0) return discardAttempts--;
-  if (Date.now() - startTime > RUN_TIME) post();
-
-  const region = REGIONS[regionIndex];
-  const index = --indices[region];
-  const id = groups[region][index];
-  const stats = await getTankStats(region, id, {
-    fields: 'all,battle_life_time,tank_id',
-  });
-
-  if (stats !== null) players++;
-
-  stats?.forEach((stat) => {
-    if (!tankIds.includes(stat.tank_id)) {
-      tanks[stat.tank_id] = [];
-      tankIds.push(stat.tank_id);
-    }
-
-    // goofy ahh optimization instead of using spread syntax
-    const composite = stat.all as AverageDefinitionsAllStats;
-    composite.battle_life_time = stat.battle_life_time;
-
-    tanks[stat.tank_id].push(composite);
-  });
-
-  regionIndex = (regionIndex + 1) % REGIONS.length;
-}, 1000 / MAX_REQUESTS);
-
-retryAbleBlitzFetchEvent.on(() => discardAttempts++);
-
-function post() {
-  clearInterval(interval);
+async function postWork() {
   console.log(
-    `Out of time, computing ${tankIds.length} tanks from ${players} players...`,
+    `Generating statistics based off ${includedPlayers.toLocaleString()} players (${checkedPlayers.toLocaleString()} checked in total) and ${tankIds.length} tanks...`,
   );
 
   const averages: AverageDefinitions = {};
 
   tankIds.forEach((id) => {
-    const samples = tanks[id];
-    const n = samples.length;
+    const tanks = tanksSorted[id];
+    const samples = tanks.length;
+    const entry: AverageDefinitionsEntrySubPartial = {
+      samples,
+      mu: {},
+      sigma: {},
+      r: {},
+    };
 
-    function sum(slice: (tank: AverageDefinitionsAllStats) => number) {
-      return samples.reduce((acc, tank) => {
-        return acc + slice(tank);
-      }, 0);
-    }
+    averageDefinitionsAllStatsKeys.forEach((key) => {
+      const data: DataPoint[] = tanks.map((tank) => ({
+        w: tank.battles,
+        x: tank[key] / tank.battles,
+        y: tank.wins / tank.battles,
+      }));
 
-    const denominator = sum((tank) => tank.battles);
+      if (data.length === 0) return;
 
-    function moment(slice: (tank: AverageDefinitionsAllStats) => number) {
-      return samples.reduce((acc, tank) => {
-        return acc + tank.battles * slice(tank);
-      }, 0);
-    }
+      function sum(slicer: (data: DataPoint) => number) {
+        return data.reduce(
+          (accumulator, data) => accumulator + slicer(data),
+          0,
+        );
+      }
 
-    function weightedAverage(
-      slice: (tank: AverageDefinitionsAllStats) => number,
-    ) {
-      return moment(slice) / denominator;
-    }
+      const sum_w = sum(({ w }) => w);
+      const sum_wx = sum(({ w, x }) => w * x);
+      const sum_wy = sum(({ w, y }) => w * y);
+      const x_bar = sum_wx / sum_w;
+      const y_bar = sum_wy / sum_w;
+      const mu = x_bar;
 
-    const sumBattles = sum((tank) => tank.battles);
+      const sigma_numerator = sum(({ w, x }) => w * (x - x_bar) ** 2);
+      const sigma = Math.sqrt(sigma_numerator / sum_w);
 
-    function reduce(method: (key: keyof AverageDefinitionsAllStats) => number) {
-      return averageDefinitionsAllStatsKeys.reduce<
-        Partial<AverageDefinitionsAllStats>
-      >((acc, key) => {
-        acc[key] = method(key);
-        return acc;
-      }, {}) as AverageDefinitionsAllStats;
-    }
+      const r_numerator = sum(({ w, x, y }) => w * (x - x_bar) * (y - y_bar));
+      const r_denominator_x = sum(({ w, x }) => w * (x - x_bar) ** 2);
+      const r_denominator_y = sum(({ w, y }) => w * (y - y_bar) ** 2);
+      const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y);
 
-    const mu = reduce(
-      (key) => sum((tank) => tank.battles * tank[key]) / sumBattles,
-    );
+      entry.mu[key] = mu;
+      entry.sigma[key] = sigma;
+      entry.r[key] = isNaN(r) ? 0 : r;
+    });
 
-    // const sigma = averageDefinitionsAllStatsKeys.reduce<
-    //   Partial<AverageDefinitionsAllStats>
-    // >((acc, key) => {
-    //   acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / n);
-    //   return acc;
-    // }, {}) as AverageDefinitionsAllStats;
-    // const r = averageDefinitionsAllStatsKeys.reduce<
-    //   Partial<AverageDefinitionsAllStats>
-    // >((acc, key) => {
-    //   acc[key] =
-    //     (n *
-    //       sum(
-    //         (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
-    //       ) -
-    //       sum((tank) => tank[key] / tank.battles) *
-    //         sum((tank) => tank.wins / tank.battles)) /
-    //     Math.sqrt(
-    //       (n * sum((tank) => (tank[key] / tank.battles) ** 2) -
-    //         sum((tank) => tank[key] / tank.battles) ** 2) *
-    //         (n * sum((tank) => (tank.wins / tank.battles) ** 2) -
-    //           sum((tank) => tank.wins / tank.battles) ** 2),
-    //     );
-    //   return acc;
-    // }, {}) as AverageDefinitionsAllStats;
-
-    averages[id] = { n: n, mu, sigma, r };
+    averages[id] = entry as AverageDefinitionsEntry;
   });
+
+  commitAssets(
+    'averages',
+    [
+      {
+        path: 'definitions/averages.pb',
+        content: await encodeToBase64('blitzkit.AverageDefinitions', {
+          averages,
+        }),
+        encoding: 'base64',
+      },
+    ],
+    production,
+  );
 }

scripts/discoverIds.ts

@@ -9,7 +9,7 @@ import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
 import { DidsWriteStream } from '../src/core/streams/dids';
 
-export interface DiscoverIdsManifest {
+export export interface DiscoverIdsManifest {
   time: number;
   chunks: number;
   count: number;
@@ -28,7 +28,34 @@ console.log(`Running in ${production ? 'production' : 'development'} mode`);
 
 const startTime = Date.now();
 const indexableRegions = [...REGIONS];
-let ids = await fetchPreDiscoveredIds(!production);
+let ids: number[] = [];
+const preDiscoveredManifest = (await fetch(
+  asset('ids/manifest.json', !production),
+).then((response) => response.json())) as DiscoverIdsManifest;
+
+console.log(
+  `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
+);
+
+let chunkIndex = 0;
+while (chunkIndex < preDiscoveredManifest.chunks) {
+  const preDiscovered = await fetch(
+    asset(`ids/${chunkIndex}.dids.lz4`, !production),
+  ).then(async (response) => {
+    const buffer = await response.arrayBuffer();
+    const decompressed = decompress(new Uint8Array(buffer)).buffer;
+    return new DidsReadStream(decompressed as ArrayBuffer).dids();
+  });
+
+  // no spread syntax: https://github.com/oven-sh/bun/issues/11734
+  ids = ids.concat(preDiscovered);
+
+  console.log(
+    `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
+  );
+
+  chunkIndex++;
+}
 
 const regionalIdIndex: Record<Region, number> = {
   asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,

src/core/blitz/fetchBlitz.ts

@@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch';
 const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE'];
 
 export const retryAbleBlitzFetchEvent = new EventManager();
+export const blitzFetchQueueAvailableEvent = new EventManager();
 
 type BlitzResponse<Data extends object> =
   | {
@@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10;
 const queue: { url: string; resolve: (data: object) => void }[] = [];
 let inProgress = 0;
 
+function decrement() {
+  inProgress--;
+  blitzFetchQueueAvailableEvent.emit(undefined);
+  manageQueue();
+}
+
 async function manageQueue() {
   if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) {
     inProgress++;
 
-    const timeout = setTimeout(() => {
-      inProgress--;
-      manageQueue();
-    }, 1000);
-
+    const timeout = setTimeout(decrement, 1000);
     const request = queue.shift()!;
     const data = (await patientFetch(request.url)
       .then((response) => response.json())
@@ -42,21 +45,15 @@ async function manageQueue() {
       request.resolve(data.data);
     } else {
       if (RETRY_ERRORS.includes(data.error.message)) {
-        console.log(
-          `Encountered retry-able error ${data.error.code} ${data.error.message}, putting "${request.url}" back in queue...`,
-        );
-
         retryAbleBlitzFetchEvent.emit(undefined);
         clearTimeout(timeout);
         setTimeout(() => {
           queue.push(request);
-          inProgress--;
-          manageQueue();
+          decrement();
         }, 1000 / MAX_CALLS_PER_SECOND);
       } else {
         clearTimeout(timeout);
-        inProgress--;
-        manageQueue();
+        decrement();
 
         throw new Error(`Wargaming response error status: "${data.status}"`, {
           cause: `Message: "${data.error.message}"\nURL: "${request.url}"`,

src/core/blitzkit/averageDefinitions/constants.ts

@@ -1,5 +1,4 @@
 import { AverageDefinitionsAllStats } from '.';
-
 import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants';
 
 export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = {

src/core/blitzkit/averageDefinitions/index.ts

@@ -1,20 +1,39 @@
 import { AllStats } from '../../blitz/getAccountInfo';
+import { decode } from '../../protobuf/decode';
 import { asset } from '../asset';
-import { fetchCdonLz4 } from '../fetchCdonLz4';
 
 export interface AverageDefinitionsAllStats extends AllStats {
   battle_life_time: number;
 }
 
-export interface AverageDefinitions {
-  [id: number]: {
-    n: number; // sample size
-    r: AverageDefinitionsAllStats; // correlation coefficient
-    mu: AverageDefinitionsAllStats; // mean
-    sigma: AverageDefinitionsAllStats; // standard deviation
-  };
+export interface AverageDefinitionsEntry {
+  samples: number;
+  mu: AverageDefinitionsAllStats;
+  sigma: AverageDefinitionsAllStats;
+  r: AverageDefinitionsAllStats;
 }
 
-export const averageDefinitions = fetchCdonLz4<AverageDefinitions>(
-  asset('definitions/averages.cdon.lz4'),
-);
+export interface AverageDefinitionsEntrySubPartial {
+  samples: number;
+  mu: Partial<AverageDefinitionsAllStats>;
+  sigma: Partial<AverageDefinitionsAllStats>;
+  r: Partial<AverageDefinitionsAllStats>;
+}
+
+export interface AverageDefinitions {
+  [id: number]: AverageDefinitionsEntry;
+}
+
+interface AverageDefinitionsProto {
+  averages: AverageDefinitions;
+}
+
+export const averageDefinitions = fetch(asset('definitions/averages.pb'))
+  .then((response) => response.arrayBuffer())
+  .then((buffer) => {
+    return decode<AverageDefinitionsProto>(
+      'blitzkit.AverageDefinitions',
+      new Uint8Array(buffer),
+    );
+  })
+  .then((data) => data.averages);

src/core/blitzkit/commitAssets.ts

@@ -5,7 +5,6 @@ export async function commitAssets(
   message: string,
   changes: FileChange[],
   production: boolean,
-  verbose = true,
 ) {
   console.log(`Committing ${message}...`);
 
@@ -17,6 +16,5 @@ export async function commitAssets(
     production ? 'main' : 'dev',
     `${message} - ${new Date().toDateString()}`,
     changes,
-    verbose,
   );
 }

src/core/blitzkit/commitMultipleFiles.ts

@@ -14,7 +14,6 @@ export default async function commitMultipleFiles(
   branch: string,
   message: string,
   changes: FileChange[],
-  verbose = false,
 ) {
   const latestCommitSha = (
     await octokit.git.getRef({

src/core/blitzkit/fetchPreDiscoveredIds.ts

@@ -1,37 +1,35 @@
 import { times } from 'lodash';
 import { decompress } from 'lz4js';
+import ProgressBar from 'progress';
 import { DiscoverIdsManifest } from '../../../scripts/discoverIds';
 import { DidsReadStream } from '../streams/dids';
 import { asset } from './asset';
 
 export async function fetchPreDiscoveredIds(dev: boolean) {
-  let ids: number[] = [];
-  const preDiscoveredManifest = (await fetch(
-    asset('ids/manifest.json', dev),
-  ).then((response) => response.json())) as DiscoverIdsManifest;
-
-  console.log(
-    `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
+  const idChunks: number[][] = [];
+  const manifest = (await fetch(asset('ids/manifest.json', dev)).then(
+    (response) => response.json(),
+  )) as DiscoverIdsManifest;
+  const bar = new ProgressBar(
+    `Fetching ${manifest.count.toLocaleString()} ids in ${manifest.chunks} chunks :bar`,
+    manifest.chunks,
   );
 
+  bar.render();
   await Promise.all(
-    times(preDiscoveredManifest.chunks, async (chunkIndex) => {
+    times(manifest.chunks, async (chunkIndex) => {
       const preDiscovered = await fetch(
         asset(`ids/${chunkIndex}.dids.lz4`, dev),
       ).then(async (response) => {
         const buffer = await response.arrayBuffer();
         const decompressed = decompress(new Uint8Array(buffer)).buffer;
-        return new DidsReadStream(decompressed).dids();
+        return new DidsReadStream(decompressed as ArrayBuffer).dids();
       });
 
-      // no spread syntax: https://github.com/oven-sh/bun/issues/11734
-      ids = ids.concat(preDiscovered);
-
-      console.log(
-        `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
-      );
+      idChunks.push(preDiscovered);
+      bar.tick();
     }),
   );
 
-  return ids;
+  return idChunks.sort((a, b) => a[0] - b[0]).flat();
 }

src/core/protobuf/decode.ts

@@ -0,0 +1,6 @@
+import { lookup } from './lookup';
+
+export async function decode<Type>(proto: string, data: Uint8Array) {
+  const Message = await lookup(proto);
+  return Message.decode(data).toJSON() as Type;
+}

src/core/protobuf/encode.ts

@@ -0,0 +1,6 @@
+import { lookup } from './lookup';
+
+export async function encode(proto: string, data: any) {
+  const Message = await lookup(proto);
+  return Message.encode(Message.create(data)).finish();
+}

src/core/protobuf/encodeToBase64.ts

@@ -0,0 +1,6 @@
+import { encode } from './encode';
+
+export async function encodeToBase64(proto: string, data: any) {
+  const array = await encode(proto, data);
+  return Buffer.from(array).toString('base64');
+}

src/core/protobuf/lookup.ts

@@ -0,0 +1,10 @@
+import { Root } from 'protobufjs';
+
+export async function lookup(proto: string) {
+  const root = await new Root().load(
+    `src/protos/${proto.split('.')[0]}.proto`,
+    { keepCase: true },
+  );
+
+  return root.lookupType(proto);
+}

src/protos/blitzkit.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package blitzkit;
+
+message AverageDefinitions {
+  map<uint32, Tank> averages = 1;
+}
+
+message Tank {
+  uint32 samples = 1;
+  AverageDefinitionsAllStats mu = 2;
+  AverageDefinitionsAllStats sigma = 3;
+  AverageDefinitionsAllStats r = 4;
+}
+
+message AverageDefinitionsAllStats {
+  float battles = 1;
+  float capture_points = 2;
+  float damage_dealt = 3;
+  float damage_received = 4;
+  float dropped_capture_points = 5;
+  float frags = 6;
+  float frags8p = 7;
+  float hits = 8;
+  float losses = 9;
+  float max_frags = 10;
+  float max_xp = 11;
+  float shots = 12;
+  float spotted = 13;
+  float survived_battles = 14;
+  float win_and_survived = 15;
+  float wins = 16;
+  float xp = 17;
+  float battle_life_time = 18;
+}

src/types/tanksStats.ts

@@ -1,4 +1,5 @@
 import { AllStats } from '../core/blitz/getAccountInfo';
+import { emptyAllStats } from '../core/blitzstars/getStatsInPeriod/constants';
 
 export interface IndividualTankStats {
   all: AllStats;
@@ -23,25 +24,7 @@ export interface NormalizedTankStats {
 }
 
 export const emptyIndividualTankStats: IndividualTankStats = {
-  all: {
-    battles: 0,
-    capture_points: 0,
-    damage_dealt: 0,
-    damage_received: 0,
-    dropped_capture_points: 0,
-    frags: 0,
-    frags8p: 0,
-    hits: 0,
-    losses: 0,
-    max_frags: 0,
-    max_xp: 0,
-    shots: 0,
-    spotted: 0,
-    survived_battles: 0,
-    xp: 0,
-    win_and_survived: 0,
-    wins: 0,
-  },
+  all: emptyAllStats,
   last_battle_time: 0,
   account_id: 0,
   max_xp: 0,