Merge branch 'dev-static-updates' into dev

src/core/blitz/fetchBlitz.ts

@@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch';
 const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE'];
 
 export const retryAbleBlitzFetchEvent = new EventManager();
+export const blitzFetchQueueAvailableEvent = new EventManager();
 
 type BlitzResponse<Data extends object> =
   | {
@@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10;
 const queue: { url: string; resolve: (data: object) => void }[] = [];
 let inProgress = 0;
 
+function decrement() {
+  inProgress--;
+  blitzFetchQueueAvailableEvent.emit(undefined);
+  manageQueue();
+}
+
 async function manageQueue() {
   if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) {
     inProgress++;
 
-    const timeout = setTimeout(() => {
-      inProgress--;
-      manageQueue();
-    }, 1000);
-
+    const timeout = setTimeout(decrement, 1000);
     const request = queue.shift()!;
     const data = (await patientFetch(request.url)
       .then((response) => response.json())
@@ -42,21 +45,15 @@ async function manageQueue() {
       request.resolve(data.data);
     } else {
       if (RETRY_ERRORS.includes(data.error.message)) {
-        console.log(
-          `Encountered retry-able error ${data.error.code} ${data.error.message}, putting "${request.url}" back in queue...`,
-        );
-
         retryAbleBlitzFetchEvent.emit(undefined);
         clearTimeout(timeout);
         setTimeout(() => {
           queue.push(request);
-          inProgress--;
-          manageQueue();
+          decrement();
         }, 1000 / MAX_CALLS_PER_SECOND);
       } else {
         clearTimeout(timeout);
-        inProgress--;
-        manageQueue();
+        decrement();
 
         throw new Error(`Wargaming response error status: "${data.status}"`, {
           cause: `Message: "${data.error.message}"\nURL: "${request.url}"`,

src/core/blitzkit/averageDefinitions/constants.ts

@@ -1,5 +1,4 @@
 import { AverageDefinitionsAllStats } from '.';
-
 import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants';
 
 export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = {

src/core/blitzkit/averageDefinitions/index.ts

@@ -1,20 +1,39 @@
 import { AllStats } from '../../blitz/getAccountInfo';
+import { decode } from '../../protobuf/decode';
 import { asset } from '../asset';
-import { fetchCdonLz4 } from '../fetchCdonLz4';
 
 export interface AverageDefinitionsAllStats extends AllStats {
   battle_life_time: number;
 }
 
-export interface AverageDefinitions {
-  [id: number]: {
-    n: number; // sample size
-    r: AverageDefinitionsAllStats; // correlation coefficient
-    mu: AverageDefinitionsAllStats; // mean
-    sigma: AverageDefinitionsAllStats; // standard deviation
-  };
+export interface AverageDefinitionsEntry {
+  samples: number;
+  mu: AverageDefinitionsAllStats;
+  sigma: AverageDefinitionsAllStats;
+  r: AverageDefinitionsAllStats;
 }
 
-export const averageDefinitions = fetchCdonLz4<AverageDefinitions>(
-  asset('definitions/averages.cdon.lz4'),
-);
+export interface AverageDefinitionsEntrySubPartial {
+  samples: number;
+  mu: Partial<AverageDefinitionsAllStats>;
+  sigma: Partial<AverageDefinitionsAllStats>;
+  r: Partial<AverageDefinitionsAllStats>;
+}
+
+export interface AverageDefinitions {
+  [id: number]: AverageDefinitionsEntry;
+}
+
+interface AverageDefinitionsProto {
+  averages: AverageDefinitions;
+}
+
+export const averageDefinitions = fetch(asset('definitions/averages.pb'))
+  .then((response) => response.arrayBuffer())
+  .then((buffer) => {
+    return decode<AverageDefinitionsProto>(
+      'blitzkit.AverageDefinitions',
+      new Uint8Array(buffer),
+    );
+  })
+  .then((data) => data.averages);

src/core/blitzkit/commitAssets.ts

@@ -5,7 +5,6 @@ export async function commitAssets(
   message: string,
   changes: FileChange[],
   production: boolean,
-  verbose = true,
 ) {
   console.log(`Committing ${message}...`);
 
@@ -17,6 +16,5 @@ export async function commitAssets(
     production ? 'main' : 'dev',
     `${message} - ${new Date().toDateString()}`,
     changes,
-    verbose,
   );
 }

src/core/blitzkit/commitMultipleFiles.ts

@@ -14,7 +14,6 @@ export default async function commitMultipleFiles(
   branch: string,
   message: string,
   changes: FileChange[],
-  verbose = false,
 ) {
   const latestCommitSha = (
     await octokit.git.getRef({

src/core/blitzkit/fetchPreDiscoveredIds.ts

@@ -1,37 +1,35 @@
 import { times } from 'lodash';
 import { decompress } from 'lz4js';
+import ProgressBar from 'progress';
 import { DiscoverIdsManifest } from '../../../scripts/discoverIds';
 import { DidsReadStream } from '../streams/dids';
 import { asset } from './asset';
 
 export async function fetchPreDiscoveredIds(dev: boolean) {
-  let ids: number[] = [];
-  const preDiscoveredManifest = (await fetch(
-    asset('ids/manifest.json', dev),
-  ).then((response) => response.json())) as DiscoverIdsManifest;
-
-  console.log(
-    `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
+  const idChunks: number[][] = [];
+  const manifest = (await fetch(asset('ids/manifest.json', dev)).then(
+    (response) => response.json(),
+  )) as DiscoverIdsManifest;
+  const bar = new ProgressBar(
+    `Fetching ${manifest.count.toLocaleString()} ids in ${manifest.chunks} chunks :bar`,
+    manifest.chunks,
   );
 
+  bar.render();
   await Promise.all(
-    times(preDiscoveredManifest.chunks, async (chunkIndex) => {
+    times(manifest.chunks, async (chunkIndex) => {
       const preDiscovered = await fetch(
         asset(`ids/${chunkIndex}.dids.lz4`, dev),
       ).then(async (response) => {
         const buffer = await response.arrayBuffer();
         const decompressed = decompress(new Uint8Array(buffer)).buffer;
-        return new DidsReadStream(decompressed).dids();
+        return new DidsReadStream(decompressed as ArrayBuffer).dids();
       });
 
-      // no spread syntax: https://github.com/oven-sh/bun/issues/11734
-      ids = ids.concat(preDiscovered);
-
-      console.log(
-        `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
-      );
+      idChunks.push(preDiscovered);
+      bar.tick();
     }),
   );
 
-  return ids;
+  return idChunks.sort((a, b) => a[0] - b[0]).flat();
 }

src/core/protobuf/decode.ts

@@ -0,0 +1,6 @@
+import { lookup } from './lookup';
+
+export async function decode<Type>(proto: string, data: Uint8Array) {
+  const Message = await lookup(proto);
+  return Message.decode(data).toJSON() as Type;
+}

src/core/protobuf/encode.ts

@@ -0,0 +1,6 @@
+import { lookup } from './lookup';
+
+export async function encode(proto: string, data: any) {
+  const Message = await lookup(proto);
+  return Message.encode(Message.create(data)).finish();
+}

src/core/protobuf/encodeToBase64.ts

@@ -0,0 +1,6 @@
+import { encode } from './encode';
+
+export async function encodeToBase64(proto: string, data: any) {
+  const array = await encode(proto, data);
+  return Buffer.from(array).toString('base64');
+}

src/core/protobuf/lookup.ts

@@ -0,0 +1,10 @@
+import { Root } from 'protobufjs';
+
+export async function lookup(proto: string) {
+  const root = await new Root().load(
+    `src/protos/${proto.split('.')[0]}.proto`,
+    { keepCase: true },
+  );
+
+  return root.lookupType(proto);
+}