feat: start on fixing pre work

src/core/blitz/fetchBlitz.ts

@@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch';
 const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE'];
 
 export const retryAbleBlitzFetchEvent = new EventManager();
+export const blitzFetchQueueAvailableEvent = new EventManager();
 
 type BlitzResponse<Data extends object> =
   | {
@@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10;
 const queue: { url: string; resolve: (data: object) => void }[] = [];
 let inProgress = 0;
 
+function decrement() {
+  inProgress--;
+  blitzFetchQueueAvailableEvent.emit(undefined);
+  manageQueue();
+}
+
 async function manageQueue() {
   if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) {
     inProgress++;
 
-    const timeout = setTimeout(() => {
-      inProgress--;
-      manageQueue();
-    }, 1000);
-
+    const timeout = setTimeout(() => decrement, 1000);
     const request = queue.shift()!;
     const data = (await patientFetch(request.url)
       .then((response) => response.json())
@@ -50,13 +53,11 @@ async function manageQueue() {
         clearTimeout(timeout);
         setTimeout(() => {
           queue.push(request);
-          inProgress--;
-          manageQueue();
+          decrement();
         }, 1000 / MAX_CALLS_PER_SECOND);
       } else {
         clearTimeout(timeout);
-        inProgress--;
-        manageQueue();
+        decrement();
 
         throw new Error(`Wargaming response error status: "${data.status}"`, {
           cause: `Message: "${data.error.message}"\nURL: "${request.url}"`,

src/core/blitzkit/fetchPreDiscoveredIds.ts

@@ -0,0 +1,37 @@
+import { times } from 'lodash';
+import { decompress } from 'lz4js';
+import { DiscoverIdsManifest } from '../../../scripts/discoverIds';
+import { DidsReadStream } from '../streams/dids';
+import { asset } from './asset';
+
+export async function fetchPreDiscoveredIds(dev: boolean) {
+  const idChunks: number[][] = [];
+  const preDiscoveredManifest = (await fetch(
+    asset('ids/manifest.json', dev),
+  ).then((response) => response.json())) as DiscoverIdsManifest;
+
+  console.log(
+    `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
+  );
+
+  await Promise.all(
+    times(preDiscoveredManifest.chunks, async (chunkIndex) => {
+      const preDiscovered = await fetch(
+        asset(`ids/${chunkIndex}.dids.lz4`, dev),
+      ).then(async (response) => {
+        const buffer = await response.arrayBuffer();
+        const decompressed = decompress(new Uint8Array(buffer)).buffer;
+        return new DidsReadStream(decompressed as ArrayBuffer).dids();
+      });
+
+      // no spread syntax: https://github.com/oven-sh/bun/issues/11734
+      idChunks.push(preDiscovered);
+
+      console.log(
+        `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
+      );
+    }),
+  );
+
+  return idChunks.sort((a, b) => a[0] - b[0]).flat();
+}