more modern way of handling id discovery

package.json

@@ -6,7 +6,8 @@
   "scripts": {
     "dev": "cd packages/website && bun run dev",
     "build": "cd packages/website && bun run build",
-    "build:averages": "cd packages/scripts && bun run build:averages"
+    "build:averages": "cd packages/scripts && bun run build:averages",
+    "build:ids": "cd packages/scripts && bun run build:ids"
   },
   "type": "module",
   "workspaces": [

packages/scripts/src/discoverIds.ts

@@ -1,165 +1,87 @@
-import {
+const MAX_IDS = 100;
-  DidsReadStream,
+const API_RATE = 10;
-  DidsWriteStream,
-  DiscoveredIdsDefinitions,
-  MIN_REGION_IDS,
-  REGIONS,
-  Region,
-  asset,
-  getAccountInfo,
-  idToRegion,
-  retryAbleBlitzFetchEvent,
-} from "@blitzkit/core";
-import { chunk, times, uniq } from "lodash-es";
-import { compress, decompress } from "lz4js";
-import { AssetUploader } from "./core/github/assetUploader";
 
-const CHUNK_SIZE = 2 ** 21;
+const ACCOMMODATED_ID_COUNT = 10_000_000_000;
-const RUN_TIME = 1000 * 60 * 60 * 5.5;
+const BYTES_PER_ID = 64 / 8;
-const PROGRESS_UPDATE_FREQUENCY = 1000 * 60;
+const MAX_BYTES_PER_BATCH = 100 * 1_000_000;
-const MAX_REQUESTS = 10;
-const ACCOUNTS_PER_CALL = 100;
-const TERMINATION_THRESHOLD = 1000;
-const startTime = Date.now();
-const indexableRegions = [...REGIONS];
-let ids: number[] = [];
-const preDiscoveredManifest = (await fetch(asset("ids/manifest.json")).then(
-  (response) => response.json()
-)) as DiscoveredIdsDefinitions;
 
-console.log(
+const regions = ["eu", "com", "asia"];
-  `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`
+const seeds = [5e8, 10e8, 20e8].map(BigInt);
+
+export interface QuickInfo {
+  status: string;
+  data: Record<string, {} | null>;
+}
+
+const batches = BigInt(
+  Math.ceil((ACCOMMODATED_ID_COUNT * BYTES_PER_ID) / MAX_BYTES_PER_BATCH),
 );
+const ids: bigint[] = [];
 
-let chunkIndex = 0;
+const t0 = performance.now();
-while (chunkIndex < preDiscoveredManifest.chunks) {
+const maxT = 5 * 60 * 60 * 1000;
-  const preDiscovered = await fetch(asset(`ids/${chunkIndex}.dids.lz4`)).then(
+const t1 = t0 + maxT;
-    async (response) => {
-      const buffer = await response.arrayBuffer();
-      const decompressed = decompress(new Uint8Array(buffer)).buffer;
-      return new DidsReadStream(decompressed as ArrayBuffer).dids();
-    }
-  );
 
-  // no spread syntax: https://github.com/oven-sh/bun/issues/11734
+const unfilled = Array.from({ length: seeds.length });
-  ids = ids.concat(preDiscovered);
+const offsets = unfilled.map(() => 0n);
+const queues = unfilled.map(() => [] as bigint[]);
 
-  console.log(
+function mountIds() {
-    `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`
+  for (let i = 0; i < regions.length; i++) {
-  );
+    const queue = queues[i];
 
-  chunkIndex++;
+    if (queue.length >= MAX_IDS) continue;
+
+    const availableSpace = MAX_IDS - queue.length;
+
+    const seed = seeds[i];
+    const offset = offsets[i]++;
+
+    for (let j = 0n; j < availableSpace; j++) {
+      const id = seed + offset + j;
+      queue.push(id);
     }
 
-const regionalIdIndex: Record<Region, number> = {
+    offsets[i] += BigInt(availableSpace);
-  asia: ids.findLast((id) => idToRegion(id) === "asia") ?? MIN_REGION_IDS.asia,
-  com: ids.findLast((id) => idToRegion(id) === "com") ?? MIN_REGION_IDS.com,
-  eu: ids.findLast((id) => idToRegion(id) === "eu") ?? MIN_REGION_IDS.eu,
-};
-const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
-let regionIndex = 0;
-let outOfTimeFlagged = false;
-
-async function verify(region: Region, ids: number[]) {
-  const infos = await getAccountInfo(region, ids, undefined, {
-    // cheeky way of requesting nothing
-    fields: "account_id,-account_id",
-  });
-  const filtered = ids.filter((_, index) => {
-    const info = infos[index];
-    return info !== null;
-  });
-
-  return filtered;
-}
-
-let discardAttempts = 0;
-let lastProgressUpdate = 0;
-
-retryAbleBlitzFetchEvent.on(() => discardAttempts++);
-
-const interval = setInterval(async () => {
-  if (discardAttempts > 0) return discardAttempts--;
-
-  const region = indexableRegions[regionIndex];
-  const idsToVerify = times(
-    ACCOUNTS_PER_CALL,
-    (index) => regionalIdIndex[region] + index
-  );
-  const verified = await verify(region, idsToVerify);
-
-  if (Date.now() - lastProgressUpdate > PROGRESS_UPDATE_FREQUENCY) {
-    lastProgressUpdate = Date.now();
-    console.log(`Discovered ${ids.length} ids in ${Date.now() - startTime}ms`);
-  }
-
-  if (verified.length === 0) {
-    zeroStreak[region]++;
-
-    if (zeroStreak[region] >= TERMINATION_THRESHOLD) {
-      const sliceableIndex = indexableRegions.indexOf(region);
-
-      if (sliceableIndex !== -1) {
-        console.log(`Stopped discovery for ${region}`);
-        indexableRegions.splice(sliceableIndex, 1);
-
-        if (indexableRegions.length === 0) {
-          console.log("All players discovered exhaustively!");
-          await post();
   }
 }
-    }
+
-  } else {
+let i = 0;
-    zeroStreak[region] = 0;
+while (performance.now() < t1) {
-    ids.push(...verified);
+  mountIds();
+  await sleep();
+
+  const region = regions[i];
+  const queue = queues[i];
+
+  const accountIds = queue.join(",");
+  const fields = "account_id%2C-account_id";
+  const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
+
+  const url = `https://api.wotblitz.${
+    region
+  }/wotb/account/info/?application_id=${applicationId}&fields=${
+    fields
+  }&account_id=${accountIds}`;
+  const response = await fetch(url);
+  const data = (await response.json()) as QuickInfo;
+
+  if (data.status !== "ok") continue;
+
+  for (const key in data.data) {
+    const value = data.data[key];
+
+    if (value === null) continue;
+
+    const id = BigInt(key);
+
+    ids.push(id);
   }
 
-  if (Date.now() - startTime > RUN_TIME && !outOfTimeFlagged) {
+  queues[i] = [];
-    outOfTimeFlagged = true;
+  i = (i + 1) % regions.length;
-    console.log("Out of time");
-    await post();
 }
 
-  regionIndex = (regionIndex + 1) % indexableRegions.length;
+function sleep() {
-  regionalIdIndex[region] += ACCOUNTS_PER_CALL;
+  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
-}, 1000 / MAX_REQUESTS);
-
-async function post() {
-  clearInterval(interval);
-  ids = uniq(ids).sort((a, b) => a - b);
-  console.log(`Uploading ${ids.length} ids...`);
-
-  using uploader = new AssetUploader("discovered ids");
-  const idsChunked = chunk(ids, CHUNK_SIZE);
-
-  let index = 0;
-  for (const chunk of idsChunked) {
-    const didsWriteStream = new DidsWriteStream().dids(chunk);
-    const buffer = didsWriteStream.uint8Array;
-    const content = compress(buffer);
-
-    console.log(
-      `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`
-    );
-
-    await uploader.add({
-      content,
-      path: `ids/${index}.dids.lz4`,
-    });
-
-    index++;
-  }
-
-  await uploader.add({
-    content: new TextEncoder().encode(
-      JSON.stringify({
-        chunks: idsChunked.length,
-        count: ids.length,
-        time: Date.now(),
-      } satisfies DiscoveredIdsDefinitions)
-    ),
-    path: "ids/manifest.json",
-  });
-
-  await uploader.flush();
 }