1 sec bro

packages/scripts/package.json

@@ -41,7 +41,7 @@
     "setup:depotdownloader": "bun src/setupDepotDownloader.ts",
     "setup:patches": "bun src/applyPatches.ts",
     "build:reviews": "bun src/reviews.ts",
-    "build:ids": "bun src/discoverIds.ts",
+    "build:ids": "bun src/discoverIds/index.ts",
     "build:ratingleaderboard": "bun src/buildRatingLeaderboard.ts",
     "build:proxyclient": "bun src/buildProxyClient.ts",
     "build:catalogitemaccesssor": "bun src/buildCatalogItemAccessor.ts",

packages/scripts/src/discoverIds/idArray.ts

@@ -1,5 +1,72 @@
 export class IdArray {
   constructor(
-    private array: Uint32Array<ArrayBufferLike> = new Uint32Array(),
+    private array: Uint32Array = new Uint32Array(),
+    private _size = array.length,
   ) {}
+
+  get(index: number) {
+    return this.array[index];
+  }
+
+  push(id: number) {
+    if (this._size === this.array.length) {
+      const bigger = new Uint32Array(
+        this.array.length === 0 ? 1 : this.array.length * 2,
+      );
+
+      bigger.set(this.array);
+      this.array = bigger;
+    }
+
+    this.array[this._size++] = id;
+  }
+
+  sort() {
+    this.array.subarray(0, this._size).sort();
+  }
+
+  size() {
+    return this._size;
+  }
+
+  toBytes() {
+    return new Uint8Array(
+      this.array.buffer,
+      this.array.byteOffset,
+      this._size * Uint32Array.BYTES_PER_ELEMENT,
+    );
+  }
+
+  toArray() {
+    return Array.from(this.array.subarray(0, this._size));
+  }
+
+  static fromBytes(bytes: Uint8Array) {
+    if (bytes.byteLength % Uint32Array.BYTES_PER_ELEMENT !== 0) {
+      throw new Error("Misaligned bytes");
+    }
+
+    const size = bytes.byteLength / Uint32Array.BYTES_PER_ELEMENT;
+    const capacity = this.nextPowerOf2(size);
+
+    const values = new Uint32Array(bytes.buffer, bytes.byteOffset, size);
+
+    const uintArray = new Uint32Array(capacity);
+    uintArray.set(values);
+
+    return new IdArray(uintArray, size);
+  }
+
+  static fromArray(array: number[]) {
+    const uintArray = new Uint32Array(this.nextPowerOf2(array.length));
+
+    uintArray.set(array);
+
+    return new IdArray(uintArray, array.length);
+  }
+
+  private static nextPowerOf2(value: number): number {
+    if (value <= 1) return 1;
+    return 2 ** Math.ceil(Math.log2(value));
+  }
 }

packages/scripts/src/discoverIds/index.ts

@@ -1,5 +1,5 @@
 import { $ } from "bun";
-import { uncompress } from "lz4-napi";
+import { compress, uncompress } from "lz4-napi";
 import { mkdir, rm } from "node:fs/promises";
 import { IdArray } from "./idArray";
 
@@ -16,9 +16,10 @@ const REPO = "blitzkit/ids";
 const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
 const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
 
-const chunkCount = Math.ceil(
-  (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
-);
+// const chunkCount = Math.ceil(
+//   (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
+// );
+const chunkCount = 2;
 
 console.log(
   `${chunkCount.toLocaleString()} will be needed for ${ACCOMMODATED_ID_COUNT.toLocaleString()} ids`,
@@ -27,11 +28,14 @@ console.log(
 await mkdir(TEMP, { recursive: true });
 await rm(WORKING_DIR, { recursive: true, force: true });
 
-await $`git clone https://github.com/${REPO} ${WORKING_DIR}`;
+console.log(`Cloning ${REPO} to ${WORKING_DIR}`);
 
+await $`git clone --depth 1 https://github.com/${REPO} ${WORKING_DIR}`;
 await mkdir(CHUNKS_DIR, { recursive: true });
 
-const discoveredChunks: IdArray[] = [];
+console.log(`Discovering pre-existing chunks...`);
+
+const discoveredChunks: (IdArray | null)[] = [];
 
 for (let i = 0; true; i++) {
   const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`;
@@ -42,22 +46,54 @@ for (let i = 0; true; i++) {
   const bytes = await file.bytes();
   const uncompressed = await uncompress(bytes);
 
-  const array = new Uint32Array(
-    uncompressed.buffer,
-    uncompressed.byteOffset,
-    uncompressed.byteLength / Uint32Array.BYTES_PER_ELEMENT,
-  );
-  const ids = new IdArray(array);
+  const ids = IdArray.fromBytes(new Uint8Array(uncompressed));
 
   discoveredChunks.push(ids);
 }
 
+console.log(`Found ${discoveredChunks.length} pre-existing chunks`);
+
 let chunks: IdArray[] = [];
 
 if (discoveredChunks.length === chunkCount) {
-  chunks = discoveredChunks;
+  console.log("Pre-existing chunking is consistent, passing as-is...");
+
+  chunks = discoveredChunks as IdArray[];
 } else {
+  console.log("Pre-existing chunking is not consistent, re-chunking...");
+
   chunks = Array.from({ length: chunkCount }).map(() => new IdArray());
+
+  for (let i = 0; i < discoveredChunks.length; i++) {
+    const ids = discoveredChunks[i];
+    const size = ids!.size();
+
+    for (let j = 0; j < size; j++) {
+      const id = ids!.get(j);
+      const chunk = chunks[id % chunkCount];
+
+      chunk.push(id);
+    }
+
+    discoveredChunks[i] = null;
+  }
+
+  await rm(`${CHUNKS_DIR}/*`, { force: true });
+}
+
+console.log("Sorting chunks...");
+
+for (let i = 0; i < chunks.length; i++) chunks[i].sort();
+
+console.log("Saving chunks...");
+
+for (let i = 0; i < chunks.length; i++) {
+  const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`;
+  const chunk = chunks[i];
+  const bytes = chunk.toBytes();
+  const compressed = await compress(bytes);
+
+  await Bun.write(path, compressed);
 }
 
 // const MAX_IDS = 100;