diff --git a/packages/scripts/package.json b/packages/scripts/package.json index ca48d6942..8f3dac71f 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -41,7 +41,7 @@ "setup:depotdownloader": "bun src/setupDepotDownloader.ts", "setup:patches": "bun src/applyPatches.ts", "build:reviews": "bun src/reviews.ts", - "build:ids": "bun src/discoverIds.ts", + "build:ids": "bun src/discoverIds/index.ts", "build:ratingleaderboard": "bun src/buildRatingLeaderboard.ts", "build:proxyclient": "bun src/buildProxyClient.ts", "build:catalogitemaccesssor": "bun src/buildCatalogItemAccessor.ts", diff --git a/packages/scripts/src/discoverIds/idArray.ts b/packages/scripts/src/discoverIds/idArray.ts index a9800c71a..05a043891 100644 --- a/packages/scripts/src/discoverIds/idArray.ts +++ b/packages/scripts/src/discoverIds/idArray.ts @@ -1,5 +1,72 @@ export class IdArray { constructor( - private array: Uint32Array = new Uint32Array(), + private array: Uint32Array = new Uint32Array(), + private _size = array.length, ) {} + + get(index: number) { + return this.array[index]; + } + + push(id: number) { + if (this._size === this.array.length) { + const bigger = new Uint32Array( + this.array.length === 0 ? 1 : this.array.length * 2, + ); + + bigger.set(this.array); + this.array = bigger; + } + + this.array[this._size++] = id; + } + + sort() { + this.array.subarray(0, this._size).sort(); + } + + size() { + return this._size; + } + + toBytes() { + return new Uint8Array( + this.array.buffer, + this.array.byteOffset, + this._size * Uint32Array.BYTES_PER_ELEMENT, + ); + } + + toArray() { + return Array.from(this.array.subarray(0, this._size)); + } + + static fromBytes(bytes: Uint8Array) { + if (bytes.byteLength % Uint32Array.BYTES_PER_ELEMENT !== 0) { + throw new Error("Misaligned bytes"); + } + + const size = bytes.byteLength / Uint32Array.BYTES_PER_ELEMENT; + const capacity = this.nextPowerOf2(size); + + const values = new Uint32Array(bytes.buffer, bytes.byteOffset, size); + + const uintArray = new Uint32Array(capacity); + uintArray.set(values); + + return new IdArray(uintArray, size); + } + + static fromArray(array: number[]) { + const uintArray = new Uint32Array(this.nextPowerOf2(array.length)); + + uintArray.set(array); + + return new IdArray(uintArray, array.length); + } + + private static nextPowerOf2(value: number): number { + if (value <= 1) return 1; + return 2 ** Math.ceil(Math.log2(value)); + } } diff --git a/packages/scripts/src/discoverIds/index.ts b/packages/scripts/src/discoverIds/index.ts index 9eb6e6603..f6a19b301 100644 --- a/packages/scripts/src/discoverIds/index.ts +++ b/packages/scripts/src/discoverIds/index.ts @@ -1,5 +1,5 @@ import { $ } from "bun"; -import { uncompress } from "lz4-napi"; +import { compress, uncompress } from "lz4-napi"; import { mkdir, rm } from "node:fs/promises"; import { IdArray } from "./idArray"; @@ -16,9 +16,10 @@ const REPO = "blitzkit/ids"; const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB -const chunkCount = Math.ceil( - (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK, -); +// const chunkCount = Math.ceil( +// (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK, +// ); +const chunkCount = 2; console.log( `${chunkCount.toLocaleString()} will be needed for ${ACCOMMODATED_ID_COUNT.toLocaleString()} ids`, @@ -27,11 +28,14 @@ console.log( await mkdir(TEMP, { recursive: true }); await rm(WORKING_DIR, { recursive: true, force: true }); -await $`git clone https://github.com/${REPO} ${WORKING_DIR}`; +console.log(`Cloning ${REPO} to ${WORKING_DIR}`); +await $`git clone --depth 1 https://github.com/${REPO} ${WORKING_DIR}`; await mkdir(CHUNKS_DIR, { recursive: true }); -const discoveredChunks: IdArray[] = []; +console.log(`Discovering pre-existing chunks...`); + +const discoveredChunks: (IdArray | null)[] = []; for (let i = 0; true; i++) { const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`; @@ -42,22 +46,54 @@ for (let i = 0; true; i++) { const bytes = await file.bytes(); const uncompressed = await uncompress(bytes); - const array = new Uint32Array( - uncompressed.buffer, - uncompressed.byteOffset, - uncompressed.byteLength / Uint32Array.BYTES_PER_ELEMENT, - ); - const ids = new IdArray(array); + const ids = IdArray.fromBytes(new Uint8Array(uncompressed)); discoveredChunks.push(ids); } +console.log(`Found ${discoveredChunks.length} pre-existing chunks`); + let chunks: IdArray[] = []; if (discoveredChunks.length === chunkCount) { - chunks = discoveredChunks; + console.log("Pre-existing chunking is consistent, passing as-is..."); + + chunks = discoveredChunks as IdArray[]; } else { + console.log("Pre-existing chunking is not consistent, re-chunking..."); + chunks = Array.from({ length: chunkCount }).map(() => new IdArray()); + + for (let i = 0; i < discoveredChunks.length; i++) { + const ids = discoveredChunks[i]; + const size = ids!.size(); + + for (let j = 0; j < size; j++) { + const id = ids!.get(j); + const chunk = chunks[id % chunkCount]; + + chunk.push(id); + } + + discoveredChunks[i] = null; + } + + await rm(`${CHUNKS_DIR}/*`, { force: true }); +} + +console.log("Sorting chunks..."); + +for (let i = 0; i < chunks.length; i++) chunks[i].sort(); + +console.log("Saving chunks..."); + +for (let i = 0; i < chunks.length; i++) { + const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`; + const chunk = chunks[i]; + const bytes = chunk.toBytes(); + const compressed = await compress(bytes); + + await Bun.write(path, compressed); } // const MAX_IDS = 100;