start of recording

packages/scripts/src/core/blitzkit/idArray.ts

@@ -1,3 +1,5 @@
+import { decompress } from "lz4js";
+
 export class IdArray {
   constructor(
     private array: Uint32Array = new Uint32Array(),
@@ -57,6 +59,11 @@ export class IdArray {
     return new IdArray(uintArray, size);
   }
 
+  static fromCompressed(bytes: Uint8Array) {
+    const decompressed = decompress(bytes);
+    return this.fromBytes(decompressed);
+  }
+
   static fromArray(array: number[]) {
     const uintArray = new Uint32Array(this.nextPowerOf2(array.length));
 

packages/scripts/src/discoverIds/index.ts

@@ -1,5 +1,5 @@
 import { range } from "lodash-es";
-import { compress, decompress } from "lz4js";
+import { compress } from "lz4js";
 import { exit } from "node:process";
 import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
 import { IdArray } from "../core/blitzkit/idArray";
@@ -28,8 +28,7 @@ const manifest = await storage.json<IdsManifest>("manifest.json");
 
 for (let i = 0; i < manifest.chunks; i++) {
   const bytes = await storage.bytes(`chunks/chunk-${i}.dat.lz4`);
-  const uncompressed = decompress(bytes);
+  const ids = IdArray.fromCompressed(bytes);
-  const ids = IdArray.fromBytes(uncompressed);
 
   discoveredChunks.push(ids);
 }

packages/scripts/src/recordPerformance/index.ts

@@ -1,4 +1,6 @@
 import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
+import { IdArray } from "../core/blitzkit/idArray";
+import { API_RATE } from "../discoverIds/constants";
 import { sleep, withinTimeLimit } from "../discoverIds/time";
 import { IdsManifest } from "../discoverIds/types";
 import { IDS_API_BASE } from "./constants";
@@ -16,7 +18,24 @@ const idsManifest = await fetch(`${IDS_API_BASE}/manifest.json`).then(
 );
 
 while (withinTimeLimit()) {
-  await sleep();
+  manifest.last_updated = (manifest.last_updated + 1) % idsManifest.chunks;
+
+  const url = `${IDS_API_BASE}/chunks/chunk-${manifest.last_updated}.dat.lz4`;
+
+  const response = await fetch(url);
+  const buffer = await response.arrayBuffer();
+  const bytes = new Uint8Array(buffer);
+
+  const ids = IdArray.fromCompressed(bytes);
+  const size = ids.size();
+
+  console.log(
+    `Chunk ${manifest.last_updated} contains ${size} ids, estimated to take ${(size / API_RATE / 60).toFixed(0)}min`,
+  );
+
+  for (let i = 0; i < size; i++) {
+    await sleep();
+  }
 }
 
 // storage.commit();