start of performance

package.json

@@ -7,7 +7,8 @@
     "dev": "cd packages/website && bun run dev",
     "build": "cd packages/website && bun run build",
     "discover-ids": "cd packages/scripts && bun run discover-ids",
-    "build-protos": "cd packages/scripts && bun run build-protos"
+    "build-protos": "cd packages/scripts && bun run build-protos",
+    "record-performance": "cd packages/scripts && bun run record-performance"
   },
   "type": "module",
   "workspaces": [

packages/core/src/streams/dids.ts

@@ -1,51 +0,0 @@
-import { times } from 'lodash-es';
-import { ReadStream, WriteStream } from './buffer';
-
-export class DidsReadStream extends ReadStream {
-  dids() {
-    this.magic();
-    this.header();
-    return this.body();
-  }
-
-  magic() {
-    const magic = this.utf8(4);
-    if (magic !== 'DIDS') {
-      throw new Error(`Invalid DIDS magic number: "${magic}"`);
-    }
-  }
-
-  header() {
-    return {
-      version: this.uint16(),
-    };
-  }
-
-  body() {
-    const count = this.uint32();
-    return times(count, () => this.uint32());
-  }
-}
-
-export class DidsWriteStream extends WriteStream {
-  dids(ids: number[]) {
-    this.magic();
-    this.header();
-    this.body(ids);
-
-    return this;
-  }
-
-  magic() {
-    this.utf8('DIDS');
-  }
-
-  header() {
-    this.uint16(1);
-  }
-
-  body(ids: number[]) {
-    this.uint32(ids.length);
-    ids.forEach((id) => this.uint32(id));
-  }
-}

packages/core/src/streams/index.ts

@@ -1,11 +1,9 @@
-export * from './buffer';
-export * from './bufferToBigInt';
-export * from './dds';
-export * from './dids';
-export * from './pvr';
-export * from './rts';
-export * from './sc2';
-export * from './scg';
-export * from './scpg';
-export * from './windows';
-
+export * from "./buffer";
+export * from "./bufferToBigInt";
+export * from "./dds";
+export * from "./pvr";
+export * from "./rts";
+export * from "./sc2";
+export * from "./scg";
+export * from "./scpg";
+export * from "./windows";

packages/scripts/package.json

@@ -3,7 +3,8 @@
   "private": true,
   "scripts": {
     "discover-ids": "bun src/discoverIds/index.ts",
-    "build-protos": "bun src/protos.ts"
+    "build-protos": "bun src/protos.ts",
+    "record-performance": "bun src/recordPerformance/index.ts"
   },
   "dependencies": {
     "@blitzkit/closed": "workspace:*",

packages/scripts/src/discoverIds/constants.ts

@@ -1,6 +1,3 @@
-export const WORKING_DIR = `../../temp/ids`;
-export const REPO = "blitzkit/data-ids";
-
 export const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
 export const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
 

packages/scripts/src/discoverIds/index.ts

@@ -5,32 +5,26 @@ import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
 import { IdArray } from "../core/blitzkit/idArray";
 import {
   ACCOMMODATED_ID_COUNT,
-  API_RATE,
   MAX_BYTES_PER_CHUNK,
   MAX_DRY_STREAK,
   MAX_IDS_PER_CALL,
-  REPO,
-  WORKING_DIR,
 } from "./constants";
-import { ManifestV1, QuickInfo, RegionDescriptor } from "./types";
+import { sleep, withinTimeLimit } from "./time";
+import { IdsManifest, QuickInfo, RegionDescriptor } from "./types";
 
 const storage = await new GitObjectStorage(
-  REPO,
-  WORKING_DIR,
+  "blitzkit/data-ids",
+  "../../temp/ids",
   import.meta.env.GH_TOKEN,
 ).init();
 
-const t0 = performance.now();
-const maxT = (5 * 60 + 55) * 60 * 1000; // 5hr 55min
-const t1 = t0 + maxT;
-
 await storage.mkdir("chunks");
 
 console.log(`Discovering pre-existing chunks...`);
 
 const discoveredChunks: (IdArray | null)[] = [];
 
-const manifest = await storage.json<ManifestV1>("manifest.json");
+const manifest = await storage.json<IdsManifest>("manifest.json");
 
 for (let i = 0; i < manifest.chunks; i++) {
   const bytes = await storage.bytes(`chunks/chunk-${i}.dat.lz4`);
@@ -189,14 +183,6 @@ function fillQueue() {
   }
 }
 
-function sleep() {
-  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
-}
-
-function withinTimeLimit() {
-  return performance.now() < t1;
-}
-
 console.log("Starting discovery loop...");
 
 while (withinTimeLimit() && regions.length > 0) {

packages/scripts/src/discoverIds/time.ts

@@ -0,0 +1,14 @@
+import { API_RATE } from "./constants";
+
+const MAX_COMPUTE_TIME = (5 * 60 + 55) * 60 * 1000; // 5hr 55min
+
+const t0 = performance.now();
+const t1 = t0 + MAX_COMPUTE_TIME;
+
+export function withinTimeLimit() {
+  return performance.now() < t1;
+}
+
+export function sleep() {
+  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
+}

packages/scripts/src/discoverIds/types.ts

@@ -1,4 +1,6 @@
-export interface ManifestV1 {
+export type IdsManifest = IdsManifestV1;
+
+interface IdsManifestV1 {
   version: 1;
   last_verified: number;
   chunks: number;

packages/scripts/src/recordPerformance/constants.ts

@@ -0,0 +1 @@
+export const IDS_API_BASE = "https://ids.blitzkit.app";

packages/scripts/src/recordPerformance/index.ts

@@ -0,0 +1,22 @@
+import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
+import { sleep, withinTimeLimit } from "../discoverIds/time";
+import { IdsManifest } from "../discoverIds/types";
+import { IDS_API_BASE } from "./constants";
+import { PerformanceManifest } from "./types";
+
+const storage = await new GitObjectStorage(
+  "blitzkit/data-performance",
+  "../../temp/performance",
+  import.meta.env.GH_TOKEN,
+).init();
+
+const manifest = await storage.json<PerformanceManifest>("manifest.json");
+const idsManifest = await fetch(`${IDS_API_BASE}/manifest.json`).then(
+  (response) => response.json() as Promise<IdsManifest>,
+);
+
+while (withinTimeLimit()) {
+  await sleep();
+}
+
+// storage.commit();

packages/scripts/src/recordPerformance/types.ts

@@ -0,0 +1,6 @@
+export type PerformanceManifest = PerformanceManifestV1;
+
+interface PerformanceManifestV1 {
+  version: 1;
+  last_updated: number;
+}