start of performance

packages/scripts/package.json

@@ -3,7 +3,8 @@
   "private": true,
   "scripts": {
     "discover-ids": "bun src/discoverIds/index.ts",
-    "build-protos": "bun src/protos.ts"
+    "build-protos": "bun src/protos.ts",
+    "record-performance": "bun src/recordPerformance/index.ts"
   },
   "dependencies": {
     "@blitzkit/closed": "workspace:*",

packages/scripts/src/discoverIds/constants.ts

@@ -1,6 +1,3 @@
-export const WORKING_DIR = `../../temp/ids`;
-export const REPO = "blitzkit/data-ids";
-
 export const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
 export const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
 

packages/scripts/src/discoverIds/index.ts

@@ -5,32 +5,26 @@ import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
 import { IdArray } from "../core/blitzkit/idArray";
 import {
   ACCOMMODATED_ID_COUNT,
-  API_RATE,
   MAX_BYTES_PER_CHUNK,
   MAX_DRY_STREAK,
   MAX_IDS_PER_CALL,
-  REPO,
-  WORKING_DIR,
 } from "./constants";
-import { ManifestV1, QuickInfo, RegionDescriptor } from "./types";
+import { sleep, withinTimeLimit } from "./time";
+import { IdsManifest, QuickInfo, RegionDescriptor } from "./types";
 
 const storage = await new GitObjectStorage(
-  REPO,
-  WORKING_DIR,
+  "blitzkit/data-ids",
+  "../../temp/ids",
   import.meta.env.GH_TOKEN,
 ).init();
 
-const t0 = performance.now();
-const maxT = (5 * 60 + 55) * 60 * 1000; // 5hr 55min
-const t1 = t0 + maxT;
-
 await storage.mkdir("chunks");
 
 console.log(`Discovering pre-existing chunks...`);
 
 const discoveredChunks: (IdArray | null)[] = [];
 
-const manifest = await storage.json<ManifestV1>("manifest.json");
+const manifest = await storage.json<IdsManifest>("manifest.json");
 
 for (let i = 0; i < manifest.chunks; i++) {
   const bytes = await storage.bytes(`chunks/chunk-${i}.dat.lz4`);
@@ -189,14 +183,6 @@ function fillQueue() {
   }
 }
 
-function sleep() {
-  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
-}
-
-function withinTimeLimit() {
-  return performance.now() < t1;
-}
-
 console.log("Starting discovery loop...");
 
 while (withinTimeLimit() && regions.length > 0) {

packages/scripts/src/discoverIds/time.ts

@@ -0,0 +1,14 @@
+import { API_RATE } from "./constants";
+
+const MAX_COMPUTE_TIME = (5 * 60 + 55) * 60 * 1000; // 5hr 55min
+
+const t0 = performance.now();
+const t1 = t0 + MAX_COMPUTE_TIME;
+
+export function withinTimeLimit() {
+  return performance.now() < t1;
+}
+
+export function sleep() {
+  return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
+}

packages/scripts/src/discoverIds/types.ts

@@ -1,4 +1,6 @@
-export interface ManifestV1 {
+export type IdsManifest = IdsManifestV1;
+
+interface IdsManifestV1 {
   version: 1;
   last_verified: number;
   chunks: number;

packages/scripts/src/recordPerformance/constants.ts

@@ -0,0 +1 @@
+export const IDS_API_BASE = "https://ids.blitzkit.app";

packages/scripts/src/recordPerformance/index.ts

@@ -0,0 +1,22 @@
+import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
+import { sleep, withinTimeLimit } from "../discoverIds/time";
+import { IdsManifest } from "../discoverIds/types";
+import { IDS_API_BASE } from "./constants";
+import { PerformanceManifest } from "./types";
+
+const storage = await new GitObjectStorage(
+  "blitzkit/data-performance",
+  "../../temp/performance",
+  import.meta.env.GH_TOKEN,
+).init();
+
+const manifest = await storage.json<PerformanceManifest>("manifest.json");
+const idsManifest = await fetch(`${IDS_API_BASE}/manifest.json`).then(
+  (response) => response.json() as Promise<IdsManifest>,
+);
+
+while (withinTimeLimit()) {
+  await sleep();
+}
+
+// storage.commit();

packages/scripts/src/recordPerformance/types.ts

@@ -0,0 +1,6 @@
+export type PerformanceManifest = PerformanceManifestV1;
+
+interface PerformanceManifestV1 {
+  version: 1;
+  last_updated: number;
+}