From b3e439669b313ee4ab91a9290c849759cae8b7c6 Mon Sep 17 00:00:00 2001 From: tresabhi <43380238+tresabhi@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:03:57 -0500 Subject: [PATCH] start of performance --- package.json | 3 +- packages/core/src/streams/dids.ts | 51 ------------------- packages/core/src/streams/index.ts | 20 ++++---- packages/scripts/package.json | 3 +- packages/scripts/src/discoverIds/constants.ts | 3 -- packages/scripts/src/discoverIds/index.ts | 24 ++------- packages/scripts/src/discoverIds/time.ts | 14 +++++ packages/scripts/src/discoverIds/types.ts | 4 +- .../src/recordPerformance/constants.ts | 1 + .../scripts/src/recordPerformance/index.ts | 22 ++++++++ .../scripts/src/recordPerformance/types.ts | 6 +++ 11 files changed, 64 insertions(+), 87 deletions(-) delete mode 100644 packages/core/src/streams/dids.ts create mode 100644 packages/scripts/src/discoverIds/time.ts create mode 100644 packages/scripts/src/recordPerformance/constants.ts create mode 100644 packages/scripts/src/recordPerformance/index.ts create mode 100644 packages/scripts/src/recordPerformance/types.ts diff --git a/package.json b/package.json index 2ed4ceb5f..340587062 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "dev": "cd packages/website && bun run dev", "build": "cd packages/website && bun run build", "discover-ids": "cd packages/scripts && bun run discover-ids", - "build-protos": "cd packages/scripts && bun run build-protos" + "build-protos": "cd packages/scripts && bun run build-protos", + "record-performance": "cd packages/scripts && bun run record-performance" }, "type": "module", "workspaces": [ diff --git a/packages/core/src/streams/dids.ts b/packages/core/src/streams/dids.ts deleted file mode 100644 index e2c70e042..000000000 --- a/packages/core/src/streams/dids.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { times } from 'lodash-es'; -import { ReadStream, WriteStream } from './buffer'; - -export class DidsReadStream extends ReadStream { - dids() { - this.magic(); - this.header(); - return this.body(); - } - - magic() { - const magic = this.utf8(4); - if (magic !== 'DIDS') { - throw new Error(`Invalid DIDS magic number: "${magic}"`); - } - } - - header() { - return { - version: this.uint16(), - }; - } - - body() { - const count = this.uint32(); - return times(count, () => this.uint32()); - } -} - -export class DidsWriteStream extends WriteStream { - dids(ids: number[]) { - this.magic(); - this.header(); - this.body(ids); - - return this; - } - - magic() { - this.utf8('DIDS'); - } - - header() { - this.uint16(1); - } - - body(ids: number[]) { - this.uint32(ids.length); - ids.forEach((id) => this.uint32(id)); - } -} diff --git a/packages/core/src/streams/index.ts b/packages/core/src/streams/index.ts index bd8949a2d..3215fdab9 100644 --- a/packages/core/src/streams/index.ts +++ b/packages/core/src/streams/index.ts @@ -1,11 +1,9 @@ -export * from './buffer'; -export * from './bufferToBigInt'; -export * from './dds'; -export * from './dids'; -export * from './pvr'; -export * from './rts'; -export * from './sc2'; -export * from './scg'; -export * from './scpg'; -export * from './windows'; - +export * from "./buffer"; +export * from "./bufferToBigInt"; +export * from "./dds"; +export * from "./pvr"; +export * from "./rts"; +export * from "./sc2"; +export * from "./scg"; +export * from "./scpg"; +export * from "./windows"; diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 393c95942..a401cc068 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -3,7 +3,8 @@ "private": true, "scripts": { "discover-ids": "bun src/discoverIds/index.ts", - "build-protos": "bun src/protos.ts" + "build-protos": "bun src/protos.ts", + "record-performance": "bun src/recordPerformance/index.ts" }, "dependencies": { "@blitzkit/closed": "workspace:*", diff --git a/packages/scripts/src/discoverIds/constants.ts b/packages/scripts/src/discoverIds/constants.ts index 022a3cce9..ee523a5e1 100644 --- a/packages/scripts/src/discoverIds/constants.ts +++ b/packages/scripts/src/discoverIds/constants.ts @@ -1,6 +1,3 @@ -export const WORKING_DIR = `../../temp/ids`; -export const REPO = "blitzkit/data-ids"; - export const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids export const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB diff --git a/packages/scripts/src/discoverIds/index.ts b/packages/scripts/src/discoverIds/index.ts index b47075e11..d7d09290d 100644 --- a/packages/scripts/src/discoverIds/index.ts +++ b/packages/scripts/src/discoverIds/index.ts @@ -5,32 +5,26 @@ import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage"; import { IdArray } from "../core/blitzkit/idArray"; import { ACCOMMODATED_ID_COUNT, - API_RATE, MAX_BYTES_PER_CHUNK, MAX_DRY_STREAK, MAX_IDS_PER_CALL, - REPO, - WORKING_DIR, } from "./constants"; -import { ManifestV1, QuickInfo, RegionDescriptor } from "./types"; +import { sleep, withinTimeLimit } from "./time"; +import { IdsManifest, QuickInfo, RegionDescriptor } from "./types"; const storage = await new GitObjectStorage( - REPO, - WORKING_DIR, + "blitzkit/data-ids", + "../../temp/ids", import.meta.env.GH_TOKEN, ).init(); -const t0 = performance.now(); -const maxT = (5 * 60 + 55) * 60 * 1000; // 5hr 55min -const t1 = t0 + maxT; - await storage.mkdir("chunks"); console.log(`Discovering pre-existing chunks...`); const discoveredChunks: (IdArray | null)[] = []; -const manifest = await storage.json("manifest.json"); +const manifest = await storage.json("manifest.json"); for (let i = 0; i < manifest.chunks; i++) { const bytes = await storage.bytes(`chunks/chunk-${i}.dat.lz4`); @@ -189,14 +183,6 @@ function fillQueue() { } } -function sleep() { - return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE)); -} - -function withinTimeLimit() { - return performance.now() < t1; -} - console.log("Starting discovery loop..."); while (withinTimeLimit() && regions.length > 0) { diff --git a/packages/scripts/src/discoverIds/time.ts b/packages/scripts/src/discoverIds/time.ts new file mode 100644 index 000000000..3f2afecad --- /dev/null +++ b/packages/scripts/src/discoverIds/time.ts @@ -0,0 +1,14 @@ +import { API_RATE } from "./constants"; + +const MAX_COMPUTE_TIME = (5 * 60 + 55) * 60 * 1000; // 5hr 55min + +const t0 = performance.now(); +const t1 = t0 + MAX_COMPUTE_TIME; + +export function withinTimeLimit() { + return performance.now() < t1; +} + +export function sleep() { + return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE)); +} diff --git a/packages/scripts/src/discoverIds/types.ts b/packages/scripts/src/discoverIds/types.ts index 31ad70b97..53eedba0b 100644 --- a/packages/scripts/src/discoverIds/types.ts +++ b/packages/scripts/src/discoverIds/types.ts @@ -1,4 +1,6 @@ -export interface ManifestV1 { +export type IdsManifest = IdsManifestV1; + +interface IdsManifestV1 { version: 1; last_verified: number; chunks: number; diff --git a/packages/scripts/src/recordPerformance/constants.ts b/packages/scripts/src/recordPerformance/constants.ts new file mode 100644 index 000000000..afbdd7c4c --- /dev/null +++ b/packages/scripts/src/recordPerformance/constants.ts @@ -0,0 +1 @@ +export const IDS_API_BASE = "https://ids.blitzkit.app"; diff --git a/packages/scripts/src/recordPerformance/index.ts b/packages/scripts/src/recordPerformance/index.ts new file mode 100644 index 000000000..fc3fba8cc --- /dev/null +++ b/packages/scripts/src/recordPerformance/index.ts @@ -0,0 +1,22 @@ +import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage"; +import { sleep, withinTimeLimit } from "../discoverIds/time"; +import { IdsManifest } from "../discoverIds/types"; +import { IDS_API_BASE } from "./constants"; +import { PerformanceManifest } from "./types"; + +const storage = await new GitObjectStorage( + "blitzkit/data-performance", + "../../temp/performance", + import.meta.env.GH_TOKEN, +).init(); + +const manifest = await storage.json("manifest.json"); +const idsManifest = await fetch(`${IDS_API_BASE}/manifest.json`).then( + (response) => response.json() as Promise, +); + +while (withinTimeLimit()) { + await sleep(); +} + +// storage.commit(); diff --git a/packages/scripts/src/recordPerformance/types.ts b/packages/scripts/src/recordPerformance/types.ts new file mode 100644 index 000000000..e5c12d10a --- /dev/null +++ b/packages/scripts/src/recordPerformance/types.ts @@ -0,0 +1,6 @@ +export type PerformanceManifest = PerformanceManifestV1; + +interface PerformanceManifestV1 { + version: 1; + last_updated: number; +}