From 0f022f8295b2a3d99e461b047347a1c1483cdb7a Mon Sep 17 00:00:00 2001
From: tresabhi <43380238+tresabhi@users.noreply.github.com>
Date: Wed, 12 Aug 2026 20:33:38 -0500
Subject: [PATCH] chunking based on fetch rate

---
 packages/scripts/src/discoverIds/constants.ts | 11 +++++++----
 packages/scripts/src/discoverIds/index.ts     | 11 ++---------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/packages/scripts/src/discoverIds/constants.ts b/packages/scripts/src/discoverIds/constants.ts
index ee523a5e1..ef26eea0c 100644
--- a/packages/scripts/src/discoverIds/constants.ts
+++ b/packages/scripts/src/discoverIds/constants.ts
@@ -1,6 +1,9 @@
-export const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
-export const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
-
-export const MAX_IDS_PER_CALL = 100;
 export const API_RATE = 10; // 10Hz
+export const MAX_IDS_PER_CALL = 100;
 export const MAX_DRY_STREAK = 10_000;
+
+export const ACCOMMODATED_ID_COUNT = 500_000_000; // 500M ids
+
+const PERFORMANCE_RUN_TIME = (5 * 60 + 30) * 60;
+export const IDS_PER_CHUNK = PERFORMANCE_RUN_TIME * API_RATE;
+export const CHUNK_COUNT = Math.ceil(ACCOMMODATED_ID_COUNT / IDS_PER_CHUNK);
diff --git a/packages/scripts/src/discoverIds/index.ts b/packages/scripts/src/discoverIds/index.ts
index c28e2e6a3..b37278829 100644
--- a/packages/scripts/src/discoverIds/index.ts
+++ b/packages/scripts/src/discoverIds/index.ts
@@ -3,12 +3,7 @@ import { compress } from "lz4js";
 import { exit } from "node:process";
 import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
 import { IdArray } from "../core/blitzkit/idArray";
-import {
-  ACCOMMODATED_ID_COUNT,
-  MAX_BYTES_PER_CHUNK,
-  MAX_DRY_STREAK,
-  MAX_IDS_PER_CALL,
-} from "./constants";
+import { CHUNK_COUNT, MAX_DRY_STREAK, MAX_IDS_PER_CALL } from "./constants";
 import { sleep, withinTimeLimit } from "./time";
 import { IdsManifest, QuickInfo, RegionDescriptor } from "./types";
 
@@ -35,9 +30,7 @@ for (let i = 0; i < manifest.chunks; i++) {
 
 console.log(`Found ${manifest.chunks} pre-existing chunks`);
 
-manifest.chunks = Math.ceil(
-  (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
-);
+manifest.chunks = CHUNK_COUNT;
 
 let chunks: IdArray[] = [];