chunking based on fetch rate

packages/scripts/src/discoverIds/constants.ts

@@ -1,6 +1,9 @@
-export const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
-export const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
-
-export const MAX_IDS_PER_CALL = 100;
 export const API_RATE = 10; // 10Hz
+export const MAX_IDS_PER_CALL = 100;
 export const MAX_DRY_STREAK = 10_000;
+
+export const ACCOMMODATED_ID_COUNT = 500_000_000; // 500M ids
+
+const PERFORMANCE_RUN_TIME = (5 * 60 + 30) * 60;
+export const IDS_PER_CHUNK = PERFORMANCE_RUN_TIME * API_RATE;
+export const CHUNK_COUNT = Math.ceil(ACCOMMODATED_ID_COUNT / IDS_PER_CHUNK);

packages/scripts/src/discoverIds/index.ts

@@ -3,12 +3,7 @@ import { compress } from "lz4js";
 import { exit } from "node:process";
 import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
 import { IdArray } from "../core/blitzkit/idArray";
-import {
+import { CHUNK_COUNT, MAX_DRY_STREAK, MAX_IDS_PER_CALL } from "./constants";
-  ACCOMMODATED_ID_COUNT,
-  MAX_BYTES_PER_CHUNK,
-  MAX_DRY_STREAK,
-  MAX_IDS_PER_CALL,
-} from "./constants";
 import { sleep, withinTimeLimit } from "./time";
 import { IdsManifest, QuickInfo, RegionDescriptor } from "./types";
 
@@ -35,9 +30,7 @@ for (let i = 0; i < manifest.chunks; i++) {
 
 console.log(`Found ${manifest.chunks} pre-existing chunks`);
 
-manifest.chunks = Math.ceil(
+manifest.chunks = CHUNK_COUNT;
-  (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
-);
 
 let chunks: IdArray[] = [];