diff --git a/package.json b/package.json index e790351de..9c533bb44 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "dev": "cd packages/website && bun run dev", "build": "cd packages/website && bun run build", - "build:averages": "cd packages/scripts && bun run build:averages" + "build:averages": "cd packages/scripts && bun run build:averages", + "build:ids": "cd packages/scripts && bun run build:ids" }, "type": "module", "workspaces": [ diff --git a/packages/scripts/src/discoverIds.ts b/packages/scripts/src/discoverIds.ts index 27d5ac36c..8bb59e588 100644 --- a/packages/scripts/src/discoverIds.ts +++ b/packages/scripts/src/discoverIds.ts @@ -1,165 +1,87 @@ -import { - DidsReadStream, - DidsWriteStream, - DiscoveredIdsDefinitions, - MIN_REGION_IDS, - REGIONS, - Region, - asset, - getAccountInfo, - idToRegion, - retryAbleBlitzFetchEvent, -} from "@blitzkit/core"; -import { chunk, times, uniq } from "lodash-es"; -import { compress, decompress } from "lz4js"; -import { AssetUploader } from "./core/github/assetUploader"; +const MAX_IDS = 100; +const API_RATE = 10; -const CHUNK_SIZE = 2 ** 21; -const RUN_TIME = 1000 * 60 * 60 * 5.5; -const PROGRESS_UPDATE_FREQUENCY = 1000 * 60; -const MAX_REQUESTS = 10; -const ACCOUNTS_PER_CALL = 100; -const TERMINATION_THRESHOLD = 1000; -const startTime = Date.now(); -const indexableRegions = [...REGIONS]; -let ids: number[] = []; -const preDiscoveredManifest = (await fetch(asset("ids/manifest.json")).then( - (response) => response.json() -)) as DiscoveredIdsDefinitions; +const ACCOMMODATED_ID_COUNT = 10_000_000_000; +const BYTES_PER_ID = 64 / 8; +const MAX_BYTES_PER_BATCH = 100 * 1_000_000; -console.log( - `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...` +const regions = ["eu", "com", "asia"]; +const seeds = [5e8, 10e8, 20e8].map(BigInt); + +export interface QuickInfo { + status: string; + data: Record; +} + +const batches = BigInt( + Math.ceil((ACCOMMODATED_ID_COUNT * BYTES_PER_ID) / MAX_BYTES_PER_BATCH), ); +const ids: bigint[] = []; -let chunkIndex = 0; -while (chunkIndex < preDiscoveredManifest.chunks) { - const preDiscovered = await fetch(asset(`ids/${chunkIndex}.dids.lz4`)).then( - async (response) => { - const buffer = await response.arrayBuffer(); - const decompressed = decompress(new Uint8Array(buffer)).buffer; - return new DidsReadStream(decompressed as ArrayBuffer).dids(); +const t0 = performance.now(); +const maxT = 5 * 60 * 60 * 1000; +const t1 = t0 + maxT; + +const unfilled = Array.from({ length: seeds.length }); +const offsets = unfilled.map(() => 0n); +const queues = unfilled.map(() => [] as bigint[]); + +function mountIds() { + for (let i = 0; i < regions.length; i++) { + const queue = queues[i]; + + if (queue.length >= MAX_IDS) continue; + + const availableSpace = MAX_IDS - queue.length; + + const seed = seeds[i]; + const offset = offsets[i]++; + + for (let j = 0n; j < availableSpace; j++) { + const id = seed + offset + j; + queue.push(id); } - ); - // no spread syntax: https://github.com/oven-sh/bun/issues/11734 - ids = ids.concat(preDiscovered); - - console.log( - `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})` - ); - - chunkIndex++; + offsets[i] += BigInt(availableSpace); + } } -const regionalIdIndex: Record = { - asia: ids.findLast((id) => idToRegion(id) === "asia") ?? MIN_REGION_IDS.asia, - com: ids.findLast((id) => idToRegion(id) === "com") ?? MIN_REGION_IDS.com, - eu: ids.findLast((id) => idToRegion(id) === "eu") ?? MIN_REGION_IDS.eu, -}; -const zeroStreak: Record = { asia: 0, com: 0, eu: 0 }; -let regionIndex = 0; -let outOfTimeFlagged = false; +let i = 0; +while (performance.now() < t1) { + mountIds(); + await sleep(); -async function verify(region: Region, ids: number[]) { - const infos = await getAccountInfo(region, ids, undefined, { - // cheeky way of requesting nothing - fields: "account_id,-account_id", - }); - const filtered = ids.filter((_, index) => { - const info = infos[index]; - return info !== null; - }); + const region = regions[i]; + const queue = queues[i]; - return filtered; + const accountIds = queue.join(","); + const fields = "account_id%2C-account_id"; + const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID; + + const url = `https://api.wotblitz.${ + region + }/wotb/account/info/?application_id=${applicationId}&fields=${ + fields + }&account_id=${accountIds}`; + const response = await fetch(url); + const data = (await response.json()) as QuickInfo; + + if (data.status !== "ok") continue; + + for (const key in data.data) { + const value = data.data[key]; + + if (value === null) continue; + + const id = BigInt(key); + + ids.push(id); + } + + queues[i] = []; + i = (i + 1) % regions.length; } -let discardAttempts = 0; -let lastProgressUpdate = 0; - -retryAbleBlitzFetchEvent.on(() => discardAttempts++); - -const interval = setInterval(async () => { - if (discardAttempts > 0) return discardAttempts--; - - const region = indexableRegions[regionIndex]; - const idsToVerify = times( - ACCOUNTS_PER_CALL, - (index) => regionalIdIndex[region] + index - ); - const verified = await verify(region, idsToVerify); - - if (Date.now() - lastProgressUpdate > PROGRESS_UPDATE_FREQUENCY) { - lastProgressUpdate = Date.now(); - console.log(`Discovered ${ids.length} ids in ${Date.now() - startTime}ms`); - } - - if (verified.length === 0) { - zeroStreak[region]++; - - if (zeroStreak[region] >= TERMINATION_THRESHOLD) { - const sliceableIndex = indexableRegions.indexOf(region); - - if (sliceableIndex !== -1) { - console.log(`Stopped discovery for ${region}`); - indexableRegions.splice(sliceableIndex, 1); - - if (indexableRegions.length === 0) { - console.log("All players discovered exhaustively!"); - await post(); - } - } - } - } else { - zeroStreak[region] = 0; - ids.push(...verified); - } - - if (Date.now() - startTime > RUN_TIME && !outOfTimeFlagged) { - outOfTimeFlagged = true; - console.log("Out of time"); - await post(); - } - - regionIndex = (regionIndex + 1) % indexableRegions.length; - regionalIdIndex[region] += ACCOUNTS_PER_CALL; -}, 1000 / MAX_REQUESTS); - -async function post() { - clearInterval(interval); - ids = uniq(ids).sort((a, b) => a - b); - console.log(`Uploading ${ids.length} ids...`); - - using uploader = new AssetUploader("discovered ids"); - const idsChunked = chunk(ids, CHUNK_SIZE); - - let index = 0; - for (const chunk of idsChunked) { - const didsWriteStream = new DidsWriteStream().dids(chunk); - const buffer = didsWriteStream.uint8Array; - const content = compress(buffer); - - console.log( - `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})` - ); - - await uploader.add({ - content, - path: `ids/${index}.dids.lz4`, - }); - - index++; - } - - await uploader.add({ - content: new TextEncoder().encode( - JSON.stringify({ - chunks: idsChunked.length, - count: ids.length, - time: Date.now(), - } satisfies DiscoveredIdsDefinitions) - ), - path: "ids/manifest.json", - }); - - await uploader.flush(); +function sleep() { + return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE)); }