From 33012f9f211f65e8ae1abdf2efc41c9e2e76804d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Wed, 26 Jun 2024 21:22:03 -0500 Subject: [PATCH] feat: start on fixing pre work --- scripts/averages.ts | 140 ++++----------------- scripts/discoverIds.ts | 4 +- src/core/blitz/fetchBlitz.ts | 19 +-- src/core/blitzkit/fetchPreDiscoveredIds.ts | 37 ++++++ 4 files changed, 72 insertions(+), 128 deletions(-) create mode 100644 src/core/blitzkit/fetchPreDiscoveredIds.ts diff --git a/scripts/averages.ts b/scripts/averages.ts index 7bb4f0a0f..efe80c7e1 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,10 +1,8 @@ import { times } from 'lodash'; import { argv } from 'process'; -import ProgressBar from 'progress'; import { Region, REGIONS } from '../src/constants/regions'; -import getTankStats from '../src/core/blitz/getTankStats'; -import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion'; -import { asset } from '../src/core/blitzkit/asset'; +import { blitzFetchQueueAvailableEvent } from '../src/core/blitz/fetchBlitz'; +import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, AverageDefinitionsAllStats, @@ -13,9 +11,8 @@ import { } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; +import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; import { superCompress } from '../src/core/blitzkit/superCompress'; -import { DidsReadStream } from '../src/core/streams/dids'; -import { IndividualTankStats } from '../src/types/tanksStats'; interface DataPoint { x: number; @@ -27,120 +24,29 @@ const RUN_TIME = 1000 * 60 * 60; const MAX_REQUESTS = 10; const production = argv.includes('--production'); -const startTime = Date.now(); -const fields = 'all,tank_id,battle_life_time'; +const preDiscoveredIds = await fetchPreDiscoveredIds(!production); +const playerIds: Record = { + asia: preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'), + com: preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), + eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), +}; +const regionalIndices: Record = { + asia: playerIds.asia.length - 1, + com: playerIds.com.length - 1, + eu: playerIds.eu.length - 1, +}; +let regionIndex = 0; +const availableRegions = [...REGIONS]; -const players: IndividualTankStats[][] = []; -const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then( - async (response) => { - if (response.status === 200) { - const buffer = await response.arrayBuffer(); - return new DidsReadStream(buffer).dids(); - } +times(MAX_REQUESTS, () => { + validate(); + blitzFetchQueueAvailableEvent.on(validate); +}); - return undefined; - }, -); -const discoveredIds: number[] = []; -let processes = 0; +function validate() { + const region = availableRegions[regionIndex]; -if (preDiscovered) { - console.log( - `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`, - ); - - const preDiscoveredBar = new ProgressBar(':bar', { - total: preDiscovered.length, - }); - let preDiscoveredIndex = 0; - - async function chainPreDiscovery() { - processes++; - - if (preDiscoveredIndex === preDiscovered!.length) { - processes--; - - if (processes !== 0) return; - - console.log( - `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`, - ); - discover(); - - return; - } - - const id = preDiscovered![preDiscoveredIndex++]; - const region = idToRegion(id); - - if (id > MAX_IDS[region]) return; - - const stats = await getTankStats(region, id, { fields }); - - if (stats !== null && stats.length > 0) { - discoveredIds.push(id); - players.push(stats); - preDiscoveredBar.tick(); - } - - processes--; - setTimeout(chainPreDiscovery); // circumvent max call stack - } - - times(MAX_REQUESTS, chainPreDiscovery); -} else { - console.log('No pre-discovered ids found :('); - discover(); -} - -function discover() { - const index: Record = { - asia: - preDiscovered?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, - com: preDiscovered?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com, - eu: preDiscovered?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu, - }; - - let regionIndex = 0; - let done = 0; - - async function chainDiscovery() { - regionIndex = (regionIndex + 1) % REGIONS.length; - const region = REGIONS[regionIndex]; - const id = index[region]++; - - if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) { - if (++done !== MAX_REQUESTS) return; - - console.log( - `Total discovery run time: ${ - Date.now() - startTime - }ms\nDiscovered ids: ${discoveredIds.length}`, - ); - - postWork(); - - return; - } - - if (id > MAX_IDS[region]) return; - - const stats = await getTankStats(region, id, { fields }); - - if (stats !== null && stats.length > 0) { - discoveredIds.push(id); - players.push(stats); - - console.log( - `discovered ${id} in ${region} (time left: ${RUN_TIME - (Date.now() - startTime)}ms)`, - ); - } - - setTimeout(chainDiscovery); // circumvent max call stack - } - - console.log(`Spawning ${MAX_REQUESTS} discovery chains...`); - times(MAX_REQUESTS, chainDiscovery); + regionIndex = (regionIndex + 1) % availableRegions.length; } function postWork() { diff --git a/scripts/discoverIds.ts b/scripts/discoverIds.ts index 320e1cc9d..e2f79a0b4 100644 --- a/scripts/discoverIds.ts +++ b/scripts/discoverIds.ts @@ -9,7 +9,7 @@ import { asset } from '../src/core/blitzkit/asset'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; -interface DiscoverIdsManifest { +export interface DiscoverIdsManifest { time: number; chunks: number; count: number; @@ -44,7 +44,7 @@ while (chunkIndex < preDiscoveredManifest.chunks) { ).then(async (response) => { const buffer = await response.arrayBuffer(); const decompressed = decompress(new Uint8Array(buffer)).buffer; - return new DidsReadStream(decompressed).dids(); + return new DidsReadStream(decompressed as ArrayBuffer).dids(); }); // no spread syntax: https://github.com/oven-sh/bun/issues/11734 diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 76d0919c9..841b658ef 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch'; const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE']; export const retryAbleBlitzFetchEvent = new EventManager(); +export const blitzFetchQueueAvailableEvent = new EventManager(); type BlitzResponse = | { @@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10; const queue: { url: string; resolve: (data: object) => void }[] = []; let inProgress = 0; +function decrement() { + inProgress--; + blitzFetchQueueAvailableEvent.emit(undefined); + manageQueue(); +} + async function manageQueue() { if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) { inProgress++; - const timeout = setTimeout(() => { - inProgress--; - manageQueue(); - }, 1000); - + const timeout = setTimeout(() => decrement, 1000); const request = queue.shift()!; const data = (await patientFetch(request.url) .then((response) => response.json()) @@ -50,13 +53,11 @@ async function manageQueue() { clearTimeout(timeout); setTimeout(() => { queue.push(request); - inProgress--; - manageQueue(); + decrement(); }, 1000 / MAX_CALLS_PER_SECOND); } else { clearTimeout(timeout); - inProgress--; - manageQueue(); + decrement(); throw new Error(`Wargaming response error status: "${data.status}"`, { cause: `Message: "${data.error.message}"\nURL: "${request.url}"`, diff --git a/src/core/blitzkit/fetchPreDiscoveredIds.ts b/src/core/blitzkit/fetchPreDiscoveredIds.ts new file mode 100644 index 000000000..4441379a7 --- /dev/null +++ b/src/core/blitzkit/fetchPreDiscoveredIds.ts @@ -0,0 +1,37 @@ +import { times } from 'lodash'; +import { decompress } from 'lz4js'; +import { DiscoverIdsManifest } from '../../../scripts/discoverIds'; +import { DidsReadStream } from '../streams/dids'; +import { asset } from './asset'; + +export async function fetchPreDiscoveredIds(dev: boolean) { + const idChunks: number[][] = []; + const preDiscoveredManifest = (await fetch( + asset('ids/manifest.json', dev), + ).then((response) => response.json())) as DiscoverIdsManifest; + + console.log( + `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + ); + + await Promise.all( + times(preDiscoveredManifest.chunks, async (chunkIndex) => { + const preDiscovered = await fetch( + asset(`ids/${chunkIndex}.dids.lz4`, dev), + ).then(async (response) => { + const buffer = await response.arrayBuffer(); + const decompressed = decompress(new Uint8Array(buffer)).buffer; + return new DidsReadStream(decompressed as ArrayBuffer).dids(); + }); + + // no spread syntax: https://github.com/oven-sh/bun/issues/11734 + idChunks.push(preDiscovered); + + console.log( + `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, + ); + }), + ); + + return idChunks.sort((a, b) => a[0] - b[0]).flat(); +}