From ffaa368f8175a548dbbd9debf68a692628853668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:12:58 -0500 Subject: [PATCH] feat: finish script --- scripts/averages.ts | 145 +++++++++++++-------- src/core/blitz/fetchBlitz.ts | 6 +- src/core/blitzkit/fetchPreDiscoveredIds.ts | 22 ++-- 3 files changed, 103 insertions(+), 70 deletions(-) diff --git a/scripts/averages.ts b/scripts/averages.ts index efe80c7e1..40ea2904b 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,7 +1,8 @@ import { times } from 'lodash'; import { argv } from 'process'; import { Region, REGIONS } from '../src/constants/regions'; -import { blitzFetchQueueAvailableEvent } from '../src/core/blitz/fetchBlitz'; +import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; +import getTankStats from '../src/core/blitz/getTankStats'; import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, @@ -10,9 +11,7 @@ import { AverageDefinitionsEntrySubPartial, } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; -import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; -import { superCompress } from '../src/core/blitzkit/superCompress'; interface DataPoint { x: number; @@ -20,9 +19,13 @@ interface DataPoint { w: number; } -const RUN_TIME = 1000 * 60 * 60; -const MAX_REQUESTS = 10; +const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 120; +const MIN_BATTLES = 5000; +const RUN_TIME = 1000 * 60; +const THREADS = 10; +const PLAYER_IDS_PER_CALL = 100; +const startTime = Date.now(); const production = argv.includes('--production'); const preDiscoveredIds = await fetchPreDiscoveredIds(!production); const playerIds: Record = { @@ -31,54 +34,88 @@ const playerIds: Record = { eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), }; const regionalIndices: Record = { - asia: playerIds.asia.length - 1, - com: playerIds.com.length - 1, - eu: playerIds.eu.length - 1, + asia: 0, + com: 0, + eu: 0, }; let regionIndex = 0; +let playerCount = 0; const availableRegions = [...REGIONS]; +const tankIds: number[] = []; +const tanksSorted: Record = {}; +let postWorkRequested = false; -times(MAX_REQUESTS, () => { - validate(); - blitzFetchQueueAvailableEvent.on(validate); -}); +times(THREADS, async () => { + while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) { + const region = availableRegions[regionIndex]; + const idIndex = regionalIndices[region]; + const ids = playerIds[region].slice(idIndex, idIndex + PLAYER_IDS_PER_CALL); + const accountInfo = await getAccountInfo(region, ids, undefined, { + fields: 'last_battle_time,statistics.all.battles', + }); -function validate() { - const region = availableRegions[regionIndex]; + /** + * Pass rates + * none: 1000 / 1000 + * activity: 55 / 1000 + * battles: 30 / 1000 + * both: 23 / 1000 + */ - regionIndex = (regionIndex + 1) % availableRegions.length; -} + const filteredIds = ids.filter((_, index) => { + const info = accountInfo[index]; + return ( + info !== null && + Date.now() - info.last_battle_time * 1000 <= MAX_ACTIVITY_TIME && + info.statistics.all.battles > MIN_BATTLES + ); + }); + playerCount += filteredIds.length; + const players = await Promise.all( + filteredIds.map((id) => getTankStats(region, id)), + ); -function postWork() { - const tankIds: number[] = []; - const sorted: Record = {}; + players.forEach((tanks) => { + tanks?.forEach((tank) => { + if (tank.all.battles === 0) return; - console.log('Sorting tanks...'); + if (!tankIds.includes(tank.tank_id)) { + tankIds.push(tank.tank_id); + tanksSorted[tank.tank_id] = []; + } - players.forEach((tanks) => { - tanks.forEach((tank) => { - if (tank.all.battles === 0) return; - - if (!tankIds.includes(tank.tank_id)) { - tankIds.push(tank.tank_id); - sorted[tank.tank_id] = []; - } - - sorted[tank.tank_id].push({ - ...tank.all, - battle_life_time: tank.battle_life_time, + tanksSorted[tank.tank_id].push({ + ...tank.all, + battle_life_time: tank.battle_life_time, + }); }); }); - }); + regionalIndices[region] += PLAYER_IDS_PER_CALL; + if ( + regionalIndices[region] >= playerIds[region].length - 1 && + availableRegions.includes(region) + ) { + availableRegions.splice(availableRegions.indexOf(region), 1); + } + regionIndex = (regionIndex + 1) % availableRegions.length; + } + + if (!postWorkRequested) { + postWorkRequested = true; + postWork(); + } +}); + +async function postWork() { console.log( - `Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`, + `Generating statistics based off ${playerCount} players and ${tankIds.length} tanks...`, ); const averages: AverageDefinitions = {}; tankIds.forEach((id) => { - const tanks = sorted[id]; + const tanks = tanksSorted[id]; const samples = tanks.length; const entry: AverageDefinitionsEntrySubPartial = { samples, @@ -94,6 +131,8 @@ function postWork() { y: tank.wins / tank.battles, })); + if (data.length === 0) return; + function sum(slicer: (data: DataPoint) => number) { return data.reduce( (accumulator, data) => accumulator + slicer(data), @@ -104,16 +143,16 @@ function postWork() { const sum_w = sum(({ w }) => w); const sum_wx = sum(({ w, x }) => w * x); const sum_wy = sum(({ w, y }) => w * y); - const mu_x = sum_wx / sum_w; - const mu_y = sum_wy / sum_w; - const mu = mu_x; + const x_bar = sum_wx / sum_w; + const y_bar = sum_wy / sum_w; + const mu = x_bar; - const sigma_numerator = sum(({ w, x }) => w * (x - mu_x) ** 2); + const sigma_numerator = sum(({ w, x }) => w * (x - x_bar) ** 2); const sigma = Math.sqrt(sigma_numerator / sum_w); - const r_numerator = sum(({ w, x, y }) => w * (x - mu_x) * (y - mu_y)); - const r_denominator_x = sum(({ w, x }) => w * (x - mu_x) ** 2); - const r_denominator_y = sum(({ w, y }) => w * (y - mu_y) ** 2); + const r_numerator = sum(({ w, x, y }) => w * (x - x_bar) * (y - y_bar)); + const r_denominator_x = sum(({ w, x }) => w * (x - x_bar) ** 2); + const r_denominator_y = sum(({ w, y }) => w * (y - y_bar) ** 2); const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y); entry.mu[key] = mu; @@ -124,15 +163,15 @@ function postWork() { averages[id] = entry as AverageDefinitionsEntry; }); - commitAssets( - 'averages', - [ - { - path: 'definitions/averages.cdon.lz4', - content: superCompress(averages), - encoding: 'base64', - }, - ], - production, - ); + // commitAssets( + // 'averages', + // [ + // { + // path: 'definitions/averages.cdon.lz4', + // content: superCompress(averages), + // encoding: 'base64', + // }, + // ], + // production, + // ); } diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 841b658ef..5194d1cbe 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -33,7 +33,7 @@ async function manageQueue() { if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) { inProgress++; - const timeout = setTimeout(() => decrement, 1000); + const timeout = setTimeout(decrement, 1000); const request = queue.shift()!; const data = (await patientFetch(request.url) .then((response) => response.json()) @@ -45,10 +45,6 @@ async function manageQueue() { request.resolve(data.data); } else { if (RETRY_ERRORS.includes(data.error.message)) { - console.log( - `Encountered retry-able error ${data.error}, putting "${request.url}" back in queue...`, - ); - retryAbleBlitzFetchEvent.emit(undefined); clearTimeout(timeout); setTimeout(() => { diff --git a/src/core/blitzkit/fetchPreDiscoveredIds.ts b/src/core/blitzkit/fetchPreDiscoveredIds.ts index 4441379a7..bb8e3d454 100644 --- a/src/core/blitzkit/fetchPreDiscoveredIds.ts +++ b/src/core/blitzkit/fetchPreDiscoveredIds.ts @@ -1,21 +1,23 @@ import { times } from 'lodash'; import { decompress } from 'lz4js'; +import ProgressBar from 'progress'; import { DiscoverIdsManifest } from '../../../scripts/discoverIds'; import { DidsReadStream } from '../streams/dids'; import { asset } from './asset'; export async function fetchPreDiscoveredIds(dev: boolean) { const idChunks: number[][] = []; - const preDiscoveredManifest = (await fetch( - asset('ids/manifest.json', dev), - ).then((response) => response.json())) as DiscoverIdsManifest; - - console.log( - `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + const manifest = (await fetch(asset('ids/manifest.json', dev)).then( + (response) => response.json(), + )) as DiscoverIdsManifest; + const bar = new ProgressBar( + `Fetching ${manifest.count.toLocaleString()} ids in ${manifest.chunks} chunks :bar`, + manifest.chunks, ); + bar.render(); await Promise.all( - times(preDiscoveredManifest.chunks, async (chunkIndex) => { + times(manifest.chunks, async (chunkIndex) => { const preDiscovered = await fetch( asset(`ids/${chunkIndex}.dids.lz4`, dev), ).then(async (response) => { @@ -24,12 +26,8 @@ export async function fetchPreDiscoveredIds(dev: boolean) { return new DidsReadStream(decompressed as ArrayBuffer).dids(); }); - // no spread syntax: https://github.com/oven-sh/bun/issues/11734 idChunks.push(preDiscovered); - - console.log( - `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, - ); + bar.tick(); }), );