From bdbc0073a86da30a5a8145933585384e692f5dbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:07:27 -0500 Subject: [PATCH 1/9] feat: start of math --- scripts/averages.ts | 120 +++++++++++++--------------------------- src/types/tanksStats.ts | 21 +------ 2 files changed, 40 insertions(+), 101 deletions(-) diff --git a/scripts/averages.ts b/scripts/averages.ts index 82b240768..592d2ebf0 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -16,6 +16,12 @@ import { superCompress } from '../src/core/blitzkit/superCompress'; import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; import { IndividualTankStats } from '../src/types/tanksStats'; +interface DataPoint { + x: number; + y: number; + w: number; +} + const RUN_TIME = 1000 * 60 * 60; const MAX_REQUESTS = 10; @@ -138,17 +144,23 @@ function discover() { function postWork() { const tankIds: number[] = []; - const sorted: Record = {}; + const sorted: Record = {}; + + console.log('Sorting tanks...'); players.forEach((tanks) => { tanks.forEach((tank) => { if (tank.all.battles === 0) return; - if (!sorted[tank.tank_id]) { + if (!tankIds.includes(tank.tank_id)) { tankIds.push(tank.tank_id); sorted[tank.tank_id] = []; } - sorted[tank.tank_id].push(tank); + + sorted[tank.tank_id].push({ + ...tank.all, + battle_life_time: tank.battle_life_time, + }); }); }); @@ -160,89 +172,33 @@ function postWork() { tankIds.forEach((id) => { const tanks = sorted[id]; - const samples = tanks.length; - function sum(slice: (tank: AverageDefinitionsAllStats) => number) { - return tanks.reduce((acc, tank) => { - return ( - acc + slice({ battle_life_time: tank.battle_life_time, ...tank.all }) + averageDefinitionsAllStatsKeys.forEach((key) => { + const data: DataPoint[] = tanks.map((tank) => ({ + w: tank.battles, + x: tank[key] / tank.battles, + y: tank.wins / tank.battles, + })); + + function sum(slicer: (data: DataPoint) => number) { + return data.reduce( + (accumulator, data) => accumulator + slicer(data), + 0, ); - }, 0); - } + } - const denominator = sum((tank) => tank.battles); + const sum_w = sum(({ w }) => w); + const sum_wx = sum(({ w, x }) => w * x); + const sum_wy = sum(({ w, y }) => w * y); + const x_w = sum_wx / sum_w; + const y_w = sum_wy / sum_w; + const r_numerator = sum(({ w, x, y }) => w * (x - x_w) * (y - y_w)); + const r_denominator_x = sum(({ w, x }) => w * (x - x_w) ** 2); + const r_denominator_y = sum(({ w, y }) => w * (y - y_w) ** 2); - function moment(slice: (tank: AverageDefinitionsAllStats) => number) { - return tanks.reduce((acc, tank) => { - return ( - acc + - tank.all.battles * - slice({ battle_life_time: tank.battle_life_time, ...tank.all }) - ); - }, 0); - } - - function weightedAverage( - slice: (tank: AverageDefinitionsAllStats) => number, - ) { - return moment(slice) / denominator; - } - - const mu = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = weightedAverage((tank) => tank[key]); - return acc; - }, {}) as AverageDefinitionsAllStats; - const sigma = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / samples); - return acc; - }, {}) as AverageDefinitionsAllStats; - const r = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = - (samples * - sum( - (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), - ) - - sum((tank) => tank[key] / tank.battles) * - sum((tank) => tank.wins / tank.battles)) / - Math.sqrt( - (samples * sum((tank) => (tank[key] / tank.battles) ** 2) - - sum((tank) => tank[key] / tank.battles) ** 2) * - (samples * sum((tank) => (tank.wins / tank.battles) ** 2) - - sum((tank) => tank.wins / tank.battles) ** 2), - ); - return acc; - }, {}) as AverageDefinitionsAllStats; - const m = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = - (samples * - sum( - (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), - ) - - sum((tank) => tank[key] / tank.battles) * - sum((tank) => tank.wins / tank.battles)) / - (samples * sum((tank) => (tank[key] / tank.battles) ** 2) - - sum((tank) => tank[key] / tank.battles) ** 2); - return acc; - }, {}) as AverageDefinitionsAllStats; - const b = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = - (sum((tank) => tank.wins / tank.battles) - - m[key] * sum((tank) => tank[key] / tank.battles)) / - samples; - return acc; - }, {}) as AverageDefinitionsAllStats; - - averages[id] = { samples, mu, sigma, r, m, b }; + const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y); + const mu = x_w; + }); }); commitAssets( diff --git a/src/types/tanksStats.ts b/src/types/tanksStats.ts index 73c73fda9..dcff4a22e 100644 --- a/src/types/tanksStats.ts +++ b/src/types/tanksStats.ts @@ -1,4 +1,5 @@ import { AllStats } from '../core/blitz/getAccountInfo'; +import { emptyAllStats } from '../core/blitzstars/getStatsInPeriod/constants'; export interface IndividualTankStats { all: AllStats; @@ -23,25 +24,7 @@ export interface NormalizedTankStats { } export const emptyIndividualTankStats: IndividualTankStats = { - all: { - battles: 0, - capture_points: 0, - damage_dealt: 0, - damage_received: 0, - dropped_capture_points: 0, - frags: 0, - frags8p: 0, - hits: 0, - losses: 0, - max_frags: 0, - max_xp: 0, - shots: 0, - spotted: 0, - survived_battles: 0, - xp: 0, - win_and_survived: 0, - wins: 0, - }, + all: emptyAllStats, last_battle_time: 0, account_id: 0, max_xp: 0, From de22efabc43aae7399b57fc0da53c883ce41b8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Wed, 26 Jun 2024 21:07:54 -0500 Subject: [PATCH 2/9] feat: finish work on post work --- scripts/averages.ts | 40 ++++++++++++------- .../blitzkit/averageDefinitions/constants.ts | 1 - src/core/blitzkit/averageDefinitions/index.ts | 29 +++++++------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/scripts/averages.ts b/scripts/averages.ts index 592d2ebf0..7bb4f0a0f 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,5 +1,4 @@ import { times } from 'lodash'; -import { compress } from 'lz4js'; import { argv } from 'process'; import ProgressBar from 'progress'; import { Region, REGIONS } from '../src/constants/regions'; @@ -9,11 +8,13 @@ import { asset } from '../src/core/blitzkit/asset'; import { AverageDefinitions, AverageDefinitionsAllStats, + AverageDefinitionsEntry, + AverageDefinitionsEntrySubPartial, } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { superCompress } from '../src/core/blitzkit/superCompress'; -import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; +import { DidsReadStream } from '../src/core/streams/dids'; import { IndividualTankStats } from '../src/types/tanksStats'; interface DataPoint { @@ -172,6 +173,13 @@ function postWork() { tankIds.forEach((id) => { const tanks = sorted[id]; + const samples = tanks.length; + const entry: AverageDefinitionsEntrySubPartial = { + samples, + mu: {}, + sigma: {}, + r: {}, + }; averageDefinitionsAllStatsKeys.forEach((key) => { const data: DataPoint[] = tanks.map((tank) => ({ @@ -190,15 +198,24 @@ function postWork() { const sum_w = sum(({ w }) => w); const sum_wx = sum(({ w, x }) => w * x); const sum_wy = sum(({ w, y }) => w * y); - const x_w = sum_wx / sum_w; - const y_w = sum_wy / sum_w; - const r_numerator = sum(({ w, x, y }) => w * (x - x_w) * (y - y_w)); - const r_denominator_x = sum(({ w, x }) => w * (x - x_w) ** 2); - const r_denominator_y = sum(({ w, y }) => w * (y - y_w) ** 2); + const mu_x = sum_wx / sum_w; + const mu_y = sum_wy / sum_w; + const mu = mu_x; + const sigma_numerator = sum(({ w, x }) => w * (x - mu_x) ** 2); + const sigma = Math.sqrt(sigma_numerator / sum_w); + + const r_numerator = sum(({ w, x, y }) => w * (x - mu_x) * (y - mu_y)); + const r_denominator_x = sum(({ w, x }) => w * (x - mu_x) ** 2); + const r_denominator_y = sum(({ w, y }) => w * (y - mu_y) ** 2); const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y); - const mu = x_w; + + entry.mu[key] = mu; + entry.sigma[key] = sigma; + entry.r[key] = r; }); + + averages[id] = entry as AverageDefinitionsEntry; }); commitAssets( @@ -209,13 +226,6 @@ function postWork() { content: superCompress(averages), encoding: 'base64', }, - { - path: 'definitions/discovered.dids.lz4', - content: Buffer.from( - compress(new DidsWriteStream().dids(discoveredIds).uint8Array), - ).toString('base64'), - encoding: 'base64', - }, ], production, ); diff --git a/src/core/blitzkit/averageDefinitions/constants.ts b/src/core/blitzkit/averageDefinitions/constants.ts index 0ac8a080d..54c44b0ee 100644 --- a/src/core/blitzkit/averageDefinitions/constants.ts +++ b/src/core/blitzkit/averageDefinitions/constants.ts @@ -1,5 +1,4 @@ import { AverageDefinitionsAllStats } from '.'; - import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants'; export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = { diff --git a/src/core/blitzkit/averageDefinitions/index.ts b/src/core/blitzkit/averageDefinitions/index.ts index 1ff0c6fbd..d29b182f9 100644 --- a/src/core/blitzkit/averageDefinitions/index.ts +++ b/src/core/blitzkit/averageDefinitions/index.ts @@ -6,21 +6,22 @@ export interface AverageDefinitionsAllStats extends AllStats { battle_life_time: number; } -export interface AverageDefinitions { - [id: number]: { - samples: number; +export interface AverageDefinitionsEntry { + samples: number; + mu: AverageDefinitionsAllStats; + sigma: AverageDefinitionsAllStats; + r: AverageDefinitionsAllStats; +} - // average - mu: AverageDefinitionsAllStats; - // standard deviation - sigma: AverageDefinitionsAllStats; - // correlation coefficient - r: AverageDefinitionsAllStats; - // least squares regression line slope - m: AverageDefinitionsAllStats; - // least squares regression line y-intercept - b: AverageDefinitionsAllStats; - }; +export interface AverageDefinitionsEntrySubPartial { + samples: number; + mu: Partial; + sigma: Partial; + r: Partial; +} + +export interface AverageDefinitions { + [id: number]: AverageDefinitionsEntry; } export const averageDefinitions = fetchCdonLz4( From 33012f9f211f65e8ae1abdf2efc41c9e2e76804d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Wed, 26 Jun 2024 21:22:03 -0500 Subject: [PATCH 3/9] feat: start on fixing pre work --- scripts/averages.ts | 140 ++++----------------- scripts/discoverIds.ts | 4 +- src/core/blitz/fetchBlitz.ts | 19 +-- src/core/blitzkit/fetchPreDiscoveredIds.ts | 37 ++++++ 4 files changed, 72 insertions(+), 128 deletions(-) create mode 100644 src/core/blitzkit/fetchPreDiscoveredIds.ts diff --git a/scripts/averages.ts b/scripts/averages.ts index 7bb4f0a0f..efe80c7e1 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,10 +1,8 @@ import { times } from 'lodash'; import { argv } from 'process'; -import ProgressBar from 'progress'; import { Region, REGIONS } from '../src/constants/regions'; -import getTankStats from '../src/core/blitz/getTankStats'; -import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion'; -import { asset } from '../src/core/blitzkit/asset'; +import { blitzFetchQueueAvailableEvent } from '../src/core/blitz/fetchBlitz'; +import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, AverageDefinitionsAllStats, @@ -13,9 +11,8 @@ import { } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; +import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; import { superCompress } from '../src/core/blitzkit/superCompress'; -import { DidsReadStream } from '../src/core/streams/dids'; -import { IndividualTankStats } from '../src/types/tanksStats'; interface DataPoint { x: number; @@ -27,120 +24,29 @@ const RUN_TIME = 1000 * 60 * 60; const MAX_REQUESTS = 10; const production = argv.includes('--production'); -const startTime = Date.now(); -const fields = 'all,tank_id,battle_life_time'; +const preDiscoveredIds = await fetchPreDiscoveredIds(!production); +const playerIds: Record = { + asia: preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'), + com: preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), + eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), +}; +const regionalIndices: Record = { + asia: playerIds.asia.length - 1, + com: playerIds.com.length - 1, + eu: playerIds.eu.length - 1, +}; +let regionIndex = 0; +const availableRegions = [...REGIONS]; -const players: IndividualTankStats[][] = []; -const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then( - async (response) => { - if (response.status === 200) { - const buffer = await response.arrayBuffer(); - return new DidsReadStream(buffer).dids(); - } +times(MAX_REQUESTS, () => { + validate(); + blitzFetchQueueAvailableEvent.on(validate); +}); - return undefined; - }, -); -const discoveredIds: number[] = []; -let processes = 0; +function validate() { + const region = availableRegions[regionIndex]; -if (preDiscovered) { - console.log( - `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`, - ); - - const preDiscoveredBar = new ProgressBar(':bar', { - total: preDiscovered.length, - }); - let preDiscoveredIndex = 0; - - async function chainPreDiscovery() { - processes++; - - if (preDiscoveredIndex === preDiscovered!.length) { - processes--; - - if (processes !== 0) return; - - console.log( - `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`, - ); - discover(); - - return; - } - - const id = preDiscovered![preDiscoveredIndex++]; - const region = idToRegion(id); - - if (id > MAX_IDS[region]) return; - - const stats = await getTankStats(region, id, { fields }); - - if (stats !== null && stats.length > 0) { - discoveredIds.push(id); - players.push(stats); - preDiscoveredBar.tick(); - } - - processes--; - setTimeout(chainPreDiscovery); // circumvent max call stack - } - - times(MAX_REQUESTS, chainPreDiscovery); -} else { - console.log('No pre-discovered ids found :('); - discover(); -} - -function discover() { - const index: Record = { - asia: - preDiscovered?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, - com: preDiscovered?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com, - eu: preDiscovered?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu, - }; - - let regionIndex = 0; - let done = 0; - - async function chainDiscovery() { - regionIndex = (regionIndex + 1) % REGIONS.length; - const region = REGIONS[regionIndex]; - const id = index[region]++; - - if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) { - if (++done !== MAX_REQUESTS) return; - - console.log( - `Total discovery run time: ${ - Date.now() - startTime - }ms\nDiscovered ids: ${discoveredIds.length}`, - ); - - postWork(); - - return; - } - - if (id > MAX_IDS[region]) return; - - const stats = await getTankStats(region, id, { fields }); - - if (stats !== null && stats.length > 0) { - discoveredIds.push(id); - players.push(stats); - - console.log( - `discovered ${id} in ${region} (time left: ${RUN_TIME - (Date.now() - startTime)}ms)`, - ); - } - - setTimeout(chainDiscovery); // circumvent max call stack - } - - console.log(`Spawning ${MAX_REQUESTS} discovery chains...`); - times(MAX_REQUESTS, chainDiscovery); + regionIndex = (regionIndex + 1) % availableRegions.length; } function postWork() { diff --git a/scripts/discoverIds.ts b/scripts/discoverIds.ts index 320e1cc9d..e2f79a0b4 100644 --- a/scripts/discoverIds.ts +++ b/scripts/discoverIds.ts @@ -9,7 +9,7 @@ import { asset } from '../src/core/blitzkit/asset'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; -interface DiscoverIdsManifest { +export interface DiscoverIdsManifest { time: number; chunks: number; count: number; @@ -44,7 +44,7 @@ while (chunkIndex < preDiscoveredManifest.chunks) { ).then(async (response) => { const buffer = await response.arrayBuffer(); const decompressed = decompress(new Uint8Array(buffer)).buffer; - return new DidsReadStream(decompressed).dids(); + return new DidsReadStream(decompressed as ArrayBuffer).dids(); }); // no spread syntax: https://github.com/oven-sh/bun/issues/11734 diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 76d0919c9..841b658ef 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch'; const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE']; export const retryAbleBlitzFetchEvent = new EventManager(); +export const blitzFetchQueueAvailableEvent = new EventManager(); type BlitzResponse = | { @@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10; const queue: { url: string; resolve: (data: object) => void }[] = []; let inProgress = 0; +function decrement() { + inProgress--; + blitzFetchQueueAvailableEvent.emit(undefined); + manageQueue(); +} + async function manageQueue() { if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) { inProgress++; - const timeout = setTimeout(() => { - inProgress--; - manageQueue(); - }, 1000); - + const timeout = setTimeout(() => decrement, 1000); const request = queue.shift()!; const data = (await patientFetch(request.url) .then((response) => response.json()) @@ -50,13 +53,11 @@ async function manageQueue() { clearTimeout(timeout); setTimeout(() => { queue.push(request); - inProgress--; - manageQueue(); + decrement(); }, 1000 / MAX_CALLS_PER_SECOND); } else { clearTimeout(timeout); - inProgress--; - manageQueue(); + decrement(); throw new Error(`Wargaming response error status: "${data.status}"`, { cause: `Message: "${data.error.message}"\nURL: "${request.url}"`, diff --git a/src/core/blitzkit/fetchPreDiscoveredIds.ts b/src/core/blitzkit/fetchPreDiscoveredIds.ts new file mode 100644 index 000000000..4441379a7 --- /dev/null +++ b/src/core/blitzkit/fetchPreDiscoveredIds.ts @@ -0,0 +1,37 @@ +import { times } from 'lodash'; +import { decompress } from 'lz4js'; +import { DiscoverIdsManifest } from '../../../scripts/discoverIds'; +import { DidsReadStream } from '../streams/dids'; +import { asset } from './asset'; + +export async function fetchPreDiscoveredIds(dev: boolean) { + const idChunks: number[][] = []; + const preDiscoveredManifest = (await fetch( + asset('ids/manifest.json', dev), + ).then((response) => response.json())) as DiscoverIdsManifest; + + console.log( + `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + ); + + await Promise.all( + times(preDiscoveredManifest.chunks, async (chunkIndex) => { + const preDiscovered = await fetch( + asset(`ids/${chunkIndex}.dids.lz4`, dev), + ).then(async (response) => { + const buffer = await response.arrayBuffer(); + const decompressed = decompress(new Uint8Array(buffer)).buffer; + return new DidsReadStream(decompressed as ArrayBuffer).dids(); + }); + + // no spread syntax: https://github.com/oven-sh/bun/issues/11734 + idChunks.push(preDiscovered); + + console.log( + `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, + ); + }), + ); + + return idChunks.sort((a, b) => a[0] - b[0]).flat(); +} From ffaa368f8175a548dbbd9debf68a692628853668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:12:58 -0500 Subject: [PATCH 4/9] feat: finish script --- scripts/averages.ts | 145 +++++++++++++-------- src/core/blitz/fetchBlitz.ts | 6 +- src/core/blitzkit/fetchPreDiscoveredIds.ts | 22 ++-- 3 files changed, 103 insertions(+), 70 deletions(-) diff --git a/scripts/averages.ts b/scripts/averages.ts index efe80c7e1..40ea2904b 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,7 +1,8 @@ import { times } from 'lodash'; import { argv } from 'process'; import { Region, REGIONS } from '../src/constants/regions'; -import { blitzFetchQueueAvailableEvent } from '../src/core/blitz/fetchBlitz'; +import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; +import getTankStats from '../src/core/blitz/getTankStats'; import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, @@ -10,9 +11,7 @@ import { AverageDefinitionsEntrySubPartial, } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; -import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; -import { superCompress } from '../src/core/blitzkit/superCompress'; interface DataPoint { x: number; @@ -20,9 +19,13 @@ interface DataPoint { w: number; } -const RUN_TIME = 1000 * 60 * 60; -const MAX_REQUESTS = 10; +const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 120; +const MIN_BATTLES = 5000; +const RUN_TIME = 1000 * 60; +const THREADS = 10; +const PLAYER_IDS_PER_CALL = 100; +const startTime = Date.now(); const production = argv.includes('--production'); const preDiscoveredIds = await fetchPreDiscoveredIds(!production); const playerIds: Record = { @@ -31,54 +34,88 @@ const playerIds: Record = { eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), }; const regionalIndices: Record = { - asia: playerIds.asia.length - 1, - com: playerIds.com.length - 1, - eu: playerIds.eu.length - 1, + asia: 0, + com: 0, + eu: 0, }; let regionIndex = 0; +let playerCount = 0; const availableRegions = [...REGIONS]; +const tankIds: number[] = []; +const tanksSorted: Record = {}; +let postWorkRequested = false; -times(MAX_REQUESTS, () => { - validate(); - blitzFetchQueueAvailableEvent.on(validate); -}); +times(THREADS, async () => { + while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) { + const region = availableRegions[regionIndex]; + const idIndex = regionalIndices[region]; + const ids = playerIds[region].slice(idIndex, idIndex + PLAYER_IDS_PER_CALL); + const accountInfo = await getAccountInfo(region, ids, undefined, { + fields: 'last_battle_time,statistics.all.battles', + }); -function validate() { - const region = availableRegions[regionIndex]; + /** + * Pass rates + * none: 1000 / 1000 + * activity: 55 / 1000 + * battles: 30 / 1000 + * both: 23 / 1000 + */ - regionIndex = (regionIndex + 1) % availableRegions.length; -} + const filteredIds = ids.filter((_, index) => { + const info = accountInfo[index]; + return ( + info !== null && + Date.now() - info.last_battle_time * 1000 <= MAX_ACTIVITY_TIME && + info.statistics.all.battles > MIN_BATTLES + ); + }); + playerCount += filteredIds.length; + const players = await Promise.all( + filteredIds.map((id) => getTankStats(region, id)), + ); -function postWork() { - const tankIds: number[] = []; - const sorted: Record = {}; + players.forEach((tanks) => { + tanks?.forEach((tank) => { + if (tank.all.battles === 0) return; - console.log('Sorting tanks...'); + if (!tankIds.includes(tank.tank_id)) { + tankIds.push(tank.tank_id); + tanksSorted[tank.tank_id] = []; + } - players.forEach((tanks) => { - tanks.forEach((tank) => { - if (tank.all.battles === 0) return; - - if (!tankIds.includes(tank.tank_id)) { - tankIds.push(tank.tank_id); - sorted[tank.tank_id] = []; - } - - sorted[tank.tank_id].push({ - ...tank.all, - battle_life_time: tank.battle_life_time, + tanksSorted[tank.tank_id].push({ + ...tank.all, + battle_life_time: tank.battle_life_time, + }); }); }); - }); + regionalIndices[region] += PLAYER_IDS_PER_CALL; + if ( + regionalIndices[region] >= playerIds[region].length - 1 && + availableRegions.includes(region) + ) { + availableRegions.splice(availableRegions.indexOf(region), 1); + } + regionIndex = (regionIndex + 1) % availableRegions.length; + } + + if (!postWorkRequested) { + postWorkRequested = true; + postWork(); + } +}); + +async function postWork() { console.log( - `Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`, + `Generating statistics based off ${playerCount} players and ${tankIds.length} tanks...`, ); const averages: AverageDefinitions = {}; tankIds.forEach((id) => { - const tanks = sorted[id]; + const tanks = tanksSorted[id]; const samples = tanks.length; const entry: AverageDefinitionsEntrySubPartial = { samples, @@ -94,6 +131,8 @@ function postWork() { y: tank.wins / tank.battles, })); + if (data.length === 0) return; + function sum(slicer: (data: DataPoint) => number) { return data.reduce( (accumulator, data) => accumulator + slicer(data), @@ -104,16 +143,16 @@ function postWork() { const sum_w = sum(({ w }) => w); const sum_wx = sum(({ w, x }) => w * x); const sum_wy = sum(({ w, y }) => w * y); - const mu_x = sum_wx / sum_w; - const mu_y = sum_wy / sum_w; - const mu = mu_x; + const x_bar = sum_wx / sum_w; + const y_bar = sum_wy / sum_w; + const mu = x_bar; - const sigma_numerator = sum(({ w, x }) => w * (x - mu_x) ** 2); + const sigma_numerator = sum(({ w, x }) => w * (x - x_bar) ** 2); const sigma = Math.sqrt(sigma_numerator / sum_w); - const r_numerator = sum(({ w, x, y }) => w * (x - mu_x) * (y - mu_y)); - const r_denominator_x = sum(({ w, x }) => w * (x - mu_x) ** 2); - const r_denominator_y = sum(({ w, y }) => w * (y - mu_y) ** 2); + const r_numerator = sum(({ w, x, y }) => w * (x - x_bar) * (y - y_bar)); + const r_denominator_x = sum(({ w, x }) => w * (x - x_bar) ** 2); + const r_denominator_y = sum(({ w, y }) => w * (y - y_bar) ** 2); const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y); entry.mu[key] = mu; @@ -124,15 +163,15 @@ function postWork() { averages[id] = entry as AverageDefinitionsEntry; }); - commitAssets( - 'averages', - [ - { - path: 'definitions/averages.cdon.lz4', - content: superCompress(averages), - encoding: 'base64', - }, - ], - production, - ); + // commitAssets( + // 'averages', + // [ + // { + // path: 'definitions/averages.cdon.lz4', + // content: superCompress(averages), + // encoding: 'base64', + // }, + // ], + // production, + // ); } diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 841b658ef..5194d1cbe 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -33,7 +33,7 @@ async function manageQueue() { if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) { inProgress++; - const timeout = setTimeout(() => decrement, 1000); + const timeout = setTimeout(decrement, 1000); const request = queue.shift()!; const data = (await patientFetch(request.url) .then((response) => response.json()) @@ -45,10 +45,6 @@ async function manageQueue() { request.resolve(data.data); } else { if (RETRY_ERRORS.includes(data.error.message)) { - console.log( - `Encountered retry-able error ${data.error}, putting "${request.url}" back in queue...`, - ); - retryAbleBlitzFetchEvent.emit(undefined); clearTimeout(timeout); setTimeout(() => { diff --git a/src/core/blitzkit/fetchPreDiscoveredIds.ts b/src/core/blitzkit/fetchPreDiscoveredIds.ts index 4441379a7..bb8e3d454 100644 --- a/src/core/blitzkit/fetchPreDiscoveredIds.ts +++ b/src/core/blitzkit/fetchPreDiscoveredIds.ts @@ -1,21 +1,23 @@ import { times } from 'lodash'; import { decompress } from 'lz4js'; +import ProgressBar from 'progress'; import { DiscoverIdsManifest } from '../../../scripts/discoverIds'; import { DidsReadStream } from '../streams/dids'; import { asset } from './asset'; export async function fetchPreDiscoveredIds(dev: boolean) { const idChunks: number[][] = []; - const preDiscoveredManifest = (await fetch( - asset('ids/manifest.json', dev), - ).then((response) => response.json())) as DiscoverIdsManifest; - - console.log( - `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + const manifest = (await fetch(asset('ids/manifest.json', dev)).then( + (response) => response.json(), + )) as DiscoverIdsManifest; + const bar = new ProgressBar( + `Fetching ${manifest.count.toLocaleString()} ids in ${manifest.chunks} chunks :bar`, + manifest.chunks, ); + bar.render(); await Promise.all( - times(preDiscoveredManifest.chunks, async (chunkIndex) => { + times(manifest.chunks, async (chunkIndex) => { const preDiscovered = await fetch( asset(`ids/${chunkIndex}.dids.lz4`, dev), ).then(async (response) => { @@ -24,12 +26,8 @@ export async function fetchPreDiscoveredIds(dev: boolean) { return new DidsReadStream(decompressed as ArrayBuffer).dids(); }); - // no spread syntax: https://github.com/oven-sh/bun/issues/11734 idChunks.push(preDiscovered); - - console.log( - `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, - ); + bar.tick(); }), ); From ed173663656f5b22320888ead1a066b4f54e35ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Fri, 28 Jun 2024 20:08:50 -0500 Subject: [PATCH 5/9] feat: add cron --- .github/workflows/cd-averages.yml | 4 +-- bun.lockb | Bin 385249 -> 385249 bytes scripts/averages.ts | 38 +++++++++++++---------- src/core/blitzkit/commitAssets.ts | 2 -- src/core/blitzkit/commitMultipleFiles.ts | 1 - src/core/protobuf/decode.ts | 6 ++++ src/core/protobuf/encode.ts | 6 ++++ src/core/protobuf/encodeToBase64.ts | 6 ++++ src/core/protobuf/lookup.ts | 10 ++++++ src/protos/blitzkit.proto | 35 +++++++++++++++++++++ 10 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 src/core/protobuf/decode.ts create mode 100644 src/core/protobuf/encode.ts create mode 100644 src/core/protobuf/encodeToBase64.ts create mode 100644 src/core/protobuf/lookup.ts create mode 100644 src/protos/blitzkit.proto diff --git a/.github/workflows/cd-averages.yml b/.github/workflows/cd-averages.yml index dea3e9c9b..eb5e36002 100644 --- a/.github/workflows/cd-averages.yml +++ b/.github/workflows/cd-averages.yml @@ -2,8 +2,8 @@ name: CD Averages on: workflow_dispatch: - # schedule: - # - cron: '0 12 * * 0' + schedule: + - cron: '0 0 * * 2' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/bun.lockb b/bun.lockb index 4ea53cfd8fc762e81d6b9106bad5a3fa7bfaaef7..bf9fad3c2618bfec5aaa4df31d84f3f34a91423e 100644 GIT binary patch delta 33 pcmaEOQ2gOR@rD-07N#xCE8ns+#u@1u>KV3gc+0$f!&?@^&j9m24nzO| delta 33 mcmaEOQ2gOR@rD-07N#xCE8ns+F@Qn)hPTYyH@syr{0so$Jq-5% diff --git a/scripts/averages.ts b/scripts/averages.ts index 40ea2904b..7dcf32463 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -11,7 +11,9 @@ import { AverageDefinitionsEntrySubPartial, } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; +import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; +import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64'; interface DataPoint { x: number; @@ -21,7 +23,7 @@ interface DataPoint { const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 120; const MIN_BATTLES = 5000; -const RUN_TIME = 1000 * 60; +const RUN_TIME = 1000 * 60 * 60 * 5.9; const THREADS = 10; const PLAYER_IDS_PER_CALL = 100; @@ -39,7 +41,8 @@ const regionalIndices: Record = { eu: 0, }; let regionIndex = 0; -let playerCount = 0; +let checkedPlayers = 0; +let includedPlayers = 0; const availableRegions = [...REGIONS]; const tankIds: number[] = []; const tanksSorted: Record = {}; @@ -50,6 +53,7 @@ times(THREADS, async () => { const region = availableRegions[regionIndex]; const idIndex = regionalIndices[region]; const ids = playerIds[region].slice(idIndex, idIndex + PLAYER_IDS_PER_CALL); + checkedPlayers += PLAYER_IDS_PER_CALL; const accountInfo = await getAccountInfo(region, ids, undefined, { fields: 'last_battle_time,statistics.all.battles', }); @@ -70,7 +74,7 @@ times(THREADS, async () => { info.statistics.all.battles > MIN_BATTLES ); }); - playerCount += filteredIds.length; + includedPlayers += filteredIds.length; const players = await Promise.all( filteredIds.map((id) => getTankStats(region, id)), ); @@ -109,7 +113,7 @@ times(THREADS, async () => { async function postWork() { console.log( - `Generating statistics based off ${playerCount} players and ${tankIds.length} tanks...`, + `Generating statistics based off ${includedPlayers.toLocaleString()} players (${checkedPlayers.toLocaleString()} checked in total) and ${tankIds.length} tanks...`, ); const averages: AverageDefinitions = {}; @@ -157,21 +161,23 @@ async function postWork() { entry.mu[key] = mu; entry.sigma[key] = sigma; - entry.r[key] = r; + entry.r[key] = isNaN(r) ? 0 : r; }); averages[id] = entry as AverageDefinitionsEntry; }); - // commitAssets( - // 'averages', - // [ - // { - // path: 'definitions/averages.cdon.lz4', - // content: superCompress(averages), - // encoding: 'base64', - // }, - // ], - // production, - // ); + commitAssets( + 'averages', + [ + { + path: 'definitions/averages.pb', + content: await encodeToBase64('blitzkit.AverageDefinitions', { + averages, + }), + encoding: 'base64', + }, + ], + production, + ); } diff --git a/src/core/blitzkit/commitAssets.ts b/src/core/blitzkit/commitAssets.ts index d00446a28..a769a0799 100644 --- a/src/core/blitzkit/commitAssets.ts +++ b/src/core/blitzkit/commitAssets.ts @@ -5,7 +5,6 @@ export async function commitAssets( message: string, changes: FileChange[], production: boolean, - verbose = true, ) { console.log(`Committing ${message}...`); @@ -17,6 +16,5 @@ export async function commitAssets( production ? 'main' : 'dev', `${message} - ${new Date().toDateString()}`, changes, - verbose, ); } diff --git a/src/core/blitzkit/commitMultipleFiles.ts b/src/core/blitzkit/commitMultipleFiles.ts index 0b75f6cf8..36f561149 100644 --- a/src/core/blitzkit/commitMultipleFiles.ts +++ b/src/core/blitzkit/commitMultipleFiles.ts @@ -14,7 +14,6 @@ export default async function commitMultipleFiles( branch: string, message: string, changes: FileChange[], - verbose = false, ) { const latestCommitSha = ( await octokit.git.getRef({ diff --git a/src/core/protobuf/decode.ts b/src/core/protobuf/decode.ts new file mode 100644 index 000000000..42c5749fb --- /dev/null +++ b/src/core/protobuf/decode.ts @@ -0,0 +1,6 @@ +import { lookup } from './lookup'; + +export async function decode(proto: string, data: Uint8Array) { + const Message = await lookup(proto); + return Message.decode(data).toJSON(); +} diff --git a/src/core/protobuf/encode.ts b/src/core/protobuf/encode.ts new file mode 100644 index 000000000..a7c3034c6 --- /dev/null +++ b/src/core/protobuf/encode.ts @@ -0,0 +1,6 @@ +import { lookup } from './lookup'; + +export async function encode(proto: string, data: any) { + const Message = await lookup(proto); + return Message.encode(Message.create(data)).finish(); +} diff --git a/src/core/protobuf/encodeToBase64.ts b/src/core/protobuf/encodeToBase64.ts new file mode 100644 index 000000000..86a35166f --- /dev/null +++ b/src/core/protobuf/encodeToBase64.ts @@ -0,0 +1,6 @@ +import { encode } from './encode'; + +export async function encodeToBase64(proto: string, data: any) { + const array = await encode(proto, data); + return Buffer.from(array).toString('base64'); +} diff --git a/src/core/protobuf/lookup.ts b/src/core/protobuf/lookup.ts new file mode 100644 index 000000000..d9c6418b6 --- /dev/null +++ b/src/core/protobuf/lookup.ts @@ -0,0 +1,10 @@ +import { Root } from 'protobufjs'; + +export async function lookup(proto: string) { + const root = await new Root().load( + `src/protos/${proto.split('.')[0]}.proto`, + { keepCase: true }, + ); + + return root.lookupType(proto); +} diff --git a/src/protos/blitzkit.proto b/src/protos/blitzkit.proto new file mode 100644 index 000000000..99cb35b6f --- /dev/null +++ b/src/protos/blitzkit.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package blitzkit; + +message AverageDefinitions { + map averages = 1; +} + +message Tank { + uint32 samples = 1; + AverageDefinitionsAllStats mu = 2; + AverageDefinitionsAllStats sigma = 3; + AverageDefinitionsAllStats r = 4; +} + +message AverageDefinitionsAllStats { + float battles = 1; + float capture_points = 2; + float damage_dealt = 3; + float damage_received = 4; + float dropped_capture_points = 5; + float frags = 6; + float frags8p = 7; + float hits = 8; + float losses = 9; + float max_frags = 10; + float max_xp = 11; + float shots = 12; + float spotted = 13; + float survived_battles = 14; + float win_and_survived = 15; + float wins = 16; + float xp = 17; + float battle_life_time = 18; +} \ No newline at end of file From 8abdc3f82472c4a68ceff376c42b0655836d3a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Fri, 28 Jun 2024 20:11:34 -0500 Subject: [PATCH 6/9] feat: trust protobuf post install --- bun.lockb | Bin 385249 -> 385308 bytes package.json | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bun.lockb b/bun.lockb index bf9fad3c2618bfec5aaa4df31d84f3f34a91423e..77df4ca1b8517f3965fdbb35cd343c653b7014c1 100644 GIT binary patch delta 64 zcmaEOP<+lI@d delta 34 qcmbPpNc`bJ@d Date: Sat, 29 Jun 2024 10:49:32 -0500 Subject: [PATCH 7/9] feat: random sampling --- scripts/averages.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/averages.ts b/scripts/averages.ts index 7dcf32463..44ecaa034 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -35,11 +35,6 @@ const playerIds: Record = { com: preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), }; -const regionalIndices: Record = { - asia: 0, - com: 0, - eu: 0, -}; let regionIndex = 0; let checkedPlayers = 0; let includedPlayers = 0; @@ -51,8 +46,13 @@ let postWorkRequested = false; times(THREADS, async () => { while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) { const region = availableRegions[regionIndex]; - const idIndex = regionalIndices[region]; - const ids = playerIds[region].slice(idIndex, idIndex + PLAYER_IDS_PER_CALL); + const regionIds = playerIds[region]; + const initialLength = regionIds.length; + const idIndex = Math.min( + Math.floor(initialLength * Math.random()), + initialLength - 2, + ); + const ids = playerIds[region].splice(idIndex, PLAYER_IDS_PER_CALL); checkedPlayers += PLAYER_IDS_PER_CALL; const accountInfo = await getAccountInfo(region, ids, undefined, { fields: 'last_battle_time,statistics.all.battles', @@ -95,11 +95,7 @@ times(THREADS, async () => { }); }); - regionalIndices[region] += PLAYER_IDS_PER_CALL; - if ( - regionalIndices[region] >= playerIds[region].length - 1 && - availableRegions.includes(region) - ) { + if (regionIds.length === 0 && availableRegions.includes(region)) { availableRegions.splice(availableRegions.indexOf(region), 1); } regionIndex = (regionIndex + 1) % availableRegions.length; From 88d8f4f14b995be46b62c6dba86c1265d822055b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Sat, 29 Jun 2024 19:43:46 -0500 Subject: [PATCH 8/9] feat: efficient pre-chunking --- scripts/averages.ts | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/averages.ts b/scripts/averages.ts index 44ecaa034..1e8d5fe4b 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,4 +1,4 @@ -import { times } from 'lodash'; +import { chunk, times } from 'lodash'; import { argv } from 'process'; import { Region, REGIONS } from '../src/constants/regions'; import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; @@ -30,10 +30,19 @@ const PLAYER_IDS_PER_CALL = 100; const startTime = Date.now(); const production = argv.includes('--production'); const preDiscoveredIds = await fetchPreDiscoveredIds(!production); -const playerIds: Record = { - asia: preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'), - com: preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), - eu: preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), +const playerIds: Record = { + asia: chunk( + preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'), + PLAYER_IDS_PER_CALL, + ), + com: chunk( + preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), + PLAYER_IDS_PER_CALL, + ), + eu: chunk( + preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), + PLAYER_IDS_PER_CALL, + ), }; let regionIndex = 0; let checkedPlayers = 0; @@ -47,16 +56,14 @@ times(THREADS, async () => { while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) { const region = availableRegions[regionIndex]; const regionIds = playerIds[region]; - const initialLength = regionIds.length; - const idIndex = Math.min( - Math.floor(initialLength * Math.random()), - initialLength - 2, + const [ids] = regionIds.splice( + Math.floor(Math.random() * regionIds.length), + 1, ); - const ids = playerIds[region].splice(idIndex, PLAYER_IDS_PER_CALL); - checkedPlayers += PLAYER_IDS_PER_CALL; const accountInfo = await getAccountInfo(region, ids, undefined, { fields: 'last_battle_time,statistics.all.battles', }); + checkedPlayers += ids.length; /** * Pass rates From 0dc6af440b40a8fe61592a2837a695b7106044cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Sat, 29 Jun 2024 20:58:38 -0500 Subject: [PATCH 9/9] feat: fetch average definitions --- .github/workflows/cd-averages.yml | 2 +- src/core/blitzkit/averageDefinitions/index.ts | 18 ++++++++++++++---- src/core/protobuf/decode.ts | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cd-averages.yml b/.github/workflows/cd-averages.yml index eb5e36002..6e37946c1 100644 --- a/.github/workflows/cd-averages.yml +++ b/.github/workflows/cd-averages.yml @@ -3,7 +3,7 @@ name: CD Averages on: workflow_dispatch: schedule: - - cron: '0 0 * * 2' + - cron: '0 8 * * *' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/src/core/blitzkit/averageDefinitions/index.ts b/src/core/blitzkit/averageDefinitions/index.ts index d29b182f9..5f37ae1d0 100644 --- a/src/core/blitzkit/averageDefinitions/index.ts +++ b/src/core/blitzkit/averageDefinitions/index.ts @@ -1,6 +1,6 @@ import { AllStats } from '../../blitz/getAccountInfo'; +import { decode } from '../../protobuf/decode'; import { asset } from '../asset'; -import { fetchCdonLz4 } from '../fetchCdonLz4'; export interface AverageDefinitionsAllStats extends AllStats { battle_life_time: number; @@ -24,6 +24,16 @@ export interface AverageDefinitions { [id: number]: AverageDefinitionsEntry; } -export const averageDefinitions = fetchCdonLz4( - asset('definitions/averages.cdon.lz4'), -); +interface AverageDefinitionsProto { + averages: AverageDefinitions; +} + +export const averageDefinitions = fetch(asset('definitions/averages.pb')) + .then((response) => response.arrayBuffer()) + .then((buffer) => { + return decode( + 'blitzkit.AverageDefinitions', + new Uint8Array(buffer), + ); + }) + .then((data) => data.averages); diff --git a/src/core/protobuf/decode.ts b/src/core/protobuf/decode.ts index 42c5749fb..86684640c 100644 --- a/src/core/protobuf/decode.ts +++ b/src/core/protobuf/decode.ts @@ -1,6 +1,6 @@ import { lookup } from './lookup'; -export async function decode(proto: string, data: Uint8Array) { +export async function decode(proto: string, data: Uint8Array) { const Message = await lookup(proto); - return Message.decode(data).toJSON(); + return Message.decode(data).toJSON() as Type; }