From 243bc1104e2967cf169f8c7a6c418f6ec2b5ed2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Tue, 18 Jun 2024 12:55:48 -0500 Subject: [PATCH] feat: start on new discover ids script --- docs/technical/whss.md | 6 +- scripts/averages.ts | 320 ++++++------------ scripts/discoverIds.ts | 41 +-- src/core/blitzkit/averageDefinitions/index.ts | 16 +- src/core/blitzkit/fetchPreDiscoveredIds.ts | 37 ++ 5 files changed, 150 insertions(+), 270 deletions(-) create mode 100644 src/core/blitzkit/fetchPreDiscoveredIds.ts diff --git a/docs/technical/whss.md b/docs/technical/whss.md index a69317a07..354ebfff9 100644 --- a/docs/technical/whss.md +++ b/docs/technical/whss.md @@ -36,6 +36,8 @@ $$ $r$ is the corelation coefficient determined by analysis between the effect of $x_{i}$, any individual statistic like damage dealt normalized by the number of battles, against $y_{i}$, the wins also normalized by the number of battles. Normalizing this way makes $x_{i}$ the average of said statistics over all battles and $y_{i}$ the average winrate. Each player will have their own average $x_{i}$ and $y_{i}$ values which can be used to determine $r$. $n$ here is the number of players sampled to determine $r$. +TODO: ADD WEIGHTS TO R VALUE + $$ r=\frac{n\sum x_{i}y_{i}-\sum x_{i}\sum y_{i}}{\sqrt{\left(n\sum x_{i}^{2}-\left(\sum x_{i}\right)^{2}\right)\left(n\sum y_{i}^{2}-\left(\sum y_{i}\right)^{2}\right)}} $$ @@ -43,13 +45,13 @@ $$ Though not discussed yet, we will also be needing the two constants of a normal distribution. $\mu$ is the mean of the distribution. $$ -\mu=\frac{\sum x_{i}}{n} +\mu=\frac{\sum w_{i}x_{i}}{w_{i}} $$ In a similar fashion, we will also need the standard deviation $\sigma$. $$ -\sigma=\sqrt{\frac{\sum \left(x_{i}-\mu\right)^{2}}{n}} +\sigma=\sqrt{\frac{\sum w_{i}\left(x_{i}-\mu\right)^{2}}{\sum w_{i}}} $$ Note that both $r$ and $\mu$ use many of the same summations just in different places. During real computation, I would highly recommend pre-computing all the summations and deploying them to use to avoid excessive computation. This makes a world of difference when sampling hundreds of millions of players. diff --git a/scripts/averages.ts b/scripts/averages.ts index 82b240768..bee4ffb32 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,184 +1,95 @@ -import { times } from 'lodash'; -import { compress } from 'lz4js'; import { argv } from 'process'; -import ProgressBar from 'progress'; -import { Region, REGIONS } from '../src/constants/regions'; +import { REGIONS, Region } from '../src/constants/regions'; +import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz'; import getTankStats from '../src/core/blitz/getTankStats'; -import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion'; -import { asset } from '../src/core/blitzkit/asset'; +import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, AverageDefinitionsAllStats, } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; -import { commitAssets } from '../src/core/blitzkit/commitAssets'; -import { superCompress } from '../src/core/blitzkit/superCompress'; -import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; -import { IndividualTankStats } from '../src/types/tanksStats'; +import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; -const RUN_TIME = 1000 * 60 * 60; const MAX_REQUESTS = 10; +const RUN_TIME = 1000 * 20; -const production = argv.includes('--production'); const startTime = Date.now(); -const fields = 'all,tank_id,battle_life_time'; +const production = argv.includes('--production'); -const players: IndividualTankStats[][] = []; -const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then( - async (response) => { - if (response.status === 200) { - const buffer = await response.arrayBuffer(); - return new DidsReadStream(buffer).dids(); - } +console.log(`Running in ${production ? 'production' : 'development'} mode`); - return undefined; - }, -); -const discoveredIds: number[] = []; -let processes = 0; +const ids = await fetchPreDiscoveredIds(!production); +const groups: Record = { asia: [], com: [], eu: [] }; -if (preDiscovered) { - console.log( - `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`, - ); +ids.forEach((id) => { + groups[idToRegion(id)].push(id); +}); - const preDiscoveredBar = new ProgressBar(':bar', { - total: preDiscovered.length, - }); - let preDiscoveredIndex = 0; +const tankIds: number[] = []; +const tanks: Record = {}; +const indices: Record = { + asia: groups.asia.length, + com: groups.com.length, + eu: groups.eu.length, +}; +let players = 0; +let regionIndex = 0; +let discardAttempts = 0; - async function chainPreDiscovery() { - processes++; +const interval = setInterval(async () => { + if (discardAttempts > 0) return discardAttempts--; + if (Date.now() - startTime > RUN_TIME) post(); - if (preDiscoveredIndex === preDiscovered!.length) { - processes--; - - if (processes !== 0) return; - - console.log( - `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`, - ); - discover(); - - return; - } - - const id = preDiscovered![preDiscoveredIndex++]; - const region = idToRegion(id); - - if (id > MAX_IDS[region]) return; - - const stats = await getTankStats(region, id, { fields }); - - if (stats !== null && stats.length > 0) { - discoveredIds.push(id); - players.push(stats); - preDiscoveredBar.tick(); - } - - processes--; - setTimeout(chainPreDiscovery); // circumvent max call stack - } - - times(MAX_REQUESTS, chainPreDiscovery); -} else { - console.log('No pre-discovered ids found :('); - discover(); -} - -function discover() { - const index: Record = { - asia: - preDiscovered?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, - com: preDiscovered?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com, - eu: preDiscovered?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu, - }; - - let regionIndex = 0; - let done = 0; - - async function chainDiscovery() { - regionIndex = (regionIndex + 1) % REGIONS.length; - const region = REGIONS[regionIndex]; - const id = index[region]++; - - if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) { - if (++done !== MAX_REQUESTS) return; - - console.log( - `Total discovery run time: ${ - Date.now() - startTime - }ms\nDiscovered ids: ${discoveredIds.length}`, - ); - - postWork(); - - return; - } - - if (id > MAX_IDS[region]) return; - - const stats = await getTankStats(region, id, { fields }); - - if (stats !== null && stats.length > 0) { - discoveredIds.push(id); - players.push(stats); - - console.log( - `discovered ${id} in ${region} (time left: ${RUN_TIME - (Date.now() - startTime)}ms)`, - ); - } - - setTimeout(chainDiscovery); // circumvent max call stack - } - - console.log(`Spawning ${MAX_REQUESTS} discovery chains...`); - times(MAX_REQUESTS, chainDiscovery); -} - -function postWork() { - const tankIds: number[] = []; - const sorted: Record = {}; - - players.forEach((tanks) => { - tanks.forEach((tank) => { - if (tank.all.battles === 0) return; - - if (!sorted[tank.tank_id]) { - tankIds.push(tank.tank_id); - sorted[tank.tank_id] = []; - } - sorted[tank.tank_id].push(tank); - }); + const region = REGIONS[regionIndex]; + const index = --indices[region]; + const id = groups[region][index]; + const stats = await getTankStats(region, id, { + fields: 'all,battle_life_time,tank_id', }); + if (stats !== null) players++; + + stats?.forEach((stat) => { + if (!tankIds.includes(stat.tank_id)) { + tanks[stat.tank_id] = []; + tankIds.push(stat.tank_id); + } + + // goofy ahh optimization instead of using spread syntax + const composite = stat.all as AverageDefinitionsAllStats; + composite.battle_life_time = stat.battle_life_time; + + tanks[stat.tank_id].push(composite); + }); + + regionIndex = (regionIndex + 1) % REGIONS.length; +}, 1000 / MAX_REQUESTS); + +retryAbleBlitzFetchEvent.on(() => discardAttempts++); + +function post() { + clearInterval(interval); console.log( - `Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`, + `Out of time, computing ${tankIds.length} tanks from ${players} players...`, ); const averages: AverageDefinitions = {}; tankIds.forEach((id) => { - const tanks = sorted[id]; - const samples = tanks.length; + const samples = tanks[id]; + const n = samples.length; function sum(slice: (tank: AverageDefinitionsAllStats) => number) { - return tanks.reduce((acc, tank) => { - return ( - acc + slice({ battle_life_time: tank.battle_life_time, ...tank.all }) - ); + return samples.reduce((acc, tank) => { + return acc + slice(tank); }, 0); } const denominator = sum((tank) => tank.battles); function moment(slice: (tank: AverageDefinitionsAllStats) => number) { - return tanks.reduce((acc, tank) => { - return ( - acc + - tank.all.battles * - slice({ battle_life_time: tank.battle_life_time, ...tank.all }) - ); + return samples.reduce((acc, tank) => { + return acc + tank.battles * slice(tank); }, 0); } @@ -188,79 +99,46 @@ function postWork() { return moment(slice) / denominator; } - const mu = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = weightedAverage((tank) => tank[key]); - return acc; - }, {}) as AverageDefinitionsAllStats; - const sigma = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / samples); - return acc; - }, {}) as AverageDefinitionsAllStats; - const r = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = - (samples * - sum( - (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), - ) - - sum((tank) => tank[key] / tank.battles) * - sum((tank) => tank.wins / tank.battles)) / - Math.sqrt( - (samples * sum((tank) => (tank[key] / tank.battles) ** 2) - - sum((tank) => tank[key] / tank.battles) ** 2) * - (samples * sum((tank) => (tank.wins / tank.battles) ** 2) - - sum((tank) => tank.wins / tank.battles) ** 2), - ); - return acc; - }, {}) as AverageDefinitionsAllStats; - const m = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = - (samples * - sum( - (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), - ) - - sum((tank) => tank[key] / tank.battles) * - sum((tank) => tank.wins / tank.battles)) / - (samples * sum((tank) => (tank[key] / tank.battles) ** 2) - - sum((tank) => tank[key] / tank.battles) ** 2); - return acc; - }, {}) as AverageDefinitionsAllStats; - const b = averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = - (sum((tank) => tank.wins / tank.battles) - - m[key] * sum((tank) => tank[key] / tank.battles)) / - samples; - return acc; - }, {}) as AverageDefinitionsAllStats; + const sumBattles = sum((tank) => tank.battles); - averages[id] = { samples, mu, sigma, r, m, b }; + function reduce(method: (key: keyof AverageDefinitionsAllStats) => number) { + return averageDefinitionsAllStatsKeys.reduce< + Partial + >((acc, key) => { + acc[key] = method(key); + return acc; + }, {}) as AverageDefinitionsAllStats; + } + + const mu = reduce( + (key) => sum((tank) => tank.battles * tank[key]) / sumBattles, + ); + + // const sigma = averageDefinitionsAllStatsKeys.reduce< + // Partial + // >((acc, key) => { + // acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / n); + // return acc; + // }, {}) as AverageDefinitionsAllStats; + // const r = averageDefinitionsAllStatsKeys.reduce< + // Partial + // >((acc, key) => { + // acc[key] = + // (n * + // sum( + // (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), + // ) - + // sum((tank) => tank[key] / tank.battles) * + // sum((tank) => tank.wins / tank.battles)) / + // Math.sqrt( + // (n * sum((tank) => (tank[key] / tank.battles) ** 2) - + // sum((tank) => tank[key] / tank.battles) ** 2) * + // (n * sum((tank) => (tank.wins / tank.battles) ** 2) - + // sum((tank) => tank.wins / tank.battles) ** 2), + // ); + // return acc; + // }, {}) as AverageDefinitionsAllStats; + + averages[id] = { n: n, mu, sigma, r }; }); - - commitAssets( - 'averages', - [ - { - path: 'definitions/averages.cdon.lz4', - content: superCompress(averages), - encoding: 'base64', - }, - { - path: 'definitions/discovered.dids.lz4', - content: Buffer.from( - compress(new DidsWriteStream().dids(discoveredIds).uint8Array), - ).toString('base64'), - encoding: 'base64', - }, - ], - production, - ); } diff --git a/scripts/discoverIds.ts b/scripts/discoverIds.ts index 320e1cc9d..35db6fce1 100644 --- a/scripts/discoverIds.ts +++ b/scripts/discoverIds.ts @@ -1,15 +1,15 @@ import { chunk, times, uniq } from 'lodash'; -import { compress, decompress } from 'lz4js'; +import { compress } from 'lz4js'; import { argv } from 'process'; import { REGIONS, Region } from '../src/constants/regions'; import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz'; import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion'; -import { asset } from '../src/core/blitzkit/asset'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; -import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; +import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; +import { DidsWriteStream } from '../src/core/streams/dids'; -interface DiscoverIdsManifest { +export interface DiscoverIdsManifest { time: number; chunks: number; count: number; @@ -28,34 +28,7 @@ console.log(`Running in ${production ? 'production' : 'development'} mode`); const startTime = Date.now(); const indexableRegions = [...REGIONS]; -let ids: number[] = []; -const preDiscoveredManifest = (await fetch( - asset('ids/manifest.json', !production), -).then((response) => response.json())) as DiscoverIdsManifest; - -console.log( - `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, -); - -let chunkIndex = 0; -while (chunkIndex < preDiscoveredManifest.chunks) { - const preDiscovered = await fetch( - asset(`ids/${chunkIndex}.dids.lz4`, !production), - ).then(async (response) => { - const buffer = await response.arrayBuffer(); - const decompressed = decompress(new Uint8Array(buffer)).buffer; - return new DidsReadStream(decompressed).dids(); - }); - - // no spread syntax: https://github.com/oven-sh/bun/issues/11734 - ids = ids.concat(preDiscovered); - - console.log( - `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, - ); - - chunkIndex++; -} +let ids = await fetchPreDiscoveredIds(!production); const regionalIdIndex: Record = { asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, @@ -82,9 +55,7 @@ async function verify(region: Region, ids: number[]) { let discardAttempts = 0; let lastProgressUpdate = 0; -retryAbleBlitzFetchEvent.on(() => { - discardAttempts++; -}); +retryAbleBlitzFetchEvent.on(() => discardAttempts++); const interval = setInterval(async () => { if (discardAttempts > 0) return discardAttempts--; diff --git a/src/core/blitzkit/averageDefinitions/index.ts b/src/core/blitzkit/averageDefinitions/index.ts index 1ff0c6fbd..e6fe5e82f 100644 --- a/src/core/blitzkit/averageDefinitions/index.ts +++ b/src/core/blitzkit/averageDefinitions/index.ts @@ -8,18 +8,10 @@ export interface AverageDefinitionsAllStats extends AllStats { export interface AverageDefinitions { [id: number]: { - samples: number; - - // average - mu: AverageDefinitionsAllStats; - // standard deviation - sigma: AverageDefinitionsAllStats; - // correlation coefficient - r: AverageDefinitionsAllStats; - // least squares regression line slope - m: AverageDefinitionsAllStats; - // least squares regression line y-intercept - b: AverageDefinitionsAllStats; + n: number; // sample size + r: AverageDefinitionsAllStats; // correlation coefficient + mu: AverageDefinitionsAllStats; // mean + sigma: AverageDefinitionsAllStats; // standard deviation }; } diff --git a/src/core/blitzkit/fetchPreDiscoveredIds.ts b/src/core/blitzkit/fetchPreDiscoveredIds.ts new file mode 100644 index 000000000..855ee12fa --- /dev/null +++ b/src/core/blitzkit/fetchPreDiscoveredIds.ts @@ -0,0 +1,37 @@ +import { times } from 'lodash'; +import { decompress } from 'lz4js'; +import { DiscoverIdsManifest } from '../../../scripts/discoverIds'; +import { DidsReadStream } from '../streams/dids'; +import { asset } from './asset'; + +export async function fetchPreDiscoveredIds(dev: boolean) { + let ids: number[] = []; + const preDiscoveredManifest = (await fetch( + asset('ids/manifest.json', dev), + ).then((response) => response.json())) as DiscoverIdsManifest; + + console.log( + `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + ); + + await Promise.all( + times(preDiscoveredManifest.chunks, async (chunkIndex) => { + const preDiscovered = await fetch( + asset(`ids/${chunkIndex}.dids.lz4`, dev), + ).then(async (response) => { + const buffer = await response.arrayBuffer(); + const decompressed = decompress(new Uint8Array(buffer)).buffer; + return new DidsReadStream(decompressed).dids(); + }); + + // no spread syntax: https://github.com/oven-sh/bun/issues/11734 + ids = ids.concat(preDiscovered); + + console.log( + `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, + ); + }), + ); + + return ids; +}