diff --git a/docs/technical/bkpm.md b/docs/technical/bkni.md similarity index 99% rename from docs/technical/bkpm.md rename to docs/technical/bkni.md index f8403ad27..2f7b6f507 100644 --- a/docs/technical/bkpm.md +++ b/docs/technical/bkni.md @@ -58,7 +58,7 @@ $$ The weight of an atomic statistic is $w$ where $r$ is the correlation coefficient between the statistic and win rate, $\mu$ is the mean of the statistic across all players, and $m$ is the slope of the least squares regression line. $$ -w=\mu mr +w=\left|r\right|\mu m $$ The raw metric is the weighted average of all atomic statistics. diff --git a/package.json b/package.json index 5650355f6..7a82a8d70 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "parse-dds": "^1.2.1", "pickleparser": "^0.2.1", "postprocessing": "^6.34.1", + "progress": "^2.0.3", "protobufjs": "^7.2.6", "react": "^18.2.0", "react-device-detect": "^2.2.3", @@ -95,6 +96,7 @@ "@types/node": "^20.6.5", "@types/parse-dds": "^1.0.3", "@types/pngjs": "^6.0.4", + "@types/progress": "^2.0.7", "@types/react": "^18.2.22", "@vanilla-extract/next-plugin": "^2.3.1", "compare-versions": "^6.1.0", diff --git a/scripts/averages.ts b/scripts/averages.ts index 5d4a98316..7e931a805 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,29 +1,31 @@ -import { mkdir, writeFile } from 'fs/promises'; import { times } from 'lodash'; import { compress } from 'lz4js'; +import { argv } from 'process'; +import ProgressBar from 'progress'; import { Region, REGIONS } from '../src/constants/regions'; import getTankStats from '../src/core/blitz/getTankStats'; import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion'; import { asset } from '../src/core/blitzkit/asset'; +import { + AverageDefinitions, + AverageDefinitionsAllStats, +} from '../src/core/blitzkit/averageDefinitions'; +import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; +import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { superCompress } from '../src/core/blitzkit/superCompress'; import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; import { IndividualTankStats } from '../src/types/tanksStats'; -const RUN_TIME = 1000 * 60; +const RUN_TIME = 1000 * 0; const MAX_REQUESTS = 10; -const startTime = Date.now(); -const list: IndividualTankStats[][] = []; +const production = argv.includes('--production'); +const startTime = Date.now(); +const fields = 'all,tank_id,battle_life_time'; + +const players: IndividualTankStats[][] = []; const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then( async (response) => { - return [ - 1000000048, 500000058, 500000076, 2000000096, 2000000097, 2000000102, - 2000000103, 2000000109, 1000000116, 500000131, 2000000138, 2000000141, - 2000000140, 2000000143, 2000000144, 2000000145, 500000147, 2000000150, - 2000000173, 2000000174, 2000000178, 2000000177, 2000000182, 2000000189, - 500000191, 2000000191, 2000000192, 2000000196, - ]; - if (response.status === 200) { const buffer = await response.arrayBuffer(); return new DidsReadStream(buffer).dids(); @@ -40,17 +42,22 @@ if (preDiscovered) { `Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`, ); + const preDiscoveredBar = new ProgressBar(':bar', { + total: preDiscovered.length, + }); let preDiscoveredIndex = 0; async function chainPreDiscovery() { processes++; - if (preDiscoveredIndex >= preDiscovered!.length) { + if (preDiscoveredIndex === preDiscovered!.length) { processes--; if (processes !== 0) return; - console.log(`Rediscovered ids in ${Date.now() - startTime}ms`); + console.log( + `Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`, + ); discover(); return; @@ -61,13 +68,12 @@ if (preDiscovered) { if (id > MAX_IDS[region]) return; - const stats = await getTankStats(region, id, { felids: 'all' }); + const stats = await getTankStats(region, id, { fields }); if (stats !== null && stats.length > 0) { discoveredIds.push(id); - list.push(stats); - - console.log(`re-discovered ${id} in ${region}`); + players.push(stats); + preDiscoveredBar.tick(); } processes--; @@ -89,51 +95,170 @@ function discover() { }; let regionIndex = 0; + let done = 0; async function chainDiscovery() { - processes++; - - if (Date.now() - startTime > RUN_TIME) { - processes--; - - if (processes !== 0) return; - - const discoveredIdsContent = compress( - new DidsWriteStream().dids(discoveredIds).uint8Array, - ); - - await mkdir('temp/averages', { recursive: true }); - writeFile('temp/averages/discovered.dids.lz4', discoveredIdsContent); - writeFile('temp/averages/list.cdon.lz4', superCompress(list)); - - console.log( - `Total run time: ${ - Date.now() - startTime - }ms\nDiscovered ids: ${discoveredIds.length}`, - ); - - return; - } - regionIndex = (regionIndex + 1) % REGIONS.length; const region = REGIONS[regionIndex]; const id = index[region]++; + if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) { + if (++done !== MAX_REQUESTS) return; + + console.log( + `Total discovery run time: ${ + Date.now() - startTime + }ms\nDiscovered ids: ${discoveredIds.length}`, + ); + + postWork(); + + return; + } + if (id > MAX_IDS[region]) return; - const stats = await getTankStats(region, id, { felids: 'all' }); + const stats = await getTankStats(region, id, { fields }); if (stats !== null && stats.length > 0) { discoveredIds.push(id); - list.push(stats); + players.push(stats); console.log(`discovered ${id} in ${region}`); } - processes--; setTimeout(chainDiscovery); // circumvent max call stack } console.log(`Spawning ${MAX_REQUESTS} discovery chains...`); times(MAX_REQUESTS, chainDiscovery); } + +function postWork() { + const tankIds: number[] = []; + const sorted: Record = {}; + + players.forEach((tanks) => { + tanks.forEach((tank) => { + if (tank.all.battles === 0) return; + + if (!sorted[tank.tank_id]) { + tankIds.push(tank.tank_id); + sorted[tank.tank_id] = []; + } + sorted[tank.tank_id].push(tank); + }); + }); + + console.log( + `Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`, + ); + + const averages: AverageDefinitions = {}; + + tankIds.forEach((id) => { + const tanks = sorted[id]; + const samples = tanks.length; + + function sum(slice: (tank: AverageDefinitionsAllStats) => number) { + return tanks.reduce((acc, tank) => { + return ( + acc + slice({ battle_life_time: tank.battle_life_time, ...tank.all }) + ); + }, 0); + } + + const denominator = sum((tank) => tank.battles); + + function moment(slice: (tank: AverageDefinitionsAllStats) => number) { + return tanks.reduce((acc, tank) => { + return ( + acc + + tank.all.battles * + slice({ battle_life_time: tank.battle_life_time, ...tank.all }) + ); + }, 0); + } + + function weightedAverage( + slice: (tank: AverageDefinitionsAllStats) => number, + ) { + return moment(slice) / denominator; + } + + const mu = averageDefinitionsAllStatsKeys.reduce< + Partial + >((acc, key) => { + acc[key] = weightedAverage((tank) => tank[key]); + return acc; + }, {}) as AverageDefinitionsAllStats; + const sigma = averageDefinitionsAllStatsKeys.reduce< + Partial + >((acc, key) => { + acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / samples); + return acc; + }, {}) as AverageDefinitionsAllStats; + const r = averageDefinitionsAllStatsKeys.reduce< + Partial + >((acc, key) => { + acc[key] = + (samples * + sum( + (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), + ) - + sum((tank) => tank[key] / tank.battles) * + sum((tank) => tank.wins / tank.battles)) / + Math.sqrt( + (samples * sum((tank) => (tank[key] / tank.battles) ** 2) - + sum((tank) => tank[key] / tank.battles) ** 2) * + (samples * sum((tank) => (tank.wins / tank.battles) ** 2) - + sum((tank) => tank.wins / tank.battles) ** 2), + ); + return acc; + }, {}) as AverageDefinitionsAllStats; + const m = averageDefinitionsAllStatsKeys.reduce< + Partial + >((acc, key) => { + acc[key] = + (samples * + sum( + (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), + ) - + sum((tank) => tank[key] / tank.battles) * + sum((tank) => tank.wins / tank.battles)) / + (samples * sum((tank) => (tank[key] / tank.battles) ** 2) - + sum((tank) => tank[key] / tank.battles) ** 2); + return acc; + }, {}) as AverageDefinitionsAllStats; + const b = averageDefinitionsAllStatsKeys.reduce< + Partial + >((acc, key) => { + acc[key] = + (sum((tank) => tank.wins / tank.battles) - + m[key] * sum((tank) => tank[key] / tank.battles)) / + samples; + return acc; + }, {}) as AverageDefinitionsAllStats; + + averages[id] = { samples, mu, sigma, r, m, b }; + }); + + commitAssets( + 'averages', + [ + { + path: 'definitions/averages.cdon.lz4', + content: superCompress(averages), + encoding: 'base64', + }, + { + path: 'definitions/discovered.dids.lz4', + content: Buffer.from( + compress(new DidsWriteStream().dids(discoveredIds).uint8Array), + ).toString('base64'), + encoding: 'base64', + }, + ], + production, + ); +} diff --git a/src/core/blitzkit/averageDefinitions/constants.ts b/src/core/blitzkit/averageDefinitions/constants.ts new file mode 100644 index 000000000..0ac8a080d --- /dev/null +++ b/src/core/blitzkit/averageDefinitions/constants.ts @@ -0,0 +1,12 @@ +import { AverageDefinitionsAllStats } from '.'; + +import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants'; + +export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = { + ...emptyAllStats, + battle_life_time: 0, +}; + +export const averageDefinitionsAllStatsKeys = Object.keys( + emptyAverageDefinitionsAllStats, +) as (keyof AverageDefinitionsAllStats)[]; diff --git a/src/core/blitzkit/averageDefinitions/index.ts b/src/core/blitzkit/averageDefinitions/index.ts new file mode 100644 index 000000000..1ff0c6fbd --- /dev/null +++ b/src/core/blitzkit/averageDefinitions/index.ts @@ -0,0 +1,28 @@ +import { AllStats } from '../../blitz/getAccountInfo'; +import { asset } from '../asset'; +import { fetchCdonLz4 } from '../fetchCdonLz4'; + +export interface AverageDefinitionsAllStats extends AllStats { + battle_life_time: number; +} + +export interface AverageDefinitions { + [id: number]: { + samples: number; + + // average + mu: AverageDefinitionsAllStats; + // standard deviation + sigma: AverageDefinitionsAllStats; + // correlation coefficient + r: AverageDefinitionsAllStats; + // least squares regression line slope + m: AverageDefinitionsAllStats; + // least squares regression line y-intercept + b: AverageDefinitionsAllStats; + }; +} + +export const averageDefinitions = fetchCdonLz4( + asset('definitions/averages.cdon.lz4'), +); diff --git a/yarn.lock b/yarn.lock index b38063eca..819d563e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2311,6 +2311,13 @@ dependencies: "@types/node" "*" +"@types/progress@^2.0.7": + version "2.0.7" + resolved "https://registry.yarnpkg.com/@types/progress/-/progress-2.0.7.tgz#798b309935ef1cf5bef3b3f7bb8da7b0335bc67e" + integrity sha512-iadjw02vte8qWx7U0YM++EybBha2CQLPGu9iJ97whVgJUT5Zq9MjAPYUnbfRI2Kpehimf1QjFJYxD0t8nqzu5w== + dependencies: + "@types/node" "*" + "@types/prop-types@*": version "15.7.11" resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.11.tgz#2596fb352ee96a1379c657734d4b913a613ad563" @@ -4944,6 +4951,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +progress@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"