From d34bc4db06ea888b0e76154dbbfdebb8d3b2c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:41:06 -0500 Subject: [PATCH] feat: historical averages --- .vscode/settings.json | 2 +- public/protos/blitzkit.proto | 26 +++-- scripts/averages.ts | 102 ++++++++++++------ .../blitzkit/averageDefinitions/constants.ts | 12 ++- src/core/blitzkit/averageDefinitions/index.ts | 50 ++++++--- 5 files changed, 134 insertions(+), 58 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 61f4ce672..d9be76451 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -191,6 +191,6 @@ ], "deno.enable": false, "deno.unstable": false, - "cSpell.enableFiletypes": ["glsl"], + "cSpell.enableFiletypes": ["glsl", "proto3"], "typescript.tsdk": "node_modules\\typescript\\lib" } diff --git a/public/protos/blitzkit.proto b/public/protos/blitzkit.proto index 9e6edf050..4070567ad 100644 --- a/public/protos/blitzkit.proto +++ b/public/protos/blitzkit.proto @@ -3,16 +3,16 @@ syntax = "proto3"; package blitzkit; message AverageDefinitions { - map averages = 1; - uint32 sampled_players = 2; - uint32 scanned_players = 3; + map averages = 1; + Samples samples = 2; + uint64 time = 3; } -message Tank { - uint32 samples = 1; - AverageDefinitionsAllStats mu = 2; - AverageDefinitionsAllStats sigma = 3; - AverageDefinitionsAllStats r = 4; +message AverageDefinitionsEntry { + AverageDefinitionsAllStats mu = 1; + AverageDefinitionsAllStats sigma = 2; + AverageDefinitionsAllStats r = 3; + Samples samples = 4; } message AverageDefinitionsAllStats { @@ -34,4 +34,14 @@ message AverageDefinitionsAllStats { float wins = 16; float xp = 17; float battle_life_time = 18; +} + +message Samples { + uint32 d_1 = 1; + uint32 d_7 = 2; + uint32 d_30 = 3; + uint32 d_60 = 4; + uint32 d_90 = 5; + uint32 d_120 = 6; + uint32 total = 7; } \ No newline at end of file diff --git a/scripts/averages.ts b/scripts/averages.ts index f2cf10141..42d443e1d 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -6,14 +6,19 @@ import getTankStats from '../src/core/blitz/getTankStats'; import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, - AverageDefinitionsAllStats, AverageDefinitionsEntry, AverageDefinitionsEntrySubPartial, + AverageDefinitionsManifest, + Samples, } from '../src/core/blitzkit/averageDefinitions'; -import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; +import { + averageDefinitionsAllStatsKeys, + emptySamples, +} from '../src/core/blitzkit/averageDefinitions/constants'; import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64'; +import { IndividualTankStats } from '../src/types/tanksStats'; interface DataPoint { x: number; @@ -21,9 +26,12 @@ interface DataPoint { w: number; } -const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 30; +const MINUTE = 60 * 1000; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +const MAX_ACTIVITY_TIME = 120 * DAY; const MIN_BATTLES = 5000; -const RUN_TIME = 1000 * 60 * 60 * 5.9; +const RUN_TIME = 5 * HOUR + 55 * MINUTE; const THREADS = 10; const PLAYER_IDS_PER_CALL = 100; @@ -45,11 +53,10 @@ const playerIds: Record = { ), }; let regionIndex = 0; -let scanned_players = 0; -let sampled_players = 0; +const samples: Samples = { ...emptySamples }; const availableRegions = [...REGIONS]; const tankIds: number[] = []; -const tanksSorted: Record = {}; +const tanksSorted: Record = {}; let postWorkRequested = false; times(THREADS, async () => { @@ -63,27 +70,32 @@ times(THREADS, async () => { const accountInfo = await getAccountInfo(region, ids, undefined, { fields: 'last_battle_time,statistics.all.battles', }); - scanned_players += ids.length; - - /** - * Pass rates - * none: 1000 / 1000 - * activity: 55 / 1000 - * battles: 30 / 1000 - * both: 23 / 1000 - */ const filteredIds = ids.filter((_, index) => { const info = accountInfo[index]; + const timeSinceLastActivity = Date.now() - info.last_battle_time * 1000; + + samples.total++; + if (timeSinceLastActivity <= 120 * DAY) samples.d_120++; + if (timeSinceLastActivity <= 90 * DAY) samples.d_90++; + if (timeSinceLastActivity <= 60 * DAY) samples.d_60++; + if (timeSinceLastActivity <= 30 * DAY) samples.d_30++; + if (timeSinceLastActivity <= 7 * DAY) samples.d_7++; + if (timeSinceLastActivity <= 1 * DAY) samples.d_1++; + return ( info !== null && - Date.now() - info.last_battle_time * 1000 <= MAX_ACTIVITY_TIME && + timeSinceLastActivity <= MAX_ACTIVITY_TIME && info.statistics.all.battles > MIN_BATTLES ); }); - sampled_players += filteredIds.length; + const players = await Promise.all( - filteredIds.map((id) => getTankStats(region, id)), + filteredIds.map((id) => + getTankStats(region, id, { + fields: 'last_battle_time,battle_life_time,all', + }), + ), ); players.forEach((tanks) => { @@ -95,10 +107,7 @@ times(THREADS, async () => { tanksSorted[tank.tank_id] = []; } - tanksSorted[tank.tank_id].push({ - ...tank.all, - battle_life_time: tank.battle_life_time, - }); + tanksSorted[tank.tank_id].push(tank); }); }); @@ -116,29 +125,44 @@ times(THREADS, async () => { async function postWork() { console.log( - `Generating statistics based off ${sampled_players.toLocaleString()} players (${scanned_players.toLocaleString()} checked in total) and ${tankIds.length} tanks...`, + `Generating statistics based off ${samples.d_120.toLocaleString()} players (${samples.total.toLocaleString()} checked in total) and ${tankIds.length} tanks...`, ); const averages: Record = {}; tankIds.forEach((id) => { const tanks = tanksSorted[id]; - const samples = tanks.length; const entry: AverageDefinitionsEntrySubPartial = { - samples, mu: {}, sigma: {}, r: {}, + samples: { ...emptySamples }, }; + + tanks.forEach((tank) => { + const timeSinceLastActivity = Date.now() - tank.last_battle_time * 1000; + + entry.samples.total++; + if (timeSinceLastActivity <= 120 * DAY) entry.samples.d_120++; + if (timeSinceLastActivity <= 90 * DAY) entry.samples.d_90++; + if (timeSinceLastActivity <= 60 * DAY) entry.samples.d_60++; + if (timeSinceLastActivity <= 30 * DAY) entry.samples.d_30++; + if (timeSinceLastActivity <= 7 * DAY) entry.samples.d_7++; + if (timeSinceLastActivity <= 1 * DAY) entry.samples.d_1++; + }); + const dataWY = tanks.map((tank) => ({ - w: tank.battles, - y: tank.wins / tank.battles, + w: tank.all.battles, + y: tank.all.wins / tank.all.battles, })); averageDefinitionsAllStatsKeys.forEach((key) => { const data: DataPoint[] = dataWY.map(({ w, y }, index) => ({ w, - x: tanks[index][key] / w, + x: + (key === 'battle_life_time' + ? tanks[index].battle_life_time + : tanks[index].all[key]) / w, y, })); @@ -157,7 +181,7 @@ async function postWork() { const x_bar = sum_wx / sum_w; const y_bar = sum_wy / sum_w; // undo normalization for mu to get a simple average - const mu = sum(({ w, x }) => w * x) / samples; + const mu = sum(({ w, x }) => w * x) / samples.d_120; const sigma_numerator = sum(({ w, x }) => w * (x - x_bar) ** 2); const sigma = Math.sqrt(sigma_numerator / sum_w); @@ -175,23 +199,35 @@ async function postWork() { averages[id] = entry as AverageDefinitionsEntry; }); + const time = Date.now(); + const latest = Math.round(time / DAY); const averageDefinitions = { averages, - sampled_players, - scanned_players, + samples, + time, } satisfies AverageDefinitions; + const manifest: AverageDefinitionsManifest = { + version: 1, + latest, + }; + commitAssets( 'averages', [ { - path: 'definitions/averages.pb', + path: `averages/${latest}.pb`, content: await encodeToBase64( 'blitzkit.AverageDefinitions', averageDefinitions, ), encoding: 'base64', }, + { + path: 'averages/manifest.json', + content: JSON.stringify(manifest), + encoding: 'utf-8', + }, ], production, ); diff --git a/src/core/blitzkit/averageDefinitions/constants.ts b/src/core/blitzkit/averageDefinitions/constants.ts index 54c44b0ee..035ae11be 100644 --- a/src/core/blitzkit/averageDefinitions/constants.ts +++ b/src/core/blitzkit/averageDefinitions/constants.ts @@ -1,4 +1,4 @@ -import { AverageDefinitionsAllStats } from '.'; +import { AverageDefinitionsAllStats, Samples } from '.'; import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants'; export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = { @@ -9,3 +9,13 @@ export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = { export const averageDefinitionsAllStatsKeys = Object.keys( emptyAverageDefinitionsAllStats, ) as (keyof AverageDefinitionsAllStats)[]; + +export const emptySamples: Samples = { + d_1: 0, + d_7: 0, + d_30: 0, + d_60: 0, + d_90: 0, + d_120: 0, + total: 0, +}; diff --git a/src/core/blitzkit/averageDefinitions/index.ts b/src/core/blitzkit/averageDefinitions/index.ts index 4ff8a8c04..bf05ec559 100644 --- a/src/core/blitzkit/averageDefinitions/index.ts +++ b/src/core/blitzkit/averageDefinitions/index.ts @@ -7,37 +7,57 @@ export interface AverageDefinitionsAllStats extends AllStats { } export interface AverageDefinitionsEntry { - samples: number; mu: AverageDefinitionsAllStats; sigma: AverageDefinitionsAllStats; r: AverageDefinitionsAllStats; + samples: Samples; +} + +export interface AverageDefinitionsEntrySubPartial { + mu: Partial; + sigma: Partial; + r: Partial; + samples: Samples; } export interface AverageDefinitionsEntryWithId extends AverageDefinitionsEntry { id: number; } -export interface AverageDefinitionsEntrySubPartial { - samples: number; - mu: Partial; - sigma: Partial; - r: Partial; -} - export interface AverageDefinitions { averages: Record; - sampled_players: number; - scanned_players: number; + samples: Samples; + time: number; } -export const averageDefinitions = fetch(asset('definitions/averages.pb')) +export interface Samples { + d_1: number; + d_7: number; + d_30: number; + d_60: number; + d_90: number; + d_120: number; + total: number; +} + +export interface AverageDefinitionsManifest { + version: 1; + /** + * epoch in days + */ + latest: number; +} + +export const averageDefinitions = fetch(asset('averages/manifest.json')) + .then((response) => response.json() as Promise) + .then((manifest) => fetch(asset(`averages/${manifest.latest}.pb`))) .then((response) => response.arrayBuffer()) - .then((buffer) => { - return decode( + .then((buffer) => + decode( 'blitzkit.AverageDefinitions', new Uint8Array(buffer), - ); - }); + ), + ); export const averageDefinitionsArray = averageDefinitions.then((data) => Object.entries(data.averages).map(