diff --git a/.github/workflows/cd-averages.yml b/.github/workflows/cd-averages.yml index dea3e9c9b..6e37946c1 100644 --- a/.github/workflows/cd-averages.yml +++ b/.github/workflows/cd-averages.yml @@ -2,8 +2,8 @@ name: CD Averages on: workflow_dispatch: - # schedule: - # - cron: '0 12 * * 0' + schedule: + - cron: '0 8 * * *' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/bun.lockb b/bun.lockb index 4ea53cfd8..77df4ca1b 100644 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 13e6262b7..68dade06a 100644 --- a/package.json +++ b/package.json @@ -106,5 +106,8 @@ "prisma": "^5.4.1", "typescript": "^5.4.5" }, - "packageManager": "yarn@1.22.1" + "packageManager": "yarn@1.22.1", + "trustedDependencies": [ + "protobufjs" + ] } diff --git a/scripts/averages.ts b/scripts/averages.ts index bee4ffb32..1e8d5fe4b 100644 --- a/scripts/averages.ts +++ b/scripts/averages.ts @@ -1,144 +1,186 @@ +import { chunk, times } from 'lodash'; import { argv } from 'process'; -import { REGIONS, Region } from '../src/constants/regions'; -import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz'; +import { Region, REGIONS } from '../src/constants/regions'; +import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; import getTankStats from '../src/core/blitz/getTankStats'; import { idToRegion } from '../src/core/blitz/idToRegion'; import { AverageDefinitions, AverageDefinitionsAllStats, + AverageDefinitionsEntry, + AverageDefinitionsEntrySubPartial, } from '../src/core/blitzkit/averageDefinitions'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; +import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; +import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64'; -const MAX_REQUESTS = 10; -const RUN_TIME = 1000 * 20; +interface DataPoint { + x: number; + y: number; + w: number; +} + +const MAX_ACTIVITY_TIME = 1000 * 60 * 60 * 24 * 120; +const MIN_BATTLES = 5000; +const RUN_TIME = 1000 * 60 * 60 * 5.9; +const THREADS = 10; +const PLAYER_IDS_PER_CALL = 100; const startTime = Date.now(); const production = argv.includes('--production'); +const preDiscoveredIds = await fetchPreDiscoveredIds(!production); +const playerIds: Record = { + asia: chunk( + preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'), + PLAYER_IDS_PER_CALL, + ), + com: chunk( + preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), + PLAYER_IDS_PER_CALL, + ), + eu: chunk( + preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), + PLAYER_IDS_PER_CALL, + ), +}; +let regionIndex = 0; +let checkedPlayers = 0; +let includedPlayers = 0; +const availableRegions = [...REGIONS]; +const tankIds: number[] = []; +const tanksSorted: Record = {}; +let postWorkRequested = false; -console.log(`Running in ${production ? 'production' : 'development'} mode`); +times(THREADS, async () => { + while (availableRegions.length > 0 && startTime + RUN_TIME > Date.now()) { + const region = availableRegions[regionIndex]; + const regionIds = playerIds[region]; + const [ids] = regionIds.splice( + Math.floor(Math.random() * regionIds.length), + 1, + ); + const accountInfo = await getAccountInfo(region, ids, undefined, { + fields: 'last_battle_time,statistics.all.battles', + }); + checkedPlayers += ids.length; -const ids = await fetchPreDiscoveredIds(!production); -const groups: Record = { asia: [], com: [], eu: [] }; + /** + * Pass rates + * none: 1000 / 1000 + * activity: 55 / 1000 + * battles: 30 / 1000 + * both: 23 / 1000 + */ -ids.forEach((id) => { - groups[idToRegion(id)].push(id); + const filteredIds = ids.filter((_, index) => { + const info = accountInfo[index]; + return ( + info !== null && + Date.now() - info.last_battle_time * 1000 <= MAX_ACTIVITY_TIME && + info.statistics.all.battles > MIN_BATTLES + ); + }); + includedPlayers += filteredIds.length; + const players = await Promise.all( + filteredIds.map((id) => getTankStats(region, id)), + ); + + players.forEach((tanks) => { + tanks?.forEach((tank) => { + if (tank.all.battles === 0) return; + + if (!tankIds.includes(tank.tank_id)) { + tankIds.push(tank.tank_id); + tanksSorted[tank.tank_id] = []; + } + + tanksSorted[tank.tank_id].push({ + ...tank.all, + battle_life_time: tank.battle_life_time, + }); + }); + }); + + if (regionIds.length === 0 && availableRegions.includes(region)) { + availableRegions.splice(availableRegions.indexOf(region), 1); + } + regionIndex = (regionIndex + 1) % availableRegions.length; + } + + if (!postWorkRequested) { + postWorkRequested = true; + postWork(); + } }); -const tankIds: number[] = []; -const tanks: Record = {}; -const indices: Record = { - asia: groups.asia.length, - com: groups.com.length, - eu: groups.eu.length, -}; -let players = 0; -let regionIndex = 0; -let discardAttempts = 0; - -const interval = setInterval(async () => { - if (discardAttempts > 0) return discardAttempts--; - if (Date.now() - startTime > RUN_TIME) post(); - - const region = REGIONS[regionIndex]; - const index = --indices[region]; - const id = groups[region][index]; - const stats = await getTankStats(region, id, { - fields: 'all,battle_life_time,tank_id', - }); - - if (stats !== null) players++; - - stats?.forEach((stat) => { - if (!tankIds.includes(stat.tank_id)) { - tanks[stat.tank_id] = []; - tankIds.push(stat.tank_id); - } - - // goofy ahh optimization instead of using spread syntax - const composite = stat.all as AverageDefinitionsAllStats; - composite.battle_life_time = stat.battle_life_time; - - tanks[stat.tank_id].push(composite); - }); - - regionIndex = (regionIndex + 1) % REGIONS.length; -}, 1000 / MAX_REQUESTS); - -retryAbleBlitzFetchEvent.on(() => discardAttempts++); - -function post() { - clearInterval(interval); +async function postWork() { console.log( - `Out of time, computing ${tankIds.length} tanks from ${players} players...`, + `Generating statistics based off ${includedPlayers.toLocaleString()} players (${checkedPlayers.toLocaleString()} checked in total) and ${tankIds.length} tanks...`, ); const averages: AverageDefinitions = {}; tankIds.forEach((id) => { - const samples = tanks[id]; - const n = samples.length; + const tanks = tanksSorted[id]; + const samples = tanks.length; + const entry: AverageDefinitionsEntrySubPartial = { + samples, + mu: {}, + sigma: {}, + r: {}, + }; - function sum(slice: (tank: AverageDefinitionsAllStats) => number) { - return samples.reduce((acc, tank) => { - return acc + slice(tank); - }, 0); - } + averageDefinitionsAllStatsKeys.forEach((key) => { + const data: DataPoint[] = tanks.map((tank) => ({ + w: tank.battles, + x: tank[key] / tank.battles, + y: tank.wins / tank.battles, + })); - const denominator = sum((tank) => tank.battles); + if (data.length === 0) return; - function moment(slice: (tank: AverageDefinitionsAllStats) => number) { - return samples.reduce((acc, tank) => { - return acc + tank.battles * slice(tank); - }, 0); - } + function sum(slicer: (data: DataPoint) => number) { + return data.reduce( + (accumulator, data) => accumulator + slicer(data), + 0, + ); + } - function weightedAverage( - slice: (tank: AverageDefinitionsAllStats) => number, - ) { - return moment(slice) / denominator; - } + const sum_w = sum(({ w }) => w); + const sum_wx = sum(({ w, x }) => w * x); + const sum_wy = sum(({ w, y }) => w * y); + const x_bar = sum_wx / sum_w; + const y_bar = sum_wy / sum_w; + const mu = x_bar; - const sumBattles = sum((tank) => tank.battles); + const sigma_numerator = sum(({ w, x }) => w * (x - x_bar) ** 2); + const sigma = Math.sqrt(sigma_numerator / sum_w); - function reduce(method: (key: keyof AverageDefinitionsAllStats) => number) { - return averageDefinitionsAllStatsKeys.reduce< - Partial - >((acc, key) => { - acc[key] = method(key); - return acc; - }, {}) as AverageDefinitionsAllStats; - } + const r_numerator = sum(({ w, x, y }) => w * (x - x_bar) * (y - y_bar)); + const r_denominator_x = sum(({ w, x }) => w * (x - x_bar) ** 2); + const r_denominator_y = sum(({ w, y }) => w * (y - y_bar) ** 2); + const r = r_numerator / Math.sqrt(r_denominator_x * r_denominator_y); - const mu = reduce( - (key) => sum((tank) => tank.battles * tank[key]) / sumBattles, - ); + entry.mu[key] = mu; + entry.sigma[key] = sigma; + entry.r[key] = isNaN(r) ? 0 : r; + }); - // const sigma = averageDefinitionsAllStatsKeys.reduce< - // Partial - // >((acc, key) => { - // acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / n); - // return acc; - // }, {}) as AverageDefinitionsAllStats; - // const r = averageDefinitionsAllStatsKeys.reduce< - // Partial - // >((acc, key) => { - // acc[key] = - // (n * - // sum( - // (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles), - // ) - - // sum((tank) => tank[key] / tank.battles) * - // sum((tank) => tank.wins / tank.battles)) / - // Math.sqrt( - // (n * sum((tank) => (tank[key] / tank.battles) ** 2) - - // sum((tank) => tank[key] / tank.battles) ** 2) * - // (n * sum((tank) => (tank.wins / tank.battles) ** 2) - - // sum((tank) => tank.wins / tank.battles) ** 2), - // ); - // return acc; - // }, {}) as AverageDefinitionsAllStats; - - averages[id] = { n: n, mu, sigma, r }; + averages[id] = entry as AverageDefinitionsEntry; }); + + commitAssets( + 'averages', + [ + { + path: 'definitions/averages.pb', + content: await encodeToBase64('blitzkit.AverageDefinitions', { + averages, + }), + encoding: 'base64', + }, + ], + production, + ); } diff --git a/scripts/discoverIds.ts b/scripts/discoverIds.ts index 35db6fce1..444139c34 100644 --- a/scripts/discoverIds.ts +++ b/scripts/discoverIds.ts @@ -9,7 +9,7 @@ import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds'; import { DidsWriteStream } from '../src/core/streams/dids'; -export interface DiscoverIdsManifest { +export export interface DiscoverIdsManifest { time: number; chunks: number; count: number; @@ -28,7 +28,34 @@ console.log(`Running in ${production ? 'production' : 'development'} mode`); const startTime = Date.now(); const indexableRegions = [...REGIONS]; -let ids = await fetchPreDiscoveredIds(!production); +let ids: number[] = []; +const preDiscoveredManifest = (await fetch( + asset('ids/manifest.json', !production), +).then((response) => response.json())) as DiscoverIdsManifest; + +console.log( + `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, +); + +let chunkIndex = 0; +while (chunkIndex < preDiscoveredManifest.chunks) { + const preDiscovered = await fetch( + asset(`ids/${chunkIndex}.dids.lz4`, !production), + ).then(async (response) => { + const buffer = await response.arrayBuffer(); + const decompressed = decompress(new Uint8Array(buffer)).buffer; + return new DidsReadStream(decompressed as ArrayBuffer).dids(); + }); + + // no spread syntax: https://github.com/oven-sh/bun/issues/11734 + ids = ids.concat(preDiscovered); + + console.log( + `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, + ); + + chunkIndex++; +} const regionalIdIndex: Record = { asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 71b45ae5c..5194d1cbe 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -6,6 +6,7 @@ import { patientFetch } from '../blitzkit/patientFetch'; const RETRY_ERRORS = ['REQUEST_LIMIT_EXCEEDED', 'SOURCE_NOT_AVAILABLE']; export const retryAbleBlitzFetchEvent = new EventManager(); +export const blitzFetchQueueAvailableEvent = new EventManager(); type BlitzResponse = | { @@ -22,15 +23,17 @@ const MAX_CALLS_PER_SECOND = 10; const queue: { url: string; resolve: (data: object) => void }[] = []; let inProgress = 0; +function decrement() { + inProgress--; + blitzFetchQueueAvailableEvent.emit(undefined); + manageQueue(); +} + async function manageQueue() { if (queue.length > 0 && inProgress < MAX_CALLS_PER_SECOND) { inProgress++; - const timeout = setTimeout(() => { - inProgress--; - manageQueue(); - }, 1000); - + const timeout = setTimeout(decrement, 1000); const request = queue.shift()!; const data = (await patientFetch(request.url) .then((response) => response.json()) @@ -42,21 +45,15 @@ async function manageQueue() { request.resolve(data.data); } else { if (RETRY_ERRORS.includes(data.error.message)) { - console.log( - `Encountered retry-able error ${data.error.code} ${data.error.message}, putting "${request.url}" back in queue...`, - ); - retryAbleBlitzFetchEvent.emit(undefined); clearTimeout(timeout); setTimeout(() => { queue.push(request); - inProgress--; - manageQueue(); + decrement(); }, 1000 / MAX_CALLS_PER_SECOND); } else { clearTimeout(timeout); - inProgress--; - manageQueue(); + decrement(); throw new Error(`Wargaming response error status: "${data.status}"`, { cause: `Message: "${data.error.message}"\nURL: "${request.url}"`, diff --git a/src/core/blitzkit/averageDefinitions/constants.ts b/src/core/blitzkit/averageDefinitions/constants.ts index 0ac8a080d..54c44b0ee 100644 --- a/src/core/blitzkit/averageDefinitions/constants.ts +++ b/src/core/blitzkit/averageDefinitions/constants.ts @@ -1,5 +1,4 @@ import { AverageDefinitionsAllStats } from '.'; - import { emptyAllStats } from '../../blitzstars/getStatsInPeriod/constants'; export const emptyAverageDefinitionsAllStats: AverageDefinitionsAllStats = { diff --git a/src/core/blitzkit/averageDefinitions/index.ts b/src/core/blitzkit/averageDefinitions/index.ts index e6fe5e82f..5f37ae1d0 100644 --- a/src/core/blitzkit/averageDefinitions/index.ts +++ b/src/core/blitzkit/averageDefinitions/index.ts @@ -1,20 +1,39 @@ import { AllStats } from '../../blitz/getAccountInfo'; +import { decode } from '../../protobuf/decode'; import { asset } from '../asset'; -import { fetchCdonLz4 } from '../fetchCdonLz4'; export interface AverageDefinitionsAllStats extends AllStats { battle_life_time: number; } -export interface AverageDefinitions { - [id: number]: { - n: number; // sample size - r: AverageDefinitionsAllStats; // correlation coefficient - mu: AverageDefinitionsAllStats; // mean - sigma: AverageDefinitionsAllStats; // standard deviation - }; +export interface AverageDefinitionsEntry { + samples: number; + mu: AverageDefinitionsAllStats; + sigma: AverageDefinitionsAllStats; + r: AverageDefinitionsAllStats; } -export const averageDefinitions = fetchCdonLz4( - asset('definitions/averages.cdon.lz4'), -); +export interface AverageDefinitionsEntrySubPartial { + samples: number; + mu: Partial; + sigma: Partial; + r: Partial; +} + +export interface AverageDefinitions { + [id: number]: AverageDefinitionsEntry; +} + +interface AverageDefinitionsProto { + averages: AverageDefinitions; +} + +export const averageDefinitions = fetch(asset('definitions/averages.pb')) + .then((response) => response.arrayBuffer()) + .then((buffer) => { + return decode( + 'blitzkit.AverageDefinitions', + new Uint8Array(buffer), + ); + }) + .then((data) => data.averages); diff --git a/src/core/blitzkit/commitAssets.ts b/src/core/blitzkit/commitAssets.ts index d00446a28..a769a0799 100644 --- a/src/core/blitzkit/commitAssets.ts +++ b/src/core/blitzkit/commitAssets.ts @@ -5,7 +5,6 @@ export async function commitAssets( message: string, changes: FileChange[], production: boolean, - verbose = true, ) { console.log(`Committing ${message}...`); @@ -17,6 +16,5 @@ export async function commitAssets( production ? 'main' : 'dev', `${message} - ${new Date().toDateString()}`, changes, - verbose, ); } diff --git a/src/core/blitzkit/commitMultipleFiles.ts b/src/core/blitzkit/commitMultipleFiles.ts index 0b75f6cf8..36f561149 100644 --- a/src/core/blitzkit/commitMultipleFiles.ts +++ b/src/core/blitzkit/commitMultipleFiles.ts @@ -14,7 +14,6 @@ export default async function commitMultipleFiles( branch: string, message: string, changes: FileChange[], - verbose = false, ) { const latestCommitSha = ( await octokit.git.getRef({ diff --git a/src/core/blitzkit/fetchPreDiscoveredIds.ts b/src/core/blitzkit/fetchPreDiscoveredIds.ts index 855ee12fa..bb8e3d454 100644 --- a/src/core/blitzkit/fetchPreDiscoveredIds.ts +++ b/src/core/blitzkit/fetchPreDiscoveredIds.ts @@ -1,37 +1,35 @@ import { times } from 'lodash'; import { decompress } from 'lz4js'; +import ProgressBar from 'progress'; import { DiscoverIdsManifest } from '../../../scripts/discoverIds'; import { DidsReadStream } from '../streams/dids'; import { asset } from './asset'; export async function fetchPreDiscoveredIds(dev: boolean) { - let ids: number[] = []; - const preDiscoveredManifest = (await fetch( - asset('ids/manifest.json', dev), - ).then((response) => response.json())) as DiscoverIdsManifest; - - console.log( - `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + const idChunks: number[][] = []; + const manifest = (await fetch(asset('ids/manifest.json', dev)).then( + (response) => response.json(), + )) as DiscoverIdsManifest; + const bar = new ProgressBar( + `Fetching ${manifest.count.toLocaleString()} ids in ${manifest.chunks} chunks :bar`, + manifest.chunks, ); + bar.render(); await Promise.all( - times(preDiscoveredManifest.chunks, async (chunkIndex) => { + times(manifest.chunks, async (chunkIndex) => { const preDiscovered = await fetch( asset(`ids/${chunkIndex}.dids.lz4`, dev), ).then(async (response) => { const buffer = await response.arrayBuffer(); const decompressed = decompress(new Uint8Array(buffer)).buffer; - return new DidsReadStream(decompressed).dids(); + return new DidsReadStream(decompressed as ArrayBuffer).dids(); }); - // no spread syntax: https://github.com/oven-sh/bun/issues/11734 - ids = ids.concat(preDiscovered); - - console.log( - `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, - ); + idChunks.push(preDiscovered); + bar.tick(); }), ); - return ids; + return idChunks.sort((a, b) => a[0] - b[0]).flat(); } diff --git a/src/core/protobuf/decode.ts b/src/core/protobuf/decode.ts new file mode 100644 index 000000000..86684640c --- /dev/null +++ b/src/core/protobuf/decode.ts @@ -0,0 +1,6 @@ +import { lookup } from './lookup'; + +export async function decode(proto: string, data: Uint8Array) { + const Message = await lookup(proto); + return Message.decode(data).toJSON() as Type; +} diff --git a/src/core/protobuf/encode.ts b/src/core/protobuf/encode.ts new file mode 100644 index 000000000..a7c3034c6 --- /dev/null +++ b/src/core/protobuf/encode.ts @@ -0,0 +1,6 @@ +import { lookup } from './lookup'; + +export async function encode(proto: string, data: any) { + const Message = await lookup(proto); + return Message.encode(Message.create(data)).finish(); +} diff --git a/src/core/protobuf/encodeToBase64.ts b/src/core/protobuf/encodeToBase64.ts new file mode 100644 index 000000000..86a35166f --- /dev/null +++ b/src/core/protobuf/encodeToBase64.ts @@ -0,0 +1,6 @@ +import { encode } from './encode'; + +export async function encodeToBase64(proto: string, data: any) { + const array = await encode(proto, data); + return Buffer.from(array).toString('base64'); +} diff --git a/src/core/protobuf/lookup.ts b/src/core/protobuf/lookup.ts new file mode 100644 index 000000000..d9c6418b6 --- /dev/null +++ b/src/core/protobuf/lookup.ts @@ -0,0 +1,10 @@ +import { Root } from 'protobufjs'; + +export async function lookup(proto: string) { + const root = await new Root().load( + `src/protos/${proto.split('.')[0]}.proto`, + { keepCase: true }, + ); + + return root.lookupType(proto); +} diff --git a/src/protos/blitzkit.proto b/src/protos/blitzkit.proto new file mode 100644 index 000000000..99cb35b6f --- /dev/null +++ b/src/protos/blitzkit.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package blitzkit; + +message AverageDefinitions { + map averages = 1; +} + +message Tank { + uint32 samples = 1; + AverageDefinitionsAllStats mu = 2; + AverageDefinitionsAllStats sigma = 3; + AverageDefinitionsAllStats r = 4; +} + +message AverageDefinitionsAllStats { + float battles = 1; + float capture_points = 2; + float damage_dealt = 3; + float damage_received = 4; + float dropped_capture_points = 5; + float frags = 6; + float frags8p = 7; + float hits = 8; + float losses = 9; + float max_frags = 10; + float max_xp = 11; + float shots = 12; + float spotted = 13; + float survived_battles = 14; + float win_and_survived = 15; + float wins = 16; + float xp = 17; + float battle_life_time = 18; +} \ No newline at end of file diff --git a/src/types/tanksStats.ts b/src/types/tanksStats.ts index 73c73fda9..dcff4a22e 100644 --- a/src/types/tanksStats.ts +++ b/src/types/tanksStats.ts @@ -1,4 +1,5 @@ import { AllStats } from '../core/blitz/getAccountInfo'; +import { emptyAllStats } from '../core/blitzstars/getStatsInPeriod/constants'; export interface IndividualTankStats { all: AllStats; @@ -23,25 +24,7 @@ export interface NormalizedTankStats { } export const emptyIndividualTankStats: IndividualTankStats = { - all: { - battles: 0, - capture_points: 0, - damage_dealt: 0, - damage_received: 0, - dropped_capture_points: 0, - frags: 0, - frags8p: 0, - hits: 0, - losses: 0, - max_frags: 0, - max_xp: 0, - shots: 0, - spotted: 0, - survived_battles: 0, - xp: 0, - win_and_survived: 0, - wins: 0, - }, + all: emptyAllStats, last_battle_time: 0, account_id: 0, max_xp: 0,