TresAbhi/blitzkit
1
0 Fork 0
mirror of https://github.com/blitzkit/blitzkit.git synced 2026-10-23 19:49:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: start on new discover ids script

Browse Source
This commit is contained in:
TrèsAbhi committed 2024-06-18 12:55:48 -05:00
1 parent e96ac14770
commit 243bc1104e
5 files changed
+150 -270
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/technical/whss.md
+4 -2
View File
@@ -36,6 +36,8 @@ $$
$r$ is the corelation coefficient determined by analysis between the effect of $x_{i}$, any individual statistic like damage dealt normalized by the number of battles, against $y_{i}$, the wins also normalized by the number of battles. Normalizing this way makes $x_{i}$ the average of said statistics over all battles and $y_{i}$ the average winrate. Each player will have their own average $x_{i}$ and $y_{i}$ values which can be used to determine $r$. $n$ here is the number of players sampled to determine $r$. $r$ is the corelation coefficient determined by analysis between the effect of $x_{i}$, any individual statistic like damage dealt normalized by the number of battles, against $y_{i}$, the wins also normalized by the number of battles. Normalizing this way makes $x_{i}$ the average of said statistics over all battles and $y_{i}$ the average winrate. Each player will have their own average $x_{i}$ and $y_{i}$ values which can be used to determine $r$. $n$ here is the number of players sampled to determine $r$.
TODO: ADD WEIGHTS TO R VALUE
$$ $$
r=\frac{n\sum x_{i}y_{i}-\sum x_{i}\sum y_{i}}{\sqrt{\left(n\sum x_{i}^{2}-\left(\sum x_{i}\right)^{2}\right)\left(n\sum y_{i}^{2}-\left(\sum y_{i}\right)^{2}\right)}} r=\frac{n\sum x_{i}y_{i}-\sum x_{i}\sum y_{i}}{\sqrt{\left(n\sum x_{i}^{2}-\left(\sum x_{i}\right)^{2}\right)\left(n\sum y_{i}^{2}-\left(\sum y_{i}\right)^{2}\right)}}
$$ $$
@@ -43,13 +45,13 @@ $$
Though not discussed yet, we will also be needing the two constants of a normal distribution. $\mu$ is the mean of the distribution. Though not discussed yet, we will also be needing the two constants of a normal distribution. $\mu$ is the mean of the distribution.
$$ $$
\mu=\frac{\sum x_{i}}{n} \mu=\frac{\sum w_{i}x_{i}}{w_{i}}
$$ $$
In a similar fashion, we will also need the standard deviation $\sigma$. In a similar fashion, we will also need the standard deviation $\sigma$.
$$ $$
\sigma=\sqrt{\frac{\sum \left(x_{i}-\mu\right)^{2}}{n}} \sigma=\sqrt{\frac{\sum w_{i}\left(x_{i}-\mu\right)^{2}}{\sum w_{i}}}
$$ $$
Note that both $r$ and $\mu$ use many of the same summations just in different places. During real computation, I would highly recommend pre-computing all the summations and deploying them to use to avoid excessive computation. This makes a world of difference when sampling hundreds of millions of players. Note that both $r$ and $\mu$ use many of the same summations just in different places. During real computation, I would highly recommend pre-computing all the summations and deploying them to use to avoid excessive computation. This makes a world of difference when sampling hundreds of millions of players.
scripts/averages.ts
+99 -221
View File
@@ -1,184 +1,95 @@
import { times } from 'lodash';
import { compress } from 'lz4js';
import { argv } from 'process'; import { argv } from 'process';
import ProgressBar from 'progress'; import { REGIONS, Region } from '../src/constants/regions';
import { Region, REGIONS } from '../src/constants/regions'; import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
import getTankStats from '../src/core/blitz/getTankStats'; import getTankStats from '../src/core/blitz/getTankStats';
import { idToRegion, MAX_IDS, MIN_IDS } from '../src/core/blitz/idToRegion'; import { idToRegion } from '../src/core/blitz/idToRegion';
import { asset } from '../src/core/blitzkit/asset';
import { import {
AverageDefinitions, AverageDefinitions,
AverageDefinitionsAllStats, AverageDefinitionsAllStats,
} from '../src/core/blitzkit/averageDefinitions'; } from '../src/core/blitzkit/averageDefinitions';
import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants'; import { averageDefinitionsAllStatsKeys } from '../src/core/blitzkit/averageDefinitions/constants';
import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
import { superCompress } from '../src/core/blitzkit/superCompress';
import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
import { IndividualTankStats } from '../src/types/tanksStats';
const RUN_TIME = 1000 * 60 * 60;
const MAX_REQUESTS = 10; const MAX_REQUESTS = 10;
const RUN_TIME = 1000 * 20;
const production = argv.includes('--production');
const startTime = Date.now(); const startTime = Date.now();
const fields = 'all,tank_id,battle_life_time'; const production = argv.includes('--production');
const players: IndividualTankStats[][] = []; console.log(`Running in ${production ? 'production' : 'development'} mode`);
const preDiscovered = await fetch(asset('averages/discovered.dids.lz4')).then(
async (response) => {
if (response.status === 200) {
const buffer = await response.arrayBuffer();
return new DidsReadStream(buffer).dids();
}
return undefined; const ids = await fetchPreDiscoveredIds(!production);
}, const groups: Record<Region, number[]> = { asia: [], com: [], eu: [] };
);
const discoveredIds: number[] = [];
let processes = 0;
if (preDiscovered) { ids.forEach((id) => {
console.log( groups[idToRegion(id)].push(id);
`Revalidating ${preDiscovered.length} pre-discovered ids with ${MAX_REQUESTS} chains...`, });
);
const preDiscoveredBar = new ProgressBar(':bar', { const tankIds: number[] = [];
total: preDiscovered.length, const tanks: Record<number, AverageDefinitionsAllStats[]> = {};
}); const indices: Record<Region, number> = {
let preDiscoveredIndex = 0; asia: groups.asia.length,
com: groups.com.length,
eu: groups.eu.length,
};
let players = 0;
let regionIndex = 0;
let discardAttempts = 0;
async function chainPreDiscovery() { const interval = setInterval(async () => {
processes++; if (discardAttempts > 0) return discardAttempts--;
if (Date.now() - startTime > RUN_TIME) post();
if (preDiscoveredIndex === preDiscovered!.length) { const region = REGIONS[regionIndex];
processes--; const index = --indices[region];
const id = groups[region][index];
if (processes !== 0) return; const stats = await getTankStats(region, id, {
fields: 'all,battle_life_time,tank_id',
console.log(
`Rediscovered ${discoveredIds.length} / ${preDiscovered!.length} ids in ${Date.now() - startTime}ms`,
);
discover();
return;
}
const id = preDiscovered![preDiscoveredIndex++];
const region = idToRegion(id);
if (id > MAX_IDS[region]) return;
const stats = await getTankStats(region, id, { fields });
if (stats !== null && stats.length > 0) {
discoveredIds.push(id);
players.push(stats);
preDiscoveredBar.tick();
}
processes--;
setTimeout(chainPreDiscovery); // circumvent max call stack
}
times(MAX_REQUESTS, chainPreDiscovery);
} else {
console.log('No pre-discovered ids found :(');
discover();
}
function discover() {
const index: Record<Region, number> = {
asia:
preDiscovered?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
com: preDiscovered?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com,
eu: preDiscovered?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu,
};
let regionIndex = 0;
let done = 0;
async function chainDiscovery() {
regionIndex = (regionIndex + 1) % REGIONS.length;
const region = REGIONS[regionIndex];
const id = index[region]++;
if (Date.now() - startTime > RUN_TIME || id === MAX_IDS[region]) {
if (++done !== MAX_REQUESTS) return;
console.log(
`Total discovery run time: ${
Date.now() - startTime
}ms\nDiscovered ids: ${discoveredIds.length}`,
);
postWork();
return;
}
if (id > MAX_IDS[region]) return;
const stats = await getTankStats(region, id, { fields });
if (stats !== null && stats.length > 0) {
discoveredIds.push(id);
players.push(stats);
console.log(
`discovered ${id} in ${region} (time left: ${RUN_TIME - (Date.now() - startTime)}ms)`,
);
}
setTimeout(chainDiscovery); // circumvent max call stack
}
console.log(`Spawning ${MAX_REQUESTS} discovery chains...`);
times(MAX_REQUESTS, chainDiscovery);
}
function postWork() {
const tankIds: number[] = [];
const sorted: Record<number, IndividualTankStats[]> = {};
players.forEach((tanks) => {
tanks.forEach((tank) => {
if (tank.all.battles === 0) return;
if (!sorted[tank.tank_id]) {
tankIds.push(tank.tank_id);
sorted[tank.tank_id] = [];
}
sorted[tank.tank_id].push(tank);
});
}); });
if (stats !== null) players++;
stats?.forEach((stat) => {
if (!tankIds.includes(stat.tank_id)) {
tanks[stat.tank_id] = [];
tankIds.push(stat.tank_id);
}
// goofy ahh optimization instead of using spread syntax
const composite = stat.all as AverageDefinitionsAllStats;
composite.battle_life_time = stat.battle_life_time;
tanks[stat.tank_id].push(composite);
});
regionIndex = (regionIndex + 1) % REGIONS.length;
}, 1000 / MAX_REQUESTS);
retryAbleBlitzFetchEvent.on(() => discardAttempts++);
function post() {
clearInterval(interval);
console.log( console.log(
`Generating statistics based off ${players.length} players and ${tankIds.length} tanks...`, `Out of time, computing ${tankIds.length} tanks from ${players} players...`,
); );
const averages: AverageDefinitions = {}; const averages: AverageDefinitions = {};
tankIds.forEach((id) => { tankIds.forEach((id) => {
const tanks = sorted[id]; const samples = tanks[id];
const samples = tanks.length; const n = samples.length;
function sum(slice: (tank: AverageDefinitionsAllStats) => number) { function sum(slice: (tank: AverageDefinitionsAllStats) => number) {
return tanks.reduce((acc, tank) => { return samples.reduce((acc, tank) => {
return ( return acc + slice(tank);
acc + slice({ battle_life_time: tank.battle_life_time, ...tank.all })
);
}, 0); }, 0);
} }
const denominator = sum((tank) => tank.battles); const denominator = sum((tank) => tank.battles);
function moment(slice: (tank: AverageDefinitionsAllStats) => number) { function moment(slice: (tank: AverageDefinitionsAllStats) => number) {
return tanks.reduce((acc, tank) => { return samples.reduce((acc, tank) => {
return ( return acc + tank.battles * slice(tank);
acc +
tank.all.battles *
slice({ battle_life_time: tank.battle_life_time, ...tank.all })
);
}, 0); }, 0);
} }
@@ -188,79 +99,46 @@ function postWork() {
return moment(slice) / denominator; return moment(slice) / denominator;
} }
const mu = averageDefinitionsAllStatsKeys.reduce< const sumBattles = sum((tank) => tank.battles);
Partial<AverageDefinitionsAllStats>
>((acc, key) => {
acc[key] = weightedAverage((tank) => tank[key]);
return acc;
}, {}) as AverageDefinitionsAllStats;
const sigma = averageDefinitionsAllStatsKeys.reduce<
Partial<AverageDefinitionsAllStats>
>((acc, key) => {
acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / samples);
return acc;
}, {}) as AverageDefinitionsAllStats;
const r = averageDefinitionsAllStatsKeys.reduce<
Partial<AverageDefinitionsAllStats>
>((acc, key) => {
acc[key] =
(samples *
sum(
(tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
) -
sum((tank) => tank[key] / tank.battles) *
sum((tank) => tank.wins / tank.battles)) /
Math.sqrt(
(samples * sum((tank) => (tank[key] / tank.battles) ** 2) -
sum((tank) => tank[key] / tank.battles) ** 2) *
(samples * sum((tank) => (tank.wins / tank.battles) ** 2) -
sum((tank) => tank.wins / tank.battles) ** 2),
);
return acc;
}, {}) as AverageDefinitionsAllStats;
const m = averageDefinitionsAllStatsKeys.reduce<
Partial<AverageDefinitionsAllStats>
>((acc, key) => {
acc[key] =
(samples *
sum(
(tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
) -
sum((tank) => tank[key] / tank.battles) *
sum((tank) => tank.wins / tank.battles)) /
(samples * sum((tank) => (tank[key] / tank.battles) ** 2) -
sum((tank) => tank[key] / tank.battles) ** 2);
return acc;
}, {}) as AverageDefinitionsAllStats;
const b = averageDefinitionsAllStatsKeys.reduce<
Partial<AverageDefinitionsAllStats>
>((acc, key) => {
acc[key] =
(sum((tank) => tank.wins / tank.battles) -
m[key] * sum((tank) => tank[key] / tank.battles)) /
samples;
return acc;
}, {}) as AverageDefinitionsAllStats;
averages[id] = { samples, mu, sigma, r, m, b }; function reduce(method: (key: keyof AverageDefinitionsAllStats) => number) {
return averageDefinitionsAllStatsKeys.reduce<
Partial<AverageDefinitionsAllStats>
>((acc, key) => {
acc[key] = method(key);
return acc;
}, {}) as AverageDefinitionsAllStats;
}
const mu = reduce(
(key) => sum((tank) => tank.battles * tank[key]) / sumBattles,
);
// const sigma = averageDefinitionsAllStatsKeys.reduce<
// Partial<AverageDefinitionsAllStats>
// >((acc, key) => {
// acc[key] = Math.sqrt(sum((tank) => (tank[key] - mu[key]) ** 2) / n);
// return acc;
// }, {}) as AverageDefinitionsAllStats;
// const r = averageDefinitionsAllStatsKeys.reduce<
// Partial<AverageDefinitionsAllStats>
// >((acc, key) => {
// acc[key] =
// (n *
// sum(
// (tank) => (tank[key] / tank.battles) * (tank.wins / tank.battles),
// ) -
// sum((tank) => tank[key] / tank.battles) *
// sum((tank) => tank.wins / tank.battles)) /
// Math.sqrt(
// (n * sum((tank) => (tank[key] / tank.battles) ** 2) -
// sum((tank) => tank[key] / tank.battles) ** 2) *
// (n * sum((tank) => (tank.wins / tank.battles) ** 2) -
// sum((tank) => tank.wins / tank.battles) ** 2),
// );
// return acc;
// }, {}) as AverageDefinitionsAllStats;
averages[id] = { n: n, mu, sigma, r };
}); });
commitAssets(
'averages',
[
{
path: 'definitions/averages.cdon.lz4',
content: superCompress(averages),
encoding: 'base64',
},
{
path: 'definitions/discovered.dids.lz4',
content: Buffer.from(
compress(new DidsWriteStream().dids(discoveredIds).uint8Array),
).toString('base64'),
encoding: 'base64',
},
],
production,
);
} }
scripts/discoverIds.ts
+6 -35
View File
@@ -1,15 +1,15 @@
import { chunk, times, uniq } from 'lodash'; import { chunk, times, uniq } from 'lodash';
import { compress, decompress } from 'lz4js'; import { compress } from 'lz4js';
import { argv } from 'process'; import { argv } from 'process';
import { REGIONS, Region } from '../src/constants/regions'; import { REGIONS, Region } from '../src/constants/regions';
import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz'; import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion'; import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion';
import { asset } from '../src/core/blitzkit/asset';
import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { commitAssets } from '../src/core/blitzkit/commitAssets';
import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
import { DidsWriteStream } from '../src/core/streams/dids';
interface DiscoverIdsManifest { export interface DiscoverIdsManifest {
time: number; time: number;
chunks: number; chunks: number;
count: number; count: number;
@@ -28,34 +28,7 @@ console.log(`Running in ${production ? 'production' : 'development'} mode`);
const startTime = Date.now(); const startTime = Date.now();
const indexableRegions = [...REGIONS]; const indexableRegions = [...REGIONS];
let ids: number[] = []; let ids = await fetchPreDiscoveredIds(!production);
const preDiscoveredManifest = (await fetch(
asset('ids/manifest.json', !production),
).then((response) => response.json())) as DiscoverIdsManifest;
console.log(
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
);
let chunkIndex = 0;
while (chunkIndex < preDiscoveredManifest.chunks) {
const preDiscovered = await fetch(
asset(`ids/${chunkIndex}.dids.lz4`, !production),
).then(async (response) => {
const buffer = await response.arrayBuffer();
const decompressed = decompress(new Uint8Array(buffer)).buffer;
return new DidsReadStream(decompressed).dids();
});
// no spread syntax: https://github.com/oven-sh/bun/issues/11734
ids = ids.concat(preDiscovered);
console.log(
`Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
);
chunkIndex++;
}
const regionalIdIndex: Record<Region, number> = { const regionalIdIndex: Record<Region, number> = {
asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
@@ -82,9 +55,7 @@ async function verify(region: Region, ids: number[]) {
let discardAttempts = 0; let discardAttempts = 0;
let lastProgressUpdate = 0; let lastProgressUpdate = 0;
retryAbleBlitzFetchEvent.on(() => { retryAbleBlitzFetchEvent.on(() => discardAttempts++);
discardAttempts++;
});
const interval = setInterval(async () => { const interval = setInterval(async () => {
if (discardAttempts > 0) return discardAttempts--; if (discardAttempts > 0) return discardAttempts--;
src/core/blitzkit/averageDefinitions/index.ts
+4 -12
View File
@@ -8,18 +8,10 @@ export interface AverageDefinitionsAllStats extends AllStats {
export interface AverageDefinitions { export interface AverageDefinitions {
[id: number]: { [id: number]: {
samples: number; n: number; // sample size
r: AverageDefinitionsAllStats; // correlation coefficient
// average mu: AverageDefinitionsAllStats; // mean
mu: AverageDefinitionsAllStats; sigma: AverageDefinitionsAllStats; // standard deviation
// standard deviation
sigma: AverageDefinitionsAllStats;
// correlation coefficient
r: AverageDefinitionsAllStats;
// least squares regression line slope
m: AverageDefinitionsAllStats;
// least squares regression line y-intercept
b: AverageDefinitionsAllStats;
}; };
} }
src/core/blitzkit/fetchPreDiscoveredIds.ts
+37
View File
@@ -0,0 +1,37 @@
import { times } from 'lodash';
import { decompress } from 'lz4js';
import { DiscoverIdsManifest } from '../../../scripts/discoverIds';
import { DidsReadStream } from '../streams/dids';
import { asset } from './asset';
export async function fetchPreDiscoveredIds(dev: boolean) {
let ids: number[] = [];
const preDiscoveredManifest = (await fetch(
asset('ids/manifest.json', dev),
).then((response) => response.json())) as DiscoverIdsManifest;
console.log(
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
);
await Promise.all(
times(preDiscoveredManifest.chunks, async (chunkIndex) => {
const preDiscovered = await fetch(
asset(`ids/${chunkIndex}.dids.lz4`, dev),
).then(async (response) => {
const buffer = await response.arrayBuffer();
const decompressed = decompress(new Uint8Array(buffer)).buffer;
return new DidsReadStream(decompressed).dids();
});
// no spread syntax: https://github.com/oven-sh/bun/issues/11734
ids = ids.concat(preDiscovered);
console.log(
`Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
);
}),
);
return ids;
}