From 74828e3c4516b1f7fb6efdc6cdf91ac856e6ca8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Sat, 13 Apr 2024 22:13:57 -0500 Subject: [PATCH] feat: rts --- .vscode/settings.json | 4 + docs/guide/rts.md | 70 +++++++++++ scripts/buildTest.ts | 120 +++++++++++++++--- src/core/blitz/fetchBlitz.ts | 36 ++++-- src/core/blitzkrieg/commitAssets.ts | 3 +- src/core/streams/rts.ts | 181 ++++++++++++++++++++++++++++ 6 files changed, 381 insertions(+), 33 deletions(-) create mode 100644 docs/guide/rts.md create mode 100644 src/core/streams/rts.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 04866d79e..f83e0197c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,8 @@ "BINORMAL", "Binormals", "BKRL", + "BKRTS", + "BKRTSMANIFEST", "bkwd", "blitzstars", "Camo", @@ -108,6 +110,8 @@ "researchable", "Resvg", "roboto", + "rtsc", + "RTSM", "SCPG", "scroller", "searchclans", diff --git a/docs/guide/rts.md b/docs/guide/rts.md new file mode 100644 index 000000000..2cdc62175 --- /dev/null +++ b/docs/guide/rts.md @@ -0,0 +1,70 @@ +# Blitzkrieg Regular Tank Stats + +A straightforward way of storing the entire World of Tanks Blitz tank statistics for players to disk. + +## Manifest `manifest.rtsm` + +```cpp +primary RegularTankStatisticsManifest { + utf8"RTSM" magic; + Header header; + primary Body body; +} + +Header { + uint16 version; // always 1 +} + +Body { + primary uint8 base; +} +``` + +## Chunk `*.rtsc` + +```cpp +primary RegularTankStatisticsChunk { + utf8"RTSC" magic; + Header header; + primary Body body; +} + +Header { + uint16 version; // always 1 + uint32 playerCount; +} + +Body { + Player[playerCount] players; +} + +Player { + uint32 id; + uint16 tankCount; + Tank[tankCount] tanks; +} + +Tank { + uint32 id; + uint32 battleLifeTime; + uint32 lastBattleTime; + uint8 markOfMastery; + uint32 battles; + uint32 capturePoints; + uint32 damageDealt; + uint32 damageReceived; + uint32 droppedCapturePoints; + uint32 frags; + uint32 frags8p; + uint32 hits; + uint32 losses; + uint8 maxFrags; + uint16 maxXp; + uint32 shots; + uint32 spotted; + uint32 survivedBattles; + uint32 winAndSurvived; + uint32 wins; + uint32 xp; +} +``` diff --git a/scripts/buildTest.ts b/scripts/buildTest.ts index 82f96880e..06bfc5d52 100644 --- a/scripts/buildTest.ts +++ b/scripts/buildTest.ts @@ -1,32 +1,112 @@ import { Locale } from 'discord.js'; -import { REGIONS } from '../src/constants/regions'; +import { chunk, times } from 'lodash'; +import { compress } from 'lz4js'; +import { argv } from 'process'; +import { Region, REGIONS } from '../src/constants/regions'; import getTankStats from '../src/core/blitz/getTankStats'; import { idToRegion } from '../src/core/blitz/idToRegion'; +import { commitAssets } from '../src/core/blitzkrieg/commitAssets'; +import { FileChange } from '../src/core/blitzkrieg/commitMultipleFiles'; +import { + RtscWritePlayer, + RtscWriteStream, + RtsmWriteStream, +} from '../src/core/streams/rts'; import usersRaw from '../test.users.json'; +const PLAYERS_PER_CHUNK = 256; +const CHUNKS_PER_BATCH = 128; + +const production = argv.includes('--production'); +const users = usersRaw; //.slice(0, 10); + let done = 0; +const regionSortedUsers = REGIONS.map((region) => ({ + region, + users: users.filter((user) => idToRegion(user.blitz) === region), +})); -const users = usersRaw.slice(0, 100); -const regionUsers = REGIONS.map((region) => - users.filter((user) => idToRegion(user.blitz) === region), -); -const regionChunkedUsers = await Promise.all( - regionUsers.map(async (regionUsers, index) => { - return await Promise.all( - regionUsers.map(async (user) => { - const region = REGIONS[index]; - const stats = await getTankStats(region, user.blitz, Locale.EnglishUS); +const today = new Date(); +const todayPath = `${today.getFullYear()}/${today.getMonth() + 1}/${today.getDate()}`; +const bases: Partial> = {}; +const chunks = regionSortedUsers + .map(({ region, users }) => { + const base = Math.ceil(users.length / PLAYERS_PER_CHUNK); + bases[region] = base; - console.log(`Fetch user ${++done} / ${users.length}`); + return times(base, (offset) => { + return { + region, + offset, + users: users.filter((user) => user.blitz % base === offset), + }; + }); + }) + .flat(); +const batches = chunk(chunks, CHUNKS_PER_BATCH); - return stats; - }), - ); - }), -); +const manifests = Object.entries(bases).map(([region, base]) => { + const writeStream = new RtsmWriteStream(); -regionChunkedUsers.map((users, regionIndex) => { - const region = REGIONS[regionIndex]; + writeStream.rtsm(base); - console.log(region, users.length); + return { + encoding: 'base64', + path: `regions/${region}/regular/tanks/${todayPath}/manifest.rtsm`, + content: Buffer.from(compress(writeStream.uint8Array)).toString('base64'), + } satisfies FileChange; }); + +await commitAssets( + `regular tank stats manifests ${todayPath}`, + manifests, + production, +); + +let batchesDone = 0; +for (const batch of batches) { + console.log(`Batch ${++batchesDone} of ${batches.length}`); + + const changes: FileChange[] = []; + + let chunksDone = 0; + for (const chunk of batch) { + console.log(`Chunk ${++chunksDone} of ${chunks.length}`); + + let playersDone = 0; + const players = await Promise.all( + chunk.users.map(async (user) => { + const tanks = await getTankStats( + chunk.region, + user.blitz, + Locale.EnglishUS, + ); + + console.log(`Player ${++playersDone} of ${chunk.users.length}`); + + if (tanks === null) return null; + + return { + id: user.blitz, + tanks, + } satisfies RtscWritePlayer; + }), + ).then((players) => players.filter((player) => player !== null)); + const writeStream = new RtscWriteStream(); + + writeStream.rtsc(players); + changes.push({ + encoding: 'base64', + content: Buffer.from(compress(writeStream.uint8Array)).toString('base64'), + path: `regions/${chunk.region}/regular/tanks/${todayPath}/${chunk.offset}.rtsc.lz4`, + }); + } + + // do not await; commit in background + commitAssets( + `regular tank stats ${todayPath} batch ${batches.indexOf(batch) + 1}`, + changes, + production, + false, + ); +} diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index cc631e376..5c3a17bcb 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -10,8 +10,7 @@ type BlitzResponse = data: Data; }; -// 1 less than 10 to ensure not bad things lol -const MAX_CALLS_PER_SECOND = 9; +const MAX_CALLS_PER_SECOND = 10; const queue: { url: string; resolve: (data: object) => void }[] = []; let inProgress = 0; @@ -25,23 +24,36 @@ async function manageQueue() { manageQueue(); }, 1000); - const { url, resolve } = queue.shift()!; - const data = (await patientFetch(url) + const request = queue.shift()!; + const data = (await patientFetch(request.url) .then((response) => response.json()) .catch((error) => ({ status: 'error', error }))) as BlitzResponse< - typeof resolve + typeof request.resolve >; if (data.status === 'ok') { - resolve(data.data); + request.resolve(data.data); } else { - clearTimeout(timeout); - inProgress--; - manageQueue(); + if (data.error.message === 'REQUEST_LIMIT_EXCEEDED') { + console.log( + `Rate limit exceeded, putting "${request.url}" back in queue...`, + ); - throw new Error(`Wargaming response error status: "${data.status}"`, { - cause: `Message: "${data.error.message}"\nURL: "${url}"`, - }); + clearTimeout(timeout); + setTimeout(() => { + queue.push(request); + inProgress--; + manageQueue(); + }, 1000 / MAX_CALLS_PER_SECOND); + } else { + clearTimeout(timeout); + inProgress--; + manageQueue(); + + throw new Error(`Wargaming response error status: "${data.status}"`, { + cause: `Message: "${data.error.message}"\nURL: "${request.url}"`, + }); + } } } } diff --git a/src/core/blitzkrieg/commitAssets.ts b/src/core/blitzkrieg/commitAssets.ts index 2c53e8814..9b5b93be8 100644 --- a/src/core/blitzkrieg/commitAssets.ts +++ b/src/core/blitzkrieg/commitAssets.ts @@ -4,6 +4,7 @@ export async function commitAssets( message: string, changes: FileChange[], production: boolean, + verbose = true, ) { console.log(`Committing ${message}...`); @@ -15,6 +16,6 @@ export async function commitAssets( production ? 'main' : 'dev', `${message} - ${new Date().toDateString()}`, changes, - true, + verbose, ); } diff --git a/src/core/streams/rts.ts b/src/core/streams/rts.ts new file mode 100644 index 000000000..438ca1d18 --- /dev/null +++ b/src/core/streams/rts.ts @@ -0,0 +1,181 @@ +import { times } from 'lodash'; +import { IndividualTankStats } from '../../types/tanksStats'; +import { AllStats } from '../blitz/getAccountInfo'; +import { ReadStream, WriteStream } from './buffer'; + +export interface BlitzkriegTankStats extends AllStats { + id: number; + battle_life_time: number; + last_battle_time: number; + mark_of_mastery: number; +} + +export class RtsmReadStream extends ReadStream { + rtsm() { + this.magic(); + this.header(); + return this.body(); + } + + magic() { + const magic = this.utf8(4); + + if (magic !== 'RTSM') { + throw new Error(`Invalid RTSM magic number: "${magic}"`); + } + } + + header() { + return { + version: this.uint16(), + }; + } + + body() { + return { + base: this.uint8(), + }; + } +} + +export class RtsmWriteStream extends WriteStream { + rtsm(base: number) { + this.magic(); + this.header(); + this.body(base); + + return this; + } + + magic() { + this.utf8('RTSM'); + } + + header() { + this.uint16(1); + } + + body(base: number) { + this.uint8(base); + } +} + +export class RtscReadStream extends ReadStream { + rtsc() { + this.magic(); + const header = this.header(); + return this.body(header.playerCount); + } + + magic() { + const magic = this.utf8(4); + + if (magic !== 'RTSC') { + throw new Error(`Invalid RTSC magic number: "${magic}"`); + } + } + + header() { + return { + version: this.uint16(), + playerCount: this.uint32(), + }; + } + + body(playerCount: number) { + return times(playerCount, () => this.player()); + } + + player() { + const id = this.uint32(); + const tankCount = this.uint16(); + const tanks = times(tankCount, () => this.tank()); + + return { + id, + tankCount, + tanks, + }; + } + + tank() { + return { + id: this.uint32(), + battle_life_time: this.uint32(), + last_battle_time: this.uint32(), + mark_of_mastery: this.uint8(), + battles: this.uint32(), + capture_points: this.uint32(), + damage_dealt: this.uint32(), + damage_received: this.uint32(), + dropped_capture_points: this.uint32(), + frags: this.uint32(), + frags8p: this.uint32(), + hits: this.uint32(), + losses: this.uint32(), + max_frags: this.uint8(), + max_xp: this.uint16(), + shots: this.uint32(), + spotted: this.uint32(), + survived_battles: this.uint32(), + win_and_survived: this.uint32(), + wins: this.uint32(), + xp: this.uint32(), + } satisfies BlitzkriegTankStats; + } +} + +export interface RtscWritePlayer { + id: number; + tanks: IndividualTankStats[]; +} + +export class RtscWriteStream extends WriteStream { + rtsc(players: RtscWritePlayer[]) { + this.magic(); + this.header(players.length); + this.body(players); + + return this; + } + + magic() { + this.utf8('RTSC'); + } + + header(playerCount: number) { + this.uint16(1); + this.uint32(playerCount); + } + + body(players: RtscWritePlayer[]) { + players.forEach((player) => { + this.uint32(player.id); + this.uint16(player.tanks.length); + + player.tanks.forEach((tank) => { + this.uint32(tank.tank_id); + this.uint32(tank.battle_life_time); + this.uint32(tank.last_battle_time); + this.uint8(tank.mark_of_mastery); + this.uint32(tank.all.battles); + this.uint32(tank.all.capture_points); + this.uint32(tank.all.damage_dealt); + this.uint32(tank.all.damage_received); + this.uint32(tank.all.dropped_capture_points); + this.uint32(tank.all.frags); + this.uint32(tank.all.frags8p); + this.uint32(tank.all.hits); + this.uint32(tank.all.losses); + this.uint8(tank.all.max_frags); + this.uint16(tank.all.max_xp); + this.uint32(tank.all.shots); + this.uint32(tank.all.spotted); + this.uint32(tank.all.survived_battles); + this.uint32(tank.all.win_and_survived); + this.uint32(tank.all.wins); + this.uint32(tank.all.xp); + }); + }); + } +}