feat: rts

.vscode/settings.json

@@ -18,6 +18,8 @@
     "BINORMAL",
     "Binormals",
     "BKRL",
+    "BKRTS",
+    "BKRTSMANIFEST",
     "bkwd",
     "blitzstars",
     "Camo",
@@ -108,6 +110,8 @@
     "researchable",
     "Resvg",
     "roboto",
+    "rtsc",
+    "RTSM",
     "SCPG",
     "scroller",
     "searchclans",

docs/guide/rts.md

@@ -0,0 +1,70 @@
+# Blitzkrieg Regular Tank Stats
+
+A straightforward way of storing the entire World of Tanks Blitz tank statistics for players to disk.
+
+## Manifest `manifest.rtsm`
+
+```cpp
+primary RegularTankStatisticsManifest {
+  utf8"RTSM" magic;
+  Header header;
+  primary Body body;
+}
+
+Header {
+  uint16 version; // always 1
+}
+
+Body {
+  primary uint8 base;
+}
+```
+
+## Chunk `*.rtsc`
+
+```cpp
+primary RegularTankStatisticsChunk {
+  utf8"RTSC" magic;
+  Header header;
+  primary Body body<header.playerCount>;
+}
+
+Header {
+  uint16 version; // always 1
+  uint32 playerCount;
+}
+
+Body<uint32 playerCount> {
+  Player[playerCount] players;
+}
+
+Player {
+  uint32 id;
+  uint16 tankCount;
+  Tank[tankCount] tanks;
+}
+
+Tank {
+  uint32 id;
+  uint32 battleLifeTime;
+  uint32 lastBattleTime;
+  uint8 markOfMastery;
+  uint32 battles;
+  uint32 capturePoints;
+  uint32 damageDealt;
+  uint32 damageReceived;
+  uint32 droppedCapturePoints;
+  uint32 frags;
+  uint32 frags8p;
+  uint32 hits;
+  uint32 losses;
+  uint8 maxFrags;
+  uint16 maxXp;
+  uint32 shots;
+  uint32 spotted;
+  uint32 survivedBattles;
+  uint32 winAndSurvived;
+  uint32 wins;
+  uint32 xp;
+}
+```

scripts/buildTest.ts

@@ -1,32 +1,112 @@
 import { Locale } from 'discord.js';
-import { REGIONS } from '../src/constants/regions';
+import { chunk, times } from 'lodash';
+import { compress } from 'lz4js';
+import { argv } from 'process';
+import { Region, REGIONS } from '../src/constants/regions';
 import getTankStats from '../src/core/blitz/getTankStats';
 import { idToRegion } from '../src/core/blitz/idToRegion';
+import { commitAssets } from '../src/core/blitzkrieg/commitAssets';
+import { FileChange } from '../src/core/blitzkrieg/commitMultipleFiles';
+import {
+  RtscWritePlayer,
+  RtscWriteStream,
+  RtsmWriteStream,
+} from '../src/core/streams/rts';
 import usersRaw from '../test.users.json';
 
+const PLAYERS_PER_CHUNK = 256;
+const CHUNKS_PER_BATCH = 128;
+
+const production = argv.includes('--production');
+const users = usersRaw; //.slice(0, 10);
+
 let done = 0;
+const regionSortedUsers = REGIONS.map((region) => ({
+  region,
+  users: users.filter((user) => idToRegion(user.blitz) === region),
+}));
 
-const users = usersRaw.slice(0, 100);
-const regionUsers = REGIONS.map((region) =>
-  users.filter((user) => idToRegion(user.blitz) === region),
-);
-const regionChunkedUsers = await Promise.all(
-  regionUsers.map(async (regionUsers, index) => {
-    return await Promise.all(
-      regionUsers.map(async (user) => {
-        const region = REGIONS[index];
-        const stats = await getTankStats(region, user.blitz, Locale.EnglishUS);
+const today = new Date();
+const todayPath = `${today.getFullYear()}/${today.getMonth() + 1}/${today.getDate()}`;
+const bases: Partial<Record<Region, number>> = {};
+const chunks = regionSortedUsers
+  .map(({ region, users }) => {
+    const base = Math.ceil(users.length / PLAYERS_PER_CHUNK);
+    bases[region] = base;
 
-        console.log(`Fetch user ${++done} / ${users.length}`);
+    return times(base, (offset) => {
+      return {
+        region,
+        offset,
+        users: users.filter((user) => user.blitz % base === offset),
+      };
+    });
+  })
+  .flat();
+const batches = chunk(chunks, CHUNKS_PER_BATCH);
 
-        return stats;
-      }),
-    );
-  }),
-);
+const manifests = Object.entries(bases).map(([region, base]) => {
+  const writeStream = new RtsmWriteStream();
 
-regionChunkedUsers.map((users, regionIndex) => {
-  const region = REGIONS[regionIndex];
+  writeStream.rtsm(base);
 
-  console.log(region, users.length);
+  return {
+    encoding: 'base64',
+    path: `regions/${region}/regular/tanks/${todayPath}/manifest.rtsm`,
+    content: Buffer.from(compress(writeStream.uint8Array)).toString('base64'),
+  } satisfies FileChange;
 });
+
+await commitAssets(
+  `regular tank stats manifests ${todayPath}`,
+  manifests,
+  production,
+);
+
+let batchesDone = 0;
+for (const batch of batches) {
+  console.log(`Batch ${++batchesDone} of ${batches.length}`);
+
+  const changes: FileChange[] = [];
+
+  let chunksDone = 0;
+  for (const chunk of batch) {
+    console.log(`Chunk ${++chunksDone} of ${chunks.length}`);
+
+    let playersDone = 0;
+    const players = await Promise.all(
+      chunk.users.map(async (user) => {
+        const tanks = await getTankStats(
+          chunk.region,
+          user.blitz,
+          Locale.EnglishUS,
+        );
+
+        console.log(`Player ${++playersDone} of ${chunk.users.length}`);
+
+        if (tanks === null) return null;
+
+        return {
+          id: user.blitz,
+          tanks,
+        } satisfies RtscWritePlayer;
+      }),
+    ).then((players) => players.filter((player) => player !== null));
+    const writeStream = new RtscWriteStream();
+
+    writeStream.rtsc(players);
+    changes.push({
+      encoding: 'base64',
+      content: Buffer.from(compress(writeStream.uint8Array)).toString('base64'),
+      path: `regions/${chunk.region}/regular/tanks/${todayPath}/${chunk.offset}.rtsc.lz4`,
+    });
+  }
+
+  // do not await; commit in background
+  commitAssets(
+    `regular tank stats ${todayPath} batch ${batches.indexOf(batch) + 1}`,
+    changes,
+    production,
+    false,
+  );
+}

src/core/blitz/fetchBlitz.ts

@@ -10,8 +10,7 @@ type BlitzResponse<Data extends object> =
       data: Data;
     };
 
-// 1 less than 10 to ensure not bad things lol
-const MAX_CALLS_PER_SECOND = 9;
+const MAX_CALLS_PER_SECOND = 10;
 
 const queue: { url: string; resolve: (data: object) => void }[] = [];
 let inProgress = 0;
@@ -25,23 +24,36 @@ async function manageQueue() {
       manageQueue();
     }, 1000);
 
-    const { url, resolve } = queue.shift()!;
-    const data = (await patientFetch(url)
+    const request = queue.shift()!;
+    const data = (await patientFetch(request.url)
       .then((response) => response.json())
       .catch((error) => ({ status: 'error', error }))) as BlitzResponse<
-      typeof resolve
+      typeof request.resolve
     >;
 
     if (data.status === 'ok') {
-      resolve(data.data);
+      request.resolve(data.data);
     } else {
-      clearTimeout(timeout);
-      inProgress--;
-      manageQueue();
+      if (data.error.message === 'REQUEST_LIMIT_EXCEEDED') {
+        console.log(
+          `Rate limit exceeded, putting "${request.url}" back in queue...`,
+        );
 
-      throw new Error(`Wargaming response error status: "${data.status}"`, {
-        cause: `Message: "${data.error.message}"\nURL: "${url}"`,
-      });
+        clearTimeout(timeout);
+        setTimeout(() => {
+          queue.push(request);
+          inProgress--;
+          manageQueue();
+        }, 1000 / MAX_CALLS_PER_SECOND);
+      } else {
+        clearTimeout(timeout);
+        inProgress--;
+        manageQueue();
+
+        throw new Error(`Wargaming response error status: "${data.status}"`, {
+          cause: `Message: "${data.error.message}"\nURL: "${request.url}"`,
+        });
+      }
     }
   }
 }

src/core/blitzkrieg/commitAssets.ts

@@ -4,6 +4,7 @@ export async function commitAssets(
   message: string,
   changes: FileChange[],
   production: boolean,
+  verbose = true,
 ) {
   console.log(`Committing ${message}...`);
 
@@ -15,6 +16,6 @@ export async function commitAssets(
     production ? 'main' : 'dev',
     `${message} - ${new Date().toDateString()}`,
     changes,
-    true,
+    verbose,
   );
 }

src/core/streams/rts.ts

@@ -0,0 +1,181 @@
+import { times } from 'lodash';
+import { IndividualTankStats } from '../../types/tanksStats';
+import { AllStats } from '../blitz/getAccountInfo';
+import { ReadStream, WriteStream } from './buffer';
+
+export interface BlitzkriegTankStats extends AllStats {
+  id: number;
+  battle_life_time: number;
+  last_battle_time: number;
+  mark_of_mastery: number;
+}
+
+export class RtsmReadStream extends ReadStream {
+  rtsm() {
+    this.magic();
+    this.header();
+    return this.body();
+  }
+
+  magic() {
+    const magic = this.utf8(4);
+
+    if (magic !== 'RTSM') {
+      throw new Error(`Invalid RTSM magic number: "${magic}"`);
+    }
+  }
+
+  header() {
+    return {
+      version: this.uint16(),
+    };
+  }
+
+  body() {
+    return {
+      base: this.uint8(),
+    };
+  }
+}
+
+export class RtsmWriteStream extends WriteStream {
+  rtsm(base: number) {
+    this.magic();
+    this.header();
+    this.body(base);
+
+    return this;
+  }
+
+  magic() {
+    this.utf8('RTSM');
+  }
+
+  header() {
+    this.uint16(1);
+  }
+
+  body(base: number) {
+    this.uint8(base);
+  }
+}
+
+export class RtscReadStream extends ReadStream {
+  rtsc() {
+    this.magic();
+    const header = this.header();
+    return this.body(header.playerCount);
+  }
+
+  magic() {
+    const magic = this.utf8(4);
+
+    if (magic !== 'RTSC') {
+      throw new Error(`Invalid RTSC magic number: "${magic}"`);
+    }
+  }
+
+  header() {
+    return {
+      version: this.uint16(),
+      playerCount: this.uint32(),
+    };
+  }
+
+  body(playerCount: number) {
+    return times(playerCount, () => this.player());
+  }
+
+  player() {
+    const id = this.uint32();
+    const tankCount = this.uint16();
+    const tanks = times(tankCount, () => this.tank());
+
+    return {
+      id,
+      tankCount,
+      tanks,
+    };
+  }
+
+  tank() {
+    return {
+      id: this.uint32(),
+      battle_life_time: this.uint32(),
+      last_battle_time: this.uint32(),
+      mark_of_mastery: this.uint8(),
+      battles: this.uint32(),
+      capture_points: this.uint32(),
+      damage_dealt: this.uint32(),
+      damage_received: this.uint32(),
+      dropped_capture_points: this.uint32(),
+      frags: this.uint32(),
+      frags8p: this.uint32(),
+      hits: this.uint32(),
+      losses: this.uint32(),
+      max_frags: this.uint8(),
+      max_xp: this.uint16(),
+      shots: this.uint32(),
+      spotted: this.uint32(),
+      survived_battles: this.uint32(),
+      win_and_survived: this.uint32(),
+      wins: this.uint32(),
+      xp: this.uint32(),
+    } satisfies BlitzkriegTankStats;
+  }
+}
+
+export interface RtscWritePlayer {
+  id: number;
+  tanks: IndividualTankStats[];
+}
+
+export class RtscWriteStream extends WriteStream {
+  rtsc(players: RtscWritePlayer[]) {
+    this.magic();
+    this.header(players.length);
+    this.body(players);
+
+    return this;
+  }
+
+  magic() {
+    this.utf8('RTSC');
+  }
+
+  header(playerCount: number) {
+    this.uint16(1);
+    this.uint32(playerCount);
+  }
+
+  body(players: RtscWritePlayer[]) {
+    players.forEach((player) => {
+      this.uint32(player.id);
+      this.uint16(player.tanks.length);
+
+      player.tanks.forEach((tank) => {
+        this.uint32(tank.tank_id);
+        this.uint32(tank.battle_life_time);
+        this.uint32(tank.last_battle_time);
+        this.uint8(tank.mark_of_mastery);
+        this.uint32(tank.all.battles);
+        this.uint32(tank.all.capture_points);
+        this.uint32(tank.all.damage_dealt);
+        this.uint32(tank.all.damage_received);
+        this.uint32(tank.all.dropped_capture_points);
+        this.uint32(tank.all.frags);
+        this.uint32(tank.all.frags8p);
+        this.uint32(tank.all.hits);
+        this.uint32(tank.all.losses);
+        this.uint8(tank.all.max_frags);
+        this.uint16(tank.all.max_xp);
+        this.uint32(tank.all.shots);
+        this.uint32(tank.all.spotted);
+        this.uint32(tank.all.survived_battles);
+        this.uint32(tank.all.win_and_survived);
+        this.uint32(tank.all.wins);
+        this.uint32(tank.all.xp);
+      });
+    });
+  }
+}