feat: extended ratings stats

.github/workflows/cd-ratings-asia.yml

@@ -31,4 +31,4 @@ jobs:
         run: yarn
 
       - name: Generate leaderboard
-        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=midnight
+        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=midnight --production

.github/workflows/cd-ratings-eu.yml

@@ -31,4 +31,4 @@ jobs:
         run: yarn
 
       - name: Generate leaderboard
-        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=midnight
+        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=midnight --production

.github/workflows/cd-ratings-latest.yml

@@ -31,13 +31,13 @@ jobs:
         run: yarn
 
       - name: Generate NA leaderboard
-        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=latest
+        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=latest --production
         if: always()
 
       - name: Generate EU leaderboard
-        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=latest
+        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=latest --production
         if: always()
 
       - name: Generate Asia leaderboard
-        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=latest
+        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=latest --production
         if: always()

.github/workflows/cd-ratings-na.yml

@@ -31,4 +31,4 @@ jobs:
         run: yarn
 
       - name: Generate leaderboard
-        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=midnight
+        run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=midnight --production

scripts/buildRatingsLeaderboard.ts

@@ -1,38 +1,47 @@
-import { rm, writeFile } from 'fs/promises';
+import { chunk } from 'lodash';
 import { argv } from 'process';
 import {
   RatingsInfo,
   RatingsNeighbors,
   RatingsPlayer,
 } from '../src/commands/ratings';
-import commitMultipleFiles from '../src/core/blitzkrieg/commitMultipleFiles.js';
+import { RegionSubdomain } from '../src/constants/regions';
+import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
+import regionSubdomainToRegion from '../src/core/blitz/regionSubdomainToRegion';
+import { commitAssets } from '../src/core/blitzkrieg/commitAssets';
+import { FileChange } from '../src/core/blitzkrieg/commitMultipleFiles.js';
 import { patientFetchJSON } from '../src/core/blitzkrieg/patientFetchJSON';
+import { superCompress } from '../src/core/blitzkrieg/superCompress';
+import {
+  BkrlComprehensive1Entry,
+  BkrlFormat,
+  BkrlMinimalEntry,
+  BkrlWriteStream,
+} from '../src/core/streams/bkrl';
+import { CdonValue, CdonWriteStream } from '../src/core/streams/cdon';
 
 /*
  * Central North American Time (UTC-5): use 0 5 * * *.
  * Central European Time (UTC+1): use 0 23 * * *.
  * Central Asia Standard Time (UTC+7): use 0 17 * * *.
- *
- * [region]/ratings/[season]/info.json: season info like rewards, start & end time, etc.
- * [region]/ratings/[season]/latest.json: the absolute latest leaderboard recorded at UTC+0 midnight
- * [region]/ratings/[season]/midnight.json: last midnight's leaderboard relative to the region
  */
 
-export type BlitzkriegRatingsLeaderboardEntry = { id: number; score: number };
+const production = argv.includes('--production');
-export type BlitzkriegRatingsLeaderboard = BlitzkriegRatingsLeaderboardEntry[];
-
-const publish = argv.includes('--publish');
 const target = argv
   .find((arg) => arg.startsWith('--target='))
   ?.split('=')[1] as 'midnight' | 'latest' | undefined;
-const server = argv.find((arg) => arg.startsWith('--region='))?.split('=')[1];
+const regionSubdomain = argv
+  .find((arg) => arg.startsWith('--region='))
+  ?.split('=')[1] as RegionSubdomain | undefined;
 
 if (!target) throw new Error('Target parameter not specified');
-if (!server) throw new Error('Region parameter not specified');
+if (!regionSubdomain) throw new Error('Region parameter not specified');
+
+const region = regionSubdomainToRegion(regionSubdomain);
 
 const NEIGHBORS = 2 ** 8;
 const PLAYERS = await patientFetchJSON<RatingsInfo>(
-  `https://${server}.wotblitz.com/en/api/rating-leaderboards/season/`,
+  `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`,
 ).then((data) => (data.detail === undefined ? data.count : undefined));
 
 if (PLAYERS === undefined) {
@@ -40,12 +49,12 @@ if (PLAYERS === undefined) {
 }
 
 const players: Record<number, RatingsPlayer> = {};
-const leaderboard: BlitzkriegRatingsLeaderboard = [];
+const leaderboard: BkrlMinimalEntry[] = [];
 let registered = 0;
 
 async function branchFromPlayer(id: number) {
   const { neighbors } = await patientFetchJSON<RatingsNeighbors>(
-    `https://${server}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`,
+    `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`,
   );
 
   const firstPlayer = neighbors[0];
@@ -76,11 +85,11 @@ async function branchFromPlayer(id: number) {
 
 async function branchFromLeague(id: number) {
   const { result } = await patientFetchJSON<{ result: RatingsPlayer[] }>(
-    `https://${server}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`,
+    `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`,
   );
 
   const firstPlayer = result[0];
-  const lastPlayer = result[result.length - 1];
+  const lastPlayer = result.at(-1)!;
 
   await Promise.all([
     branchFromPlayer(firstPlayer.spa_id),
@@ -100,41 +109,90 @@ await Promise.all([
 console.log('Converting to an array...');
 for (let index = 0; index < PLAYERS; index++) {
   const listing = players[index + 1];
-  if (!listing) {
+  if (listing) {
+    leaderboard[index] = { id: listing.spa_id, score: listing.score };
+  } else {
     console.warn(`Position ${index + 1} had no player`);
-    continue;
+    leaderboard[index] = { id: 0, score: 0 };
   }
-
-  leaderboard[index] = {
-    id: listing.spa_id,
-    score: listing.score,
-  };
 }
 
-const leaderboardJSON = JSON.stringify(leaderboard);
+console.log('Requesting further stats...');
-
+const comprehensiveLeaderboard: BkrlComprehensive1Entry[] = [];
-if (publish) {
+let chunkIndex = 0;
-  const info = await patientFetchJSON<RatingsInfo & { detail: undefined }>(
+for (const leaderboardChunk of chunk(leaderboard, 100)) {
-    `https://${server}.wotblitz.com/en/api/rating-leaderboards/season/`,
+  console.log(
+    `Chunk ${chunkIndex + 1} of ${Math.ceil(leaderboard.length / 100)}`,
   );
-  const normalizedServer = server === 'na' ? 'com' : server;
-  const infoPath = `regions/${normalizedServer}/ratings/${info.current_season}/info.json`;
-  const leaderboardPath = `regions/${normalizedServer}/ratings/${info.current_season}/${target}.json`;
-  const infoJSON = JSON.stringify(info);
 
-  console.log(`Publishing to season ${info.current_season}...`);
+  const stats = await getAccountInfo(
-  commitMultipleFiles(
+    region,
-    'tresabhi',
+    leaderboardChunk.map(({ id }) => id),
-    'blitzkrieg-assets',
+    ['statistics.rating'],
-    'main',
-    new Date().toString(),
-    [
-      { path: infoPath, content: infoJSON, encoding: 'utf-8' },
-      { path: leaderboardPath, content: leaderboardJSON, encoding: 'utf-8' },
-    ],
   );
-} else {
+
-  console.log('Writing to disk...');
+  stats.forEach((stat, index) => {
-  await rm('dist.json', { force: true });
+    const trueIndex = chunkIndex * 100 + index;
-  await writeFile('dist.json', leaderboardJSON);
+    comprehensiveLeaderboard[trueIndex] =
+      leaderboardChunk[index].id === 0
+        ? {
+            battles: 0,
+            damageDealt: 0,
+            damageReceived: 0,
+            hits: 0,
+            id: 0,
+            kills: 0,
+            score: 0,
+            shots: 0,
+            survived: 0,
+            wins: 0,
+          }
+        : {
+            id: leaderboardChunk[index].id,
+            score: leaderboardChunk[index].score,
+            battles: stat.statistics.rating!.battles,
+            damageDealt: stat.statistics.rating!.damage_dealt,
+            damageReceived: stat.statistics.rating!.damage_received,
+            hits: stat.statistics.rating!.hits,
+            kills: stat.statistics.rating!.frags,
+            shots: stat.statistics.rating!.shots,
+            survived: stat.statistics.rating!.win_and_survived,
+            wins: stat.statistics.rating!.wins,
+          };
+  });
+
+  chunkIndex++;
 }
+
+const info = await patientFetchJSON<RatingsInfo & { detail: undefined }>(
+  `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`,
+);
+const normalizedServer = regionSubdomain === 'na' ? 'com' : regionSubdomain;
+const infoContent = new CdonWriteStream().cdon(
+  info as unknown as CdonValue,
+).uint8Array;
+const leaderboardWriteStream = new BkrlWriteStream();
+
+leaderboardWriteStream.bkrl({
+  format: BkrlFormat.Comprehensive1,
+  entries: comprehensiveLeaderboard,
+});
+
+const changes: FileChange[] = [
+  {
+    path: `regions/${normalizedServer}/ratings/${info.current_season}/info.cdon.lz4`,
+    content: superCompress(infoContent),
+    encoding: 'base64',
+  },
+  {
+    path: `regions/${normalizedServer}/ratings/${info.current_season}/${target}.bkrl`,
+    content: Buffer.from(leaderboardWriteStream.uint8Array).toString('base64'),
+    encoding: 'base64',
+  },
+];
+
+commitAssets(
+  `ratings leaderboard ${regionSubdomain} ${target}`,
+  changes,
+  production,
+);

scripts/buildRatingsPlot.ts

@@ -1,3 +1,5 @@
+// what the hell is this script???
+
 import { writeFile } from 'fs/promises';
 import { Region } from '../src/constants/regions';
 import { getRatingsNeighbors } from '../src/core/blitz/getRatingsNeighbors';

src/core/blitz/fetchBlitz.ts

@@ -1,3 +1,5 @@
+import { patientFetch } from '../blitzkrieg/patientFetch';
+
 type BlitzResponse<Data extends object> =
   | {
       status: 'error';
@@ -23,7 +25,7 @@ async function manageQueue() {
     }, 1000);
 
     const { url, resolve } = queue.shift()!;
-    const data = (await fetch(url)
+    const data = (await patientFetch(url)
       .then((response) => response.json())
       .catch((error) => ({ status: 'error', error }))) as BlitzResponse<
       typeof resolve

src/core/blitz/getAccountInfo.ts

@@ -51,6 +51,28 @@ export interface ClanStats {
   dropped_capture_points: number;
 }
 
+interface RatingStats {
+  spotted: number;
+  calibration_battles_left: number;
+  hits: number;
+  frags: number;
+  recalibration_start_time: number;
+  mm_rating: number;
+  wins: number;
+  losses: number;
+  is_recalibration: boolean;
+  capture_points: number;
+  battles: number;
+  damage_dealt: number;
+  damage_received: number;
+  shots: number;
+  frags8p: number;
+  xp: number;
+  win_and_survived: number;
+  survived_battles: number;
+  dropped_capture_points: number;
+}
+
 interface IndividualAccountInfo {
   account_id: number;
   created_at: number;
@@ -61,6 +83,7 @@ interface IndividualAccountInfo {
   statistics: {
     all: AllStats;
     clan: ClanStats;
+    rating?: RatingStats;
     frags: null | number;
   };
 }
@@ -74,11 +97,11 @@ export async function getAccountInfo<
   ReturnType = Ids extends number
     ? IndividualAccountInfo
     : IndividualAccountInfo[],
->(region: Region, ids: Ids) {
+>(region: Region, ids: Ids, extra: string[] = []) {
   const object = await fetchBlitz<AccountInfo>(
     `https://api.wotblitz.${region}/wotb/account/info/?application_id=${WARGAMING_APPLICATION_ID}&account_id=${normalizeIds(
       ids,
-    )}`,
+    )}${extra.length === 0 ? '' : `&extra=${extra.join(',')}`}`,
   );
 
   if (typeof ids === 'number') {

src/core/blitz/regionSubdomainToRegion.ts

@@ -0,0 +1,7 @@
+import { Region, RegionSubdomain } from '../../constants/regions';
+
+export default function regionSubdomainToRegion(
+  regionDomain: RegionSubdomain,
+): Region {
+  return regionDomain === 'na' ? 'com' : regionDomain;
+}

src/core/streams/bkrl.ts

@@ -1,17 +1,17 @@
 import { times } from 'lodash';
 import { ReadStream, WriteStream } from './buffer';
 
-enum Format {
+export enum BkrlFormat {
   Minimal,
   Comprehensive1,
 }
 
-interface BkrlMinimalEntry {
+export interface BkrlMinimalEntry {
   id: number;
   score: number;
 }
 
-interface BkrlComprehensive1Entry {
+export interface BkrlComprehensive1Entry {
   id: number;
   score: number;
 
@@ -30,11 +30,11 @@ interface BkrlComprehensive1Entry {
 
 type DiscriminatedEntries =
   | {
-      format: Format.Minimal;
+      format: BkrlFormat.Minimal;
       entries: BkrlMinimalEntry[];
     }
   | {
-      format: Format.Comprehensive1;
+      format: BkrlFormat.Comprehensive1;
       entries: BkrlComprehensive1Entry[];
     };
 
@@ -47,9 +47,9 @@ export class BkrlReadStream extends ReadStream {
 
   body(header: ReturnType<typeof this.header>): DiscriminatedEntries {
     switch (header.format) {
-      case Format.Minimal: {
+      case BkrlFormat.Minimal: {
         return {
-          format: Format.Minimal,
+          format: BkrlFormat.Minimal,
           entries: times(
             header.count,
             () =>
@@ -61,9 +61,9 @@ export class BkrlReadStream extends ReadStream {
         };
       }
 
-      case Format.Comprehensive1: {
+      case BkrlFormat.Comprehensive1: {
         return {
-          format: Format.Comprehensive1,
+          format: BkrlFormat.Comprehensive1,
           entries: times(
             header.count,
             () =>
@@ -92,7 +92,7 @@ export class BkrlReadStream extends ReadStream {
   header() {
     return {
       version: this.uint16(),
-      format: this.uint8() as Format,
+      format: this.uint8() as BkrlFormat,
       count: this.uint32(),
     };
   }
@@ -115,7 +115,7 @@ export class BkrlWriteStream extends WriteStream {
 
   body(discriminatedEntries: DiscriminatedEntries) {
     switch (discriminatedEntries.format) {
-      case Format.Minimal: {
+      case BkrlFormat.Minimal: {
         discriminatedEntries.entries.forEach((entry) => {
           this.uint32(entry.id);
           this.uint16(entry.score);
@@ -123,7 +123,7 @@ export class BkrlWriteStream extends WriteStream {
         break;
       }
 
-      case Format.Comprehensive1: {
+      case BkrlFormat.Comprehensive1: {
         discriminatedEntries.entries.forEach((entry) => {
           this.uint32(entry.id);
           this.uint16(entry.score);