feat: resurrect rating leaderboard builder

.github/workflows/rating-asia.yml

@@ -23,4 +23,4 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
-        run: bun scripts/buildRatingLeaderboard.ts --region=asia --target=midnight --production
+        run: bun build:ratingleaderboard --region=asia

.github/workflows/rating-eu.yml

@@ -23,4 +23,4 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
-        run: bun scripts/buildRatingLeaderboard.ts --region=eu --target=midnight --production
+        run: bun build:ratingleaderboard --region=eu

.github/workflows/rating-na.yml

@@ -23,4 +23,4 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
-        run: bun scripts/buildRatingLeaderboard.ts --region=na --target=midnight --production
+        run: bun build:ratingleaderboard --region=com

package.json

@@ -16,7 +16,7 @@
     "build:prisma": "prisma generate",
     "build:sitemap": "bun scripts/sitemap.ts",
     "build:docs": "vitepress build docs",
-    "build:ratingleaderboard": "bun --script scripts/buildRatingLeaderboard.ts",
+    "build:ratingleaderboard": "cd packages/scripts && bun run build:ratingleaderboard",
     "dev:bot": "bun run build:bot --dev && bun start:bot",
     "start:bot": "bun dist/bot/main.js",
     "dev:docs": "vitepress dev docs",

packages/core/src/streams/bkrl.ts

@@ -1,142 +0,0 @@
-import { times } from 'lodash-es';
-import { ReadStream, WriteStream } from './buffer';
-
-export enum BkrlFormat {
-  Base,
-  Superset1,
-}
-
-export interface BkrlBaseEntry {
-  id: number;
-  score: number;
-}
-
-export interface BkrlSuperset1Entry {
-  id: number;
-  score: number;
-
-  battles: number;
-  wins: number;
-  survived: number;
-
-  damage: number;
-  kills: number;
-}
-
-export type BkrlDiscriminatedEntries =
-  | {
-      format: BkrlFormat.Base;
-      entries: BkrlBaseEntry[];
-    }
-  | {
-      format: BkrlFormat.Superset1;
-      entries: BkrlSuperset1Entry[];
-    };
-
-export class BkrlReadStream extends ReadStream {
-  bkrl() {
-    this.magic();
-    const header = this.header();
-    return this.body(header);
-  }
-
-  body(header: ReturnType<typeof this.header>): BkrlDiscriminatedEntries {
-    switch (header.format) {
-      case BkrlFormat.Base: {
-        return {
-          format: BkrlFormat.Base,
-          entries: times(
-            header.count,
-            () =>
-              ({
-                id: this.uint32(),
-                score: this.uint16(),
-              }) satisfies BkrlBaseEntry,
-          ),
-        };
-      }
-
-      case BkrlFormat.Superset1: {
-        return {
-          format: BkrlFormat.Superset1,
-          entries: times(
-            header.count,
-            () =>
-              ({
-                id: this.uint32(),
-                score: this.uint16(),
-
-                battles: this.uint32(),
-                wins: this.uint32(),
-                survived: this.uint32(),
-
-                damage: this.uint32(),
-                kills: this.uint32(),
-              }) satisfies BkrlSuperset1Entry,
-          ),
-        };
-      }
-    }
-  }
-
-  header() {
-    return {
-      version: this.uint16(),
-      format: this.uint8() as BkrlFormat,
-      count: this.uint32(),
-    };
-  }
-
-  magic() {
-    const magic = this.utf8(4);
-
-    if (magic !== 'BKRL') {
-      throw new Error(`Invalid BKRL magic number: "${magic}"`);
-    }
-  }
-}
-
-export class BkrlWriteStream extends WriteStream {
-  bkrl(discriminatedEntries: BkrlDiscriminatedEntries) {
-    this.magic();
-    this.header(discriminatedEntries);
-    this.body(discriminatedEntries);
-  }
-
-  body(discriminatedEntries: BkrlDiscriminatedEntries) {
-    switch (discriminatedEntries.format) {
-      case BkrlFormat.Base: {
-        discriminatedEntries.entries.forEach((entry) => {
-          this.uint32(entry.id);
-          this.uint16(entry.score);
-        });
-        break;
-      }
-
-      case BkrlFormat.Superset1: {
-        discriminatedEntries.entries.forEach((entry) => {
-          this.uint32(entry.id);
-          this.uint16(entry.score);
-
-          this.uint32(entry.battles);
-          this.uint32(entry.wins);
-          this.uint32(entry.survived);
-
-          this.uint32(entry.damage);
-          this.uint32(entry.kills);
-        });
-        break;
-      }
-    }
-  }
-
-  header(discriminatedEntries: BkrlDiscriminatedEntries) {
-    this.uint16(1);
-    this.uint8(discriminatedEntries.format);
-    this.uint32(discriminatedEntries.entries.length);
-  }
-
-  magic() {
-    this.utf8('BKRL');
-  }
-}

packages/scripts/package.json

@@ -9,7 +9,8 @@
     "setup:patches": "bun src/applyPatches.ts",
     "build:reviews": "bun src/reviews.ts",
     "build:ids": "bun src/discoverIds.ts",
-    "build:bot": "bun src/buildBot.ts"
+    "build:bot": "bun src/buildBot.ts",
+    "build:ratingleaderboard": "bun src/buildRatingLeaderboard.ts"
   },
   "dependencies": {
     "esbuild": "^0.19.12",

packages/scripts/src/buildBot.ts

@@ -23,6 +23,7 @@ build({
   loader: {
     '.ttf': 'file',
     '.node': 'empty',
+    '.json': 'json',
   },
   minifyIdentifiers: false, // causes goofy ahh issues
   minifySyntax: !dev,

packages/scripts/src/buildRatingLeaderboard.ts

@@ -0,0 +1,171 @@
+import {
+  getAccountInfo,
+  patientFetchJSON,
+  RatingInfo,
+  RatingLeaderboard,
+  RatingLeaderboardEntryV1,
+  RatingLeaderboardEntryV2,
+  RatingNeighbors,
+  RatingPlayer,
+  Region,
+  REGIONS,
+  regionToRegionSubdomain,
+} from '@blitzkit/core';
+import { chunk } from 'lodash';
+import { argv } from 'process';
+import { commitAssets } from './core/github/commitAssets';
+import { FileChange } from './core/github/commitMultipleFiles';
+
+const region = argv
+  .find((arg) => arg.startsWith('--region='))
+  ?.split('=')[1] as Region | undefined;
+
+if (region === undefined || !REGIONS.includes(region)) {
+  throw new Error('No valid region specified');
+}
+
+console.log(`Building rating leaderboard for ${region}`);
+
+const regionSubdomain = regionToRegionSubdomain(region);
+const NEIGHBORS = 2 ** 8;
+const PLAYERS = await patientFetchJSON<RatingInfo>(
+  `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`,
+).then((data) => (data.detail === undefined ? data.count : undefined));
+
+if (PLAYERS === undefined) {
+  throw new Error('No current rating season running');
+}
+
+const players: Record<number, RatingPlayer> = {};
+const entriesV1: RatingLeaderboardEntryV1[] = [];
+
+let registered = 0;
+
+async function branchFromPlayer(id: number) {
+  const { neighbors } = await patientFetchJSON<RatingNeighbors>(
+    `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`,
+  );
+
+  const firstPlayer = neighbors[0];
+  const lastPlayer = neighbors[neighbors.length - 1];
+
+  const branches: Promise<void>[] = [];
+  if (!players[firstPlayer.number])
+    branches.push(branchFromPlayer(firstPlayer.spa_id));
+  if (!players[lastPlayer.number])
+    branches.push(branchFromPlayer(lastPlayer.spa_id));
+
+  neighbors.forEach((neighbor) => {
+    if (players[neighbor.number]) return;
+
+    players[neighbor.number] = neighbor;
+    registered++;
+  });
+
+  console.log(
+    `Registered ${registered} / ${PLAYERS} (${(
+      (registered / PLAYERS!) *
+      100
+    ).toFixed(2)}%)`,
+  );
+
+  await Promise.all(branches);
+}
+
+async function branchFromLeague(id: number) {
+  const { result } = await patientFetchJSON<{ result: RatingPlayer[] }>(
+    `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`,
+  );
+
+  const firstPlayer = result[0];
+  const lastPlayer = result.at(-1)!;
+
+  await Promise.all([
+    branchFromPlayer(firstPlayer.spa_id),
+    branchFromPlayer(lastPlayer.spa_id),
+  ]);
+}
+
+console.log('Branching from leagues...');
+await Promise.all([
+  branchFromLeague(0),
+  branchFromLeague(1),
+  branchFromLeague(2),
+  branchFromLeague(3),
+  branchFromLeague(4),
+]);
+
+console.log('Converting to an array...');
+for (let index = 0; index < PLAYERS; index++) {
+  const listing = players[index + 1];
+  if (listing) {
+    entriesV1[index] = { id: listing.spa_id, score: listing.score };
+  } else {
+    console.warn(`Position ${index + 1} had no player`);
+    entriesV1[index] = { id: 0, score: 0 };
+  }
+}
+
+console.log('Requesting further stats...');
+const entries: RatingLeaderboardEntryV2[] = [];
+let chunkIndex = 0;
+for (const leaderboardChunk of chunk(entriesV1, 100)) {
+  console.log(
+    `Chunk ${chunkIndex + 1} of ${Math.ceil(entriesV1.length / 100)}`,
+  );
+
+  const stats = await getAccountInfo(
+    region,
+    leaderboardChunk.map(({ id }) => id),
+    ['statistics.rating'],
+  );
+
+  stats.forEach((stat, index) => {
+    const trueIndex = chunkIndex * 100 + index;
+
+    if (
+      leaderboardChunk[index].id === 0 ||
+      stat?.statistics.rating === undefined
+    ) {
+      entries[trueIndex] = {
+        battles: 0,
+        damage: 0,
+        id: 0,
+        kills: 0,
+        score: 0,
+        survived: 0,
+        wins: 0,
+      };
+    } else {
+      entries[trueIndex] = {
+        id: leaderboardChunk[index].id,
+        score: leaderboardChunk[index].score,
+        battles: stat.statistics.rating.battles,
+        damage: stat.statistics.rating.damage_dealt,
+        kills: stat.statistics.rating.frags,
+        survived: stat.statistics.rating.survived_battles,
+        wins: stat.statistics.rating.wins,
+      };
+    }
+  });
+
+  chunkIndex++;
+}
+
+const info = await patientFetchJSON<RatingInfo & { detail: undefined }>(
+  `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`,
+);
+const normalizedServer = regionSubdomain === 'na' ? 'com' : regionSubdomain;
+
+const content = RatingLeaderboard.encode({
+  version: { $case: 'v2', value: { entries } },
+}).finish();
+
+const changes: FileChange[] = [
+  {
+    path: `regions/${normalizedServer}/rating/${info.current_season}.pb`,
+    content: Buffer.from(content),
+  },
+];
+
+commitAssets(`rating leaderboard ${regionSubdomain}`, changes);