diff --git a/.github/workflows/cd-ratings-asia.yml b/.github/workflows/cd-ratings-asia.yml index a4bb34d20..5aaedffaf 100644 --- a/.github/workflows/cd-ratings-asia.yml +++ b/.github/workflows/cd-ratings-asia.yml @@ -31,4 +31,4 @@ jobs: run: yarn - name: Generate leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=midnight + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=midnight --production diff --git a/.github/workflows/cd-ratings-eu.yml b/.github/workflows/cd-ratings-eu.yml index 66f331e95..63efa1d89 100644 --- a/.github/workflows/cd-ratings-eu.yml +++ b/.github/workflows/cd-ratings-eu.yml @@ -31,4 +31,4 @@ jobs: run: yarn - name: Generate leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=midnight + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=midnight --production diff --git a/.github/workflows/cd-ratings-latest.yml b/.github/workflows/cd-ratings-latest.yml index 0aea10c1e..91bd8f8dd 100644 --- a/.github/workflows/cd-ratings-latest.yml +++ b/.github/workflows/cd-ratings-latest.yml @@ -31,13 +31,13 @@ jobs: run: yarn - name: Generate NA leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=latest + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=latest --production if: always() - name: Generate EU leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=latest + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=eu --target=latest --production if: always() - name: Generate Asia leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=latest + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=asia --target=latest --production if: always() diff --git a/.github/workflows/cd-ratings-na.yml b/.github/workflows/cd-ratings-na.yml index 65156fae8..18c771840 100644 --- a/.github/workflows/cd-ratings-na.yml +++ b/.github/workflows/cd-ratings-na.yml @@ -31,4 +31,4 @@ jobs: run: yarn - name: Generate leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=midnight + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingsleaderboard --region=na --target=midnight --production diff --git a/scripts/buildRatingsLeaderboard.ts b/scripts/buildRatingsLeaderboard.ts index 891ea6edf..0c2d25c16 100644 --- a/scripts/buildRatingsLeaderboard.ts +++ b/scripts/buildRatingsLeaderboard.ts @@ -1,38 +1,47 @@ -import { rm, writeFile } from 'fs/promises'; +import { chunk } from 'lodash'; import { argv } from 'process'; import { RatingsInfo, RatingsNeighbors, RatingsPlayer, } from '../src/commands/ratings'; -import commitMultipleFiles from '../src/core/blitzkrieg/commitMultipleFiles.js'; +import { RegionSubdomain } from '../src/constants/regions'; +import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; +import regionSubdomainToRegion from '../src/core/blitz/regionSubdomainToRegion'; +import { commitAssets } from '../src/core/blitzkrieg/commitAssets'; +import { FileChange } from '../src/core/blitzkrieg/commitMultipleFiles.js'; import { patientFetchJSON } from '../src/core/blitzkrieg/patientFetchJSON'; +import { superCompress } from '../src/core/blitzkrieg/superCompress'; +import { + BkrlComprehensive1Entry, + BkrlFormat, + BkrlMinimalEntry, + BkrlWriteStream, +} from '../src/core/streams/bkrl'; +import { CdonValue, CdonWriteStream } from '../src/core/streams/cdon'; /* * Central North American Time (UTC-5): use 0 5 * * *. * Central European Time (UTC+1): use 0 23 * * *. * Central Asia Standard Time (UTC+7): use 0 17 * * *. - * - * [region]/ratings/[season]/info.json: season info like rewards, start & end time, etc. - * [region]/ratings/[season]/latest.json: the absolute latest leaderboard recorded at UTC+0 midnight - * [region]/ratings/[season]/midnight.json: last midnight's leaderboard relative to the region */ -export type BlitzkriegRatingsLeaderboardEntry = { id: number; score: number }; -export type BlitzkriegRatingsLeaderboard = BlitzkriegRatingsLeaderboardEntry[]; - -const publish = argv.includes('--publish'); +const production = argv.includes('--production'); const target = argv .find((arg) => arg.startsWith('--target=')) ?.split('=')[1] as 'midnight' | 'latest' | undefined; -const server = argv.find((arg) => arg.startsWith('--region='))?.split('=')[1]; +const regionSubdomain = argv + .find((arg) => arg.startsWith('--region=')) + ?.split('=')[1] as RegionSubdomain | undefined; if (!target) throw new Error('Target parameter not specified'); -if (!server) throw new Error('Region parameter not specified'); +if (!regionSubdomain) throw new Error('Region parameter not specified'); + +const region = regionSubdomainToRegion(regionSubdomain); const NEIGHBORS = 2 ** 8; const PLAYERS = await patientFetchJSON( - `https://${server}.wotblitz.com/en/api/rating-leaderboards/season/`, + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`, ).then((data) => (data.detail === undefined ? data.count : undefined)); if (PLAYERS === undefined) { @@ -40,12 +49,12 @@ if (PLAYERS === undefined) { } const players: Record = {}; -const leaderboard: BlitzkriegRatingsLeaderboard = []; +const leaderboard: BkrlMinimalEntry[] = []; let registered = 0; async function branchFromPlayer(id: number) { const { neighbors } = await patientFetchJSON( - `https://${server}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`, + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`, ); const firstPlayer = neighbors[0]; @@ -76,11 +85,11 @@ async function branchFromPlayer(id: number) { async function branchFromLeague(id: number) { const { result } = await patientFetchJSON<{ result: RatingsPlayer[] }>( - `https://${server}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`, + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`, ); const firstPlayer = result[0]; - const lastPlayer = result[result.length - 1]; + const lastPlayer = result.at(-1)!; await Promise.all([ branchFromPlayer(firstPlayer.spa_id), @@ -100,41 +109,90 @@ await Promise.all([ console.log('Converting to an array...'); for (let index = 0; index < PLAYERS; index++) { const listing = players[index + 1]; - if (!listing) { + if (listing) { + leaderboard[index] = { id: listing.spa_id, score: listing.score }; + } else { console.warn(`Position ${index + 1} had no player`); - continue; + leaderboard[index] = { id: 0, score: 0 }; } - - leaderboard[index] = { - id: listing.spa_id, - score: listing.score, - }; } -const leaderboardJSON = JSON.stringify(leaderboard); - -if (publish) { - const info = await patientFetchJSON( - `https://${server}.wotblitz.com/en/api/rating-leaderboards/season/`, +console.log('Requesting further stats...'); +const comprehensiveLeaderboard: BkrlComprehensive1Entry[] = []; +let chunkIndex = 0; +for (const leaderboardChunk of chunk(leaderboard, 100)) { + console.log( + `Chunk ${chunkIndex + 1} of ${Math.ceil(leaderboard.length / 100)}`, ); - const normalizedServer = server === 'na' ? 'com' : server; - const infoPath = `regions/${normalizedServer}/ratings/${info.current_season}/info.json`; - const leaderboardPath = `regions/${normalizedServer}/ratings/${info.current_season}/${target}.json`; - const infoJSON = JSON.stringify(info); - console.log(`Publishing to season ${info.current_season}...`); - commitMultipleFiles( - 'tresabhi', - 'blitzkrieg-assets', - 'main', - new Date().toString(), - [ - { path: infoPath, content: infoJSON, encoding: 'utf-8' }, - { path: leaderboardPath, content: leaderboardJSON, encoding: 'utf-8' }, - ], + const stats = await getAccountInfo( + region, + leaderboardChunk.map(({ id }) => id), + ['statistics.rating'], ); -} else { - console.log('Writing to disk...'); - await rm('dist.json', { force: true }); - await writeFile('dist.json', leaderboardJSON); + + stats.forEach((stat, index) => { + const trueIndex = chunkIndex * 100 + index; + comprehensiveLeaderboard[trueIndex] = + leaderboardChunk[index].id === 0 + ? { + battles: 0, + damageDealt: 0, + damageReceived: 0, + hits: 0, + id: 0, + kills: 0, + score: 0, + shots: 0, + survived: 0, + wins: 0, + } + : { + id: leaderboardChunk[index].id, + score: leaderboardChunk[index].score, + battles: stat.statistics.rating!.battles, + damageDealt: stat.statistics.rating!.damage_dealt, + damageReceived: stat.statistics.rating!.damage_received, + hits: stat.statistics.rating!.hits, + kills: stat.statistics.rating!.frags, + shots: stat.statistics.rating!.shots, + survived: stat.statistics.rating!.win_and_survived, + wins: stat.statistics.rating!.wins, + }; + }); + + chunkIndex++; } + +const info = await patientFetchJSON( + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`, +); +const normalizedServer = regionSubdomain === 'na' ? 'com' : regionSubdomain; +const infoContent = new CdonWriteStream().cdon( + info as unknown as CdonValue, +).uint8Array; +const leaderboardWriteStream = new BkrlWriteStream(); + +leaderboardWriteStream.bkrl({ + format: BkrlFormat.Comprehensive1, + entries: comprehensiveLeaderboard, +}); + +const changes: FileChange[] = [ + { + path: `regions/${normalizedServer}/ratings/${info.current_season}/info.cdon.lz4`, + content: superCompress(infoContent), + encoding: 'base64', + }, + { + path: `regions/${normalizedServer}/ratings/${info.current_season}/${target}.bkrl`, + content: Buffer.from(leaderboardWriteStream.uint8Array).toString('base64'), + encoding: 'base64', + }, +]; + +commitAssets( + `ratings leaderboard ${regionSubdomain} ${target}`, + changes, + production, +); diff --git a/scripts/buildRatingsPlot.ts b/scripts/buildRatingsPlot.ts index a0352a9ab..aac576f4d 100644 --- a/scripts/buildRatingsPlot.ts +++ b/scripts/buildRatingsPlot.ts @@ -1,3 +1,5 @@ +// what the hell is this script??? + import { writeFile } from 'fs/promises'; import { Region } from '../src/constants/regions'; import { getRatingsNeighbors } from '../src/core/blitz/getRatingsNeighbors'; diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 973c7a204..48bd1f211 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -1,3 +1,5 @@ +import { patientFetch } from '../blitzkrieg/patientFetch'; + type BlitzResponse = | { status: 'error'; @@ -23,7 +25,7 @@ async function manageQueue() { }, 1000); const { url, resolve } = queue.shift()!; - const data = (await fetch(url) + const data = (await patientFetch(url) .then((response) => response.json()) .catch((error) => ({ status: 'error', error }))) as BlitzResponse< typeof resolve diff --git a/src/core/blitz/getAccountInfo.ts b/src/core/blitz/getAccountInfo.ts index efb795c41..8d63a8693 100644 --- a/src/core/blitz/getAccountInfo.ts +++ b/src/core/blitz/getAccountInfo.ts @@ -51,6 +51,28 @@ export interface ClanStats { dropped_capture_points: number; } +interface RatingStats { + spotted: number; + calibration_battles_left: number; + hits: number; + frags: number; + recalibration_start_time: number; + mm_rating: number; + wins: number; + losses: number; + is_recalibration: boolean; + capture_points: number; + battles: number; + damage_dealt: number; + damage_received: number; + shots: number; + frags8p: number; + xp: number; + win_and_survived: number; + survived_battles: number; + dropped_capture_points: number; +} + interface IndividualAccountInfo { account_id: number; created_at: number; @@ -61,6 +83,7 @@ interface IndividualAccountInfo { statistics: { all: AllStats; clan: ClanStats; + rating?: RatingStats; frags: null | number; }; } @@ -74,11 +97,11 @@ export async function getAccountInfo< ReturnType = Ids extends number ? IndividualAccountInfo : IndividualAccountInfo[], ->(region: Region, ids: Ids) { +>(region: Region, ids: Ids, extra: string[] = []) { const object = await fetchBlitz( `https://api.wotblitz.${region}/wotb/account/info/?application_id=${WARGAMING_APPLICATION_ID}&account_id=${normalizeIds( ids, - )}`, + )}${extra.length === 0 ? '' : `&extra=${extra.join(',')}`}`, ); if (typeof ids === 'number') { diff --git a/src/core/blitz/regionSubdomainToRegion.ts b/src/core/blitz/regionSubdomainToRegion.ts new file mode 100644 index 000000000..066f772cd --- /dev/null +++ b/src/core/blitz/regionSubdomainToRegion.ts @@ -0,0 +1,7 @@ +import { Region, RegionSubdomain } from '../../constants/regions'; + +export default function regionSubdomainToRegion( + regionDomain: RegionSubdomain, +): Region { + return regionDomain === 'na' ? 'com' : regionDomain; +} diff --git a/src/core/streams/bkrl.ts b/src/core/streams/bkrl.ts index 09150be31..120f30cf1 100644 --- a/src/core/streams/bkrl.ts +++ b/src/core/streams/bkrl.ts @@ -1,17 +1,17 @@ import { times } from 'lodash'; import { ReadStream, WriteStream } from './buffer'; -enum Format { +export enum BkrlFormat { Minimal, Comprehensive1, } -interface BkrlMinimalEntry { +export interface BkrlMinimalEntry { id: number; score: number; } -interface BkrlComprehensive1Entry { +export interface BkrlComprehensive1Entry { id: number; score: number; @@ -30,11 +30,11 @@ interface BkrlComprehensive1Entry { type DiscriminatedEntries = | { - format: Format.Minimal; + format: BkrlFormat.Minimal; entries: BkrlMinimalEntry[]; } | { - format: Format.Comprehensive1; + format: BkrlFormat.Comprehensive1; entries: BkrlComprehensive1Entry[]; }; @@ -47,9 +47,9 @@ export class BkrlReadStream extends ReadStream { body(header: ReturnType): DiscriminatedEntries { switch (header.format) { - case Format.Minimal: { + case BkrlFormat.Minimal: { return { - format: Format.Minimal, + format: BkrlFormat.Minimal, entries: times( header.count, () => @@ -61,9 +61,9 @@ export class BkrlReadStream extends ReadStream { }; } - case Format.Comprehensive1: { + case BkrlFormat.Comprehensive1: { return { - format: Format.Comprehensive1, + format: BkrlFormat.Comprehensive1, entries: times( header.count, () => @@ -92,7 +92,7 @@ export class BkrlReadStream extends ReadStream { header() { return { version: this.uint16(), - format: this.uint8() as Format, + format: this.uint8() as BkrlFormat, count: this.uint32(), }; } @@ -115,7 +115,7 @@ export class BkrlWriteStream extends WriteStream { body(discriminatedEntries: DiscriminatedEntries) { switch (discriminatedEntries.format) { - case Format.Minimal: { + case BkrlFormat.Minimal: { discriminatedEntries.entries.forEach((entry) => { this.uint32(entry.id); this.uint16(entry.score); @@ -123,7 +123,7 @@ export class BkrlWriteStream extends WriteStream { break; } - case Format.Comprehensive1: { + case BkrlFormat.Comprehensive1: { discriminatedEntries.entries.forEach((entry) => { this.uint32(entry.id); this.uint16(entry.score);