From f083c18dd0a51fff80d89473298458a8cb5476f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:31:36 -0500 Subject: [PATCH] feat: more optimal fetching --- .github/workflows/cd-rating-latest.yml | 13 +- scripts/buildRatingLeaderboard.ts | 297 +++++++++++++------------ 2 files changed, 154 insertions(+), 156 deletions(-) diff --git a/.github/workflows/cd-rating-latest.yml b/.github/workflows/cd-rating-latest.yml index 57245a61f..78ad34fad 100644 --- a/.github/workflows/cd-rating-latest.yml +++ b/.github/workflows/cd-rating-latest.yml @@ -30,14 +30,5 @@ jobs: - name: Install dependencies run: yarn - - name: Generate NA leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingleaderboard --region=na --target=latest --production - if: always() - - - name: Generate EU leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingleaderboard --region=eu --target=latest --production - if: always() - - - name: Generate Asia leaderboard - run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingleaderboard --region=asia --target=latest --production - if: always() + - name: Generate all leaderboards + run: GH_TOKEN=${{ secrets.GH_TOKEN }} DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} DISCORD_BLITZ_DB=${{ secrets.DISCORD_BLITZ_DB }} yarn publish:ratingleaderboard --region=na,asia,eu --target=latest --production diff --git a/scripts/buildRatingLeaderboard.ts b/scripts/buildRatingLeaderboard.ts index 7c4d2b698..2a731485e 100644 --- a/scripts/buildRatingLeaderboard.ts +++ b/scripts/buildRatingLeaderboard.ts @@ -30,163 +30,170 @@ const production = argv.includes('--production'); const target = argv .find((arg) => arg.startsWith('--target=')) ?.split('=')[1] as 'midnight' | 'latest' | undefined; -const regionSubdomain = argv +const regionSubdomains = argv .find((arg) => arg.startsWith('--region=')) - ?.split('=')[1] as RegionSubdomain | undefined; + ?.split('=')[1]; if (!target) throw new Error('Target parameter not specified'); -if (!regionSubdomain) throw new Error('Region parameter not specified'); +if (!regionSubdomains) throw new Error('Region parameter not specified'); -const region = regionSubdomainToRegion(regionSubdomain); +regionSubdomains.split(',').forEach(async (regionSubdomain) => { + const region = regionSubdomainToRegion(regionSubdomain as RegionSubdomain); -const NEIGHBORS = 2 ** 8; -const PLAYERS = await patientFetchJSON( - `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`, -).then((data) => (data.detail === undefined ? data.count : undefined)); + console.log(`Building ${target} rating leaderboard for ${region}`); -if (PLAYERS === undefined) { - throw new Error('No current rating season running'); -} + const NEIGHBORS = 2 ** 8; + const PLAYERS = await patientFetchJSON( + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`, + ).then((data) => (data.detail === undefined ? data.count : undefined)); -const players: Record = {}; -const leaderboard: BkrlBaseEntry[] = []; -let registered = 0; - -async function branchFromPlayer(id: number) { - const { neighbors } = await patientFetchJSON( - `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`, - ); - - const firstPlayer = neighbors[0]; - const lastPlayer = neighbors[neighbors.length - 1]; - - const branches: Promise[] = []; - if (!players[firstPlayer.number]) - branches.push(branchFromPlayer(firstPlayer.spa_id)); - if (!players[lastPlayer.number]) - branches.push(branchFromPlayer(lastPlayer.spa_id)); - - neighbors.forEach((neighbor) => { - if (players[neighbor.number]) return; - - players[neighbor.number] = neighbor; - registered++; - }); - - console.log( - `Registered ${registered} / ${PLAYERS} (${( - (registered / PLAYERS!) * - 100 - ).toFixed(2)}%)`, - ); - - await Promise.all(branches); -} - -async function branchFromLeague(id: number) { - const { result } = await patientFetchJSON<{ result: RatingPlayer[] }>( - `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`, - ); - - const firstPlayer = result[0]; - const lastPlayer = result.at(-1)!; - - await Promise.all([ - branchFromPlayer(firstPlayer.spa_id), - branchFromPlayer(lastPlayer.spa_id), - ]); -} - -console.log('Branching from leagues...'); -await Promise.all([ - branchFromLeague(0), - branchFromLeague(1), - branchFromLeague(2), - branchFromLeague(3), - branchFromLeague(4), -]); - -console.log('Converting to an array...'); -for (let index = 0; index < PLAYERS; index++) { - const listing = players[index + 1]; - if (listing) { - leaderboard[index] = { id: listing.spa_id, score: listing.score }; - } else { - console.warn(`Position ${index + 1} had no player`); - leaderboard[index] = { id: 0, score: 0 }; + if (PLAYERS === undefined) { + throw new Error('No current rating season running'); } -} -console.log('Requesting further stats...'); -const comprehensiveLeaderboard: BkrlSuperset1Entry[] = []; -let chunkIndex = 0; -for (const leaderboardChunk of chunk(leaderboard, 100)) { - console.log( - `Chunk ${chunkIndex + 1} of ${Math.ceil(leaderboard.length / 100)}`, + const players: Record = {}; + const leaderboard: BkrlBaseEntry[] = []; + let registered = 0; + + async function branchFromPlayer(id: number) { + const { neighbors } = await patientFetchJSON( + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/user/${id}/?neighbors=${NEIGHBORS}`, + ); + + const firstPlayer = neighbors[0]; + const lastPlayer = neighbors[neighbors.length - 1]; + + const branches: Promise[] = []; + if (!players[firstPlayer.number]) + branches.push(branchFromPlayer(firstPlayer.spa_id)); + if (!players[lastPlayer.number]) + branches.push(branchFromPlayer(lastPlayer.spa_id)); + + neighbors.forEach((neighbor) => { + if (players[neighbor.number]) return; + + players[neighbor.number] = neighbor; + registered++; + }); + + console.log( + `Registered ${registered} / ${PLAYERS} (${( + (registered / PLAYERS!) * + 100 + ).toFixed(2)}%)`, + ); + + await Promise.all(branches); + } + + async function branchFromLeague(id: number) { + const { result } = await patientFetchJSON<{ result: RatingPlayer[] }>( + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/league/${id}/top/`, + ); + + const firstPlayer = result[0]; + const lastPlayer = result.at(-1)!; + + await Promise.all([ + branchFromPlayer(firstPlayer.spa_id), + branchFromPlayer(lastPlayer.spa_id), + ]); + } + + console.log('Branching from leagues...'); + await Promise.all([ + branchFromLeague(0), + branchFromLeague(1), + branchFromLeague(2), + branchFromLeague(3), + branchFromLeague(4), + ]); + + console.log('Converting to an array...'); + for (let index = 0; index < PLAYERS; index++) { + const listing = players[index + 1]; + if (listing) { + leaderboard[index] = { id: listing.spa_id, score: listing.score }; + } else { + console.warn(`Position ${index + 1} had no player`); + leaderboard[index] = { id: 0, score: 0 }; + } + } + + console.log('Requesting further stats...'); + const comprehensiveLeaderboard: BkrlSuperset1Entry[] = []; + let chunkIndex = 0; + for (const leaderboardChunk of chunk(leaderboard, 100)) { + console.log( + `Chunk ${chunkIndex + 1} of ${Math.ceil(leaderboard.length / 100)}`, + ); + + const stats = await getAccountInfo( + region, + leaderboardChunk.map(({ id }) => id), + ['statistics.rating'], + ); + + stats.forEach((stat, index) => { + const trueIndex = chunkIndex * 100 + index; + comprehensiveLeaderboard[trueIndex] = + leaderboardChunk[index].id === 0 || + stat?.statistics.rating === undefined + ? { + battles: 0, + damage: 0, + id: 0, + kills: 0, + score: 0, + survived: 0, + wins: 0, + } + : { + id: leaderboardChunk[index].id, + score: leaderboardChunk[index].score, + battles: stat.statistics.rating.battles, + damage: stat.statistics.rating.damage_dealt, + kills: stat.statistics.rating.frags, + survived: stat.statistics.rating.survived_battles, + wins: stat.statistics.rating.wins, + }; + }); + + chunkIndex++; + } + + const info = await patientFetchJSON( + `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`, ); + const normalizedServer = regionSubdomain === 'na' ? 'com' : regionSubdomain; + const infoContent = new CdonWriteStream().cdon( + info as unknown as CdonValue, + ).uint8Array; + const leaderboardWriteStream = new BkrlWriteStream(); - const stats = await getAccountInfo( - region, - leaderboardChunk.map(({ id }) => id), - ['statistics.rating'], - ); - - stats.forEach((stat, index) => { - const trueIndex = chunkIndex * 100 + index; - comprehensiveLeaderboard[trueIndex] = - leaderboardChunk[index].id === 0 || stat?.statistics.rating === undefined - ? { - battles: 0, - damage: 0, - id: 0, - kills: 0, - score: 0, - survived: 0, - wins: 0, - } - : { - id: leaderboardChunk[index].id, - score: leaderboardChunk[index].score, - battles: stat.statistics.rating.battles, - damage: stat.statistics.rating.damage_dealt, - kills: stat.statistics.rating.frags, - survived: stat.statistics.rating.survived_battles, - wins: stat.statistics.rating.wins, - }; + leaderboardWriteStream.bkrl({ + format: BkrlFormat.Superset1, + entries: comprehensiveLeaderboard, }); - chunkIndex++; -} + const changes: FileChange[] = [ + { + path: `regions/${normalizedServer}/rating/${info.current_season}/info.cdon.lz4`, + content: superCompress(infoContent), + encoding: 'base64', + }, + { + path: `regions/${normalizedServer}/rating/${info.current_season}/${target}.bkrl`, + content: Buffer.from(leaderboardWriteStream.uint8Array).toString( + 'base64', + ), + encoding: 'base64', + }, + ]; -const info = await patientFetchJSON( - `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`, -); -const normalizedServer = regionSubdomain === 'na' ? 'com' : regionSubdomain; -const infoContent = new CdonWriteStream().cdon( - info as unknown as CdonValue, -).uint8Array; -const leaderboardWriteStream = new BkrlWriteStream(); - -leaderboardWriteStream.bkrl({ - format: BkrlFormat.Superset1, - entries: comprehensiveLeaderboard, + commitAssets( + `rating leaderboard ${regionSubdomain} ${target}`, + changes, + production, + ); }); - -const changes: FileChange[] = [ - { - path: `regions/${normalizedServer}/rating/${info.current_season}/info.cdon.lz4`, - content: superCompress(infoContent), - encoding: 'base64', - }, - { - path: `regions/${normalizedServer}/rating/${info.current_season}/${target}.bkrl`, - content: Buffer.from(leaderboardWriteStream.uint8Array).toString('base64'), - encoding: 'base64', - }, -]; - -commitAssets( - `rating leaderboard ${regionSubdomain} ${target}`, - changes, - production, -);