feat: detached cdn branch

This commit is contained in:
TrèsAbhi committed 2024-08-27 14:40:43 -05:00
1 parent ed041b78bb
commit 48dcbbc227
43 files changed
+267 -397

No files matched your search

+12 -9
View File
@@ -5,7 +5,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
import { readStringDVPL } from '../src/core/blitz/readStringDVPL';
import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL';
import { writeDVPL } from '../src/core/blitz/writeDVPL';
import { secrets } from '../src/core/blitzkit/secrets';
import { assertSecrete } from '../src/core/blitzkit/secrete';
import { dvp } from '../submodules/blitzkit-closed/src/dvp';
import { DATA } from './buildAssets/constants';
@@ -21,18 +21,18 @@ console.log(`Installing patches for ${currentVersion}...`);
let patchIndex = 1;
while (true) {
const response = await fetch(
`${secrets.WOTB_DLC_CDN}/dlc/s${currentVersion}_${patchIndex}.yaml`,
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/s${currentVersion}_${patchIndex}.yaml`,
);
if (response.status === 200) {
console.log(`Applying patch ${patchIndex}...`);
const data = parseYaml(await response.text());
const dvpm = await fetch(`${secrets.WOTB_DLC_CDN}/dlc/${data.dx11}`).then(
(response) => response.arrayBuffer(),
);
const dvpm = await fetch(
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11}`,
).then((response) => response.arrayBuffer());
const dvpd = await fetch(
`${secrets.WOTB_DLC_CDN}/dlc/${data.dx11.replace('.dvpm', '.dvpd')}`,
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11.replace('.dvpm', '.dvpd')}`,
).then((response) => response.arrayBuffer());
const files = await dvp(dvpm, dvpd);
const bar = new ProgressBar(
@@ -45,7 +45,10 @@ while (true) {
const { dir } = parsePath(path);
await mkdir(`${DATA}/${dir}`, { recursive: true });
await writeFile(`${DATA}/${path}.dvpl`, writeDVPL(Buffer.from(data)));
await writeFile(
`${DATA}/${path}.dvpl`,
new Uint8Array(writeDVPL(Buffer.from(data))),
);
bar.tick();
}),
@@ -55,7 +58,7 @@ while (true) {
console.log('Found dynamic content localizations; patching...');
const localizationsResponse = await fetch(
`${secrets.WOTB_DLC_CDN}/dlc/${data.dynamicContentLocalizationsDir}/en.yaml`,
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/${data.dynamicContentLocalizationsDir}/en.yaml`,
);
const newStrings = parseYaml(await localizationsResponse.text());
const oldStrings = await readYAMLDVPL<Record<string, string>>(
@@ -66,7 +69,7 @@ while (true) {
await writeFile(
`${DATA}/Strings/en.yaml.dvpl`,
writeDVPL(Buffer.from(patchedContent)),
new Uint8Array(writeDVPL(Buffer.from(patchedContent))),
);
}
+16 -22
View File
@@ -1,5 +1,4 @@
import { chunk, times } from 'lodash';
import { argv } from 'process';
import { Region, REGIONS } from '../src/constants/regions';
import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
import getTankStats from '../src/core/blitz/getTankStats';
@@ -36,8 +35,7 @@ const THREADS = 10;
const PLAYER_IDS_PER_CALL = 100;
const startTime = Date.now();
const production = argv.includes('--production');
const preDiscoveredIds = await fetchPreDiscoveredIds(!production);
const preDiscoveredIds = await fetchPreDiscoveredIds();
const playerIds: Record<Region, number[][]> = {
asia: chunk(
preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'),
@@ -223,23 +221,19 @@ async function postWork() {
latest,
};
commitAssets(
'averages',
[
{
path: `averages/${latest}.pb`,
content: await encodeToBase64(
'blitzkit/blitzkit.AverageDefinitions',
averageDefinitions,
),
encoding: 'base64',
},
{
path: 'averages/manifest.json',
content: JSON.stringify(manifest),
encoding: 'utf-8',
},
],
production,
);
commitAssets('averages', [
{
path: `averages/${latest}.pb`,
content: await encodeToBase64(
'blitzkit/blitzkit.AverageDefinitions',
averageDefinitions,
),
encoding: 'base64',
},
{
path: 'averages/manifest.json',
content: JSON.stringify(manifest),
encoding: 'utf-8',
},
]);
}
+2 -2
View File
@@ -6,7 +6,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
export async function boosterIcons(production: boolean) {
export async function boosterIcons() {
const boosterFiles = (await readdir(`${DATA}/Gfx/Shared/boosters`)).filter(
(file) => !file.endsWith('@2x.txt.dvpl') && !file.startsWith('texture0'),
);
@@ -47,5 +47,5 @@ export async function boosterIcons(production: boolean) {
}),
);
await commitAssets('booster icons', changes, production);
await commitAssets('booster icons', changes);
}
+2 -2
View File
@@ -3,7 +3,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
export async function camouflageIcons(production: boolean) {
export async function camouflageIcons() {
console.log('Building camouflage icons...');
const content = await readDVPLFile(
`${DATA}/Gfx/UI/Hangar/IconCamouflage.packed.webp.dvpl`,
@@ -16,5 +16,5 @@ export async function camouflageIcons(production: boolean) {
},
];
await commitAssets('camo icons', changes, production);
await commitAssets('camo icons', changes);
}
@@ -19,7 +19,7 @@ interface Mappings {
const listItemsPattern = /<items path="(.+)\.xml"\/>/g;
export async function consumableProvisionIcons(production: boolean) {
export async function consumableProvisionIcons() {
console.log('Building consumable and provision icons...');
const changes: FileChange[] = [];
@@ -188,5 +188,5 @@ export async function consumableProvisionIcons(production: boolean) {
);
}
await commitAssets('consumable and provision icons', changes, production);
await commitAssets('consumable and provision icons', changes);
}
+4 -3
View File
@@ -2,6 +2,7 @@ import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { commitAssets } from '../../src/core/blitzkit/commitAssets';
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { assertSecrete } from '../../src/core/blitzkit/secrete';
import { DATA } from './constants';
const ICONS = [
@@ -21,7 +22,7 @@ type BlitzGlossary = Record<
}
>;
export async function currencies(production: boolean) {
export async function currencies() {
console.log('Building currency icons...');
const changes: FileChange[] = [];
@@ -44,7 +45,7 @@ export async function currencies(production: boolean) {
}),
);
const glossary = await fetch(process.env.WOTB_GLOSSARY!).then(
const glossary = await fetch(assertSecrete(process.env.WOTB_GLOSSARY)).then(
(response) => response.json() as Promise<BlitzGlossary>,
);
@@ -74,5 +75,5 @@ export async function currencies(production: boolean) {
}),
);
await commitAssets('currency icons', changes, production);
await commitAssets('currency icons', changes);
}
+48 -52
View File
@@ -433,7 +433,7 @@ const missingStrings: Record<string, string> = {
};
export const botPattern = /^.+((tutorial_bot(\d+)?)|(TU(R?)))$/;
export async function definitions(production: boolean) {
export async function definitions() {
console.log('Building definitions...');
const wargamingTankopedia = (await fetch(
@@ -1460,55 +1460,51 @@ export async function definitions(production: boolean) {
};
});
await commitAssets(
'definitions',
[
{
content: superCompress(gameDefinitions),
encoding: 'base64',
path: 'definitions/game.cdon.lz4',
},
{
content: superCompress(tankDefinitions),
encoding: 'base64',
path: 'definitions/tanks.cdon.lz4',
},
{
content: superCompress(modelDefinitions),
encoding: 'base64',
path: 'definitions/models.cdon.lz4',
},
{
content: superCompress(equipmentDefinitions),
encoding: 'base64',
path: 'definitions/equipment.cdon.lz4',
},
{
content: superCompress(consumableDefinitions),
encoding: 'base64',
path: 'definitions/consumables.cdon.lz4',
},
{
content: superCompress(provisionDefinitions),
encoding: 'base64',
path: 'definitions/provisions.cdon.lz4',
},
{
content: superCompress(skillDefinitions),
encoding: 'base64',
path: 'definitions/skills.cdon.lz4',
},
{
content: superCompress(mapDefinitions),
encoding: 'base64',
path: 'definitions/maps.cdon.lz4',
},
{
content: superCompress(camouflageDefinitions),
encoding: 'base64',
path: 'definitions/camouflages.cdon.lz4',
},
],
production,
);
await commitAssets('definitions', [
{
content: superCompress(gameDefinitions),
encoding: 'base64',
path: 'definitions/game.cdon.lz4',
},
{
content: superCompress(tankDefinitions),
encoding: 'base64',
path: 'definitions/tanks.cdon.lz4',
},
{
content: superCompress(modelDefinitions),
encoding: 'base64',
path: 'definitions/models.cdon.lz4',
},
{
content: superCompress(equipmentDefinitions),
encoding: 'base64',
path: 'definitions/equipment.cdon.lz4',
},
{
content: superCompress(consumableDefinitions),
encoding: 'base64',
path: 'definitions/consumables.cdon.lz4',
},
{
content: superCompress(provisionDefinitions),
encoding: 'base64',
path: 'definitions/provisions.cdon.lz4',
},
{
content: superCompress(skillDefinitions),
encoding: 'base64',
path: 'definitions/skills.cdon.lz4',
},
{
content: superCompress(mapDefinitions),
encoding: 'base64',
path: 'definitions/maps.cdon.lz4',
},
{
content: superCompress(camouflageDefinitions),
encoding: 'base64',
path: 'definitions/camouflages.cdon.lz4',
},
]);
}
+2 -2
View File
@@ -17,7 +17,7 @@ interface Mappings {
}[];
}
export async function equipmentIcons(production: boolean) {
export async function equipmentIcons() {
console.log('Building equipment icons...');
const changes: FileChange[] = [];
@@ -91,5 +91,5 @@ export async function equipmentIcons(production: boolean) {
),
);
await commitAssets('equipment icons', changes, production);
await commitAssets('equipment icons', changes);
}
+2 -2
View File
@@ -6,7 +6,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
export async function flags(production: boolean) {
export async function flags() {
console.log('Building flags...');
const changes = [
@@ -82,5 +82,5 @@ export async function flags(production: boolean) {
)),
];
await commitAssets('flags', changes, production);
await commitAssets('flags', changes);
}
+2 -2
View File
@@ -6,7 +6,7 @@ import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
import { SquadBattleTypeStyles } from './definitions';
export async function gameModeBanners(production: boolean) {
export async function gameModeBanners() {
console.log('Building game mode banners...');
const changes: FileChange[] = [];
@@ -50,5 +50,5 @@ export async function gameModeBanners(production: boolean) {
});
}
await commitAssets('game mode banners', changes, production);
await commitAssets('game mode banners', changes);
}
+1 -2
View File
@@ -8,7 +8,6 @@ const targets = argv
.find((argument) => argument.startsWith('--target'))
?.split('=')[1]
.split(',');
const production = argv.includes('--production');
import { boosterIcons } from './boosterIcons';
import { camouflageIcons } from './camouflageIcons';
@@ -45,5 +44,5 @@ const methods = [
];
for (const method of methods) {
if (allTargets || targets?.includes(method.name)) await method(production);
if (allTargets || targets?.includes(method.name)) await method();
}
+2 -2
View File
@@ -5,7 +5,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
export async function moduleIcons(production: boolean) {
export async function moduleIcons() {
console.log('Building module icons...');
const changes = await Promise.all(
@@ -33,5 +33,5 @@ export async function moduleIcons(production: boolean) {
}),
);
await commitAssets('module icons', changes, production);
await commitAssets('module icons', changes);
}
+2 -2
View File
@@ -6,7 +6,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
export async function shellIcons(production: boolean) {
export async function shellIcons() {
console.log('Building shell icons...');
const image = sharp(
@@ -47,5 +47,5 @@ export async function shellIcons(production: boolean) {
}),
);
await commitAssets('shell icons', changes, production);
await commitAssets('shell icons', changes);
}
+2 -2
View File
@@ -25,7 +25,7 @@ export interface Avatar {
};
}
export async function skillIcons(production: boolean) {
export async function skillIcons() {
console.log('Building skill icons...');
const avatar = await readXMLDVPL<{ root: Avatar }>(
@@ -47,5 +47,5 @@ export async function skillIcons(production: boolean) {
}),
);
commitAssets('skill icons', changes, production);
commitAssets('skill icons', changes);
}
+2 -2
View File
@@ -8,7 +8,7 @@ import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
import { DATA } from './constants';
import { VehicleDefinitionList } from './definitions';
export async function tankArmors(production: boolean) {
export async function tankArmors() {
console.log('Building tank armors...');
const changes: FileChange[] = [];
@@ -42,5 +42,5 @@ export async function tankArmors(production: boolean) {
}),
);
await commitAssets('tank armor', changes, production);
await commitAssets('tank armor', changes);
}
+2 -2
View File
@@ -42,7 +42,7 @@ export interface TankParameters {
};
}
export async function tankIcons(production: boolean) {
export async function tankIcons() {
console.log('Building tank icons...');
const changes: FileChange[] = [];
@@ -97,5 +97,5 @@ export async function tankIcons(production: boolean) {
}),
);
await commitAssets('tank icons', changes, production);
await commitAssets('tank icons', changes);
}
+2 -2
View File
@@ -10,7 +10,7 @@ import { DATA } from './constants';
import { VehicleDefinitionList } from './definitions';
import { TankParameters } from './tankIcons';
export async function tankModels(production: boolean) {
export async function tankModels() {
console.log('Building tank models...');
const nodeIO = new NodeIO();
@@ -50,6 +50,6 @@ export async function tankModels(production: boolean) {
}),
);
await commitAssets(`tank models ${nation}`, changes, production);
await commitAssets(`tank models ${nation}`, changes);
}
}
+1 -5
View File
@@ -191,9 +191,5 @@ regionSubdomains.split(',').forEach(async (regionSubdomain) => {
},
];
commitAssets(
`rating leaderboard ${regionSubdomain} ${target}`,
changes,
production,
);
commitAssets(`rating leaderboard ${regionSubdomain} ${target}`, changes);
});
+23 -33
View File
@@ -1,13 +1,12 @@
import { chunk, times, uniq } from 'lodash';
import { compress, decompress } from 'lz4js';
import { argv } from 'process';
import { REGIONS, Region } from '../src/constants/regions';
import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion';
import { asset } from '../src/core/blitzkit/asset';
import { commitAssets } from '../src/core/blitzkit/commitAssets';
import { DiscoverIdsDefinitions } from '../src/core/blitzkit/discoveredIdDefinitions';
import { DiscoveredIdsDefinitions } from '../src/core/blitzkit/discoveredIdDefinitions';
import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
const CHUNK_SIZE = 2 ** 21;
@@ -16,17 +15,12 @@ const PROGRESS_UPDATE_FREQUENCY = 1000 * 60;
const MAX_REQUESTS = 10;
const ACCOUNTS_PER_CALL = 100;
const TERMINATION_THRESHOLD = 1000;
const production = argv.includes('--production');
console.log(`Running in ${production ? 'production' : 'development'} mode`);
const startTime = Date.now();
const indexableRegions = [...REGIONS];
let ids: number[] = [];
const preDiscoveredManifest = (await fetch(
asset('ids/manifest.json', !production),
).then((response) => response.json())) as DiscoverIdsDefinitions;
const preDiscoveredManifest = (await fetch(asset('ids/manifest.json')).then(
(response) => response.json(),
)) as DiscoveredIdsDefinitions;
console.log(
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
@@ -34,13 +28,13 @@ console.log(
let chunkIndex = 0;
while (chunkIndex < preDiscoveredManifest.chunks) {
const preDiscovered = await fetch(
asset(`ids/${chunkIndex}.dids.lz4`, !production),
).then(async (response) => {
const buffer = await response.arrayBuffer();
const decompressed = decompress(new Uint8Array(buffer)).buffer;
return new DidsReadStream(decompressed as ArrayBuffer).dids();
});
const preDiscovered = await fetch(asset(`ids/${chunkIndex}.dids.lz4`)).then(
async (response) => {
const buffer = await response.arrayBuffer();
const decompressed = decompress(new Uint8Array(buffer)).buffer;
return new DidsReadStream(decompressed as ArrayBuffer).dids();
},
);
// no spread syntax: https://github.com/oven-sh/bun/issues/11734
ids = ids.concat(preDiscovered);
@@ -149,20 +143,16 @@ function post() {
};
});
commitAssets(
'discovered ids',
[
...files,
{
content: JSON.stringify({
chunks: idsChunked.length,
count: ids.length,
time: Date.now(),
} satisfies DiscoverIdsDefinitions),
encoding: 'utf-8',
path: 'ids/manifest.json',
},
],
production,
);
commitAssets('discovered ids', [
...files,
{
content: JSON.stringify({
chunks: idsChunked.length,
count: ids.length,
time: Date.now(),
} satisfies DiscoveredIdsDefinitions),
encoding: 'utf-8',
path: 'ids/manifest.json',
},
]);
}
+4 -6
View File
@@ -22,7 +22,7 @@ const MAX_QUERIES = 128;
console.log('Finding reviews...');
const currentReviews = await fetch(asset('definitions/reviews.pb', false))
const currentReviews = await fetch(asset('definitions/reviews.pb'))
.then((response) => response.arrayBuffer())
.then((buffer) =>
decode<Reviews>('reviews/blitzkit.Reviews', new Uint8Array(buffer)),
@@ -128,8 +128,6 @@ const content = await encode('reviews/blitzkit.Reviews', reviews).then(
(array) => Buffer.from(array).toString('base64'),
);
await commitAssets(
'reviews',
[{ content, encoding: 'base64', path: 'definitions/reviews.pb' }],
true,
);
await commitAssets('reviews', [
{ content, encoding: 'base64', path: 'definitions/reviews.pb' },
]);
-96
View File
@@ -1,96 +0,0 @@
import { chunk, times } from 'lodash';
import { compress } from 'lz4js';
import { argv } from 'process';
import { Region, REGIONS } from '../src/constants/regions';
import getTankStats from '../src/core/blitz/getTankStats';
import { idToRegion } from '../src/core/blitz/idToRegion';
import { commitAssets } from '../src/core/blitzkit/commitAssets';
import { FileChange } from '../src/core/blitzkit/commitMultipleFiles';
import {
RtscWritePlayer,
RtscWriteStream,
RtsmWriteStream,
} from '../src/core/streams/rts';
import usersRaw from '../test.users.json';
const PLAYERS_PER_CHUNK = 256;
const CHUNKS_PER_BATCH = 128;
const production = argv.includes('--production');
const region = argv
.find((argument) => argument.startsWith('--region'))
?.split('=')[1] as Region | undefined;
if (!region || !REGIONS.includes(region)) {
throw new Error('No valid region specified');
}
const today = new Date();
const todayPath = `${today.getUTCFullYear()}/${today.getUTCMonth() + 1}/${today.getUTCDate()}`;
const users = usersRaw.filter(({ blitz }) => idToRegion(blitz) === region);
const base = Math.ceil(users.length / PLAYERS_PER_CHUNK);
const chunks = times(base, (offset) => ({
offset,
users: users.filter((user) => user.blitz % base === offset),
}));
const batches = chunk(chunks, CHUNKS_PER_BATCH);
const manifestWriteStream = new RtsmWriteStream();
console.log(`Found ${users.length} players in ${region}`);
await commitAssets(
`regular tank stats manifests ${todayPath}`,
[
{
encoding: 'base64',
path: `regions/${region}/regular/tanks/${todayPath}/manifest.rtsm`,
content: Buffer.from(
compress(manifestWriteStream.rtsm(base).uint8Array),
).toString('base64'),
},
],
production,
);
let batchesDone = 0;
for (const batch of batches) {
console.log(`Batch ${++batchesDone} of ${batches.length}`);
const changes: FileChange[] = [];
let chunksDone = 0;
for (const chunk of batch) {
console.log(`Chunk ${++chunksDone} of ${chunks.length}`);
let playersDone = 0;
const players = await Promise.all(
chunk.users.map(async (user) => {
const tanks = await getTankStats(region, user.blitz);
console.log(`Player ${++playersDone} of ${chunk.users.length}`);
if (tanks === null) return null;
return {
id: user.blitz,
tanks,
} satisfies RtscWritePlayer;
}),
).then((players) => players.filter((player) => player !== null));
const writeStream = new RtscWriteStream();
writeStream.rtsc(players);
changes.push({
encoding: 'base64',
content: Buffer.from(compress(writeStream.uint8Array)).toString('base64'),
path: `regions/${region}/regular/tanks/${todayPath}/${chunk.offset}.rtsc.lz4`,
});
}
// do not await; commit in background
commitAssets(
`regular tank stats ${todayPath} batch ${batches.indexOf(batch) + 1}`,
changes,
production,
);
}
-36
View File
@@ -1,36 +0,0 @@
import AdmZip from 'adm-zip';
import { readFile } from 'fs/promises';
import { BattleResultsReadStream } from '../src/core/streams/battleResults';
interface WoTBReplayMeta {
version: string;
title: string;
dbid: string;
playerName: string;
battleStartTime: string;
playerVehicleName: string;
mapName: string;
arenaUniqueId: string;
battleDuration: number;
vehicleCompDescriptor: number;
camouflageId: number;
mapId: number;
arenaBonusType: number;
}
const file = await readFile('temp/test.wotbreplay');
const zip = new AdmZip(file);
const entries = zip.getEntries();
const metaRaw = entries.find((entry) => entry.name === 'meta.json');
const battleResultsRaw = entries.find(
(entry) => entry.name === 'battle_results.dat',
);
if (!metaRaw) throw new Error('No meta.json found');
if (!battleResultsRaw) throw new Error('No battle_results.dat found');
const meta = JSON.parse(metaRaw.getData().toString()) as WoTBReplayMeta;
const battleResults = new BattleResultsReadStream(
battleResultsRaw.getData().buffer,
).battleResults();