mirror of
https://github.com/blitzkit/blitzkit.git
synced 2026-10-21 18:57:03 +02:00
feat: detached cdn branch
This commit is contained in:
1 parent
ed041b78bb
commit
48dcbbc227
43 files changed
+267
-397
No files matched your search
+12
-9
@@ -5,7 +5,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
|
||||
import { readStringDVPL } from '../src/core/blitz/readStringDVPL';
|
||||
import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL';
|
||||
import { writeDVPL } from '../src/core/blitz/writeDVPL';
|
||||
import { secrets } from '../src/core/blitzkit/secrets';
|
||||
import { assertSecrete } from '../src/core/blitzkit/secrete';
|
||||
import { dvp } from '../submodules/blitzkit-closed/src/dvp';
|
||||
import { DATA } from './buildAssets/constants';
|
||||
|
||||
@@ -21,18 +21,18 @@ console.log(`Installing patches for ${currentVersion}...`);
|
||||
let patchIndex = 1;
|
||||
while (true) {
|
||||
const response = await fetch(
|
||||
`${secrets.WOTB_DLC_CDN}/dlc/s${currentVersion}_${patchIndex}.yaml`,
|
||||
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/s${currentVersion}_${patchIndex}.yaml`,
|
||||
);
|
||||
|
||||
if (response.status === 200) {
|
||||
console.log(`Applying patch ${patchIndex}...`);
|
||||
|
||||
const data = parseYaml(await response.text());
|
||||
const dvpm = await fetch(`${secrets.WOTB_DLC_CDN}/dlc/${data.dx11}`).then(
|
||||
(response) => response.arrayBuffer(),
|
||||
);
|
||||
const dvpm = await fetch(
|
||||
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11}`,
|
||||
).then((response) => response.arrayBuffer());
|
||||
const dvpd = await fetch(
|
||||
`${secrets.WOTB_DLC_CDN}/dlc/${data.dx11.replace('.dvpm', '.dvpd')}`,
|
||||
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11.replace('.dvpm', '.dvpd')}`,
|
||||
).then((response) => response.arrayBuffer());
|
||||
const files = await dvp(dvpm, dvpd);
|
||||
const bar = new ProgressBar(
|
||||
@@ -45,7 +45,10 @@ while (true) {
|
||||
const { dir } = parsePath(path);
|
||||
|
||||
await mkdir(`${DATA}/${dir}`, { recursive: true });
|
||||
await writeFile(`${DATA}/${path}.dvpl`, writeDVPL(Buffer.from(data)));
|
||||
await writeFile(
|
||||
`${DATA}/${path}.dvpl`,
|
||||
new Uint8Array(writeDVPL(Buffer.from(data))),
|
||||
);
|
||||
|
||||
bar.tick();
|
||||
}),
|
||||
@@ -55,7 +58,7 @@ while (true) {
|
||||
console.log('Found dynamic content localizations; patching...');
|
||||
|
||||
const localizationsResponse = await fetch(
|
||||
`${secrets.WOTB_DLC_CDN}/dlc/${data.dynamicContentLocalizationsDir}/en.yaml`,
|
||||
`${assertSecrete(process.env.WOTB_DLC_CDN)}/dlc/${data.dynamicContentLocalizationsDir}/en.yaml`,
|
||||
);
|
||||
const newStrings = parseYaml(await localizationsResponse.text());
|
||||
const oldStrings = await readYAMLDVPL<Record<string, string>>(
|
||||
@@ -66,7 +69,7 @@ while (true) {
|
||||
|
||||
await writeFile(
|
||||
`${DATA}/Strings/en.yaml.dvpl`,
|
||||
writeDVPL(Buffer.from(patchedContent)),
|
||||
new Uint8Array(writeDVPL(Buffer.from(patchedContent))),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+16
-22
@@ -1,5 +1,4 @@
|
||||
import { chunk, times } from 'lodash';
|
||||
import { argv } from 'process';
|
||||
import { Region, REGIONS } from '../src/constants/regions';
|
||||
import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
|
||||
import getTankStats from '../src/core/blitz/getTankStats';
|
||||
@@ -36,8 +35,7 @@ const THREADS = 10;
|
||||
const PLAYER_IDS_PER_CALL = 100;
|
||||
|
||||
const startTime = Date.now();
|
||||
const production = argv.includes('--production');
|
||||
const preDiscoveredIds = await fetchPreDiscoveredIds(!production);
|
||||
const preDiscoveredIds = await fetchPreDiscoveredIds();
|
||||
const playerIds: Record<Region, number[][]> = {
|
||||
asia: chunk(
|
||||
preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'),
|
||||
@@ -223,23 +221,19 @@ async function postWork() {
|
||||
latest,
|
||||
};
|
||||
|
||||
commitAssets(
|
||||
'averages',
|
||||
[
|
||||
{
|
||||
path: `averages/${latest}.pb`,
|
||||
content: await encodeToBase64(
|
||||
'blitzkit/blitzkit.AverageDefinitions',
|
||||
averageDefinitions,
|
||||
),
|
||||
encoding: 'base64',
|
||||
},
|
||||
{
|
||||
path: 'averages/manifest.json',
|
||||
content: JSON.stringify(manifest),
|
||||
encoding: 'utf-8',
|
||||
},
|
||||
],
|
||||
production,
|
||||
);
|
||||
commitAssets('averages', [
|
||||
{
|
||||
path: `averages/${latest}.pb`,
|
||||
content: await encodeToBase64(
|
||||
'blitzkit/blitzkit.AverageDefinitions',
|
||||
averageDefinitions,
|
||||
),
|
||||
encoding: 'base64',
|
||||
},
|
||||
{
|
||||
path: 'averages/manifest.json',
|
||||
content: JSON.stringify(manifest),
|
||||
encoding: 'utf-8',
|
||||
},
|
||||
]);
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
|
||||
export async function boosterIcons(production: boolean) {
|
||||
export async function boosterIcons() {
|
||||
const boosterFiles = (await readdir(`${DATA}/Gfx/Shared/boosters`)).filter(
|
||||
(file) => !file.endsWith('@2x.txt.dvpl') && !file.startsWith('texture0'),
|
||||
);
|
||||
@@ -47,5 +47,5 @@ export async function boosterIcons(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets('booster icons', changes, production);
|
||||
await commitAssets('booster icons', changes);
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
|
||||
export async function camouflageIcons(production: boolean) {
|
||||
export async function camouflageIcons() {
|
||||
console.log('Building camouflage icons...');
|
||||
const content = await readDVPLFile(
|
||||
`${DATA}/Gfx/UI/Hangar/IconCamouflage.packed.webp.dvpl`,
|
||||
@@ -16,5 +16,5 @@ export async function camouflageIcons(production: boolean) {
|
||||
},
|
||||
];
|
||||
|
||||
await commitAssets('camo icons', changes, production);
|
||||
await commitAssets('camo icons', changes);
|
||||
}
|
||||
@@ -19,7 +19,7 @@ interface Mappings {
|
||||
|
||||
const listItemsPattern = /<items path="(.+)\.xml"\/>/g;
|
||||
|
||||
export async function consumableProvisionIcons(production: boolean) {
|
||||
export async function consumableProvisionIcons() {
|
||||
console.log('Building consumable and provision icons...');
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
@@ -188,5 +188,5 @@ export async function consumableProvisionIcons(production: boolean) {
|
||||
);
|
||||
}
|
||||
|
||||
await commitAssets('consumable and provision icons', changes, production);
|
||||
await commitAssets('consumable and provision icons', changes);
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import sharp from 'sharp';
|
||||
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
|
||||
import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { assertSecrete } from '../../src/core/blitzkit/secrete';
|
||||
import { DATA } from './constants';
|
||||
|
||||
const ICONS = [
|
||||
@@ -21,7 +22,7 @@ type BlitzGlossary = Record<
|
||||
}
|
||||
>;
|
||||
|
||||
export async function currencies(production: boolean) {
|
||||
export async function currencies() {
|
||||
console.log('Building currency icons...');
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
@@ -44,7 +45,7 @@ export async function currencies(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
const glossary = await fetch(process.env.WOTB_GLOSSARY!).then(
|
||||
const glossary = await fetch(assertSecrete(process.env.WOTB_GLOSSARY)).then(
|
||||
(response) => response.json() as Promise<BlitzGlossary>,
|
||||
);
|
||||
|
||||
@@ -74,5 +75,5 @@ export async function currencies(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets('currency icons', changes, production);
|
||||
await commitAssets('currency icons', changes);
|
||||
}
|
||||
@@ -433,7 +433,7 @@ const missingStrings: Record<string, string> = {
|
||||
};
|
||||
export const botPattern = /^.+((tutorial_bot(\d+)?)|(TU(R?)))$/;
|
||||
|
||||
export async function definitions(production: boolean) {
|
||||
export async function definitions() {
|
||||
console.log('Building definitions...');
|
||||
|
||||
const wargamingTankopedia = (await fetch(
|
||||
@@ -1460,55 +1460,51 @@ export async function definitions(production: boolean) {
|
||||
};
|
||||
});
|
||||
|
||||
await commitAssets(
|
||||
'definitions',
|
||||
[
|
||||
{
|
||||
content: superCompress(gameDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/game.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(tankDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/tanks.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(modelDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/models.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(equipmentDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/equipment.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(consumableDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/consumables.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(provisionDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/provisions.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(skillDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/skills.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(mapDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/maps.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(camouflageDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/camouflages.cdon.lz4',
|
||||
},
|
||||
],
|
||||
production,
|
||||
);
|
||||
await commitAssets('definitions', [
|
||||
{
|
||||
content: superCompress(gameDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/game.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(tankDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/tanks.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(modelDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/models.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(equipmentDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/equipment.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(consumableDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/consumables.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(provisionDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/provisions.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(skillDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/skills.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(mapDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/maps.cdon.lz4',
|
||||
},
|
||||
{
|
||||
content: superCompress(camouflageDefinitions),
|
||||
encoding: 'base64',
|
||||
path: 'definitions/camouflages.cdon.lz4',
|
||||
},
|
||||
]);
|
||||
}
|
||||
@@ -17,7 +17,7 @@ interface Mappings {
|
||||
}[];
|
||||
}
|
||||
|
||||
export async function equipmentIcons(production: boolean) {
|
||||
export async function equipmentIcons() {
|
||||
console.log('Building equipment icons...');
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
@@ -91,5 +91,5 @@ export async function equipmentIcons(production: boolean) {
|
||||
),
|
||||
);
|
||||
|
||||
await commitAssets('equipment icons', changes, production);
|
||||
await commitAssets('equipment icons', changes);
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
|
||||
export async function flags(production: boolean) {
|
||||
export async function flags() {
|
||||
console.log('Building flags...');
|
||||
|
||||
const changes = [
|
||||
@@ -82,5 +82,5 @@ export async function flags(production: boolean) {
|
||||
)),
|
||||
];
|
||||
|
||||
await commitAssets('flags', changes, production);
|
||||
await commitAssets('flags', changes);
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
import { SquadBattleTypeStyles } from './definitions';
|
||||
|
||||
export async function gameModeBanners(production: boolean) {
|
||||
export async function gameModeBanners() {
|
||||
console.log('Building game mode banners...');
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
@@ -50,5 +50,5 @@ export async function gameModeBanners(production: boolean) {
|
||||
});
|
||||
}
|
||||
|
||||
await commitAssets('game mode banners', changes, production);
|
||||
await commitAssets('game mode banners', changes);
|
||||
}
|
||||
@@ -8,7 +8,6 @@ const targets = argv
|
||||
.find((argument) => argument.startsWith('--target'))
|
||||
?.split('=')[1]
|
||||
.split(',');
|
||||
const production = argv.includes('--production');
|
||||
|
||||
import { boosterIcons } from './boosterIcons';
|
||||
import { camouflageIcons } from './camouflageIcons';
|
||||
@@ -45,5 +44,5 @@ const methods = [
|
||||
];
|
||||
|
||||
for (const method of methods) {
|
||||
if (allTargets || targets?.includes(method.name)) await method(production);
|
||||
if (allTargets || targets?.includes(method.name)) await method();
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
|
||||
export async function moduleIcons(production: boolean) {
|
||||
export async function moduleIcons() {
|
||||
console.log('Building module icons...');
|
||||
|
||||
const changes = await Promise.all(
|
||||
@@ -33,5 +33,5 @@ export async function moduleIcons(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets('module icons', changes, production);
|
||||
await commitAssets('module icons', changes);
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
|
||||
export async function shellIcons(production: boolean) {
|
||||
export async function shellIcons() {
|
||||
console.log('Building shell icons...');
|
||||
|
||||
const image = sharp(
|
||||
@@ -47,5 +47,5 @@ export async function shellIcons(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets('shell icons', changes, production);
|
||||
await commitAssets('shell icons', changes);
|
||||
}
|
||||
@@ -25,7 +25,7 @@ export interface Avatar {
|
||||
};
|
||||
}
|
||||
|
||||
export async function skillIcons(production: boolean) {
|
||||
export async function skillIcons() {
|
||||
console.log('Building skill icons...');
|
||||
|
||||
const avatar = await readXMLDVPL<{ root: Avatar }>(
|
||||
@@ -47,5 +47,5 @@ export async function skillIcons(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
commitAssets('skill icons', changes, production);
|
||||
commitAssets('skill icons', changes);
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
|
||||
import { DATA } from './constants';
|
||||
import { VehicleDefinitionList } from './definitions';
|
||||
|
||||
export async function tankArmors(production: boolean) {
|
||||
export async function tankArmors() {
|
||||
console.log('Building tank armors...');
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
@@ -42,5 +42,5 @@ export async function tankArmors(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets('tank armor', changes, production);
|
||||
await commitAssets('tank armor', changes);
|
||||
}
|
||||
@@ -42,7 +42,7 @@ export interface TankParameters {
|
||||
};
|
||||
}
|
||||
|
||||
export async function tankIcons(production: boolean) {
|
||||
export async function tankIcons() {
|
||||
console.log('Building tank icons...');
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
@@ -97,5 +97,5 @@ export async function tankIcons(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets('tank icons', changes, production);
|
||||
await commitAssets('tank icons', changes);
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import { DATA } from './constants';
|
||||
import { VehicleDefinitionList } from './definitions';
|
||||
import { TankParameters } from './tankIcons';
|
||||
|
||||
export async function tankModels(production: boolean) {
|
||||
export async function tankModels() {
|
||||
console.log('Building tank models...');
|
||||
|
||||
const nodeIO = new NodeIO();
|
||||
@@ -50,6 +50,6 @@ export async function tankModels(production: boolean) {
|
||||
}),
|
||||
);
|
||||
|
||||
await commitAssets(`tank models ${nation}`, changes, production);
|
||||
await commitAssets(`tank models ${nation}`, changes);
|
||||
}
|
||||
}
|
||||
@@ -191,9 +191,5 @@ regionSubdomains.split(',').forEach(async (regionSubdomain) => {
|
||||
},
|
||||
];
|
||||
|
||||
commitAssets(
|
||||
`rating leaderboard ${regionSubdomain} ${target}`,
|
||||
changes,
|
||||
production,
|
||||
);
|
||||
commitAssets(`rating leaderboard ${regionSubdomain} ${target}`, changes);
|
||||
});
|
||||
+23
-33
@@ -1,13 +1,12 @@
|
||||
import { chunk, times, uniq } from 'lodash';
|
||||
import { compress, decompress } from 'lz4js';
|
||||
import { argv } from 'process';
|
||||
import { REGIONS, Region } from '../src/constants/regions';
|
||||
import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
|
||||
import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
|
||||
import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion';
|
||||
import { asset } from '../src/core/blitzkit/asset';
|
||||
import { commitAssets } from '../src/core/blitzkit/commitAssets';
|
||||
import { DiscoverIdsDefinitions } from '../src/core/blitzkit/discoveredIdDefinitions';
|
||||
import { DiscoveredIdsDefinitions } from '../src/core/blitzkit/discoveredIdDefinitions';
|
||||
import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
|
||||
|
||||
const CHUNK_SIZE = 2 ** 21;
|
||||
@@ -16,17 +15,12 @@ const PROGRESS_UPDATE_FREQUENCY = 1000 * 60;
|
||||
const MAX_REQUESTS = 10;
|
||||
const ACCOUNTS_PER_CALL = 100;
|
||||
const TERMINATION_THRESHOLD = 1000;
|
||||
|
||||
const production = argv.includes('--production');
|
||||
|
||||
console.log(`Running in ${production ? 'production' : 'development'} mode`);
|
||||
|
||||
const startTime = Date.now();
|
||||
const indexableRegions = [...REGIONS];
|
||||
let ids: number[] = [];
|
||||
const preDiscoveredManifest = (await fetch(
|
||||
asset('ids/manifest.json', !production),
|
||||
).then((response) => response.json())) as DiscoverIdsDefinitions;
|
||||
const preDiscoveredManifest = (await fetch(asset('ids/manifest.json')).then(
|
||||
(response) => response.json(),
|
||||
)) as DiscoveredIdsDefinitions;
|
||||
|
||||
console.log(
|
||||
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
|
||||
@@ -34,13 +28,13 @@ console.log(
|
||||
|
||||
let chunkIndex = 0;
|
||||
while (chunkIndex < preDiscoveredManifest.chunks) {
|
||||
const preDiscovered = await fetch(
|
||||
asset(`ids/${chunkIndex}.dids.lz4`, !production),
|
||||
).then(async (response) => {
|
||||
const buffer = await response.arrayBuffer();
|
||||
const decompressed = decompress(new Uint8Array(buffer)).buffer;
|
||||
return new DidsReadStream(decompressed as ArrayBuffer).dids();
|
||||
});
|
||||
const preDiscovered = await fetch(asset(`ids/${chunkIndex}.dids.lz4`)).then(
|
||||
async (response) => {
|
||||
const buffer = await response.arrayBuffer();
|
||||
const decompressed = decompress(new Uint8Array(buffer)).buffer;
|
||||
return new DidsReadStream(decompressed as ArrayBuffer).dids();
|
||||
},
|
||||
);
|
||||
|
||||
// no spread syntax: https://github.com/oven-sh/bun/issues/11734
|
||||
ids = ids.concat(preDiscovered);
|
||||
@@ -149,20 +143,16 @@ function post() {
|
||||
};
|
||||
});
|
||||
|
||||
commitAssets(
|
||||
'discovered ids',
|
||||
[
|
||||
...files,
|
||||
{
|
||||
content: JSON.stringify({
|
||||
chunks: idsChunked.length,
|
||||
count: ids.length,
|
||||
time: Date.now(),
|
||||
} satisfies DiscoverIdsDefinitions),
|
||||
encoding: 'utf-8',
|
||||
path: 'ids/manifest.json',
|
||||
},
|
||||
],
|
||||
production,
|
||||
);
|
||||
commitAssets('discovered ids', [
|
||||
...files,
|
||||
{
|
||||
content: JSON.stringify({
|
||||
chunks: idsChunked.length,
|
||||
count: ids.length,
|
||||
time: Date.now(),
|
||||
} satisfies DiscoveredIdsDefinitions),
|
||||
encoding: 'utf-8',
|
||||
path: 'ids/manifest.json',
|
||||
},
|
||||
]);
|
||||
}
|
||||
+4
-6
@@ -22,7 +22,7 @@ const MAX_QUERIES = 128;
|
||||
|
||||
console.log('Finding reviews...');
|
||||
|
||||
const currentReviews = await fetch(asset('definitions/reviews.pb', false))
|
||||
const currentReviews = await fetch(asset('definitions/reviews.pb'))
|
||||
.then((response) => response.arrayBuffer())
|
||||
.then((buffer) =>
|
||||
decode<Reviews>('reviews/blitzkit.Reviews', new Uint8Array(buffer)),
|
||||
@@ -128,8 +128,6 @@ const content = await encode('reviews/blitzkit.Reviews', reviews).then(
|
||||
(array) => Buffer.from(array).toString('base64'),
|
||||
);
|
||||
|
||||
await commitAssets(
|
||||
'reviews',
|
||||
[{ content, encoding: 'base64', path: 'definitions/reviews.pb' }],
|
||||
true,
|
||||
);
|
||||
await commitAssets('reviews', [
|
||||
{ content, encoding: 'base64', path: 'definitions/reviews.pb' },
|
||||
]);
|
||||
@@ -1,96 +0,0 @@
|
||||
import { chunk, times } from 'lodash';
|
||||
import { compress } from 'lz4js';
|
||||
import { argv } from 'process';
|
||||
import { Region, REGIONS } from '../src/constants/regions';
|
||||
import getTankStats from '../src/core/blitz/getTankStats';
|
||||
import { idToRegion } from '../src/core/blitz/idToRegion';
|
||||
import { commitAssets } from '../src/core/blitzkit/commitAssets';
|
||||
import { FileChange } from '../src/core/blitzkit/commitMultipleFiles';
|
||||
import {
|
||||
RtscWritePlayer,
|
||||
RtscWriteStream,
|
||||
RtsmWriteStream,
|
||||
} from '../src/core/streams/rts';
|
||||
import usersRaw from '../test.users.json';
|
||||
|
||||
const PLAYERS_PER_CHUNK = 256;
|
||||
const CHUNKS_PER_BATCH = 128;
|
||||
|
||||
const production = argv.includes('--production');
|
||||
const region = argv
|
||||
.find((argument) => argument.startsWith('--region'))
|
||||
?.split('=')[1] as Region | undefined;
|
||||
|
||||
if (!region || !REGIONS.includes(region)) {
|
||||
throw new Error('No valid region specified');
|
||||
}
|
||||
|
||||
const today = new Date();
|
||||
const todayPath = `${today.getUTCFullYear()}/${today.getUTCMonth() + 1}/${today.getUTCDate()}`;
|
||||
const users = usersRaw.filter(({ blitz }) => idToRegion(blitz) === region);
|
||||
const base = Math.ceil(users.length / PLAYERS_PER_CHUNK);
|
||||
const chunks = times(base, (offset) => ({
|
||||
offset,
|
||||
users: users.filter((user) => user.blitz % base === offset),
|
||||
}));
|
||||
const batches = chunk(chunks, CHUNKS_PER_BATCH);
|
||||
const manifestWriteStream = new RtsmWriteStream();
|
||||
|
||||
console.log(`Found ${users.length} players in ${region}`);
|
||||
|
||||
await commitAssets(
|
||||
`regular tank stats manifests ${todayPath}`,
|
||||
[
|
||||
{
|
||||
encoding: 'base64',
|
||||
path: `regions/${region}/regular/tanks/${todayPath}/manifest.rtsm`,
|
||||
content: Buffer.from(
|
||||
compress(manifestWriteStream.rtsm(base).uint8Array),
|
||||
).toString('base64'),
|
||||
},
|
||||
],
|
||||
production,
|
||||
);
|
||||
|
||||
let batchesDone = 0;
|
||||
for (const batch of batches) {
|
||||
console.log(`Batch ${++batchesDone} of ${batches.length}`);
|
||||
|
||||
const changes: FileChange[] = [];
|
||||
|
||||
let chunksDone = 0;
|
||||
for (const chunk of batch) {
|
||||
console.log(`Chunk ${++chunksDone} of ${chunks.length}`);
|
||||
|
||||
let playersDone = 0;
|
||||
const players = await Promise.all(
|
||||
chunk.users.map(async (user) => {
|
||||
const tanks = await getTankStats(region, user.blitz);
|
||||
|
||||
console.log(`Player ${++playersDone} of ${chunk.users.length}`);
|
||||
|
||||
if (tanks === null) return null;
|
||||
|
||||
return {
|
||||
id: user.blitz,
|
||||
tanks,
|
||||
} satisfies RtscWritePlayer;
|
||||
}),
|
||||
).then((players) => players.filter((player) => player !== null));
|
||||
const writeStream = new RtscWriteStream();
|
||||
|
||||
writeStream.rtsc(players);
|
||||
changes.push({
|
||||
encoding: 'base64',
|
||||
content: Buffer.from(compress(writeStream.uint8Array)).toString('base64'),
|
||||
path: `regions/${region}/regular/tanks/${todayPath}/${chunk.offset}.rtsc.lz4`,
|
||||
});
|
||||
}
|
||||
|
||||
// do not await; commit in background
|
||||
commitAssets(
|
||||
`regular tank stats ${todayPath} batch ${batches.indexOf(batch) + 1}`,
|
||||
changes,
|
||||
production,
|
||||
);
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import AdmZip from 'adm-zip';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { BattleResultsReadStream } from '../src/core/streams/battleResults';
|
||||
|
||||
interface WoTBReplayMeta {
|
||||
version: string;
|
||||
title: string;
|
||||
dbid: string;
|
||||
playerName: string;
|
||||
battleStartTime: string;
|
||||
playerVehicleName: string;
|
||||
mapName: string;
|
||||
arenaUniqueId: string;
|
||||
battleDuration: number;
|
||||
vehicleCompDescriptor: number;
|
||||
camouflageId: number;
|
||||
mapId: number;
|
||||
arenaBonusType: number;
|
||||
}
|
||||
|
||||
const file = await readFile('temp/test.wotbreplay');
|
||||
const zip = new AdmZip(file);
|
||||
const entries = zip.getEntries();
|
||||
|
||||
const metaRaw = entries.find((entry) => entry.name === 'meta.json');
|
||||
const battleResultsRaw = entries.find(
|
||||
(entry) => entry.name === 'battle_results.dat',
|
||||
);
|
||||
|
||||
if (!metaRaw) throw new Error('No meta.json found');
|
||||
if (!battleResultsRaw) throw new Error('No battle_results.dat found');
|
||||
|
||||
const meta = JSON.parse(metaRaw.getData().toString()) as WoTBReplayMeta;
|
||||
const battleResults = new BattleResultsReadStream(
|
||||
battleResultsRaw.getData().buffer,
|
||||
).battleResults();
|
||||
Reference in new issue
Block a user