diff --git a/scripts/buildTankopedia.ts b/scripts/buildTankopedia.ts index 28cbd490e..d62c1cc24 100644 --- a/scripts/buildTankopedia.ts +++ b/scripts/buildTankopedia.ts @@ -9,38 +9,36 @@ import { TreeTypeString } from '../src/components/Tanks'; import { NATION_IDS } from '../src/constants/nations'; import commitMultipleFiles, { FileChange } from './commitMultipleFiles'; +const LANGUAGE = 'en'; +const TEMP = 'temp/blitz'; + const publish = argv.includes('--publish'); const apkFile = argv .find((argument) => argument.startsWith('--file')) ?.split('=')[1]; const noRewrite = argv.includes('--no-rewrite'); -const VEHICLE_DEFS = 'assets/Data/XML/item_defs/vehicles'; -const STRINGS = 'assets/Data/Strings'; -const LANGUAGE = 'en'; -const TANK_PARAMETERS = 'assets/Data/3d/Tanks/Parameters'; -const SMALL_ICONS = 'assets/Data/Gfx/UI/BattleScreenHUD/SmallTankIcons'; -const BIG_ICONS = 'assets/Data/Gfx/UI/BigTankIcons'; -const DIRECTORIES_OF_INTEREST = [ - VEHICLE_DEFS, - STRINGS, - TANK_PARAMETERS, - SMALL_ICONS, - BIG_ICONS, -]; - if (!apkFile) throw new Error('No file specified'); +const directoriesOfInterest = { + vehicleDefinitions: 'assets/Data/XML/item_defs/vehicles', + strings: 'assets/Data/Strings', + tankParameters: 'assets/Data/3d/Tanks/Parameters', + smallIcons: 'assets/Data/Gfx/UI/BattleScreenHUD/SmallTankIcons', + bigIcons: 'assets/Data/Gfx/UI/BigTankIcons', +}; +const directoriesOfInterestArray = Object.values(directoriesOfInterest); + console.log('Removing old tankopedia files...'); await rm('dist/tankopedia', { recursive: true, force: true }); if (!noRewrite) { console.log('Removing old temp files...'); - await rm('temp/blitz', { recursive: true, force: true }); + await rm(TEMP, { recursive: true, force: true }); console.log('Extracting Blitz files...'); - await decompress(apkFile, 'temp/blitz', { + await decompress(apkFile, TEMP, { filter: (file) => - DIRECTORIES_OF_INTEREST.some((dir) => file.path.startsWith(dir)), + directoriesOfInterestArray.some((dir) => file.path.startsWith(dir)), }); } @@ -58,20 +56,22 @@ interface BlitzkriegTankopedia { } const strings = await readFile( - `temp/blitz/${STRINGS}/${LANGUAGE}.yaml`, + `${TEMP}/${directoriesOfInterest.strings}/${LANGUAGE}.yaml`, 'utf-8', ).then((data) => load(data) as Record); const tankopedia: Record = {}; const tankIds: number[] = []; const images: Record = {}; -const nations = await readdir(`temp/blitz/${VEHICLE_DEFS}`); +const nations = await readdir( + `${TEMP}/${directoriesOfInterest.vehicleDefinitions}`, +); Promise.all( nations .filter((nation) => nation !== 'common') .map(async (nation) => { const listXML = await readFile( - `temp/blitz/${VEHICLE_DEFS}/${nation}/list.xml`, + `${TEMP}/${directoriesOfInterest.vehicleDefinitions}/${nation}/list.xml`, 'utf-8', ); const list = xml2js(listXML, { compact: true }) as ElementCompact; @@ -109,15 +109,15 @@ Promise.all( }; // Tank images - const parameterPath = `temp/blitz/${TANK_PARAMETERS}/${nation}/${vehicle}.yaml`; + const parameterPath = `${TEMP}/${directoriesOfInterest.tankParameters}/${nation}/${vehicle}.yaml`; const parameterString = await readFile(parameterPath, 'utf-8'); const { smallIconPath, bigIconPath } = parse(parameterString).resourcesPath; images[id] = { - small: `temp/blitz/assets/Data/${(smallIconPath as string) + small: `${TEMP}/assets/Data/${(smallIconPath as string) .replace(/~res:\//, '') .replace(/\..+/, '')}.packed.webp`, - big: `temp/blitz/assets/Data/${(bigIconPath as string) + big: `${TEMP}/assets/Data/${(bigIconPath as string) .replace(/~res:\//, '') .replace(/\..+/, '')}.packed.webp`, }; @@ -133,48 +133,49 @@ Promise.all( ).then(async () => { if (publish) { const octokit = new Octokit({ auth: env.GH_TOKEN }); + const tankopediaChange: FileChange = { + content: JSON.stringify(tankopedia), + path: 'tankopedia.json', + encoding: 'utf-8', + }; + const iconsChange = ( + await Promise.all( + tankIds.map(async (id) => { + try { + const [contentSmall, contentBig] = await Promise.all([ + readFile(images[id].small, { encoding: 'base64' }), + readFile(images[id].big, { encoding: 'base64' }), + ]); - // console.log('Writing tankopedia.json'); - // commitMultipleFiles( - // octokit, - // 'tresabhi', - // 'blitzkrieg-assets', - // 'main', - // new Date().toString(), - // [ - // { - // content: JSON.stringify(tankopedia), - // path: 'tankopedia.json', - // encoding: 'utf-8', - // }, - // ], - // ); + return [ + { + content: contentBig, + path: `icons/big/${id}.webp`, + encoding: 'base64', + }, + { + content: contentSmall, + path: `icons/small/${id}.webp`, + encoding: 'base64', + }, + ]; + } catch (error) { + console.log(error); + return undefined; + } + }), + ) + ).flat() as FileChange[]; - console.log('Writing big icons...'); + console.log('Writing files...'); await commitMultipleFiles( octokit, 'tresabhi', 'blitzkrieg-assets', 'main', new Date().toString(), - (await Promise.all( - tankIds.map(async (id) => { - try { - const content = await readFile(images[id].big, { - encoding: 'base64', - }); - - return { - content, - path: `icons/big/${id}.webp`, - encoding: 'base64', - }; - } catch (error) { - console.log(error); - return undefined; - } - }), - )) as FileChange[], + iconsChange, + true, ); } else { writeFile( diff --git a/scripts/commitMultipleFiles.ts b/scripts/commitMultipleFiles.ts index fe1b73a25..d24eb2a42 100644 --- a/scripts/commitMultipleFiles.ts +++ b/scripts/commitMultipleFiles.ts @@ -1,5 +1,7 @@ import { Octokit } from '@octokit/rest'; +const TIME_PER_BLOB = 2 ** 4 * 1000; + export interface FileChange { path: string; content: string; @@ -13,41 +15,71 @@ export default async function commitMultipleFiles( branch: string, message: string, changes: FileChange[], + logBlobCreation = false, ) { // Get the latest commit SHA on the specified branch - const { data: refData } = await octokit.git.getRef({ - owner, - repo, - ref: `heads/${branch}`, - }); - const latestCommitSha = refData.object.sha; + const latestCommitSha = ( + await octokit.git.getRef({ + owner, + repo, + ref: `heads/${branch}`, + }) + ).data.object.sha; // Get the tree SHA for the latest commit - const { data: commitData } = await octokit.git.getCommit({ - owner, - repo, - commit_sha: latestCommitSha, - }); - const treeSha = commitData.tree.sha; + const treeSha = ( + await octokit.git.getCommit({ + owner, + repo, + commit_sha: latestCommitSha, + }) + ).data.tree.sha; - // Create blobs for each file - const blobs = await Promise.all( - changes.map(async (change) => { - const { data: blobData } = await octokit.git.createBlob({ - owner, - repo, - content: change.content, - encoding: change.encoding, - }); + const blobs: { + sha: string; + path: string; + mode: '100644'; + type: 'blob'; + }[] = []; - return { - sha: blobData.sha, - path: change.path, - mode: '100644', - type: 'blob', - } as const; - }), - ); + for (const changeIndexString in changes) { + const changeIndex = parseInt(changeIndexString); + const change = changes[changeIndex]; + let done = false; + + while (!done) { + try { + const { sha } = ( + await octokit.git.createBlob({ + owner, + repo, + content: change.content, + encoding: change.encoding, + }) + ).data; + + blobs.push({ sha, path: change.path, mode: '100644', type: 'blob' }); + done = true; + } catch (error) { + console.warn( + `Failed blob ${changeIndex + 1} / ${ + changes.length + }; retrying in ${TIME_PER_BLOB}ms...`, + ); + + await new Promise((resolve) => setTimeout(resolve, TIME_PER_BLOB)); + } + } + + if (logBlobCreation) { + console.log( + `Created blob ${changeIndex + 1} / ${changes.length} (${( + (100 * (changeIndex + 1)) / + changes.length + ).toFixed(2)}%)`, + ); + } + } // Create a new tree with the new blobs const { data: treeData } = await octokit.git.createTree({