diff --git a/.vscode/settings.json b/.vscode/settings.json index fd33836a2..5d988642b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -64,6 +64,7 @@ "deburr", "decalmask", "decompressor", + "dedup", "defence", "degressive", "depotdownloader", diff --git a/packages/scripts/src/averages.ts b/packages/scripts/src/averages.ts index d174b875e..38853e1bb 100644 --- a/packages/scripts/src/averages.ts +++ b/packages/scripts/src/averages.ts @@ -11,10 +11,10 @@ import { getAccountInfo, getTankStats, idToRegion, -} from '@blitzkit/core'; -import { chunk, times } from 'lodash-es'; -import { commitAssets } from './core/github/commitAssets'; -import { fetchPreDiscoveredIds } from './core/github/fetchPreDiscoveredIds'; +} from "@blitzkit/core"; +import { chunk, times } from "lodash-es"; +import { AssetUploader } from "./core/github/assetUploader"; +import { fetchPreDiscoveredIds } from "./core/github/fetchPreDiscoveredIds"; interface DataPoint { x: number; @@ -23,7 +23,7 @@ interface DataPoint { } type OptionalSecondLevel = { - [P in keyof T]: P extends 'samples' + [P in keyof T]: P extends "samples" ? T[P] : { [K in keyof T[P]]?: T[P][K]; @@ -46,16 +46,16 @@ const startTime = Date.now(); const preDiscoveredIds = await fetchPreDiscoveredIds(); const playerIds: Record = { asia: chunk( - preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'), - PLAYER_IDS_PER_CALL, + preDiscoveredIds.filter((id) => idToRegion(id) === "asia"), + PLAYER_IDS_PER_CALL ), com: chunk( - preDiscoveredIds.filter((id) => idToRegion(id) === 'com'), - PLAYER_IDS_PER_CALL, + preDiscoveredIds.filter((id) => idToRegion(id) === "com"), + PLAYER_IDS_PER_CALL ), eu: chunk( - preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'), - PLAYER_IDS_PER_CALL, + preDiscoveredIds.filter((id) => idToRegion(id) === "eu"), + PLAYER_IDS_PER_CALL ), }; let regionIndex = 0; @@ -71,10 +71,10 @@ times(THREADS, async () => { const regionIds = playerIds[region]; const [ids] = regionIds.splice( Math.floor(Math.random() * regionIds.length), - 1, + 1 ); const accountInfo = await getAccountInfo(region, ids, undefined, { - fields: 'last_battle_time,statistics.all.battles', + fields: "last_battle_time,statistics.all.battles", }); const filteredIds = ids.filter((_, index) => { @@ -103,9 +103,9 @@ times(THREADS, async () => { const players = await Promise.all( filteredIds.map((id) => getTankStats(region, id, { - fields: 'last_battle_time,battle_life_time,all,tank_id', - }), - ), + fields: "last_battle_time,battle_life_time,all,tank_id", + }) + ) ); players.forEach((tanks) => { @@ -114,7 +114,7 @@ times(THREADS, async () => { if (!tankIds.includes(tank.tank_id)) { console.log( - `Found new tank: ${tank.tank_id} (${tankIds.length + 1})`, + `Found new tank: ${tank.tank_id} (${tankIds.length + 1})` ); tankIds.push(tank.tank_id); tanksSorted[tank.tank_id] = []; @@ -140,9 +140,10 @@ async function postWork() { console.log( `Generating statistics based on ${samples.d_120.toLocaleString()} players (${samples.total.toLocaleString()} checked in total) and ${ tankIds.length - } tanks...`, + } tanks...` ); + using uploader = new AssetUploader("averages"); const averages: Record = {}; tankIds.forEach((id) => { @@ -179,7 +180,7 @@ async function postWork() { const data: DataPoint[] = dataWY.map(({ w, y }, index) => ({ w, x: - (key === 'battle_life_time' + (key === "battle_life_time" ? tanks[index].battle_life_time : tanks[index].all[key]) / w, y, @@ -190,7 +191,7 @@ async function postWork() { function sum(slicer: (data: DataPoint) => number) { return data.reduce( (accumulator, data) => accumulator + slicer(data), - 0, + 0 ); } @@ -231,14 +232,14 @@ async function postWork() { latest, }; - commitAssets('averages', [ - { - path: `averages/${latest}.pb`, - content: AverageDefinitions.encode(averageDefinitions).finish(), - }, - { - path: 'averages/manifest.json', - content: new TextEncoder().encode(JSON.stringify(manifest)), - }, - ]); + await uploader.add({ + path: `averages/${latest}.pb`, + content: AverageDefinitions.encode(averageDefinitions).finish(), + }); + await uploader.add({ + path: "averages/manifest.json", + content: new TextEncoder().encode(JSON.stringify(manifest)), + }); + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/blitzkitTankIcons.ts b/packages/scripts/src/buildAssets/blitzkitTankIcons.ts deleted file mode 100644 index 530df5f3f..000000000 --- a/packages/scripts/src/buildAssets/blitzkitTankIcons.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { BLITZKIT_TANK_ICON_SIZE, fetchTankDefinitions } from "@blitzkit/core"; -import { launch } from "puppeteer"; -import sharp from "sharp"; -import { commitAssets } from "../core/github/commitAssets"; -import { type FileChange } from "../core/github/commitMultipleFiles"; - -export async function blitzkitTankIcons() { - console.log("Building blitzkit tank icons..."); - - const browser = await launch(); - const page = await browser.newPage(); - const tanks = Object.values((await fetchTankDefinitions()).tanks); - const files: FileChange[] = []; - - let index = 0; - for (const { id, name } of tanks) { - await page.setViewport({ - width: BLITZKIT_TANK_ICON_SIZE.width - 1, - height: BLITZKIT_TANK_ICON_SIZE.height, - }); - - try { - await page.goto(`http://localhost:4321/api/tankopedia/tank-icon/${id}/`, { - waitUntil: "networkidle0", - }); - } catch (error) { - console.error(`${id} ${name} time out`, error); - continue; - } - - const screenshot = await page.screenshot({ - type: "webp", - omitBackground: true, - }); - const content = await sharp(screenshot).trim({ threshold: 100 }).toBuffer(); - - index++; - console.log( - `Rendered ${id} (${name}) ${((index / tanks.length) * 100).toFixed(2)}%` - ); - - files.push({ path: `icons/tanks/big/${id}.webp`, content }); - } - - await page.close(); - await browser.close(); - - await commitAssets("tank icons", files); -} diff --git a/packages/scripts/src/buildAssets/boosterIcons.ts b/packages/scripts/src/buildAssets/boosterIcons.ts index d23233290..d1cb9d316 100644 --- a/packages/scripts/src/buildAssets/boosterIcons.ts +++ b/packages/scripts/src/buildAssets/boosterIcons.ts @@ -1,48 +1,45 @@ -import { readdir } from 'fs/promises'; -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { readStringDVPL } from '../../src/core/blitz/readStringDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; +import { readdir } from "fs/promises"; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { readStringDVPL } from "../../src/core/blitz/readStringDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; export async function boosterIcons() { + console.log("Building booster icons..."); + + using uploader = new AssetUploader("booster icons"); const boosterFiles = (await readdir(`${DATA}/Gfx/Shared/boosters`)).filter( - (file) => !file.includes('@2x.txt') && !file.startsWith('texture0'), + (file) => !file.includes("@2x.txt") && !file.startsWith("texture0") ); const image = sharp( - await readDVPLFile(`${DATA}/Gfx/Shared/boosters/texture0.packed.webp`), - ); - const changes: FileChange[] = []; - - await Promise.all( - boosterFiles.map(async (file) => { - const name = file.match(/booster_(.+).txt/)?.[1]; - - if (name === undefined) return; - - const sizes = ( - await readStringDVPL(`${DATA}/Gfx/Shared/boosters/${file}`) - ) - .split('\n')[4] - .split(' ') - .map(Number); - const content = await image - .clone() - .extract({ - left: sizes[0], - top: sizes[1], - width: sizes[2], - height: sizes[3], - }) - .toBuffer(); - - changes.push({ - path: `icons/boosters/${name}.webp`, - content, - }); - }), + await readDVPLFile(`${DATA}/Gfx/Shared/boosters/texture0.packed.webp`) ); - await commitAssets('booster icons', changes); + for (const file of boosterFiles) { + const name = file.match(/booster_(.+).txt/)?.[1]; + + if (name === undefined) continue; + + const sizes = (await readStringDVPL(`${DATA}/Gfx/Shared/boosters/${file}`)) + .split("\n")[4] + .split(" ") + .map(Number); + const content = await image + .clone() + .extract({ + left: sizes[0], + top: sizes[1], + width: sizes[2], + height: sizes[3], + }) + .toBuffer(); + + await uploader.add({ + path: `icons/boosters/${name}.webp`, + content, + }); + } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/camouflageIcons.ts b/packages/scripts/src/buildAssets/camouflageIcons.ts index 88b17057a..726a0c866 100644 --- a/packages/scripts/src/buildAssets/camouflageIcons.ts +++ b/packages/scripts/src/buildAssets/camouflageIcons.ts @@ -1,19 +1,15 @@ -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; export async function camouflageIcons() { - console.log('Building camouflage icons...'); - const content = await readDVPLFile( - `${DATA}/Gfx/UI/Hangar/IconCamouflage.packed.webp`, - ); - const changes: FileChange[] = [ - { - content, - path: 'icons/camo.webp', - }, - ]; + console.log("Building camouflage icons..."); - await commitAssets('camo icons', changes); + using uploader = new AssetUploader("camo icons"); + const content = await readDVPLFile( + `${DATA}/Gfx/UI/Hangar/IconCamouflage.packed.webp` + ); + + await uploader.add({ content, path: "icons/camo.webp" }); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/consumableProvisionIcons.ts b/packages/scripts/src/buildAssets/consumableProvisionIcons.ts index 137bd3a30..5028f730c 100644 --- a/packages/scripts/src/buildAssets/consumableProvisionIcons.ts +++ b/packages/scripts/src/buildAssets/consumableProvisionIcons.ts @@ -1,178 +1,181 @@ -import { existsSync } from 'fs'; -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { readStringDVPL } from '../../src/core/blitz/readStringDVPL'; -import { readXMLDVPL } from '../core/blitz/readXMLDVPL'; -import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; -import { ConsumablesCommon, ProvisionsCommon } from './definitions'; +import { existsSync } from "fs"; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { readStringDVPL } from "../../src/core/blitz/readStringDVPL"; +import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; +import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; +import { ConsumablesCommon, ProvisionsCommon } from "./definitions"; interface Mappings { Header: { version: number }; StyleSheets: { selectors: string[]; - properties: { 'bg-sprite': string }; + properties: { "bg-sprite": string }; }[]; } const listItemsPattern = //g; export async function consumableProvisionIcons() { - console.log('Building consumable and provision icons...'); - - const changes: FileChange[] = []; + console.log("Building consumable and provision icons..."); + using uploader = new AssetUploader("consumable and provision icons"); const styles = [ - 'UI/Styles/Lobby/Inventory/Event/InventoryBigStyles.yaml', - 'UI/Styles/Lobby/Inventory/Event/InventoryNormalStyles.yaml', - 'UI/Styles/Lobby/Inventory/InventoryNormalStyles.yaml', - 'UI/Screens3/Lobby/Inventory/Equipment/EquipmentItemImage.style.yaml', - 'UI/Screens/Battle/Styles/BattleEquipmentStyles.yaml', + "UI/Styles/Lobby/Inventory/Event/InventoryBigStyles.yaml", + "UI/Styles/Lobby/Inventory/Event/InventoryNormalStyles.yaml", + "UI/Styles/Lobby/Inventory/InventoryNormalStyles.yaml", + "UI/Screens3/Lobby/Inventory/Equipment/EquipmentItemImage.style.yaml", + "UI/Screens/Battle/Styles/BattleEquipmentStyles.yaml", ]; const styleSheets = await Promise.all( styles.map(async (path) => Object.values( - (await readYAMLDVPL(`${DATA}/${path}`)).StyleSheets, - ), - ), + ( + await readYAMLDVPL(`${DATA}/${path}`) + ).StyleSheets + ) + ) ).then((array) => array.flat()); for (const match of ( await readStringDVPL( - `${DATA}/XML/item_defs/vehicles/common/consumables/list.xml`, + `${DATA}/XML/item_defs/vehicles/common/consumables/list.xml` ) ).matchAll(listItemsPattern)) { const consumablesCommon = await readXMLDVPL<{ root: ConsumablesCommon; }>(`${DATA}/XML/item_defs/vehicles/common/consumables/${match[1]}.xml`); - await Promise.all( - Object.values(consumablesCommon.root).map(async (consumable) => { - const styleSheet = styleSheets.find((styleSheet) => - styleSheet.selectors.some((selector) => - selector.includes(`${consumable.icon} `), - ), + for (const consumable of Object.values(consumablesCommon.root)) { + const styleSheet = styleSheets.find((styleSheet) => + styleSheet.selectors.some((selector) => + selector.includes(`${consumable.icon} `) + ) + ); + + if (!styleSheet) { + console.warn( + `No style sheet found for consumable ${consumable.icon}; skipping...` ); + continue; + } - if (!styleSheet) { - console.warn( - `No style sheet found for consumable ${consumable.icon}; skipping...`, - ); - return; - } + const configPath = styleSheet.properties["bg-sprite"] + .replace("~res:/", "") + .replace(".psd", "") + .replace(".txt", ""); - const configPath = styleSheet.properties['bg-sprite'] - .replace('~res:/', '') - .replace('.psd', '') - .replace('.txt', ''); + if (existsSync(`${DATA}/${configPath}.packed.webp`)) { + const image = sharp( + await readDVPLFile(`${DATA}/${configPath}.packed.webp`) + ); + const content = await image.trim({ threshold: 100 }).toBuffer(); - if (existsSync(`${DATA}/${configPath}.packed.webp`)) { - const image = sharp( - await readDVPLFile(`${DATA}/${configPath}.packed.webp`), - ); - const content = await image.trim({ threshold: 100 }).toBuffer(); + await uploader.add({ + path: `icons/consumables/${consumable.id}.webp`, + content, + }); + } else { + const consumablesTexture = sharp( + await readDVPLFile( + `${`${DATA}/${configPath}` + .split("/") + .slice(0, -1) + .join("/")}/texture0.packed.webp` + ) + ); + const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`)) + .split("\n")[4] + .split(" ") + .map(Number); + const content = await consumablesTexture + .clone() + .extract({ + left: sizes[0], + top: sizes[1], + width: sizes[2], + height: sizes[3], + }) + .toBuffer(); - changes.push({ - path: `icons/consumables/${consumable.id}.webp`, - content, - }); - } else { - const consumablesTexture = sharp( - await readDVPLFile( - `${`${DATA}/${configPath}`.split('/').slice(0, -1).join('/')}/texture0.packed.webp`, - ), - ); - const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`)) - .split('\n')[4] - .split(' ') - .map(Number); - const content = await consumablesTexture - .clone() - .extract({ - left: sizes[0], - top: sizes[1], - width: sizes[2], - height: sizes[3], - }) - .toBuffer(); - - changes.push({ - path: `icons/consumables/${consumable.id}.webp`, - content, - }); - } - }), - ); + await uploader.add({ + path: `icons/consumables/${consumable.id}.webp`, + content, + }); + } + } } for (const match of ( await readStringDVPL( - `${DATA}/XML/item_defs/vehicles/common/provisions/list.xml`, + `${DATA}/XML/item_defs/vehicles/common/provisions/list.xml` ) ).matchAll(listItemsPattern)) { const provisionsCommon = await readXMLDVPL<{ root: ProvisionsCommon; }>(`${DATA}/XML/item_defs/vehicles/common/provisions/${match[1]}.xml`); - await Promise.all( - Object.values(provisionsCommon.root).map(async (provision) => { - const styleSheet = styleSheets.find((styleSheet) => - styleSheet.selectors.some((selector) => - selector.includes(`${provision.icon} `), - ), + + for (const provision of Object.values(provisionsCommon.root)) { + const styleSheet = styleSheets.find((styleSheet) => + styleSheet.selectors.some((selector) => + selector.includes(`${provision.icon} `) + ) + ); + + if (!styleSheet) { + console.warn( + `No style sheet found for provision ${provision.icon}; skipping...` ); + continue; + } - if (!styleSheet) { - console.warn( - `No style sheet found for provision ${provision.icon}; skipping...`, - ); - return; - } + const configPath = styleSheet.properties["bg-sprite"] + .replace("~res:/", "") + .replace(".psd", "") + .replace(".txt", ""); - const configPath = styleSheet.properties['bg-sprite'] - .replace('~res:/', '') - .replace('.psd', '') - .replace('.txt', ''); + if (existsSync(`${DATA}/${configPath}.packed.webp`)) { + const image = sharp( + await readDVPLFile(`${DATA}/${configPath}.packed.webp`) + ); + const content = await image.trim({ threshold: 100 }).toBuffer(); - if (existsSync(`${DATA}/${configPath}.packed.webp`)) { - const image = sharp( - await readDVPLFile(`${DATA}/${configPath}.packed.webp`), - ); - const content = await image.trim({ threshold: 100 }).toBuffer(); + await uploader.add({ + path: `icons/provisions/${provision.id}.webp`, + content, + }); + } else { + const provisionsTexture = sharp( + await readDVPLFile( + `${`${DATA}/${configPath}` + .split("/") + .slice(0, -1) + .join("/")}/texture0.packed.webp` + ) + ); + const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`)) + .split("\n")[4] + .split(" ") + .map(Number); + const content = await provisionsTexture + .clone() + .extract({ + left: sizes[0], + top: sizes[1], + width: sizes[2], + height: sizes[3], + }) + .toBuffer(); - changes.push({ - path: `icons/provisions/${provision.id}.webp`, - content, - }); - } else { - const provisionsTexture = sharp( - await readDVPLFile( - `${`${DATA}/${configPath}`.split('/').slice(0, -1).join('/')}/texture0.packed.webp`, - ), - ); - const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`)) - .split('\n')[4] - .split(' ') - .map(Number); - const content = await provisionsTexture - .clone() - .extract({ - left: sizes[0], - top: sizes[1], - width: sizes[2], - height: sizes[3], - }) - .toBuffer(); - - changes.push({ - path: `icons/provisions/${provision.id}.webp`, - content, - }); - } - }), - ); + await uploader.add({ + path: `icons/provisions/${provision.id}.webp`, + content, + }); + } + } } - await commitAssets('consumable and provision icons', changes); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/currencies.ts b/packages/scripts/src/buildAssets/currencies.ts index ad071f090..b6a0467a8 100644 --- a/packages/scripts/src/buildAssets/currencies.ts +++ b/packages/scripts/src/buildAssets/currencies.ts @@ -1,66 +1,63 @@ -import { assertSecret, BlitzGlossary } from '@blitzkit/core'; -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; +import { assertSecret, BlitzGlossary } from "@blitzkit/core"; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; const ICONS = [ - ['currency_silver_m.packed.webp', 'silver'], - ['currency_premium_xl.packed.webp', 'premium'], - ['currency_gold_m.packed.webp', 'gold'], - ['currency_free-xp_xl.packed.webp', 'free-xp'], - ['currency_elite-xp_xl.packed.webp', 'elite-xp'], - ['currency_crew-xp_xl.packed.webp', 'crew-xp'], - ['currency_battle-xp_xl.packed.webp', 'xp'], + ["currency_silver_m.packed.webp", "silver"], + ["currency_premium_xl.packed.webp", "premium"], + ["currency_gold_m.packed.webp", "gold"], + ["currency_free-xp_xl.packed.webp", "free-xp"], + ["currency_elite-xp_xl.packed.webp", "elite-xp"], + ["currency_crew-xp_xl.packed.webp", "crew-xp"], + ["currency_battle-xp_xl.packed.webp", "xp"], ]; export async function currencies() { - console.log('Building currency icons...'); + console.log("Building currency icons..."); - const changes: FileChange[] = []; + using uploader = new AssetUploader("currency icons"); - await Promise.all( - ICONS.map(async ([file, name]) => { - const content = await sharp( - await readDVPLFile(`${DATA}/Gfx/Lobby/currency/${file}`), - ) - .trim() - .toBuffer(); + for (const [file, name] of ICONS) { + const content = await sharp( + await readDVPLFile(`${DATA}/Gfx/Lobby/currency/${file}`) + ) + .trim() + .toBuffer(); - changes.push({ - content, - path: `icons/currencies/${name}.webp`, - }); - }), - ); + await uploader.add({ + content, + path: `icons/currencies/${name}.webp`, + }); + } - const glossary = await fetch( - assertSecret(import.meta.env.WOTB_GLOSSARY), - ).then((response) => response.json() as Promise); + const glossary = await fetch(assertSecret(import.meta.env.WOTB_GLOSSARY)) + .then((response) => response.json() as Promise) + .then((glossary) => + Object.values(glossary) + .entries() + .filter(([key]) => /^prx_season_\d+$/.test(`${key}`)) + ); - await Promise.all( - Object.entries(glossary) - .filter(([key]) => /^prx_season_\d+$/.test(key)) - .map(async ([key, item]) => { - if (item.image_url === null) throw new Error(`No image_url for ${key}`); + for (const [key, item] of glossary) { + if (item.image_url === null) throw new Error(`No image_url for ${key}`); - const imageRaw = await fetch(item.image_url).then((response) => - response.arrayBuffer(), - ); - const content = await sharp(imageRaw) - .trim({ - threshold: 100, - background: { r: 0, g: 0, b: 0, alpha: 0 }, - }) - .toBuffer(); + const imageRaw = await fetch(item.image_url).then((response) => + response.arrayBuffer() + ); + const content = await sharp(imageRaw) + .trim({ + threshold: 100, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }) + .toBuffer(); - changes.push({ - content, - path: `icons/currencies/${key}.webp`, - }); - }), - ); + await uploader.add({ + content, + path: `icons/currencies/${key}.webp`, + }); + } - await commitAssets('currency icons', changes); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/definitions.ts b/packages/scripts/src/buildAssets/definitions.ts index 430f8b3e8..5ed83c7d1 100644 --- a/packages/scripts/src/buildAssets/definitions.ts +++ b/packages/scripts/src/buildAssets/definitions.ts @@ -40,7 +40,7 @@ import { parse as parseYaml } from "yaml"; import { readStringDVPL } from "../core/blitz/readStringDVPL"; import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL"; -import { commitAssets } from "../core/github/commitAssets"; +import { AssetUploader } from "../core/github/assetUploader"; import { DATA } from "./constants"; import type { Avatar } from "./skillIcons"; import type { TankParameters } from "./tankIcons"; @@ -302,7 +302,7 @@ interface VehicleDefinitions { type: "default"; enabled: false; factorsModifiers: ""; - }, + } ]; }; }; @@ -538,6 +538,7 @@ const blitzModuleTypeToBlitzkit: Record = { export async function definitions() { console.log("Building definitions..."); + using uploader = new AssetUploader("definitions"); const gameDefinitions: GameDefinitions = { version: (await readStringDVPL(`${DATA}/version.txt`)).split(" ")[0], nations: ( @@ -766,7 +767,9 @@ export async function definitions() { } console.warn( - `Multiple tanks share slug ${slug}: ${requesters.map(({ key }) => key).join(", ")}` + `Multiple tanks share slug ${slug}: ${requesters + .map(({ key }) => key) + .join(", ")}` ); if (requesters.length !== 2) { @@ -1018,8 +1021,8 @@ export async function definitions() { type: tankTags.includes("collectible") ? TankType.COLLECTOR : (typeof tank.price === "number" ? false : "gold" in tank.price) - ? TankType.PREMIUM - : TankType.RESEARCHABLE, + ? TankType.PREMIUM + : TankType.RESEARCHABLE, tier: tank.level, class: blitzTankClassToBlitzkit[tankTags[0] as BlitzTankClass], testing: tankTags.includes("testTank"), @@ -1337,24 +1340,24 @@ export async function definitions() { }, } : gunType === "autoReloader" - ? { - $case: "auto_reloader", - value: { - intra_clip: 60 / gun.clip!.rate, - shell_count: gunClipCount, - shell_reloads: gun - .pumpGunReloadTimes!.split(" ") - .map(Number), - }, - } - : { - $case: "auto_loader", - value: { - intra_clip: 60 / gun.clip!.rate, - clip_reload: gun.reloadTime, - shell_count: gunClipCount, - }, + ? { + $case: "auto_reloader", + value: { + intra_clip: 60 / gun.clip!.rate, + shell_count: gunClipCount, + shell_reloads: gun + .pumpGunReloadTimes!.split(" ") + .map(Number), }, + } + : { + $case: "auto_loader", + value: { + intra_clip: 60 / gun.clip!.rate, + clip_reload: gun.reloadTime, + shell_count: gunClipCount, + }, + }, } satisfies GunDefinition); modelDefinitions.models[tankId].turrets[turretId].guns[gunId] = { @@ -1365,8 +1368,8 @@ export async function definitions() { gun.armor.gun === undefined ? 0 : typeof gun.armor.gun === "number" - ? gun.armor.gun - : gun.armor.gun["#text"], + ? gun.armor.gun + : gun.armor.gun["#text"], pitch: { min: gunPitch[0], max: gunPitch[1], @@ -1413,7 +1416,7 @@ export async function definitions() { type: blitzShellKindToBlitzkit[shell.kind], explosion_radius: shell.kind === "HIGH_EXPLOSIVE" - ? (shell.explosionRadius ?? 0) + ? shell.explosionRadius ?? 0 : undefined, icon: shell.icon, penetration: { @@ -1783,42 +1786,42 @@ export async function definitions() { }; }); - await commitAssets("definitions", [ - { - content: GameDefinitions.encode(gameDefinitions).finish(), - path: "definitions/game.pb", - }, - { - content: TankDefinitions.encode(tankDefinitions).finish(), - path: "definitions/tanks.pb", - }, - { - content: ModelDefinitions.encode(modelDefinitions).finish(), - path: "definitions/models.pb", - }, - { - content: EquipmentDefinitions.encode(equipmentDefinitions).finish(), - path: "definitions/equipment.pb", - }, - { - content: ConsumableDefinitions.encode(consumableDefinitions).finish(), - path: "definitions/consumables.pb", - }, - { - content: ProvisionDefinitions.encode(provisionDefinitions).finish(), - path: "definitions/provisions.pb", - }, - { - content: SkillDefinitions.encode(skillDefinitions).finish(), - path: "definitions/skills.pb", - }, - { - content: MapDefinitions.encode(mapDefinitions).finish(), - path: "definitions/maps.pb", - }, - { - content: CamouflageDefinitions.encode(camouflageDefinitions).finish(), - path: "definitions/camouflages.pb", - }, - ]); + await uploader.add({ + content: GameDefinitions.encode(gameDefinitions).finish(), + path: "definitions/game.pb", + }); + await uploader.add({ + content: TankDefinitions.encode(tankDefinitions).finish(), + path: "definitions/tanks.pb", + }); + await uploader.add({ + content: ModelDefinitions.encode(modelDefinitions).finish(), + path: "definitions/models.pb", + }); + await uploader.add({ + content: EquipmentDefinitions.encode(equipmentDefinitions).finish(), + path: "definitions/equipment.pb", + }); + await uploader.add({ + content: ConsumableDefinitions.encode(consumableDefinitions).finish(), + path: "definitions/consumables.pb", + }); + await uploader.add({ + content: ProvisionDefinitions.encode(provisionDefinitions).finish(), + path: "definitions/provisions.pb", + }); + await uploader.add({ + content: SkillDefinitions.encode(skillDefinitions).finish(), + path: "definitions/skills.pb", + }); + await uploader.add({ + content: MapDefinitions.encode(mapDefinitions).finish(), + path: "definitions/maps.pb", + }); + await uploader.add({ + content: CamouflageDefinitions.encode(camouflageDefinitions).finish(), + path: "definitions/camouflages.pb", + }); + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/equipmentIcons.ts b/packages/scripts/src/buildAssets/equipmentIcons.ts index e4cebc5a8..40a6cd55b 100644 --- a/packages/scripts/src/buildAssets/equipmentIcons.ts +++ b/packages/scripts/src/buildAssets/equipmentIcons.ts @@ -1,91 +1,86 @@ -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { readStringDVPL } from '../../src/core/blitz/readStringDVPL'; -import { readXMLDVPL } from '../core/blitz/readXMLDVPL'; -import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; -import { OptionalDevices } from './definitions'; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { readStringDVPL } from "../../src/core/blitz/readStringDVPL"; +import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; +import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; +import { OptionalDevices } from "./definitions"; interface Mappings { Header: { version: number }; StyleSheets: { selectors: string[]; - properties: { 'bg-sprite': string }; + properties: { "bg-sprite": string }; }[]; } export async function equipmentIcons() { - console.log('Building equipment icons...'); + console.log("Building equipment icons..."); - const changes: FileChange[] = []; + using uploader = new AssetUploader("equipment icons"); const optionalDevices = await readXMLDVPL<{ root: OptionalDevices }>( - `${DATA}/XML/item_defs/vehicles/common/optional_devices.xml`, + `${DATA}/XML/item_defs/vehicles/common/optional_devices.xml` ); const mappings = await readYAMLDVPL( - `${DATA}/UI/Screens3/Lobby/Inventory/OptionalDevices/OptionalDevicesItemImage.style.yaml`, + `${DATA}/UI/Screens3/Lobby/Inventory/OptionalDevices/OptionalDevicesItemImage.style.yaml` ); const image = sharp( await readDVPLFile( - `${DATA}/Gfx/UI/InventoryIcons/Big/OptionalDevices/texture0.packed.webp`, - ), + `${DATA}/Gfx/UI/InventoryIcons/Big/OptionalDevices/texture0.packed.webp` + ) ); - await Promise.all( - Object.entries(optionalDevices.root).map( - async ([optionalDeviceKey, optionalDevice]) => { - if (optionalDeviceKey === 'nextAvailableId') return; + for (const optionalDeviceKey in optionalDevices.root) { + const optionalDevice = optionalDevices.root[optionalDeviceKey]; - const mapping = mappings.StyleSheets.find((mapping) => - mapping.selectors.includes( - `.optional_device_item.${optionalDevice.icon} #Img`, - ), - ); + if (optionalDeviceKey === "nextAvailableId") continue; - if (!mapping) { - console.warn( - `No mapping found for ${optionalDevice.icon}; skipping...`, - ); - return; - } + const mapping = mappings.StyleSheets.find((mapping) => + mapping.selectors.includes( + `.optional_device_item.${optionalDevice.icon} #Img` + ) + ); - const configPathRaw = mapping.properties['bg-sprite']; - const configPath = configPathRaw.replace('~res:/', ''); + if (!mapping) { + console.warn(`No mapping found for ${optionalDevice.icon}; skipping...`); + continue; + } - if (configPath.startsWith('Gfx/Lobby')) { - const configPathWebp = configPath.replace('.txt', ''); - const image = sharp( - await readDVPLFile(`${DATA}/${configPathWebp}.packed.webp`), - ); - const content = await image.trim().toBuffer(); + const configPathRaw = mapping.properties["bg-sprite"]; + const configPath = configPathRaw.replace("~res:/", ""); - changes.push({ - path: `icons/equipment/${optionalDevice.id}.webp`, - content, - }); - } else { - const config = await readStringDVPL(`${DATA}/${configPath}`); - const sizes = config.split('\n')[4].split(' ').map(Number); - const content = await image - .clone() - .extract({ - left: sizes[0], - top: sizes[1], - width: sizes[2], - height: sizes[3], - }) - .toBuffer(); + if (configPath.startsWith("Gfx/Lobby")) { + const configPathWebp = configPath.replace(".txt", ""); + const image = sharp( + await readDVPLFile(`${DATA}/${configPathWebp}.packed.webp`) + ); + const content = await image.trim().toBuffer(); - changes.push({ - path: `icons/equipment/${optionalDevice.id}.webp`, - content, - }); - } - }, - ), - ); + await uploader.add({ + path: `icons/equipment/${optionalDevice.id}.webp`, + content, + }); + } else { + const config = await readStringDVPL(`${DATA}/${configPath}`); + const sizes = config.split("\n")[4].split(" ").map(Number); + const content = await image + .clone() + .extract({ + left: sizes[0], + top: sizes[1], + width: sizes[2], + height: sizes[3], + }) + .toBuffer(); - await commitAssets('equipment icons', changes); + await uploader.add({ + path: `icons/equipment/${optionalDevice.id}.webp`, + content, + }); + } + } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/flags.ts b/packages/scripts/src/buildAssets/flags.ts index 2d1c8d134..59dc04409 100644 --- a/packages/scripts/src/buildAssets/flags.ts +++ b/packages/scripts/src/buildAssets/flags.ts @@ -1,75 +1,72 @@ -import { readdir } from 'fs/promises'; -import sharp from 'sharp'; -import { readDVPLFile } from '../core/blitz/readDVPLFile'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; +import { readdir } from "fs/promises"; +import sharp from "sharp"; +import { readDVPLFile } from "../core/blitz/readDVPLFile"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; export async function flags() { - console.log('Building flags...'); + console.log("Building flags..."); - const changes = [ - ...(await Promise.all( - (await readdir(`${DATA}/Gfx/Lobby/flags`)) - .filter( - (flag) => - flag.startsWith('flag_profile-stat_') && - !flag.endsWith('@2x.packed.webp'), - ) - .map(async (flag) => { - const image = sharp( - await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`), - ); - const content = await image.trim({ threshold: 100 }).toBuffer(); - const name = flag.match(/flag_profile-stat_(.+)\.packed\.webp/)![1]; + using uploader = new AssetUploader("flags"); + const circleFiles = await readdir(`${DATA}/Gfx/Lobby/flags`).then((files) => + files.filter( + (flag) => + flag.startsWith("flag_profile-stat_") && + !flag.endsWith("@2x.packed.webp") + ) + ); + const scratchedFiles = await readdir(`${DATA}/Gfx/Lobby/flags`).then( + (files) => + files.filter( + (flag) => + flag.startsWith("flag_tutor-tank_") && + !flag.endsWith("@2x.packed.webp") + ) + ); + const fadedFiles = await readdir(`${DATA}/Gfx/Lobby/flags`).then((files) => + files.filter( + (flag) => + flag.startsWith("flag_filter_") && flag.endsWith("@2x.packed.webp") + ) + ); - return { - content, - path: `flags/circle/${name}.webp`, - } satisfies FileChange; - }), - )), - ...(await Promise.all( - (await readdir(`${DATA}/Gfx/Lobby/flags`)) - .filter( - (flag) => - flag.startsWith('flag_tutor-tank_') && - !flag.endsWith('@2x.packed.webp'), - ) - .map(async (flag) => { - const content = await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`); - const name = flag.match(/flag_tutor-tank_(.+)\.packed\.webp/)![1]; + for (const flag of circleFiles) { + const image = sharp(await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`)); + const content = await image.trim({ threshold: 100 }).toBuffer(); + const name = flag.match(/flag_profile-stat_(.+)\.packed\.webp/)![1]; - return { - path: `flags/scratched/${name}.webp`, - content, - } satisfies FileChange; - }), - )), - ...(await Promise.all( - (await readdir(`${DATA}/Gfx/Lobby/flags`)) - .filter( - (flag) => - flag.startsWith('flag_filter_') && flag.endsWith('@2x.packed.webp'), - ) - .map(async (flag) => { - const content = await sharp( - await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`), - ) - .trim({ - threshold: 100, - background: { r: 0, g: 0, b: 0, alpha: 0 }, - }) - .toBuffer(); - const name = flag.match(/flag_filter_(.+)@2x\.packed\.webp/)![1]; + uploader.add({ + content, + path: `flags/circle/${name}.webp`, + }); + } - return { - content, - path: `flags/fade_small/${name}.webp`, - } satisfies FileChange; - }), - )), - ]; + for (const flag of scratchedFiles) { + const content = await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`); + const name = flag.match(/flag_tutor-tank_(.+)\.packed\.webp/)![1]; - await commitAssets('flags', changes); + uploader.add({ + path: `flags/scratched/${name}.webp`, + content, + }); + } + + for (const flag of fadedFiles) { + const content = await sharp( + await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`) + ) + .trim({ + threshold: 100, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }) + .toBuffer(); + const name = flag.match(/flag_filter_(.+)@2x\.packed\.webp/)![1]; + + uploader.add({ + content, + path: `flags/fade_small/${name}.webp`, + }); + } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/gameModeBanners.ts b/packages/scripts/src/buildAssets/gameModeBanners.ts index 7923e6c60..6c3bb1c10 100644 --- a/packages/scripts/src/buildAssets/gameModeBanners.ts +++ b/packages/scripts/src/buildAssets/gameModeBanners.ts @@ -1,36 +1,35 @@ -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; -import { SquadBattleTypeStyles } from './definitions'; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; +import { SquadBattleTypeStyles } from "./definitions"; export async function gameModeBanners() { - console.log('Building game mode banners...'); + console.log("Building game mode banners..."); - const changes: FileChange[] = []; + using uploader = new AssetUploader("game mode banners"); const gameTypeSelectorStyles = await readYAMLDVPL( - `${DATA}/UI/Screens/Lobby/Hangar/GameTypeSelector.yaml`, + `${DATA}/UI/Screens/Lobby/Hangar/GameTypeSelector.yaml` ); const squadBattleTypeStyles = await readYAMLDVPL( - `${DATA}/UI/Screens3/Lobby/Hangar/Squad/SquadBattleType.yaml`, + `${DATA}/UI/Screens3/Lobby/Hangar/Squad/SquadBattleType.yaml` ); const bannerMatches: { name: string; path: string }[] = []; for (const match of gameTypeSelectorStyles.Prototypes[0].components.UIDataLocalBindingsComponent.data[1][2].matchAll( - /eGameMode\.([a-zA-Z]+) -> "~res:([^"]+)"/g, + /eGameMode\.([a-zA-Z]+) -> "~res:([^"]+)"/g )) { bannerMatches.push({ name: match[1], path: match[2] }); } for (const match of squadBattleTypeStyles.Prototypes[0].components.UIDataLocalBindingsComponent.data[1][2].matchAll( - /"(\d+)" -> "battleType\/([a-zA-Z]+)"/g, + /"(\d+)" -> "battleType\/([a-zA-Z]+)"/g )) { const id = Number(match[1]); const name = match[2]; let path = bannerMatches.find( - (bannerMatch) => bannerMatch.name.toLowerCase() === name.toLowerCase(), + (bannerMatch) => bannerMatch.name.toLowerCase() === name.toLowerCase() )?.path; if (path === undefined) { @@ -38,16 +37,16 @@ export async function gameModeBanners() { } const content = await sharp( - await readDVPLFile(`${DATA}${path}.packed.webp`), + await readDVPLFile(`${DATA}${path}.packed.webp`) ) .trim({ background: { r: 0, g: 0, b: 0, alpha: 0 } }) .toBuffer(); - changes.push({ + await uploader.add({ path: `icons/game_mode_banners/${id}.webp`, content, }); } - await commitAssets('game mode banners', changes); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/glossary.ts b/packages/scripts/src/buildAssets/glossary.ts index 0048f9368..1710f658a 100644 --- a/packages/scripts/src/buildAssets/glossary.ts +++ b/packages/scripts/src/buildAssets/glossary.ts @@ -1,13 +1,13 @@ -import { Avatar, fetchGlossary, Gallery } from '@blitzkit/core'; -import locales from '@blitzkit/i18n/locales.json'; -import { extname } from 'path'; -import ProgressBar from 'progress'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; +import { Avatar, fetchGlossary, Gallery } from "@blitzkit/core"; +import locales from "@blitzkit/i18n/locales.json"; +import { extname } from "path"; +import ProgressBar from "progress"; +import { AssetUploader } from "../core/github/assetUploader"; export async function glossary() { - console.log('Building glossary...'); + console.log("Building glossary..."); + using uploader = new AssetUploader("glossary"); const avatars: Record = {}; await Promise.all( @@ -15,15 +15,15 @@ export async function glossary() { const glossary = await fetchGlossary(supported.blitz ?? supported.locale); console.log( - `Found ${Object.keys(glossary).length} things for ${supported.locale}`, + `Found ${Object.keys(glossary).length} things for ${supported.locale}` ); for (const key in glossary) { const glossaryEntry = glossary[key]; if ( - !key.startsWith('avatar') || - key.endsWith('_part') || + !key.startsWith("avatar") || + key.endsWith("_part") || glossaryEntry.image_url === null ) { continue; @@ -45,16 +45,15 @@ export async function glossary() { }; } } - }), + }) ); - const changes: FileChange[] = []; const gallery: Gallery = { avatars: [] }; const totalCount = Object.keys(avatars).length; const bar = new ProgressBar( `Fetching avatars of ${totalCount} things :bar`, - totalCount, + totalCount ); await Promise.all( @@ -72,19 +71,19 @@ export async function glossary() { .then((response) => response.arrayBuffer()) .then((buffer) => new Uint8Array(buffer)); - changes.push({ + await uploader.add({ path: `gallery/avatars/${key}${avatar.extension}`, content, }); bar.tick(); - }), + }) ); - changes.push({ - path: 'definitions/gallery.pb', + await uploader.add({ + path: "definitions/gallery.pb", content: Gallery.encode(gallery).finish(), }); - await commitAssets('glossary', changes); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/index.ts b/packages/scripts/src/buildAssets/index.ts index fc9db72c2..323d601a4 100644 --- a/packages/scripts/src/buildAssets/index.ts +++ b/packages/scripts/src/buildAssets/index.ts @@ -20,6 +20,7 @@ import { gameModeBanners } from "./gameModeBanners"; import { glossary } from "./glossary"; import { migration } from "./migration"; import { moduleIcons } from "./moduleIcons"; +import { popularTanks } from "./popularTanks"; import { shellIcons } from "./shellIcons"; import { skillIcons } from "./skillIcons"; import { tankArmors } from "./tankArmors"; @@ -29,8 +30,8 @@ import { tankModels } from "./tankModels"; if (!targets && !allTargets) throw new Error("No target(s) specified"); const methods = [ - migration, definitions, + tankIcons, tankModels, tankArmors, gameModeBanners, @@ -43,10 +44,9 @@ const methods = [ consumableProvisionIcons, camouflageIcons, boosterIcons, - tankIcons, - // blitzkitTankIcons, - // popularTanks, glossary, + migration, + popularTanks, ]; for (const method of methods) { diff --git a/packages/scripts/src/buildAssets/migration.ts b/packages/scripts/src/buildAssets/migration.ts index cc772a1b4..128f4d3d1 100644 --- a/packages/scripts/src/buildAssets/migration.ts +++ b/packages/scripts/src/buildAssets/migration.ts @@ -1,38 +1,36 @@ -import { toUniqueId } from '@blitzkit/core'; -import { readdir } from 'fs/promises'; -import { readXMLDVPL } from '../core/blitz/readXMLDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { DATA } from './constants'; -import { VehicleDefinitionList } from './definitions'; +import { toUniqueId } from "@blitzkit/core"; +import { readdir } from "fs/promises"; +import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; +import { VehicleDefinitionList } from "./definitions"; export async function migration() { - console.log('Building migration map...'); + console.log("Building migration map..."); + using uploader = new AssetUploader("migration map"); const map: Record = {}; const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then( - (nations) => nations.filter((nation) => nation !== 'common'), + (nations) => nations.filter((nation) => nation !== "common") ); - await Promise.all( - nations.map(async (nation) => { - const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>( - `${DATA}/XML/item_defs/vehicles/${nation}/list.xml`, - ); + for (const nation of nations) { + const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>( + `${DATA}/XML/item_defs/vehicles/${nation}/list.xml` + ); - for (const tankKey in tankList.root) { - const tank = tankList.root[tankKey]; - const tankId = toUniqueId(nation, tank.id); - const davaKey = `${nation}:${tankKey}`; + for (const tankKey in tankList.root) { + const tank = tankList.root[tankKey]; + const tankId = toUniqueId(nation, tank.id); + const davaKey = `${nation}:${tankKey}`; - map[davaKey] = tankId; - } - }), - ); + map[davaKey] = tankId; + } + } - await commitAssets('migration', [ - { - path: 'definitions/migration.json', - content: new TextEncoder().encode(JSON.stringify(map)), - }, - ]); + await uploader.add({ + path: "definitions/migration.json", + content: new TextEncoder().encode(JSON.stringify(map)), + }); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/moduleIcons.ts b/packages/scripts/src/buildAssets/moduleIcons.ts index b446a7314..9c8ed681c 100644 --- a/packages/scripts/src/buildAssets/moduleIcons.ts +++ b/packages/scripts/src/buildAssets/moduleIcons.ts @@ -1,34 +1,33 @@ -import { readdir } from 'fs/promises'; -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; +import { readdir } from "fs/promises"; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; export async function moduleIcons() { - console.log('Building module icons...'); + console.log("Building module icons..."); - const changes = await Promise.all( - (await readdir(`${DATA}/Gfx/UI/ModulesTechTree`)) - .filter( - (file) => - !file.endsWith('@2x.packed.webp') && file.startsWith('vehicle'), - ) - .map(async (file) => { - const nameRaw = file.match(/vehicle(.+)\.packed\.webp/)![1]; - const name = nameRaw[0].toLowerCase() + nameRaw.slice(1); - const content = await sharp( - await readDVPLFile(`${DATA}/Gfx/UI/ModulesTechTree/${file}`), - ) - .trim() - .toBuffer(); - - return { - content, - path: `icons/modules/${name}.webp`, - } satisfies FileChange; - }), + using uploader = new AssetUploader("module icons"); + const files = await readdir(`${DATA}/Gfx/UI/ModulesTechTree`).then((dir) => + dir.filter( + (file) => !file.endsWith("@2x.packed.webp") && file.startsWith("vehicle") + ) ); - await commitAssets('module icons', changes); + for (const file of files) { + const nameRaw = file.match(/vehicle(.+)\.packed\.webp/)![1]; + const name = nameRaw[0].toLowerCase() + nameRaw.slice(1); + const content = await sharp( + await readDVPLFile(`${DATA}/Gfx/UI/ModulesTechTree/${file}`) + ) + .trim() + .toBuffer(); + + await uploader.add({ + content, + path: `icons/modules/${name}.webp`, + }); + } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/popularTanks.ts b/packages/scripts/src/buildAssets/popularTanks.ts index c71709f24..9fc92c5d1 100644 --- a/packages/scripts/src/buildAssets/popularTanks.ts +++ b/packages/scripts/src/buildAssets/popularTanks.ts @@ -2,31 +2,34 @@ import { assertSecret, fetchTankDefinitions, PopularTanks, -} from '@blitzkit/core'; -import { google } from 'googleapis'; -import { commitAssets } from '../core/github/commitAssets'; +} from "@blitzkit/core"; +import { google } from "googleapis"; +import { AssetUploader } from "../core/github/assetUploader"; export async function popularTanks() { + using uploader = new AssetUploader("popular tanks"); const tankDefinitions = await fetchTankDefinitions(); const auth = await google.auth.getClient({ keyFile: import.meta.env.GOOGLE_APPLICATION_CREDENTIALS, - scopes: ['https://www.googleapis.com/auth/analytics.readonly'], + scopes: ["https://www.googleapis.com/auth/analytics.readonly"], }); - const analytics = google.analyticsdata({ version: 'v1beta', auth }); + const analytics = google.analyticsdata({ version: "v1beta", auth }); const report = await analytics.properties.runReport({ - property: `properties/${assertSecret(import.meta.env.PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID)}`, + property: `properties/${assertSecret( + import.meta.env.PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID + )}`, requestBody: { - dateRanges: [{ startDate: '7daysAgo', endDate: 'today' }], - dimensions: [{ name: 'pagePath' }], - metrics: [{ name: 'screenPageViews' }], - orderBys: [{ metric: { metricName: 'screenPageViews' }, desc: true }], + dateRanges: [{ startDate: "7daysAgo", endDate: "today" }], + dimensions: [{ name: "pagePath" }], + metrics: [{ name: "screenPageViews" }], + orderBys: [{ metric: { metricName: "screenPageViews" }, desc: true }], dimensionFilter: { filter: { - fieldName: 'pagePath', + fieldName: "pagePath", stringFilter: { - matchType: 'BEGINS_WITH', - value: '/tanks/', + matchType: "BEGINS_WITH", + value: "/tanks/", }, }, }, @@ -34,7 +37,7 @@ export async function popularTanks() { }); if (!report.data.rows) { - throw new Error('No rows in report'); + throw new Error("No rows in report"); } const tanks = report.data.rows @@ -42,21 +45,21 @@ export async function popularTanks() { (row) => row.dimensionValues && row.dimensionValues[0].value && - row.dimensionValues[0].value !== '/tanks/' && + row.dimensionValues[0].value !== "/tanks/" && row.metricValues && - row.metricValues[0].value, + row.metricValues[0].value ) .map((row) => ({ id: Number( row.dimensionValues![0].value!.match( - /\/tools\/tankopedia\/(\d+)\/?/, - )?.[1], + /\/tools\/tankopedia\/(\d+)\/?/ + )?.[1] ), views: Number(row.metricValues![0].value!), })) .filter((row) => { if (!(row.id in tankDefinitions.tanks)) { - console.warn('Skipping unknown popular tank', row.id); + console.warn("Skipping unknown popular tank", row.id); return false; } @@ -66,10 +69,9 @@ export async function popularTanks() { .map(({ id, views }) => ({ id, views })); const popularTanks: PopularTanks = { tanks }; - await commitAssets('popular tanks', [ - { - path: 'definitions/popular-tanks.pb', - content: PopularTanks.encode(popularTanks).finish(), - }, - ]); + await uploader.add({ + path: "definitions/popular-tanks.pb", + content: PopularTanks.encode(popularTanks).finish(), + }); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/shellIcons.ts b/packages/scripts/src/buildAssets/shellIcons.ts index 0e8d1f81b..c7557eca5 100644 --- a/packages/scripts/src/buildAssets/shellIcons.ts +++ b/packages/scripts/src/buildAssets/shellIcons.ts @@ -1,48 +1,47 @@ -import { readdir } from 'fs/promises'; -import sharp from 'sharp'; -import { readDVPLFile } from '../../src/core/blitz/readDVPLFile'; -import { readStringDVPL } from '../../src/core/blitz/readStringDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; +import { readdir } from "fs/promises"; +import sharp from "sharp"; +import { readDVPLFile } from "../../src/core/blitz/readDVPLFile"; +import { readStringDVPL } from "../../src/core/blitz/readStringDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; export async function shellIcons() { - console.log('Building shell icons...'); + console.log("Building shell icons..."); + using uploader = new AssetUploader("shell icons"); const image = sharp( await readDVPLFile( - `${DATA}/Gfx/Shared/tank-supply/ammunition/big/texture0.packed.webp`, - ), + `${DATA}/Gfx/Shared/tank-supply/ammunition/big/texture0.packed.webp` + ) ); + const files = await readdir( + `${DATA}/Gfx/Shared/tank-supply/ammunition/big` + ).then((files) => files.filter((file) => file.endsWith("_l.txt"))); - const changes = await Promise.all( - (await readdir(`${DATA}/Gfx/Shared/tank-supply/ammunition/big`)) - .filter((file) => file.endsWith('_l.txt')) - .map(async (file) => { - const name = file.match(/(.+)_l\.txt/)![1]; - const sizes = ( - await readStringDVPL( - `${DATA}/Gfx/Shared/tank-supply/ammunition/big/${file}`, - ) - ) - .split('\n')[4] - .split(' ') - .map(Number); + for (const file of files) { + const name = file.match(/(.+)_l\.txt/)![1]; + const sizes = ( + await readStringDVPL( + `${DATA}/Gfx/Shared/tank-supply/ammunition/big/${file}` + ) + ) + .split("\n")[4] + .split(" ") + .map(Number); - return { - content: await image - .clone() - .extract({ - left: sizes[0], - top: sizes[1], - width: sizes[2], - height: sizes[3], - }) - .toBuffer(), - path: `icons/shells/${name}.webp`, - } satisfies FileChange; - }), - ); + await uploader.add({ + content: await image + .clone() + .extract({ + left: sizes[0], + top: sizes[1], + width: sizes[2], + height: sizes[3], + }) + .toBuffer(), + path: `icons/shells/${name}.webp`, + }); + } - await commitAssets('shell icons', changes); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/skillIcons.ts b/packages/scripts/src/buildAssets/skillIcons.ts index 18aeade50..2088b31a8 100644 --- a/packages/scripts/src/buildAssets/skillIcons.ts +++ b/packages/scripts/src/buildAssets/skillIcons.ts @@ -2,8 +2,7 @@ import { TankClass } from "@blitzkit/core"; import sharp from "sharp"; import { readDVPLFile } from "../core/blitz/readDVPLFile"; import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; -import { commitAssets } from "../core/github/commitAssets"; -import type { FileChange } from "../core/github/commitMultipleFiles"; +import { AssetUploader } from "../core/github/assetUploader"; import { DATA } from "./constants"; interface SkillIcon { @@ -31,20 +30,18 @@ export async function skillIcons() { const avatar = await readXMLDVPL<{ root: Avatar }>( `${DATA}/XML/item_defs/tankmen/avatar.xml` ); - const changes = await Promise.all( - Object.values(avatar.root.skills).map(async (skill) => { - const icon = Array.isArray(skill.icon) ? skill.icon[0] : skill.icon; - const name = icon.name.split("/").at(-1)!.replace(/_\d$/, ""); - const path = `${DATA}${icon.name.replace("~res:", "")}.packed.webp`; - const image = sharp(await readDVPLFile(path)).trim(); - const content = await image.toBuffer(); + using uploader = new AssetUploader("skill icons"); - return { - content, - path: `icons/skills/${name}.webp`, - } satisfies FileChange; - }) - ); + for (const key in avatar.root.skills) { + const skill = avatar.root.skills[key]; + const icon = Array.isArray(skill.icon) ? skill.icon[0] : skill.icon; + const name = icon.name.split("/").at(-1)!.replace(/_\d$/, ""); + const path = `${DATA}${icon.name.replace("~res:", "")}.packed.webp`; + const image = sharp(await readDVPLFile(path)).trim(); + const content = await image.toBuffer(); - commitAssets("skill icons", changes); + await uploader.add({ content, path: `icons/skills/${name}.webp` }); + } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/tankArmors.ts b/packages/scripts/src/buildAssets/tankArmors.ts index 6aaefffce..e69807d70 100644 --- a/packages/scripts/src/buildAssets/tankArmors.ts +++ b/packages/scripts/src/buildAssets/tankArmors.ts @@ -1,44 +1,38 @@ -import { toUniqueId } from '@blitzkit/core'; -import { NodeIO } from '@gltf-transform/core'; -import { readdir } from 'fs/promises'; -import { extractArmor } from '../core/blitz/extractArmor'; -import { readXMLDVPL } from '../core/blitz/readXMLDVPL'; -import { commitAssets } from '../core/github/commitAssets'; -import { FileChange } from '../core/github/commitMultipleFiles'; -import { DATA } from './constants'; -import { VehicleDefinitionList } from './definitions'; +import { toUniqueId } from "@blitzkit/core"; +import { NodeIO } from "@gltf-transform/core"; +import { readdir } from "fs/promises"; +import { extractArmor } from "../core/blitz/extractArmor"; +import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; +import { AssetUploader } from "../core/github/assetUploader"; +import { DATA } from "./constants"; +import { VehicleDefinitionList } from "./definitions"; export async function tankArmors() { - console.log('Building tank armors...'); + console.log("Building tank armors..."); - const changes: FileChange[] = []; + using uploader = new AssetUploader("tank armors"); const nodeIO = new NodeIO(); const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then( - (nations) => nations.filter((nation) => nation !== 'common'), + (nations) => nations.filter((nation) => nation !== "common") ); - await Promise.all( - nations.map(async (nation) => { - const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>( - `${DATA}/XML/item_defs/vehicles/${nation}/list.xml`, - ); + for (const nation of nations) { + const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>( + `${DATA}/XML/item_defs/vehicles/${nation}/list.xml` + ); - await Promise.all( - Object.entries(tanks.root).map(async ([tankKey, tank]) => { - if (tankKey.includes('tutorial_bot')) return; + for (const tankKey in tanks.root) { + const tank = tanks.root[tankKey]; - const id = toUniqueId(nation, tank.id); - const model = await extractArmor(DATA, `${nation}-${tankKey}`); - const content = await nodeIO.writeBinary(model); + if (tankKey.includes("tutorial_bot")) continue; - changes.push({ - path: `3d/tanks/armor/${id}.glb`, - content, - }); - }), - ); - }), - ); + const id = toUniqueId(nation, tank.id); + const model = await extractArmor(DATA, `${nation}-${tankKey}`); + const content = await nodeIO.writeBinary(model); - await commitAssets('tank armor', changes); + await uploader.add({ path: `3d/tanks/armor/${id}.glb`, content }); + } + } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/tankIcons.ts b/packages/scripts/src/buildAssets/tankIcons.ts index 2fc4e1e14..7c69043f4 100644 --- a/packages/scripts/src/buildAssets/tankIcons.ts +++ b/packages/scripts/src/buildAssets/tankIcons.ts @@ -5,8 +5,7 @@ import type { Vector3Tuple } from "three"; import { readDVPLFile } from "../core/blitz/readDVPLFile"; import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL"; -import { commitAssets } from "../core/github/commitAssets"; -import type { FileChange } from "../core/github/commitMultipleFiles"; +import { AssetUploader } from "../core/github/assetUploader"; import { DATA } from "./constants"; import type { VehicleDefinitionList } from "./definitions"; @@ -45,56 +44,54 @@ export interface TankParameters { export async function tankIcons() { console.log("Building tank icons..."); - const changes: FileChange[] = []; + using uploader = new AssetUploader("tank icons"); const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then( (nations) => nations.filter((nation) => nation !== "common") ); - await Promise.all( - nations.map(async (nation) => { - const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>( - `${DATA}/XML/item_defs/vehicles/${nation}/list.xml` + for (const nation of nations) { + const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>( + `${DATA}/XML/item_defs/vehicles/${nation}/list.xml` + ); + + for (const tankKey in tanks.root) { + const tank = tanks.root[tankKey]; + + if (tankKey.includes("tutorial_bot")) continue; + + const nationVehicleId = tank.id; + const id = (nationVehicleId << 8) + (NATION_IDS[nation] << 4) + 1; + + const parameters = await readYAMLDVPL( + `${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml` ); + const smallPath = `${DATA}/${parameters.resourcesPath.smallIconPath + .replace(/~res:\//, "") + .replace(/\..+/, "")}.packed.webp`; + const bigPath = `${DATA}/${parameters.resourcesPath.bigIconPath + .replace(/~res:\//, "") + .replace(/\..+/, "")}.packed.webp`; + const big = await sharp(await readDVPLFile(bigPath)) + .trim() + .toBuffer(); + const small = await sharp(await readDVPLFile(smallPath)) + .trim() + .toBuffer(); - await Promise.all( - Object.entries(tanks.root).map(async ([tankKey, tank]) => { - if (tankKey.includes("tutorial_bot")) return; + if (big) { + await uploader.add({ + content: big, + path: `icons/tanks/big/${id}.webp`, + }); + } + if (small) { + await uploader.add({ + content: small, + path: `icons/tanks/small/${id}.webp`, + }); + } + } + } - const nationVehicleId = tank.id; - const id = (nationVehicleId << 8) + (NATION_IDS[nation] << 4) + 1; - - const parameters = await readYAMLDVPL( - `${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml` - ); - const smallPath = `${DATA}/${parameters.resourcesPath.smallIconPath - .replace(/~res:\//, "") - .replace(/\..+/, "")}.packed.webp`; - const bigPath = `${DATA}/${parameters.resourcesPath.bigIconPath - .replace(/~res:\//, "") - .replace(/\..+/, "")}.packed.webp`; - const big = await sharp(await readDVPLFile(bigPath)) - .trim() - .toBuffer(); - const small = await sharp(await readDVPLFile(smallPath)) - .trim() - .toBuffer(); - - if (big) { - changes.push({ - content: big, - path: `icons/tanks/big/${id}.webp`, - }); - } - if (small) { - changes.push({ - content: small, - path: `icons/tanks/small/${id}.webp`, - }); - } - }) - ); - }) - ); - - await commitAssets("tank icons", changes); + await uploader.flush(); } diff --git a/packages/scripts/src/buildAssets/tankModels.ts b/packages/scripts/src/buildAssets/tankModels.ts index 407d4dfbf..ff206605b 100644 --- a/packages/scripts/src/buildAssets/tankModels.ts +++ b/packages/scripts/src/buildAssets/tankModels.ts @@ -5,8 +5,7 @@ import { readdir } from "fs/promises"; import { extractModel } from "../core/blitz/extractModel"; import { readXMLDVPL } from "../core/blitz/readXMLDVPL"; import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL"; -import { commitAssets } from "../core/github/commitAssets"; -import { FileChange } from "../core/github/commitMultipleFiles"; +import { AssetUploader } from "../core/github/assetUploader"; import { DATA } from "./constants"; import { VehicleDefinitionList } from "./definitions"; import { TankParameters } from "./tankIcons"; @@ -14,6 +13,7 @@ import { TankParameters } from "./tankIcons"; export async function tankModels() { console.log("Building tank models..."); + using uploader = new AssetUploader("tank models"); const nodeIO = new NodeIO().registerExtensions(ALL_EXTENSIONS); const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then( (nations) => nations.filter((nation) => nation !== "common") @@ -21,22 +21,15 @@ export async function tankModels() { for (const nationIndex in nations) { const nation = nations[nationIndex]; - const changes: FileChange[] = []; const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>( `${DATA}/XML/item_defs/vehicles/${nation}/list.xml` ); const entries = Object.entries(tanks.root); - console.log(`Building models for ${nation}`); - - let index = 0; for (const [tankKey, tank] of entries) { if (tankKey.includes("tutorial_bot")) continue; const id = toUniqueId(nation, tank.id); - - console.log(` ${++index} / ${entries.length} : ${tankKey}`); - const parameters = await readYAMLDVPL( `${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml` ); @@ -46,9 +39,9 @@ export async function tankModels() { ); const content = Buffer.from(await nodeIO.writeBinary(model)); - changes.push({ path: `3d/tanks/models/${id}.glb`, content }); + await uploader.add({ path: `3d/tanks/models/${id}.glb`, content }); } - - await commitAssets(`tank models ${nation}`, changes); } + + await uploader.flush(); } diff --git a/packages/scripts/src/buildRatingLeaderboard.ts b/packages/scripts/src/buildRatingLeaderboard.ts index bfa23427c..4cd4081e1 100644 --- a/packages/scripts/src/buildRatingLeaderboard.ts +++ b/packages/scripts/src/buildRatingLeaderboard.ts @@ -13,8 +13,7 @@ import { } from "@blitzkit/core"; import { chunk } from "lodash-es"; import { argv } from "process"; -import { commitAssets } from "./core/github/commitAssets"; -import { FileChange } from "./core/github/commitMultipleFiles"; +import { AssetUploader } from "./core/github/assetUploader"; const region = argv .find((arg) => arg.startsWith("--region=")) @@ -156,16 +155,15 @@ const info = await patientFetchJSON( `https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/` ); const normalizedServer = regionSubdomain === "na" ? "com" : regionSubdomain; - const content = RatingLeaderboard.encode({ version: { $case: "v2", value: { entries } }, }).finish(); -const changes: FileChange[] = [ - { - path: `regions/${normalizedServer}/rating/${info.current_season}.pb`, - content, - }, -]; +using uploader = new AssetUploader(`rating leaderboard ${regionSubdomain}`); -commitAssets(`rating leaderboard ${regionSubdomain}`, changes); +await uploader.add({ + path: `regions/${normalizedServer}/rating/${info.current_season}.pb`, + content, +}); + +await uploader.flush(); diff --git a/packages/scripts/src/core/blitz/extractModel/index.ts b/packages/scripts/src/core/blitz/extractModel/index.ts index 2ee07ce58..7f888def8 100644 --- a/packages/scripts/src/core/blitz/extractModel/index.ts +++ b/packages/scripts/src/core/blitz/extractModel/index.ts @@ -67,7 +67,9 @@ export async function extractModel(data: string, path: string) { .setMimeType("image/webp") .setImage( await readBaseColor( - `${data}/3d/${dirname(path)}/${textures.baseColorMap ?? textures.albedo}`, + `${data}/3d/${dirname(path)}/${ + textures.baseColorMap ?? textures.albedo + }`, textures.miscMap ? `${data}/3d/${dirname(path)}/${textures.miscMap}` : undefined diff --git a/packages/scripts/src/core/github/assetUploader.ts b/packages/scripts/src/core/github/assetUploader.ts index 0b9fd42fe..0a064e25e 100644 --- a/packages/scripts/src/core/github/assetUploader.ts +++ b/packages/scripts/src/core/github/assetUploader.ts @@ -99,7 +99,7 @@ export class AssetUploader { } if (diff.status === DiffStatus.Unchanged) { - console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`); + console.log(`🔵 ${change.path}`); } else { console.log( `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${ diff --git a/packages/scripts/src/core/github/commitAssets.ts b/packages/scripts/src/core/github/commitAssets.ts deleted file mode 100644 index d1eab912c..000000000 --- a/packages/scripts/src/core/github/commitAssets.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { assertSecret } from "@blitzkit/core"; -import { chunk } from "lodash-es"; -import { commitMultipleFiles, FileChange } from "./commitMultipleFiles"; - -export async function commitAssets(message: string, changes: FileChange[]) { - console.log(`Committing ${message}...`); - - if (changes.length === 0) return; - - const chunks = chunk(changes, 64); - let chunkIndex = 0; - - for (const chunk of chunks) { - console.log(`Committing chunk ${chunkIndex + 1}/${chunks.length}`); - - await commitMultipleFiles( - assertSecret(import.meta.env.PUBLIC_ASSET_REPO), - assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH), - `${message} - ${new Date().toDateString()} (${chunkIndex + 1}/${ - chunks.length - })`, - chunk - ); - - chunkIndex++; - } -} diff --git a/packages/scripts/src/core/github/commitMultipleFiles.ts b/packages/scripts/src/core/github/commitMultipleFiles.ts deleted file mode 100644 index bf6f510a3..000000000 --- a/packages/scripts/src/core/github/commitMultipleFiles.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { asset } from "@blitzkit/core"; -import ProgressBar from "progress"; -import { compareUint8Arrays } from "./compareUint8Arrays"; -import { type GithubChangeBlob, createBlob } from "./createBlob"; -import { octokit } from "./octokit"; - -export interface FileChange { - path: string; - content: Uint8Array; -} - -enum DiffStatus { - New, - Changed, - Unchanged, -} - -const heuristicFormats = ["glb", "webp", "jpg", "png"]; - -export async function commitMultipleFiles( - repoRaw: string, - branch: string, - message: string, - changesRaw: FileChange[] -) { - const changes: FileChange[] = []; - - await Promise.all( - changesRaw.map(async (change) => { - const response = await fetch(asset(change.path), { - headers: { "Accept-Encoding": "identity" }, - }); - let diff: { change: number; status: DiffStatus }; - - if (response.status === 404) { - diff = { change: change.content.length, status: DiffStatus.New }; - } else if ( - response.headers.has("Content-Length") && - Number(response.headers.get("Content-Length")) !== change.content.length - ) { - diff = { - change: - change.content.length - - Number(response.headers.get("Content-Length")), - status: DiffStatus.Changed, - }; - } else if (response.status === 200) { - if ( - heuristicFormats.some((format) => change.path.endsWith(`.${format}`)) - ) { - /** - * Heuristic: if it's a large blob like a .glb, a diff = 0 might end - * up being a false positive with isDifferent = true. So, it's okay - * to assume it's unchanged. - */ - diff = { change: 0, status: DiffStatus.Unchanged }; - } else { - const buffer = new Uint8Array(await response.arrayBuffer()); - - if (compareUint8Arrays(buffer, change.content)) { - diff = { change: 0, status: DiffStatus.Unchanged }; - } else { - diff = { - change: change.content.length - buffer.length, - status: DiffStatus.Changed, - }; - } - } - } else { - throw new Error( - `Unexpected status code ${response.status} for ${change.path}` - ); - } - - if (diff.status === DiffStatus.Unchanged) { - // just don't bother - // console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`); - } else { - console.log( - `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${ - diff.change > 0 ? "+" : "" - }${diff.change.toLocaleString()}B) ${change.path}` - ); - changes.push(change); - } - }) - ); - - if (changes.length === 0) return; - - const [owner, repo] = repoRaw.split("/"); - const latestCommitSha = ( - await octokit.git.getRef({ - owner, - repo, - ref: `heads/${branch}`, - }) - ).data.object.sha; - const treeSha = ( - await octokit.git.getCommit({ - owner, - repo, - commit_sha: latestCommitSha, - }) - ).data.tree.sha; - const blobs: GithubChangeBlob[] = []; - const bar = new ProgressBar("Blobs :bar", changes.length); - - for (const change of changes) { - const createdBlob = await createBlob(owner, repo, change); - blobs.push(createdBlob!); - bar.tick(); - } - - const { data: treeData } = await octokit.git.createTree({ - owner, - repo, - base_tree: treeSha, - tree: blobs, - }); - const { data: newCommitData } = await octokit.git.createCommit({ - owner, - repo, - message, - tree: treeData.sha, - parents: [latestCommitSha], - }); - await octokit.git.updateRef({ - owner, - repo, - ref: `heads/${branch}`, - sha: newCommitData.sha, - force: true, - }); -} diff --git a/packages/scripts/src/core/github/createBlob.ts b/packages/scripts/src/core/github/createBlob.ts index 58bdf8d52..cd27ced51 100644 --- a/packages/scripts/src/core/github/createBlob.ts +++ b/packages/scripts/src/core/github/createBlob.ts @@ -1,5 +1,5 @@ import { throttle } from "lodash-es"; -import type { FileChange } from "./commitMultipleFiles"; +import { FileChange } from "./assetUploader"; import { octokit } from "./octokit"; export interface GithubChangeBlob { diff --git a/packages/scripts/src/discoverIds.ts b/packages/scripts/src/discoverIds.ts index 70a5f8c07..27d5ac36c 100644 --- a/packages/scripts/src/discoverIds.ts +++ b/packages/scripts/src/discoverIds.ts @@ -9,11 +9,10 @@ import { getAccountInfo, idToRegion, retryAbleBlitzFetchEvent, -} from '@blitzkit/core'; -import { chunk, times, uniq } from 'lodash-es'; -import { compress, decompress } from 'lz4js'; -import { commitAssets } from './core/github/commitAssets'; -import { FileChange } from './core/github/commitMultipleFiles'; +} from "@blitzkit/core"; +import { chunk, times, uniq } from "lodash-es"; +import { compress, decompress } from "lz4js"; +import { AssetUploader } from "./core/github/assetUploader"; const CHUNK_SIZE = 2 ** 21; const RUN_TIME = 1000 * 60 * 60 * 5.5; @@ -24,12 +23,12 @@ const TERMINATION_THRESHOLD = 1000; const startTime = Date.now(); const indexableRegions = [...REGIONS]; let ids: number[] = []; -const preDiscoveredManifest = (await fetch(asset('ids/manifest.json')).then( - (response) => response.json(), +const preDiscoveredManifest = (await fetch(asset("ids/manifest.json")).then( + (response) => response.json() )) as DiscoveredIdsDefinitions; console.log( - `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`, + `Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...` ); let chunkIndex = 0; @@ -39,23 +38,23 @@ while (chunkIndex < preDiscoveredManifest.chunks) { const buffer = await response.arrayBuffer(); const decompressed = decompress(new Uint8Array(buffer)).buffer; return new DidsReadStream(decompressed as ArrayBuffer).dids(); - }, + } ); // no spread syntax: https://github.com/oven-sh/bun/issues/11734 ids = ids.concat(preDiscovered); console.log( - `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`, + `Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})` ); chunkIndex++; } const regionalIdIndex: Record = { - asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_REGION_IDS.asia, - com: ids.findLast((id) => idToRegion(id) === 'com') ?? MIN_REGION_IDS.com, - eu: ids.findLast((id) => idToRegion(id) === 'eu') ?? MIN_REGION_IDS.eu, + asia: ids.findLast((id) => idToRegion(id) === "asia") ?? MIN_REGION_IDS.asia, + com: ids.findLast((id) => idToRegion(id) === "com") ?? MIN_REGION_IDS.com, + eu: ids.findLast((id) => idToRegion(id) === "eu") ?? MIN_REGION_IDS.eu, }; const zeroStreak: Record = { asia: 0, com: 0, eu: 0 }; let regionIndex = 0; @@ -64,7 +63,7 @@ let outOfTimeFlagged = false; async function verify(region: Region, ids: number[]) { const infos = await getAccountInfo(region, ids, undefined, { // cheeky way of requesting nothing - fields: 'account_id,-account_id', + fields: "account_id,-account_id", }); const filtered = ids.filter((_, index) => { const info = infos[index]; @@ -85,7 +84,7 @@ const interval = setInterval(async () => { const region = indexableRegions[regionIndex]; const idsToVerify = times( ACCOUNTS_PER_CALL, - (index) => regionalIdIndex[region] + index, + (index) => regionalIdIndex[region] + index ); const verified = await verify(region, idsToVerify); @@ -105,8 +104,8 @@ const interval = setInterval(async () => { indexableRegions.splice(sliceableIndex, 1); if (indexableRegions.length === 0) { - console.log('All players discovered exhaustively!'); - post(); + console.log("All players discovered exhaustively!"); + await post(); } } } @@ -117,46 +116,50 @@ const interval = setInterval(async () => { if (Date.now() - startTime > RUN_TIME && !outOfTimeFlagged) { outOfTimeFlagged = true; - console.log('Out of time'); - post(); + console.log("Out of time"); + await post(); } regionIndex = (regionIndex + 1) % indexableRegions.length; regionalIdIndex[region] += ACCOUNTS_PER_CALL; }, 1000 / MAX_REQUESTS); -function post() { +async function post() { clearInterval(interval); ids = uniq(ids).sort((a, b) => a - b); console.log(`Uploading ${ids.length} ids...`); + using uploader = new AssetUploader("discovered ids"); const idsChunked = chunk(ids, CHUNK_SIZE); - const files = idsChunked.map((chunk, index) => { + + let index = 0; + for (const chunk of idsChunked) { const didsWriteStream = new DidsWriteStream().dids(chunk); const buffer = didsWriteStream.uint8Array; const content = compress(buffer); console.log( - `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`, + `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})` ); - return { + await uploader.add({ content, path: `ids/${index}.dids.lz4`, - } satisfies FileChange; + }); + + index++; + } + + await uploader.add({ + content: new TextEncoder().encode( + JSON.stringify({ + chunks: idsChunked.length, + count: ids.length, + time: Date.now(), + } satisfies DiscoveredIdsDefinitions) + ), + path: "ids/manifest.json", }); - commitAssets('discovered ids', [ - ...files, - { - content: new TextEncoder().encode( - JSON.stringify({ - chunks: idsChunked.length, - count: ids.length, - time: Date.now(), - } satisfies DiscoveredIdsDefinitions), - ), - path: 'ids/manifest.json', - }, - ]); + await uploader.flush(); } diff --git a/packages/scripts/src/reviews.ts b/packages/scripts/src/reviews.ts index 1db57687a..5d3cf7571 100644 --- a/packages/scripts/src/reviews.ts +++ b/packages/scripts/src/reviews.ts @@ -4,22 +4,22 @@ import { Reviews, Video, youtubers, -} from '@blitzkit/core'; -import { google } from 'googleapis'; -import { cloneDeep, uniqBy } from 'lodash-es'; -import { commitAssets } from './core/github/commitAssets'; +} from "@blitzkit/core"; +import { google } from "googleapis"; +import { cloneDeep, uniqBy } from "lodash-es"; +import { AssetUploader } from "./core/github/assetUploader"; const MAX_QUERIES = 64; -console.log('Finding reviews...'); +console.log("Finding reviews..."); -const currentReviews = await fetch(asset('definitions/reviews.pb')) +const currentReviews = await fetch(asset("definitions/reviews.pb")) .then((response) => response.arrayBuffer()) .then((buffer) => Reviews.decode(new Uint8Array(buffer))); const auth = await google.auth.getClient({ - scopes: ['https://www.googleapis.com/auth/youtube.readonly'], + scopes: ["https://www.googleapis.com/auth/youtube.readonly"], }); -const youtube = google.youtube({ version: 'v3', auth }); +const youtube = google.youtube({ version: "v3", auth }); const tankDefinitions = await fetchTankDefinitions(); const tanks = Object.values(tankDefinitions.tanks); @@ -43,7 +43,7 @@ for (const tank of tanksSanitized) { try { const results = await youtube.search.list({ - part: ['snippet'], + part: ["snippet"], q: `World of Tanks Blitz ${tank.name}`, }); @@ -54,9 +54,9 @@ for (const tank of tanksSanitized) { (item) => item.snippet?.channelId && item.id?.videoId && - youtubers.some(({ id }) => id === item.snippet?.channelId), + youtubers.some(({ id }) => id === item.snippet?.channelId) ), - (item) => item.snippet!.channelId, + (item) => item.snippet!.channelId ); reviews.reviews[tank.id] = { @@ -66,7 +66,7 @@ for (const tank of tanksSanitized) { ({ id: item.id!.videoId!, author: item.snippet!.channelId!, - }) satisfies Video, + } satisfies Video) ), }; @@ -78,9 +78,11 @@ for (const tank of tanksSanitized) { break; } } -await commitAssets('reviews', [ - { - content: Reviews.encode(reviews).finish(), - path: 'definitions/reviews.pb', - }, -]); + +using uploader = new AssetUploader("reviews"); + +await uploader.add({ + content: Reviews.encode(reviews).finish(), + path: "definitions/reviews.pb", +}); +await uploader.flush();