feat: migrate to asset uploader

This commit is contained in:
TrèsAbhi committed 2025-10-23 21:22:21 -05:00
1 parent 797307fb81
commit a3bb023309
29 files changed
+764 -1003

No files matched your search

+1
View File
@@ -64,6 +64,7 @@
"deburr",
"decalmask",
"decompressor",
"dedup",
"defence",
"degressive",
"depotdownloader",
+31 -30
View File
@@ -11,10 +11,10 @@ import {
getAccountInfo,
getTankStats,
idToRegion,
} from '@blitzkit/core';
import { chunk, times } from 'lodash-es';
import { commitAssets } from './core/github/commitAssets';
import { fetchPreDiscoveredIds } from './core/github/fetchPreDiscoveredIds';
} from "@blitzkit/core";
import { chunk, times } from "lodash-es";
import { AssetUploader } from "./core/github/assetUploader";
import { fetchPreDiscoveredIds } from "./core/github/fetchPreDiscoveredIds";
interface DataPoint {
x: number;
@@ -23,7 +23,7 @@ interface DataPoint {
}
type OptionalSecondLevel<T> = {
[P in keyof T]: P extends 'samples'
[P in keyof T]: P extends "samples"
? T[P]
: {
[K in keyof T[P]]?: T[P][K];
@@ -46,16 +46,16 @@ const startTime = Date.now();
const preDiscoveredIds = await fetchPreDiscoveredIds();
const playerIds: Record<Region, number[][]> = {
asia: chunk(
preDiscoveredIds.filter((id) => idToRegion(id) === 'asia'),
PLAYER_IDS_PER_CALL,
preDiscoveredIds.filter((id) => idToRegion(id) === "asia"),
PLAYER_IDS_PER_CALL
),
com: chunk(
preDiscoveredIds.filter((id) => idToRegion(id) === 'com'),
PLAYER_IDS_PER_CALL,
preDiscoveredIds.filter((id) => idToRegion(id) === "com"),
PLAYER_IDS_PER_CALL
),
eu: chunk(
preDiscoveredIds.filter((id) => idToRegion(id) === 'eu'),
PLAYER_IDS_PER_CALL,
preDiscoveredIds.filter((id) => idToRegion(id) === "eu"),
PLAYER_IDS_PER_CALL
),
};
let regionIndex = 0;
@@ -71,10 +71,10 @@ times(THREADS, async () => {
const regionIds = playerIds[region];
const [ids] = regionIds.splice(
Math.floor(Math.random() * regionIds.length),
1,
1
);
const accountInfo = await getAccountInfo(region, ids, undefined, {
fields: 'last_battle_time,statistics.all.battles',
fields: "last_battle_time,statistics.all.battles",
});
const filteredIds = ids.filter((_, index) => {
@@ -103,9 +103,9 @@ times(THREADS, async () => {
const players = await Promise.all(
filteredIds.map((id) =>
getTankStats(region, id, {
fields: 'last_battle_time,battle_life_time,all,tank_id',
}),
),
fields: "last_battle_time,battle_life_time,all,tank_id",
})
)
);
players.forEach((tanks) => {
@@ -114,7 +114,7 @@ times(THREADS, async () => {
if (!tankIds.includes(tank.tank_id)) {
console.log(
`Found new tank: ${tank.tank_id} (${tankIds.length + 1})`,
`Found new tank: ${tank.tank_id} (${tankIds.length + 1})`
);
tankIds.push(tank.tank_id);
tanksSorted[tank.tank_id] = [];
@@ -140,9 +140,10 @@ async function postWork() {
console.log(
`Generating statistics based on ${samples.d_120.toLocaleString()} players (${samples.total.toLocaleString()} checked in total) and ${
tankIds.length
} tanks...`,
} tanks...`
);
using uploader = new AssetUploader("averages");
const averages: Record<number, AverageDefinitionsEntry> = {};
tankIds.forEach((id) => {
@@ -179,7 +180,7 @@ async function postWork() {
const data: DataPoint[] = dataWY.map(({ w, y }, index) => ({
w,
x:
(key === 'battle_life_time'
(key === "battle_life_time"
? tanks[index].battle_life_time
: tanks[index].all[key]) / w,
y,
@@ -190,7 +191,7 @@ async function postWork() {
function sum(slicer: (data: DataPoint) => number) {
return data.reduce(
(accumulator, data) => accumulator + slicer(data),
0,
0
);
}
@@ -231,14 +232,14 @@ async function postWork() {
latest,
};
commitAssets('averages', [
{
path: `averages/${latest}.pb`,
content: AverageDefinitions.encode(averageDefinitions).finish(),
},
{
path: 'averages/manifest.json',
content: new TextEncoder().encode(JSON.stringify(manifest)),
},
]);
await uploader.add({
path: `averages/${latest}.pb`,
content: AverageDefinitions.encode(averageDefinitions).finish(),
});
await uploader.add({
path: "averages/manifest.json",
content: new TextEncoder().encode(JSON.stringify(manifest)),
});
await uploader.flush();
}
@@ -1,49 +0,0 @@
import { BLITZKIT_TANK_ICON_SIZE, fetchTankDefinitions } from "@blitzkit/core";
import { launch } from "puppeteer";
import sharp from "sharp";
import { commitAssets } from "../core/github/commitAssets";
import { type FileChange } from "../core/github/commitMultipleFiles";
export async function blitzkitTankIcons() {
console.log("Building blitzkit tank icons...");
const browser = await launch();
const page = await browser.newPage();
const tanks = Object.values((await fetchTankDefinitions()).tanks);
const files: FileChange[] = [];
let index = 0;
for (const { id, name } of tanks) {
await page.setViewport({
width: BLITZKIT_TANK_ICON_SIZE.width - 1,
height: BLITZKIT_TANK_ICON_SIZE.height,
});
try {
await page.goto(`http://localhost:4321/api/tankopedia/tank-icon/${id}/`, {
waitUntil: "networkidle0",
});
} catch (error) {
console.error(`${id} ${name} time out`, error);
continue;
}
const screenshot = await page.screenshot({
type: "webp",
omitBackground: true,
});
const content = await sharp(screenshot).trim({ threshold: 100 }).toBuffer();
index++;
console.log(
`Rendered ${id} (${name}) ${((index / tanks.length) * 100).toFixed(2)}%`
);
files.push({ path: `icons/tanks/big/${id}.webp`, content });
}
await page.close();
await browser.close();
await commitAssets("tank icons", files);
}
@@ -1,48 +1,45 @@
import { readdir } from 'fs/promises';
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { readdir } from "fs/promises";
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { readStringDVPL } from "../../src/core/blitz/readStringDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
export async function boosterIcons() {
console.log("Building booster icons...");
using uploader = new AssetUploader("booster icons");
const boosterFiles = (await readdir(`${DATA}/Gfx/Shared/boosters`)).filter(
(file) => !file.includes('@2x.txt') && !file.startsWith('texture0'),
(file) => !file.includes("@2x.txt") && !file.startsWith("texture0")
);
const image = sharp(
await readDVPLFile(`${DATA}/Gfx/Shared/boosters/texture0.packed.webp`),
);
const changes: FileChange[] = [];
await Promise.all(
boosterFiles.map(async (file) => {
const name = file.match(/booster_(.+).txt/)?.[1];
if (name === undefined) return;
const sizes = (
await readStringDVPL(`${DATA}/Gfx/Shared/boosters/${file}`)
)
.split('\n')[4]
.split(' ')
.map(Number);
const content = await image
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
changes.push({
path: `icons/boosters/${name}.webp`,
content,
});
}),
await readDVPLFile(`${DATA}/Gfx/Shared/boosters/texture0.packed.webp`)
);
await commitAssets('booster icons', changes);
for (const file of boosterFiles) {
const name = file.match(/booster_(.+).txt/)?.[1];
if (name === undefined) continue;
const sizes = (await readStringDVPL(`${DATA}/Gfx/Shared/boosters/${file}`))
.split("\n")[4]
.split(" ")
.map(Number);
const content = await image
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
await uploader.add({
path: `icons/boosters/${name}.webp`,
content,
});
}
await uploader.flush();
}
@@ -1,19 +1,15 @@
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
export async function camouflageIcons() {
console.log('Building camouflage icons...');
const content = await readDVPLFile(
`${DATA}/Gfx/UI/Hangar/IconCamouflage.packed.webp`,
);
const changes: FileChange[] = [
{
content,
path: 'icons/camo.webp',
},
];
console.log("Building camouflage icons...");
await commitAssets('camo icons', changes);
using uploader = new AssetUploader("camo icons");
const content = await readDVPLFile(
`${DATA}/Gfx/UI/Hangar/IconCamouflage.packed.webp`
);
await uploader.add({ content, path: "icons/camo.webp" });
await uploader.flush();
}
@@ -1,178 +1,181 @@
import { existsSync } from 'fs';
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { ConsumablesCommon, ProvisionsCommon } from './definitions';
import { existsSync } from "fs";
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { readStringDVPL } from "../../src/core/blitz/readStringDVPL";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import { ConsumablesCommon, ProvisionsCommon } from "./definitions";
interface Mappings {
Header: { version: number };
StyleSheets: {
selectors: string[];
properties: { 'bg-sprite': string };
properties: { "bg-sprite": string };
}[];
}
const listItemsPattern = /<items path="(.+)\.xml"\/>/g;
export async function consumableProvisionIcons() {
console.log('Building consumable and provision icons...');
const changes: FileChange[] = [];
console.log("Building consumable and provision icons...");
using uploader = new AssetUploader("consumable and provision icons");
const styles = [
'UI/Styles/Lobby/Inventory/Event/InventoryBigStyles.yaml',
'UI/Styles/Lobby/Inventory/Event/InventoryNormalStyles.yaml',
'UI/Styles/Lobby/Inventory/InventoryNormalStyles.yaml',
'UI/Screens3/Lobby/Inventory/Equipment/EquipmentItemImage.style.yaml',
'UI/Screens/Battle/Styles/BattleEquipmentStyles.yaml',
"UI/Styles/Lobby/Inventory/Event/InventoryBigStyles.yaml",
"UI/Styles/Lobby/Inventory/Event/InventoryNormalStyles.yaml",
"UI/Styles/Lobby/Inventory/InventoryNormalStyles.yaml",
"UI/Screens3/Lobby/Inventory/Equipment/EquipmentItemImage.style.yaml",
"UI/Screens/Battle/Styles/BattleEquipmentStyles.yaml",
];
const styleSheets = await Promise.all(
styles.map(async (path) =>
Object.values(
(await readYAMLDVPL<Mappings>(`${DATA}/${path}`)).StyleSheets,
),
),
(
await readYAMLDVPL<Mappings>(`${DATA}/${path}`)
).StyleSheets
)
)
).then((array) => array.flat());
for (const match of (
await readStringDVPL(
`${DATA}/XML/item_defs/vehicles/common/consumables/list.xml`,
`${DATA}/XML/item_defs/vehicles/common/consumables/list.xml`
)
).matchAll(listItemsPattern)) {
const consumablesCommon = await readXMLDVPL<{
root: ConsumablesCommon;
}>(`${DATA}/XML/item_defs/vehicles/common/consumables/${match[1]}.xml`);
await Promise.all(
Object.values(consumablesCommon.root).map(async (consumable) => {
const styleSheet = styleSheets.find((styleSheet) =>
styleSheet.selectors.some((selector) =>
selector.includes(`${consumable.icon} `),
),
for (const consumable of Object.values(consumablesCommon.root)) {
const styleSheet = styleSheets.find((styleSheet) =>
styleSheet.selectors.some((selector) =>
selector.includes(`${consumable.icon} `)
)
);
if (!styleSheet) {
console.warn(
`No style sheet found for consumable ${consumable.icon}; skipping...`
);
continue;
}
if (!styleSheet) {
console.warn(
`No style sheet found for consumable ${consumable.icon}; skipping...`,
);
return;
}
const configPath = styleSheet.properties["bg-sprite"]
.replace("~res:/", "")
.replace(".psd", "")
.replace(".txt", "");
const configPath = styleSheet.properties['bg-sprite']
.replace('~res:/', '')
.replace('.psd', '')
.replace('.txt', '');
if (existsSync(`${DATA}/${configPath}.packed.webp`)) {
const image = sharp(
await readDVPLFile(`${DATA}/${configPath}.packed.webp`)
);
const content = await image.trim({ threshold: 100 }).toBuffer();
if (existsSync(`${DATA}/${configPath}.packed.webp`)) {
const image = sharp(
await readDVPLFile(`${DATA}/${configPath}.packed.webp`),
);
const content = await image.trim({ threshold: 100 }).toBuffer();
await uploader.add({
path: `icons/consumables/${consumable.id}.webp`,
content,
});
} else {
const consumablesTexture = sharp(
await readDVPLFile(
`${`${DATA}/${configPath}`
.split("/")
.slice(0, -1)
.join("/")}/texture0.packed.webp`
)
);
const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`))
.split("\n")[4]
.split(" ")
.map(Number);
const content = await consumablesTexture
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
changes.push({
path: `icons/consumables/${consumable.id}.webp`,
content,
});
} else {
const consumablesTexture = sharp(
await readDVPLFile(
`${`${DATA}/${configPath}`.split('/').slice(0, -1).join('/')}/texture0.packed.webp`,
),
);
const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`))
.split('\n')[4]
.split(' ')
.map(Number);
const content = await consumablesTexture
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
changes.push({
path: `icons/consumables/${consumable.id}.webp`,
content,
});
}
}),
);
await uploader.add({
path: `icons/consumables/${consumable.id}.webp`,
content,
});
}
}
}
for (const match of (
await readStringDVPL(
`${DATA}/XML/item_defs/vehicles/common/provisions/list.xml`,
`${DATA}/XML/item_defs/vehicles/common/provisions/list.xml`
)
).matchAll(listItemsPattern)) {
const provisionsCommon = await readXMLDVPL<{
root: ProvisionsCommon;
}>(`${DATA}/XML/item_defs/vehicles/common/provisions/${match[1]}.xml`);
await Promise.all(
Object.values(provisionsCommon.root).map(async (provision) => {
const styleSheet = styleSheets.find((styleSheet) =>
styleSheet.selectors.some((selector) =>
selector.includes(`${provision.icon} `),
),
for (const provision of Object.values(provisionsCommon.root)) {
const styleSheet = styleSheets.find((styleSheet) =>
styleSheet.selectors.some((selector) =>
selector.includes(`${provision.icon} `)
)
);
if (!styleSheet) {
console.warn(
`No style sheet found for provision ${provision.icon}; skipping...`
);
continue;
}
if (!styleSheet) {
console.warn(
`No style sheet found for provision ${provision.icon}; skipping...`,
);
return;
}
const configPath = styleSheet.properties["bg-sprite"]
.replace("~res:/", "")
.replace(".psd", "")
.replace(".txt", "");
const configPath = styleSheet.properties['bg-sprite']
.replace('~res:/', '')
.replace('.psd', '')
.replace('.txt', '');
if (existsSync(`${DATA}/${configPath}.packed.webp`)) {
const image = sharp(
await readDVPLFile(`${DATA}/${configPath}.packed.webp`)
);
const content = await image.trim({ threshold: 100 }).toBuffer();
if (existsSync(`${DATA}/${configPath}.packed.webp`)) {
const image = sharp(
await readDVPLFile(`${DATA}/${configPath}.packed.webp`),
);
const content = await image.trim({ threshold: 100 }).toBuffer();
await uploader.add({
path: `icons/provisions/${provision.id}.webp`,
content,
});
} else {
const provisionsTexture = sharp(
await readDVPLFile(
`${`${DATA}/${configPath}`
.split("/")
.slice(0, -1)
.join("/")}/texture0.packed.webp`
)
);
const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`))
.split("\n")[4]
.split(" ")
.map(Number);
const content = await provisionsTexture
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
changes.push({
path: `icons/provisions/${provision.id}.webp`,
content,
});
} else {
const provisionsTexture = sharp(
await readDVPLFile(
`${`${DATA}/${configPath}`.split('/').slice(0, -1).join('/')}/texture0.packed.webp`,
),
);
const sizes = (await readStringDVPL(`${DATA}/${configPath}.txt`))
.split('\n')[4]
.split(' ')
.map(Number);
const content = await provisionsTexture
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
changes.push({
path: `icons/provisions/${provision.id}.webp`,
content,
});
}
}),
);
await uploader.add({
path: `icons/provisions/${provision.id}.webp`,
content,
});
}
}
}
await commitAssets('consumable and provision icons', changes);
await uploader.flush();
}
+49 -52
View File
@@ -1,66 +1,63 @@
import { assertSecret, BlitzGlossary } from '@blitzkit/core';
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { assertSecret, BlitzGlossary } from "@blitzkit/core";
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
const ICONS = [
['currency_silver_m.packed.webp', 'silver'],
['currency_premium_xl.packed.webp', 'premium'],
['currency_gold_m.packed.webp', 'gold'],
['currency_free-xp_xl.packed.webp', 'free-xp'],
['currency_elite-xp_xl.packed.webp', 'elite-xp'],
['currency_crew-xp_xl.packed.webp', 'crew-xp'],
['currency_battle-xp_xl.packed.webp', 'xp'],
["currency_silver_m.packed.webp", "silver"],
["currency_premium_xl.packed.webp", "premium"],
["currency_gold_m.packed.webp", "gold"],
["currency_free-xp_xl.packed.webp", "free-xp"],
["currency_elite-xp_xl.packed.webp", "elite-xp"],
["currency_crew-xp_xl.packed.webp", "crew-xp"],
["currency_battle-xp_xl.packed.webp", "xp"],
];
export async function currencies() {
console.log('Building currency icons...');
console.log("Building currency icons...");
const changes: FileChange[] = [];
using uploader = new AssetUploader("currency icons");
await Promise.all(
ICONS.map(async ([file, name]) => {
const content = await sharp(
await readDVPLFile(`${DATA}/Gfx/Lobby/currency/${file}`),
)
.trim()
.toBuffer();
for (const [file, name] of ICONS) {
const content = await sharp(
await readDVPLFile(`${DATA}/Gfx/Lobby/currency/${file}`)
)
.trim()
.toBuffer();
changes.push({
content,
path: `icons/currencies/${name}.webp`,
});
}),
);
await uploader.add({
content,
path: `icons/currencies/${name}.webp`,
});
}
const glossary = await fetch(
assertSecret(import.meta.env.WOTB_GLOSSARY),
).then((response) => response.json() as Promise<BlitzGlossary>);
const glossary = await fetch(assertSecret(import.meta.env.WOTB_GLOSSARY))
.then((response) => response.json() as Promise<BlitzGlossary>)
.then((glossary) =>
Object.values(glossary)
.entries()
.filter(([key]) => /^prx_season_\d+$/.test(`${key}`))
);
await Promise.all(
Object.entries(glossary)
.filter(([key]) => /^prx_season_\d+$/.test(key))
.map(async ([key, item]) => {
if (item.image_url === null) throw new Error(`No image_url for ${key}`);
for (const [key, item] of glossary) {
if (item.image_url === null) throw new Error(`No image_url for ${key}`);
const imageRaw = await fetch(item.image_url).then((response) =>
response.arrayBuffer(),
);
const content = await sharp(imageRaw)
.trim({
threshold: 100,
background: { r: 0, g: 0, b: 0, alpha: 0 },
})
.toBuffer();
const imageRaw = await fetch(item.image_url).then((response) =>
response.arrayBuffer()
);
const content = await sharp(imageRaw)
.trim({
threshold: 100,
background: { r: 0, g: 0, b: 0, alpha: 0 },
})
.toBuffer();
changes.push({
content,
path: `icons/currencies/${key}.webp`,
});
}),
);
await uploader.add({
content,
path: `icons/currencies/${key}.webp`,
});
}
await commitAssets('currency icons', changes);
await uploader.flush();
}
+66 -63
View File
@@ -40,7 +40,7 @@ import { parse as parseYaml } from "yaml";
import { readStringDVPL } from "../core/blitz/readStringDVPL";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL";
import { commitAssets } from "../core/github/commitAssets";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import type { Avatar } from "./skillIcons";
import type { TankParameters } from "./tankIcons";
@@ -302,7 +302,7 @@ interface VehicleDefinitions {
type: "default";
enabled: false;
factorsModifiers: "";
},
}
];
};
};
@@ -538,6 +538,7 @@ const blitzModuleTypeToBlitzkit: Record<keyof BlitzModuleType, ModuleType> = {
export async function definitions() {
console.log("Building definitions...");
using uploader = new AssetUploader("definitions");
const gameDefinitions: GameDefinitions = {
version: (await readStringDVPL(`${DATA}/version.txt`)).split(" ")[0],
nations: (
@@ -766,7 +767,9 @@ export async function definitions() {
}
console.warn(
`Multiple tanks share slug ${slug}: ${requesters.map(({ key }) => key).join(", ")}`
`Multiple tanks share slug ${slug}: ${requesters
.map(({ key }) => key)
.join(", ")}`
);
if (requesters.length !== 2) {
@@ -1018,8 +1021,8 @@ export async function definitions() {
type: tankTags.includes("collectible")
? TankType.COLLECTOR
: (typeof tank.price === "number" ? false : "gold" in tank.price)
? TankType.PREMIUM
: TankType.RESEARCHABLE,
? TankType.PREMIUM
: TankType.RESEARCHABLE,
tier: tank.level,
class: blitzTankClassToBlitzkit[tankTags[0] as BlitzTankClass],
testing: tankTags.includes("testTank"),
@@ -1337,24 +1340,24 @@ export async function definitions() {
},
}
: gunType === "autoReloader"
? {
$case: "auto_reloader",
value: {
intra_clip: 60 / gun.clip!.rate,
shell_count: gunClipCount,
shell_reloads: gun
.pumpGunReloadTimes!.split(" ")
.map(Number),
},
}
: {
$case: "auto_loader",
value: {
intra_clip: 60 / gun.clip!.rate,
clip_reload: gun.reloadTime,
shell_count: gunClipCount,
},
? {
$case: "auto_reloader",
value: {
intra_clip: 60 / gun.clip!.rate,
shell_count: gunClipCount,
shell_reloads: gun
.pumpGunReloadTimes!.split(" ")
.map(Number),
},
}
: {
$case: "auto_loader",
value: {
intra_clip: 60 / gun.clip!.rate,
clip_reload: gun.reloadTime,
shell_count: gunClipCount,
},
},
} satisfies GunDefinition);
modelDefinitions.models[tankId].turrets[turretId].guns[gunId] = {
@@ -1365,8 +1368,8 @@ export async function definitions() {
gun.armor.gun === undefined
? 0
: typeof gun.armor.gun === "number"
? gun.armor.gun
: gun.armor.gun["#text"],
? gun.armor.gun
: gun.armor.gun["#text"],
pitch: {
min: gunPitch[0],
max: gunPitch[1],
@@ -1413,7 +1416,7 @@ export async function definitions() {
type: blitzShellKindToBlitzkit[shell.kind],
explosion_radius:
shell.kind === "HIGH_EXPLOSIVE"
? (shell.explosionRadius ?? 0)
? shell.explosionRadius ?? 0
: undefined,
icon: shell.icon,
penetration: {
@@ -1783,42 +1786,42 @@ export async function definitions() {
};
});
await commitAssets("definitions", [
{
content: GameDefinitions.encode(gameDefinitions).finish(),
path: "definitions/game.pb",
},
{
content: TankDefinitions.encode(tankDefinitions).finish(),
path: "definitions/tanks.pb",
},
{
content: ModelDefinitions.encode(modelDefinitions).finish(),
path: "definitions/models.pb",
},
{
content: EquipmentDefinitions.encode(equipmentDefinitions).finish(),
path: "definitions/equipment.pb",
},
{
content: ConsumableDefinitions.encode(consumableDefinitions).finish(),
path: "definitions/consumables.pb",
},
{
content: ProvisionDefinitions.encode(provisionDefinitions).finish(),
path: "definitions/provisions.pb",
},
{
content: SkillDefinitions.encode(skillDefinitions).finish(),
path: "definitions/skills.pb",
},
{
content: MapDefinitions.encode(mapDefinitions).finish(),
path: "definitions/maps.pb",
},
{
content: CamouflageDefinitions.encode(camouflageDefinitions).finish(),
path: "definitions/camouflages.pb",
},
]);
await uploader.add({
content: GameDefinitions.encode(gameDefinitions).finish(),
path: "definitions/game.pb",
});
await uploader.add({
content: TankDefinitions.encode(tankDefinitions).finish(),
path: "definitions/tanks.pb",
});
await uploader.add({
content: ModelDefinitions.encode(modelDefinitions).finish(),
path: "definitions/models.pb",
});
await uploader.add({
content: EquipmentDefinitions.encode(equipmentDefinitions).finish(),
path: "definitions/equipment.pb",
});
await uploader.add({
content: ConsumableDefinitions.encode(consumableDefinitions).finish(),
path: "definitions/consumables.pb",
});
await uploader.add({
content: ProvisionDefinitions.encode(provisionDefinitions).finish(),
path: "definitions/provisions.pb",
});
await uploader.add({
content: SkillDefinitions.encode(skillDefinitions).finish(),
path: "definitions/skills.pb",
});
await uploader.add({
content: MapDefinitions.encode(mapDefinitions).finish(),
path: "definitions/maps.pb",
});
await uploader.add({
content: CamouflageDefinitions.encode(camouflageDefinitions).finish(),
path: "definitions/camouflages.pb",
});
await uploader.flush();
}
@@ -1,91 +1,86 @@
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { OptionalDevices } from './definitions';
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { readStringDVPL } from "../../src/core/blitz/readStringDVPL";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import { OptionalDevices } from "./definitions";
interface Mappings {
Header: { version: number };
StyleSheets: {
selectors: string[];
properties: { 'bg-sprite': string };
properties: { "bg-sprite": string };
}[];
}
export async function equipmentIcons() {
console.log('Building equipment icons...');
console.log("Building equipment icons...");
const changes: FileChange[] = [];
using uploader = new AssetUploader("equipment icons");
const optionalDevices = await readXMLDVPL<{ root: OptionalDevices }>(
`${DATA}/XML/item_defs/vehicles/common/optional_devices.xml`,
`${DATA}/XML/item_defs/vehicles/common/optional_devices.xml`
);
const mappings = await readYAMLDVPL<Mappings>(
`${DATA}/UI/Screens3/Lobby/Inventory/OptionalDevices/OptionalDevicesItemImage.style.yaml`,
`${DATA}/UI/Screens3/Lobby/Inventory/OptionalDevices/OptionalDevicesItemImage.style.yaml`
);
const image = sharp(
await readDVPLFile(
`${DATA}/Gfx/UI/InventoryIcons/Big/OptionalDevices/texture0.packed.webp`,
),
`${DATA}/Gfx/UI/InventoryIcons/Big/OptionalDevices/texture0.packed.webp`
)
);
await Promise.all(
Object.entries(optionalDevices.root).map(
async ([optionalDeviceKey, optionalDevice]) => {
if (optionalDeviceKey === 'nextAvailableId') return;
for (const optionalDeviceKey in optionalDevices.root) {
const optionalDevice = optionalDevices.root[optionalDeviceKey];
const mapping = mappings.StyleSheets.find((mapping) =>
mapping.selectors.includes(
`.optional_device_item.${optionalDevice.icon} #Img`,
),
);
if (optionalDeviceKey === "nextAvailableId") continue;
if (!mapping) {
console.warn(
`No mapping found for ${optionalDevice.icon}; skipping...`,
);
return;
}
const mapping = mappings.StyleSheets.find((mapping) =>
mapping.selectors.includes(
`.optional_device_item.${optionalDevice.icon} #Img`
)
);
const configPathRaw = mapping.properties['bg-sprite'];
const configPath = configPathRaw.replace('~res:/', '');
if (!mapping) {
console.warn(`No mapping found for ${optionalDevice.icon}; skipping...`);
continue;
}
if (configPath.startsWith('Gfx/Lobby')) {
const configPathWebp = configPath.replace('.txt', '');
const image = sharp(
await readDVPLFile(`${DATA}/${configPathWebp}.packed.webp`),
);
const content = await image.trim().toBuffer();
const configPathRaw = mapping.properties["bg-sprite"];
const configPath = configPathRaw.replace("~res:/", "");
changes.push({
path: `icons/equipment/${optionalDevice.id}.webp`,
content,
});
} else {
const config = await readStringDVPL(`${DATA}/${configPath}`);
const sizes = config.split('\n')[4].split(' ').map(Number);
const content = await image
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
if (configPath.startsWith("Gfx/Lobby")) {
const configPathWebp = configPath.replace(".txt", "");
const image = sharp(
await readDVPLFile(`${DATA}/${configPathWebp}.packed.webp`)
);
const content = await image.trim().toBuffer();
changes.push({
path: `icons/equipment/${optionalDevice.id}.webp`,
content,
});
}
},
),
);
await uploader.add({
path: `icons/equipment/${optionalDevice.id}.webp`,
content,
});
} else {
const config = await readStringDVPL(`${DATA}/${configPath}`);
const sizes = config.split("\n")[4].split(" ").map(Number);
const content = await image
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer();
await commitAssets('equipment icons', changes);
await uploader.add({
path: `icons/equipment/${optionalDevice.id}.webp`,
content,
});
}
}
await uploader.flush();
}
+64 -67
View File
@@ -1,75 +1,72 @@
import { readdir } from 'fs/promises';
import sharp from 'sharp';
import { readDVPLFile } from '../core/blitz/readDVPLFile';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { readdir } from "fs/promises";
import sharp from "sharp";
import { readDVPLFile } from "../core/blitz/readDVPLFile";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
export async function flags() {
console.log('Building flags...');
console.log("Building flags...");
const changes = [
...(await Promise.all(
(await readdir(`${DATA}/Gfx/Lobby/flags`))
.filter(
(flag) =>
flag.startsWith('flag_profile-stat_') &&
!flag.endsWith('@2x.packed.webp'),
)
.map(async (flag) => {
const image = sharp(
await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`),
);
const content = await image.trim({ threshold: 100 }).toBuffer();
const name = flag.match(/flag_profile-stat_(.+)\.packed\.webp/)![1];
using uploader = new AssetUploader("flags");
const circleFiles = await readdir(`${DATA}/Gfx/Lobby/flags`).then((files) =>
files.filter(
(flag) =>
flag.startsWith("flag_profile-stat_") &&
!flag.endsWith("@2x.packed.webp")
)
);
const scratchedFiles = await readdir(`${DATA}/Gfx/Lobby/flags`).then(
(files) =>
files.filter(
(flag) =>
flag.startsWith("flag_tutor-tank_") &&
!flag.endsWith("@2x.packed.webp")
)
);
const fadedFiles = await readdir(`${DATA}/Gfx/Lobby/flags`).then((files) =>
files.filter(
(flag) =>
flag.startsWith("flag_filter_") && flag.endsWith("@2x.packed.webp")
)
);
return {
content,
path: `flags/circle/${name}.webp`,
} satisfies FileChange;
}),
)),
...(await Promise.all(
(await readdir(`${DATA}/Gfx/Lobby/flags`))
.filter(
(flag) =>
flag.startsWith('flag_tutor-tank_') &&
!flag.endsWith('@2x.packed.webp'),
)
.map(async (flag) => {
const content = await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`);
const name = flag.match(/flag_tutor-tank_(.+)\.packed\.webp/)![1];
for (const flag of circleFiles) {
const image = sharp(await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`));
const content = await image.trim({ threshold: 100 }).toBuffer();
const name = flag.match(/flag_profile-stat_(.+)\.packed\.webp/)![1];
return {
path: `flags/scratched/${name}.webp`,
content,
} satisfies FileChange;
}),
)),
...(await Promise.all(
(await readdir(`${DATA}/Gfx/Lobby/flags`))
.filter(
(flag) =>
flag.startsWith('flag_filter_') && flag.endsWith('@2x.packed.webp'),
)
.map(async (flag) => {
const content = await sharp(
await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`),
)
.trim({
threshold: 100,
background: { r: 0, g: 0, b: 0, alpha: 0 },
})
.toBuffer();
const name = flag.match(/flag_filter_(.+)@2x\.packed\.webp/)![1];
uploader.add({
content,
path: `flags/circle/${name}.webp`,
});
}
return {
content,
path: `flags/fade_small/${name}.webp`,
} satisfies FileChange;
}),
)),
];
for (const flag of scratchedFiles) {
const content = await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`);
const name = flag.match(/flag_tutor-tank_(.+)\.packed\.webp/)![1];
await commitAssets('flags', changes);
uploader.add({
path: `flags/scratched/${name}.webp`,
content,
});
}
for (const flag of fadedFiles) {
const content = await sharp(
await readDVPLFile(`${DATA}/Gfx/Lobby/flags/${flag}`)
)
.trim({
threshold: 100,
background: { r: 0, g: 0, b: 0, alpha: 0 },
})
.toBuffer();
const name = flag.match(/flag_filter_(.+)@2x\.packed\.webp/)![1];
uploader.add({
content,
path: `flags/fade_small/${name}.webp`,
});
}
await uploader.flush();
}
@@ -1,36 +1,35 @@
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { SquadBattleTypeStyles } from './definitions';
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import { SquadBattleTypeStyles } from "./definitions";
export async function gameModeBanners() {
console.log('Building game mode banners...');
console.log("Building game mode banners...");
const changes: FileChange[] = [];
using uploader = new AssetUploader("game mode banners");
const gameTypeSelectorStyles = await readYAMLDVPL<SquadBattleTypeStyles>(
`${DATA}/UI/Screens/Lobby/Hangar/GameTypeSelector.yaml`,
`${DATA}/UI/Screens/Lobby/Hangar/GameTypeSelector.yaml`
);
const squadBattleTypeStyles = await readYAMLDVPL<SquadBattleTypeStyles>(
`${DATA}/UI/Screens3/Lobby/Hangar/Squad/SquadBattleType.yaml`,
`${DATA}/UI/Screens3/Lobby/Hangar/Squad/SquadBattleType.yaml`
);
const bannerMatches: { name: string; path: string }[] = [];
for (const match of gameTypeSelectorStyles.Prototypes[0].components.UIDataLocalBindingsComponent.data[1][2].matchAll(
/eGameMode\.([a-zA-Z]+) -> "~res:([^"]+)"/g,
/eGameMode\.([a-zA-Z]+) -> "~res:([^"]+)"/g
)) {
bannerMatches.push({ name: match[1], path: match[2] });
}
for (const match of squadBattleTypeStyles.Prototypes[0].components.UIDataLocalBindingsComponent.data[1][2].matchAll(
/"(\d+)" -> "battleType\/([a-zA-Z]+)"/g,
/"(\d+)" -> "battleType\/([a-zA-Z]+)"/g
)) {
const id = Number(match[1]);
const name = match[2];
let path = bannerMatches.find(
(bannerMatch) => bannerMatch.name.toLowerCase() === name.toLowerCase(),
(bannerMatch) => bannerMatch.name.toLowerCase() === name.toLowerCase()
)?.path;
if (path === undefined) {
@@ -38,16 +37,16 @@ export async function gameModeBanners() {
}
const content = await sharp(
await readDVPLFile(`${DATA}${path}.packed.webp`),
await readDVPLFile(`${DATA}${path}.packed.webp`)
)
.trim({ background: { r: 0, g: 0, b: 0, alpha: 0 } })
.toBuffer();
changes.push({
await uploader.add({
path: `icons/game_mode_banners/${id}.webp`,
content,
});
}
await commitAssets('game mode banners', changes);
await uploader.flush();
}
+17 -18
View File
@@ -1,13 +1,13 @@
import { Avatar, fetchGlossary, Gallery } from '@blitzkit/core';
import locales from '@blitzkit/i18n/locales.json';
import { extname } from 'path';
import ProgressBar from 'progress';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { Avatar, fetchGlossary, Gallery } from "@blitzkit/core";
import locales from "@blitzkit/i18n/locales.json";
import { extname } from "path";
import ProgressBar from "progress";
import { AssetUploader } from "../core/github/assetUploader";
export async function glossary() {
console.log('Building glossary...');
console.log("Building glossary...");
using uploader = new AssetUploader("glossary");
const avatars: Record<string, { avatar: Avatar; url: string }> = {};
await Promise.all(
@@ -15,15 +15,15 @@ export async function glossary() {
const glossary = await fetchGlossary(supported.blitz ?? supported.locale);
console.log(
`Found ${Object.keys(glossary).length} things for ${supported.locale}`,
`Found ${Object.keys(glossary).length} things for ${supported.locale}`
);
for (const key in glossary) {
const glossaryEntry = glossary[key];
if (
!key.startsWith('avatar') ||
key.endsWith('_part') ||
!key.startsWith("avatar") ||
key.endsWith("_part") ||
glossaryEntry.image_url === null
) {
continue;
@@ -45,16 +45,15 @@ export async function glossary() {
};
}
}
}),
})
);
const changes: FileChange[] = [];
const gallery: Gallery = { avatars: [] };
const totalCount = Object.keys(avatars).length;
const bar = new ProgressBar(
`Fetching avatars of ${totalCount} things :bar`,
totalCount,
totalCount
);
await Promise.all(
@@ -72,19 +71,19 @@ export async function glossary() {
.then((response) => response.arrayBuffer())
.then((buffer) => new Uint8Array(buffer));
changes.push({
await uploader.add({
path: `gallery/avatars/${key}${avatar.extension}`,
content,
});
bar.tick();
}),
})
);
changes.push({
path: 'definitions/gallery.pb',
await uploader.add({
path: "definitions/gallery.pb",
content: Gallery.encode(gallery).finish(),
});
await commitAssets('glossary', changes);
await uploader.flush();
}
+4 -4
View File
@@ -20,6 +20,7 @@ import { gameModeBanners } from "./gameModeBanners";
import { glossary } from "./glossary";
import { migration } from "./migration";
import { moduleIcons } from "./moduleIcons";
import { popularTanks } from "./popularTanks";
import { shellIcons } from "./shellIcons";
import { skillIcons } from "./skillIcons";
import { tankArmors } from "./tankArmors";
@@ -29,8 +30,8 @@ import { tankModels } from "./tankModels";
if (!targets && !allTargets) throw new Error("No target(s) specified");
const methods = [
migration,
definitions,
tankIcons,
tankModels,
tankArmors,
gameModeBanners,
@@ -43,10 +44,9 @@ const methods = [
consumableProvisionIcons,
camouflageIcons,
boosterIcons,
tankIcons,
// blitzkitTankIcons,
// popularTanks,
glossary,
migration,
popularTanks,
];
for (const method of methods) {
+25 -27
View File
@@ -1,38 +1,36 @@
import { toUniqueId } from '@blitzkit/core';
import { readdir } from 'fs/promises';
import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { DATA } from './constants';
import { VehicleDefinitionList } from './definitions';
import { toUniqueId } from "@blitzkit/core";
import { readdir } from "fs/promises";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import { VehicleDefinitionList } from "./definitions";
export async function migration() {
console.log('Building migration map...');
console.log("Building migration map...");
using uploader = new AssetUploader("migration map");
const map: Record<string, number> = {};
const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
(nations) => nations.filter((nation) => nation !== 'common'),
(nations) => nations.filter((nation) => nation !== "common")
);
await Promise.all(
nations.map(async (nation) => {
const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`,
);
for (const nation of nations) {
const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`
);
for (const tankKey in tankList.root) {
const tank = tankList.root[tankKey];
const tankId = toUniqueId(nation, tank.id);
const davaKey = `${nation}:${tankKey}`;
for (const tankKey in tankList.root) {
const tank = tankList.root[tankKey];
const tankId = toUniqueId(nation, tank.id);
const davaKey = `${nation}:${tankKey}`;
map[davaKey] = tankId;
}
}),
);
map[davaKey] = tankId;
}
}
await commitAssets('migration', [
{
path: 'definitions/migration.json',
content: new TextEncoder().encode(JSON.stringify(map)),
},
]);
await uploader.add({
path: "definitions/migration.json",
content: new TextEncoder().encode(JSON.stringify(map)),
});
await uploader.flush();
}
+27 -28
View File
@@ -1,34 +1,33 @@
import { readdir } from 'fs/promises';
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { readdir } from "fs/promises";
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
export async function moduleIcons() {
console.log('Building module icons...');
console.log("Building module icons...");
const changes = await Promise.all(
(await readdir(`${DATA}/Gfx/UI/ModulesTechTree`))
.filter(
(file) =>
!file.endsWith('@2x.packed.webp') && file.startsWith('vehicle'),
)
.map(async (file) => {
const nameRaw = file.match(/vehicle(.+)\.packed\.webp/)![1];
const name = nameRaw[0].toLowerCase() + nameRaw.slice(1);
const content = await sharp(
await readDVPLFile(`${DATA}/Gfx/UI/ModulesTechTree/${file}`),
)
.trim()
.toBuffer();
return {
content,
path: `icons/modules/${name}.webp`,
} satisfies FileChange;
}),
using uploader = new AssetUploader("module icons");
const files = await readdir(`${DATA}/Gfx/UI/ModulesTechTree`).then((dir) =>
dir.filter(
(file) => !file.endsWith("@2x.packed.webp") && file.startsWith("vehicle")
)
);
await commitAssets('module icons', changes);
for (const file of files) {
const nameRaw = file.match(/vehicle(.+)\.packed\.webp/)![1];
const name = nameRaw[0].toLowerCase() + nameRaw.slice(1);
const content = await sharp(
await readDVPLFile(`${DATA}/Gfx/UI/ModulesTechTree/${file}`)
)
.trim()
.toBuffer();
await uploader.add({
content,
path: `icons/modules/${name}.webp`,
});
}
await uploader.flush();
}
@@ -2,31 +2,34 @@ import {
assertSecret,
fetchTankDefinitions,
PopularTanks,
} from '@blitzkit/core';
import { google } from 'googleapis';
import { commitAssets } from '../core/github/commitAssets';
} from "@blitzkit/core";
import { google } from "googleapis";
import { AssetUploader } from "../core/github/assetUploader";
export async function popularTanks() {
using uploader = new AssetUploader("popular tanks");
const tankDefinitions = await fetchTankDefinitions();
const auth = await google.auth.getClient({
keyFile: import.meta.env.GOOGLE_APPLICATION_CREDENTIALS,
scopes: ['https://www.googleapis.com/auth/analytics.readonly'],
scopes: ["https://www.googleapis.com/auth/analytics.readonly"],
});
const analytics = google.analyticsdata({ version: 'v1beta', auth });
const analytics = google.analyticsdata({ version: "v1beta", auth });
const report = await analytics.properties.runReport({
property: `properties/${assertSecret(import.meta.env.PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID)}`,
property: `properties/${assertSecret(
import.meta.env.PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID
)}`,
requestBody: {
dateRanges: [{ startDate: '7daysAgo', endDate: 'today' }],
dimensions: [{ name: 'pagePath' }],
metrics: [{ name: 'screenPageViews' }],
orderBys: [{ metric: { metricName: 'screenPageViews' }, desc: true }],
dateRanges: [{ startDate: "7daysAgo", endDate: "today" }],
dimensions: [{ name: "pagePath" }],
metrics: [{ name: "screenPageViews" }],
orderBys: [{ metric: { metricName: "screenPageViews" }, desc: true }],
dimensionFilter: {
filter: {
fieldName: 'pagePath',
fieldName: "pagePath",
stringFilter: {
matchType: 'BEGINS_WITH',
value: '/tanks/',
matchType: "BEGINS_WITH",
value: "/tanks/",
},
},
},
@@ -34,7 +37,7 @@ export async function popularTanks() {
});
if (!report.data.rows) {
throw new Error('No rows in report');
throw new Error("No rows in report");
}
const tanks = report.data.rows
@@ -42,21 +45,21 @@ export async function popularTanks() {
(row) =>
row.dimensionValues &&
row.dimensionValues[0].value &&
row.dimensionValues[0].value !== '/tanks/' &&
row.dimensionValues[0].value !== "/tanks/" &&
row.metricValues &&
row.metricValues[0].value,
row.metricValues[0].value
)
.map((row) => ({
id: Number(
row.dimensionValues![0].value!.match(
/\/tools\/tankopedia\/(\d+)\/?/,
)?.[1],
/\/tools\/tankopedia\/(\d+)\/?/
)?.[1]
),
views: Number(row.metricValues![0].value!),
}))
.filter((row) => {
if (!(row.id in tankDefinitions.tanks)) {
console.warn('Skipping unknown popular tank', row.id);
console.warn("Skipping unknown popular tank", row.id);
return false;
}
@@ -66,10 +69,9 @@ export async function popularTanks() {
.map(({ id, views }) => ({ id, views }));
const popularTanks: PopularTanks = { tanks };
await commitAssets('popular tanks', [
{
path: 'definitions/popular-tanks.pb',
content: PopularTanks.encode(popularTanks).finish(),
},
]);
await uploader.add({
path: "definitions/popular-tanks.pb",
content: PopularTanks.encode(popularTanks).finish(),
});
await uploader.flush();
}
+37 -38
View File
@@ -1,48 +1,47 @@
import { readdir } from 'fs/promises';
import sharp from 'sharp';
import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { readdir } from "fs/promises";
import sharp from "sharp";
import { readDVPLFile } from "../../src/core/blitz/readDVPLFile";
import { readStringDVPL } from "../../src/core/blitz/readStringDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
export async function shellIcons() {
console.log('Building shell icons...');
console.log("Building shell icons...");
using uploader = new AssetUploader("shell icons");
const image = sharp(
await readDVPLFile(
`${DATA}/Gfx/Shared/tank-supply/ammunition/big/texture0.packed.webp`,
),
`${DATA}/Gfx/Shared/tank-supply/ammunition/big/texture0.packed.webp`
)
);
const files = await readdir(
`${DATA}/Gfx/Shared/tank-supply/ammunition/big`
).then((files) => files.filter((file) => file.endsWith("_l.txt")));
const changes = await Promise.all(
(await readdir(`${DATA}/Gfx/Shared/tank-supply/ammunition/big`))
.filter((file) => file.endsWith('_l.txt'))
.map(async (file) => {
const name = file.match(/(.+)_l\.txt/)![1];
const sizes = (
await readStringDVPL(
`${DATA}/Gfx/Shared/tank-supply/ammunition/big/${file}`,
)
)
.split('\n')[4]
.split(' ')
.map(Number);
for (const file of files) {
const name = file.match(/(.+)_l\.txt/)![1];
const sizes = (
await readStringDVPL(
`${DATA}/Gfx/Shared/tank-supply/ammunition/big/${file}`
)
)
.split("\n")[4]
.split(" ")
.map(Number);
return {
content: await image
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer(),
path: `icons/shells/${name}.webp`,
} satisfies FileChange;
}),
);
await uploader.add({
content: await image
.clone()
.extract({
left: sizes[0],
top: sizes[1],
width: sizes[2],
height: sizes[3],
})
.toBuffer(),
path: `icons/shells/${name}.webp`,
});
}
await commitAssets('shell icons', changes);
await uploader.flush();
}
+13 -16
View File
@@ -2,8 +2,7 @@ import { TankClass } from "@blitzkit/core";
import sharp from "sharp";
import { readDVPLFile } from "../core/blitz/readDVPLFile";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { commitAssets } from "../core/github/commitAssets";
import type { FileChange } from "../core/github/commitMultipleFiles";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
interface SkillIcon {
@@ -31,20 +30,18 @@ export async function skillIcons() {
const avatar = await readXMLDVPL<{ root: Avatar }>(
`${DATA}/XML/item_defs/tankmen/avatar.xml`
);
const changes = await Promise.all(
Object.values(avatar.root.skills).map(async (skill) => {
const icon = Array.isArray(skill.icon) ? skill.icon[0] : skill.icon;
const name = icon.name.split("/").at(-1)!.replace(/_\d$/, "");
const path = `${DATA}${icon.name.replace("~res:", "")}.packed.webp`;
const image = sharp(await readDVPLFile(path)).trim();
const content = await image.toBuffer();
using uploader = new AssetUploader("skill icons");
return {
content,
path: `icons/skills/${name}.webp`,
} satisfies FileChange;
})
);
for (const key in avatar.root.skills) {
const skill = avatar.root.skills[key];
const icon = Array.isArray(skill.icon) ? skill.icon[0] : skill.icon;
const name = icon.name.split("/").at(-1)!.replace(/_\d$/, "");
const path = `${DATA}${icon.name.replace("~res:", "")}.packed.webp`;
const image = sharp(await readDVPLFile(path)).trim();
const content = await image.toBuffer();
commitAssets("skill icons", changes);
await uploader.add({ content, path: `icons/skills/${name}.webp` });
}
await uploader.flush();
}
+26 -32
View File
@@ -1,44 +1,38 @@
import { toUniqueId } from '@blitzkit/core';
import { NodeIO } from '@gltf-transform/core';
import { readdir } from 'fs/promises';
import { extractArmor } from '../core/blitz/extractArmor';
import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
import { commitAssets } from '../core/github/commitAssets';
import { FileChange } from '../core/github/commitMultipleFiles';
import { DATA } from './constants';
import { VehicleDefinitionList } from './definitions';
import { toUniqueId } from "@blitzkit/core";
import { NodeIO } from "@gltf-transform/core";
import { readdir } from "fs/promises";
import { extractArmor } from "../core/blitz/extractArmor";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import { VehicleDefinitionList } from "./definitions";
export async function tankArmors() {
console.log('Building tank armors...');
console.log("Building tank armors...");
const changes: FileChange[] = [];
using uploader = new AssetUploader("tank armors");
const nodeIO = new NodeIO();
const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
(nations) => nations.filter((nation) => nation !== 'common'),
(nations) => nations.filter((nation) => nation !== "common")
);
await Promise.all(
nations.map(async (nation) => {
const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`,
);
for (const nation of nations) {
const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`
);
await Promise.all(
Object.entries(tanks.root).map(async ([tankKey, tank]) => {
if (tankKey.includes('tutorial_bot')) return;
for (const tankKey in tanks.root) {
const tank = tanks.root[tankKey];
const id = toUniqueId(nation, tank.id);
const model = await extractArmor(DATA, `${nation}-${tankKey}`);
const content = await nodeIO.writeBinary(model);
if (tankKey.includes("tutorial_bot")) continue;
changes.push({
path: `3d/tanks/armor/${id}.glb`,
content,
});
}),
);
}),
);
const id = toUniqueId(nation, tank.id);
const model = await extractArmor(DATA, `${nation}-${tankKey}`);
const content = await nodeIO.writeBinary(model);
await commitAssets('tank armor', changes);
await uploader.add({ path: `3d/tanks/armor/${id}.glb`, content });
}
}
await uploader.flush();
}
+44 -47
View File
@@ -5,8 +5,7 @@ import type { Vector3Tuple } from "three";
import { readDVPLFile } from "../core/blitz/readDVPLFile";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL";
import { commitAssets } from "../core/github/commitAssets";
import type { FileChange } from "../core/github/commitMultipleFiles";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import type { VehicleDefinitionList } from "./definitions";
@@ -45,56 +44,54 @@ export interface TankParameters {
export async function tankIcons() {
console.log("Building tank icons...");
const changes: FileChange[] = [];
using uploader = new AssetUploader("tank icons");
const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
(nations) => nations.filter((nation) => nation !== "common")
);
await Promise.all(
nations.map(async (nation) => {
const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`
for (const nation of nations) {
const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`
);
for (const tankKey in tanks.root) {
const tank = tanks.root[tankKey];
if (tankKey.includes("tutorial_bot")) continue;
const nationVehicleId = tank.id;
const id = (nationVehicleId << 8) + (NATION_IDS[nation] << 4) + 1;
const parameters = await readYAMLDVPL<TankParameters>(
`${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml`
);
const smallPath = `${DATA}/${parameters.resourcesPath.smallIconPath
.replace(/~res:\//, "")
.replace(/\..+/, "")}.packed.webp`;
const bigPath = `${DATA}/${parameters.resourcesPath.bigIconPath
.replace(/~res:\//, "")
.replace(/\..+/, "")}.packed.webp`;
const big = await sharp(await readDVPLFile(bigPath))
.trim()
.toBuffer();
const small = await sharp(await readDVPLFile(smallPath))
.trim()
.toBuffer();
await Promise.all(
Object.entries(tanks.root).map(async ([tankKey, tank]) => {
if (tankKey.includes("tutorial_bot")) return;
if (big) {
await uploader.add({
content: big,
path: `icons/tanks/big/${id}.webp`,
});
}
if (small) {
await uploader.add({
content: small,
path: `icons/tanks/small/${id}.webp`,
});
}
}
}
const nationVehicleId = tank.id;
const id = (nationVehicleId << 8) + (NATION_IDS[nation] << 4) + 1;
const parameters = await readYAMLDVPL<TankParameters>(
`${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml`
);
const smallPath = `${DATA}/${parameters.resourcesPath.smallIconPath
.replace(/~res:\//, "")
.replace(/\..+/, "")}.packed.webp`;
const bigPath = `${DATA}/${parameters.resourcesPath.bigIconPath
.replace(/~res:\//, "")
.replace(/\..+/, "")}.packed.webp`;
const big = await sharp(await readDVPLFile(bigPath))
.trim()
.toBuffer();
const small = await sharp(await readDVPLFile(smallPath))
.trim()
.toBuffer();
if (big) {
changes.push({
content: big,
path: `icons/tanks/big/${id}.webp`,
});
}
if (small) {
changes.push({
content: small,
path: `icons/tanks/small/${id}.webp`,
});
}
})
);
})
);
await commitAssets("tank icons", changes);
await uploader.flush();
}
+5 -12
View File
@@ -5,8 +5,7 @@ import { readdir } from "fs/promises";
import { extractModel } from "../core/blitz/extractModel";
import { readXMLDVPL } from "../core/blitz/readXMLDVPL";
import { readYAMLDVPL } from "../core/blitz/readYAMLDVPL";
import { commitAssets } from "../core/github/commitAssets";
import { FileChange } from "../core/github/commitMultipleFiles";
import { AssetUploader } from "../core/github/assetUploader";
import { DATA } from "./constants";
import { VehicleDefinitionList } from "./definitions";
import { TankParameters } from "./tankIcons";
@@ -14,6 +13,7 @@ import { TankParameters } from "./tankIcons";
export async function tankModels() {
console.log("Building tank models...");
using uploader = new AssetUploader("tank models");
const nodeIO = new NodeIO().registerExtensions(ALL_EXTENSIONS);
const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
(nations) => nations.filter((nation) => nation !== "common")
@@ -21,22 +21,15 @@ export async function tankModels() {
for (const nationIndex in nations) {
const nation = nations[nationIndex];
const changes: FileChange[] = [];
const tanks = await readXMLDVPL<{ root: VehicleDefinitionList }>(
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml`
);
const entries = Object.entries(tanks.root);
console.log(`Building models for ${nation}`);
let index = 0;
for (const [tankKey, tank] of entries) {
if (tankKey.includes("tutorial_bot")) continue;
const id = toUniqueId(nation, tank.id);
console.log(` ${++index} / ${entries.length} : ${tankKey}`);
const parameters = await readYAMLDVPL<TankParameters>(
`${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml`
);
@@ -46,9 +39,9 @@ export async function tankModels() {
);
const content = Buffer.from(await nodeIO.writeBinary(model));
changes.push({ path: `3d/tanks/models/${id}.glb`, content });
await uploader.add({ path: `3d/tanks/models/${id}.glb`, content });
}
await commitAssets(`tank models ${nation}`, changes);
}
await uploader.flush();
}
+8 -10
View File
@@ -13,8 +13,7 @@ import {
} from "@blitzkit/core";
import { chunk } from "lodash-es";
import { argv } from "process";
import { commitAssets } from "./core/github/commitAssets";
import { FileChange } from "./core/github/commitMultipleFiles";
import { AssetUploader } from "./core/github/assetUploader";
const region = argv
.find((arg) => arg.startsWith("--region="))
@@ -156,16 +155,15 @@ const info = await patientFetchJSON<RatingInfo & { detail: undefined }>(
`https://${regionSubdomain}.wotblitz.com/en/api/rating-leaderboards/season/`
);
const normalizedServer = regionSubdomain === "na" ? "com" : regionSubdomain;
const content = RatingLeaderboard.encode({
version: { $case: "v2", value: { entries } },
}).finish();
const changes: FileChange[] = [
{
path: `regions/${normalizedServer}/rating/${info.current_season}.pb`,
content,
},
];
using uploader = new AssetUploader(`rating leaderboard ${regionSubdomain}`);
commitAssets(`rating leaderboard ${regionSubdomain}`, changes);
await uploader.add({
path: `regions/${normalizedServer}/rating/${info.current_season}.pb`,
content,
});
await uploader.flush();
@@ -67,7 +67,9 @@ export async function extractModel(data: string, path: string) {
.setMimeType("image/webp")
.setImage(
await readBaseColor(
`${data}/3d/${dirname(path)}/${textures.baseColorMap ?? textures.albedo}`,
`${data}/3d/${dirname(path)}/${
textures.baseColorMap ?? textures.albedo
}`,
textures.miscMap
? `${data}/3d/${dirname(path)}/${textures.miscMap}`
: undefined
@@ -99,7 +99,7 @@ export class AssetUploader {
}
if (diff.status === DiffStatus.Unchanged) {
console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`);
console.log(`🔵 ${change.path}`);
} else {
console.log(
`${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${
@@ -1,27 +0,0 @@
import { assertSecret } from "@blitzkit/core";
import { chunk } from "lodash-es";
import { commitMultipleFiles, FileChange } from "./commitMultipleFiles";
export async function commitAssets(message: string, changes: FileChange[]) {
console.log(`Committing ${message}...`);
if (changes.length === 0) return;
const chunks = chunk(changes, 64);
let chunkIndex = 0;
for (const chunk of chunks) {
console.log(`Committing chunk ${chunkIndex + 1}/${chunks.length}`);
await commitMultipleFiles(
assertSecret(import.meta.env.PUBLIC_ASSET_REPO),
assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH),
`${message} - ${new Date().toDateString()} (${chunkIndex + 1}/${
chunks.length
})`,
chunk
);
chunkIndex++;
}
}
@@ -1,135 +0,0 @@
import { asset } from "@blitzkit/core";
import ProgressBar from "progress";
import { compareUint8Arrays } from "./compareUint8Arrays";
import { type GithubChangeBlob, createBlob } from "./createBlob";
import { octokit } from "./octokit";
export interface FileChange {
path: string;
content: Uint8Array;
}
enum DiffStatus {
New,
Changed,
Unchanged,
}
const heuristicFormats = ["glb", "webp", "jpg", "png"];
export async function commitMultipleFiles(
repoRaw: string,
branch: string,
message: string,
changesRaw: FileChange[]
) {
const changes: FileChange[] = [];
await Promise.all(
changesRaw.map(async (change) => {
const response = await fetch(asset(change.path), {
headers: { "Accept-Encoding": "identity" },
});
let diff: { change: number; status: DiffStatus };
if (response.status === 404) {
diff = { change: change.content.length, status: DiffStatus.New };
} else if (
response.headers.has("Content-Length") &&
Number(response.headers.get("Content-Length")) !== change.content.length
) {
diff = {
change:
change.content.length -
Number(response.headers.get("Content-Length")),
status: DiffStatus.Changed,
};
} else if (response.status === 200) {
if (
heuristicFormats.some((format) => change.path.endsWith(`.${format}`))
) {
/**
* Heuristic: if it's a large blob like a .glb, a diff = 0 might end
* up being a false positive with isDifferent = true. So, it's okay
* to assume it's unchanged.
*/
diff = { change: 0, status: DiffStatus.Unchanged };
} else {
const buffer = new Uint8Array(await response.arrayBuffer());
if (compareUint8Arrays(buffer, change.content)) {
diff = { change: 0, status: DiffStatus.Unchanged };
} else {
diff = {
change: change.content.length - buffer.length,
status: DiffStatus.Changed,
};
}
}
} else {
throw new Error(
`Unexpected status code ${response.status} for ${change.path}`
);
}
if (diff.status === DiffStatus.Unchanged) {
// just don't bother
// console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`);
} else {
console.log(
`${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${
diff.change > 0 ? "+" : ""
}${diff.change.toLocaleString()}B) ${change.path}`
);
changes.push(change);
}
})
);
if (changes.length === 0) return;
const [owner, repo] = repoRaw.split("/");
const latestCommitSha = (
await octokit.git.getRef({
owner,
repo,
ref: `heads/${branch}`,
})
).data.object.sha;
const treeSha = (
await octokit.git.getCommit({
owner,
repo,
commit_sha: latestCommitSha,
})
).data.tree.sha;
const blobs: GithubChangeBlob[] = [];
const bar = new ProgressBar("Blobs :bar", changes.length);
for (const change of changes) {
const createdBlob = await createBlob(owner, repo, change);
blobs.push(createdBlob!);
bar.tick();
}
const { data: treeData } = await octokit.git.createTree({
owner,
repo,
base_tree: treeSha,
tree: blobs,
});
const { data: newCommitData } = await octokit.git.createCommit({
owner,
repo,
message,
tree: treeData.sha,
parents: [latestCommitSha],
});
await octokit.git.updateRef({
owner,
repo,
ref: `heads/${branch}`,
sha: newCommitData.sha,
force: true,
});
}
@@ -1,5 +1,5 @@
import { throttle } from "lodash-es";
import type { FileChange } from "./commitMultipleFiles";
import { FileChange } from "./assetUploader";
import { octokit } from "./octokit";
export interface GithubChangeBlob {
+40 -37
View File
@@ -9,11 +9,10 @@ import {
getAccountInfo,
idToRegion,
retryAbleBlitzFetchEvent,
} from '@blitzkit/core';
import { chunk, times, uniq } from 'lodash-es';
import { compress, decompress } from 'lz4js';
import { commitAssets } from './core/github/commitAssets';
import { FileChange } from './core/github/commitMultipleFiles';
} from "@blitzkit/core";
import { chunk, times, uniq } from "lodash-es";
import { compress, decompress } from "lz4js";
import { AssetUploader } from "./core/github/assetUploader";
const CHUNK_SIZE = 2 ** 21;
const RUN_TIME = 1000 * 60 * 60 * 5.5;
@@ -24,12 +23,12 @@ const TERMINATION_THRESHOLD = 1000;
const startTime = Date.now();
const indexableRegions = [...REGIONS];
let ids: number[] = [];
const preDiscoveredManifest = (await fetch(asset('ids/manifest.json')).then(
(response) => response.json(),
const preDiscoveredManifest = (await fetch(asset("ids/manifest.json")).then(
(response) => response.json()
)) as DiscoveredIdsDefinitions;
console.log(
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`,
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`
);
let chunkIndex = 0;
@@ -39,23 +38,23 @@ while (chunkIndex < preDiscoveredManifest.chunks) {
const buffer = await response.arrayBuffer();
const decompressed = decompress(new Uint8Array(buffer)).buffer;
return new DidsReadStream(decompressed as ArrayBuffer).dids();
},
}
);
// no spread syntax: https://github.com/oven-sh/bun/issues/11734
ids = ids.concat(preDiscovered);
console.log(
`Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`,
`Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`
);
chunkIndex++;
}
const regionalIdIndex: Record<Region, number> = {
asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_REGION_IDS.asia,
com: ids.findLast((id) => idToRegion(id) === 'com') ?? MIN_REGION_IDS.com,
eu: ids.findLast((id) => idToRegion(id) === 'eu') ?? MIN_REGION_IDS.eu,
asia: ids.findLast((id) => idToRegion(id) === "asia") ?? MIN_REGION_IDS.asia,
com: ids.findLast((id) => idToRegion(id) === "com") ?? MIN_REGION_IDS.com,
eu: ids.findLast((id) => idToRegion(id) === "eu") ?? MIN_REGION_IDS.eu,
};
const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
let regionIndex = 0;
@@ -64,7 +63,7 @@ let outOfTimeFlagged = false;
async function verify(region: Region, ids: number[]) {
const infos = await getAccountInfo(region, ids, undefined, {
// cheeky way of requesting nothing
fields: 'account_id,-account_id',
fields: "account_id,-account_id",
});
const filtered = ids.filter((_, index) => {
const info = infos[index];
@@ -85,7 +84,7 @@ const interval = setInterval(async () => {
const region = indexableRegions[regionIndex];
const idsToVerify = times(
ACCOUNTS_PER_CALL,
(index) => regionalIdIndex[region] + index,
(index) => regionalIdIndex[region] + index
);
const verified = await verify(region, idsToVerify);
@@ -105,8 +104,8 @@ const interval = setInterval(async () => {
indexableRegions.splice(sliceableIndex, 1);
if (indexableRegions.length === 0) {
console.log('All players discovered exhaustively!');
post();
console.log("All players discovered exhaustively!");
await post();
}
}
}
@@ -117,46 +116,50 @@ const interval = setInterval(async () => {
if (Date.now() - startTime > RUN_TIME && !outOfTimeFlagged) {
outOfTimeFlagged = true;
console.log('Out of time');
post();
console.log("Out of time");
await post();
}
regionIndex = (regionIndex + 1) % indexableRegions.length;
regionalIdIndex[region] += ACCOUNTS_PER_CALL;
}, 1000 / MAX_REQUESTS);
function post() {
async function post() {
clearInterval(interval);
ids = uniq(ids).sort((a, b) => a - b);
console.log(`Uploading ${ids.length} ids...`);
using uploader = new AssetUploader("discovered ids");
const idsChunked = chunk(ids, CHUNK_SIZE);
const files = idsChunked.map((chunk, index) => {
let index = 0;
for (const chunk of idsChunked) {
const didsWriteStream = new DidsWriteStream().dids(chunk);
const buffer = didsWriteStream.uint8Array;
const content = compress(buffer);
console.log(
`Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`,
`Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`
);
return {
await uploader.add({
content,
path: `ids/${index}.dids.lz4`,
} satisfies FileChange;
});
index++;
}
await uploader.add({
content: new TextEncoder().encode(
JSON.stringify({
chunks: idsChunked.length,
count: ids.length,
time: Date.now(),
} satisfies DiscoveredIdsDefinitions)
),
path: "ids/manifest.json",
});
commitAssets('discovered ids', [
...files,
{
content: new TextEncoder().encode(
JSON.stringify({
chunks: idsChunked.length,
count: ids.length,
time: Date.now(),
} satisfies DiscoveredIdsDefinitions),
),
path: 'ids/manifest.json',
},
]);
await uploader.flush();
}
+20 -18
View File
@@ -4,22 +4,22 @@ import {
Reviews,
Video,
youtubers,
} from '@blitzkit/core';
import { google } from 'googleapis';
import { cloneDeep, uniqBy } from 'lodash-es';
import { commitAssets } from './core/github/commitAssets';
} from "@blitzkit/core";
import { google } from "googleapis";
import { cloneDeep, uniqBy } from "lodash-es";
import { AssetUploader } from "./core/github/assetUploader";
const MAX_QUERIES = 64;
console.log('Finding reviews...');
console.log("Finding reviews...");
const currentReviews = await fetch(asset('definitions/reviews.pb'))
const currentReviews = await fetch(asset("definitions/reviews.pb"))
.then((response) => response.arrayBuffer())
.then((buffer) => Reviews.decode(new Uint8Array(buffer)));
const auth = await google.auth.getClient({
scopes: ['https://www.googleapis.com/auth/youtube.readonly'],
scopes: ["https://www.googleapis.com/auth/youtube.readonly"],
});
const youtube = google.youtube({ version: 'v3', auth });
const youtube = google.youtube({ version: "v3", auth });
const tankDefinitions = await fetchTankDefinitions();
const tanks = Object.values(tankDefinitions.tanks);
@@ -43,7 +43,7 @@ for (const tank of tanksSanitized) {
try {
const results = await youtube.search.list({
part: ['snippet'],
part: ["snippet"],
q: `World of Tanks Blitz ${tank.name}`,
});
@@ -54,9 +54,9 @@ for (const tank of tanksSanitized) {
(item) =>
item.snippet?.channelId &&
item.id?.videoId &&
youtubers.some(({ id }) => id === item.snippet?.channelId),
youtubers.some(({ id }) => id === item.snippet?.channelId)
),
(item) => item.snippet!.channelId,
(item) => item.snippet!.channelId
);
reviews.reviews[tank.id] = {
@@ -66,7 +66,7 @@ for (const tank of tanksSanitized) {
({
id: item.id!.videoId!,
author: item.snippet!.channelId!,
}) satisfies Video,
} satisfies Video)
),
};
@@ -78,9 +78,11 @@ for (const tank of tanksSanitized) {
break;
}
}
await commitAssets('reviews', [
{
content: Reviews.encode(reviews).finish(),
path: 'definitions/reviews.pb',
},
]);
using uploader = new AssetUploader("reviews");
await uploader.add({
content: Reviews.encode(reviews).finish(),
path: "definitions/reviews.pb",
});
await uploader.flush();