diff --git a/packages/core/src/protobuf/encodePBBuffer.ts b/packages/core/src/protobuf/encodePBBuffer.ts deleted file mode 100644 index 55828bb80..000000000 --- a/packages/core/src/protobuf/encodePBBuffer.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { MessageFns } from '../protos'; - -export function encodePBBuffer(message: MessageFns, data: Type) { - return Buffer.from(message.encode(data).finish()); -} diff --git a/packages/core/src/protobuf/index.ts b/packages/core/src/protobuf/index.ts index 773b86baa..997a198cb 100644 --- a/packages/core/src/protobuf/index.ts +++ b/packages/core/src/protobuf/index.ts @@ -1,3 +1,2 @@ -export * from './encodePBBuffer'; export * from './fetchPB'; export * from './pbMap'; diff --git a/packages/scripts/src/averages.ts b/packages/scripts/src/averages.ts index ee07cc981..a387bf8b0 100644 --- a/packages/scripts/src/averages.ts +++ b/packages/scripts/src/averages.ts @@ -8,7 +8,6 @@ import { Samples, averageDefinitionsAllStatsKeys, emptySamples, - encodePBBuffer, getAccountInfo, getTankStats, idToRegion, @@ -235,11 +234,11 @@ async function postWork() { commitAssets('averages', [ { path: `averages/${latest}.pb`, - content: encodePBBuffer(AverageDefinitions, averageDefinitions), + content: AverageDefinitions.encode(averageDefinitions).finish(), }, { path: 'averages/manifest.json', - content: Buffer.from(JSON.stringify(manifest), 'utf-8'), + content: new TextEncoder().encode(JSON.stringify(manifest)), }, ]); } diff --git a/packages/scripts/src/buildAssets/definitions.ts b/packages/scripts/src/buildAssets/definitions.ts index f794d50cf..917f94b65 100644 --- a/packages/scripts/src/buildAssets/definitions.ts +++ b/packages/scripts/src/buildAssets/definitions.ts @@ -7,7 +7,6 @@ import { ConsumableTankCategoryFilterCategory, Crew, CrewType, - encodePBBuffer, EquipmentDefinitions, EquipmentSlot, GameDefinitions, @@ -1772,39 +1771,39 @@ export async function definitions() { await commitAssets('definitions', [ { - content: encodePBBuffer(GameDefinitions, gameDefinitions), + content: GameDefinitions.encode(gameDefinitions).finish(), path: 'definitions/game.pb', }, { - content: encodePBBuffer(TankDefinitions, tankDefinitions), + content: TankDefinitions.encode(tankDefinitions).finish(), path: 'definitions/tanks.pb', }, { - content: encodePBBuffer(ModelDefinitions, modelDefinitions), + content: ModelDefinitions.encode(modelDefinitions).finish(), path: 'definitions/models.pb', }, { - content: encodePBBuffer(EquipmentDefinitions, equipmentDefinitions), + content: EquipmentDefinitions.encode(equipmentDefinitions).finish(), path: 'definitions/equipment.pb', }, { - content: encodePBBuffer(ConsumableDefinitions, consumableDefinitions), + content: ConsumableDefinitions.encode(consumableDefinitions).finish(), path: 'definitions/consumables.pb', }, { - content: encodePBBuffer(ProvisionDefinitions, provisionDefinitions), + content: ProvisionDefinitions.encode(provisionDefinitions).finish(), path: 'definitions/provisions.pb', }, { - content: encodePBBuffer(SkillDefinitions, skillDefinitions), + content: SkillDefinitions.encode(skillDefinitions).finish(), path: 'definitions/skills.pb', }, { - content: encodePBBuffer(MapDefinitions, mapDefinitions), + content: MapDefinitions.encode(mapDefinitions).finish(), path: 'definitions/maps.pb', }, { - content: encodePBBuffer(CamouflageDefinitions, camouflageDefinitions), + content: CamouflageDefinitions.encode(camouflageDefinitions).finish(), path: 'definitions/camouflages.pb', }, ]); diff --git a/packages/scripts/src/buildAssets/glossary.ts b/packages/scripts/src/buildAssets/glossary.ts index 4b50d7ff6..0048f9368 100644 --- a/packages/scripts/src/buildAssets/glossary.ts +++ b/packages/scripts/src/buildAssets/glossary.ts @@ -1,4 +1,4 @@ -import { Avatar, encodePBBuffer, fetchGlossary, Gallery } from '@blitzkit/core'; +import { Avatar, fetchGlossary, Gallery } from '@blitzkit/core'; import locales from '@blitzkit/i18n/locales.json'; import { extname } from 'path'; import ProgressBar from 'progress'; @@ -68,13 +68,13 @@ export async function glossary() { gallery.avatars.push(avatar); - const content = await fetch(url).then((response) => - response.arrayBuffer(), - ); + const content = await fetch(url) + .then((response) => response.arrayBuffer()) + .then((buffer) => new Uint8Array(buffer)); changes.push({ path: `gallery/avatars/${key}${avatar.extension}`, - content: Buffer.from(content), + content, }); bar.tick(); @@ -83,7 +83,7 @@ export async function glossary() { changes.push({ path: 'definitions/gallery.pb', - content: encodePBBuffer(Gallery, gallery), + content: Gallery.encode(gallery).finish(), }); await commitAssets('glossary', changes); diff --git a/packages/scripts/src/buildAssets/migration.ts b/packages/scripts/src/buildAssets/migration.ts index fd74496bf..cc772a1b4 100644 --- a/packages/scripts/src/buildAssets/migration.ts +++ b/packages/scripts/src/buildAssets/migration.ts @@ -32,7 +32,7 @@ export async function migration() { await commitAssets('migration', [ { path: 'definitions/migration.json', - content: Buffer.from(JSON.stringify(map)), + content: new TextEncoder().encode(JSON.stringify(map)), }, ]); } diff --git a/packages/scripts/src/buildAssets/popularTanks.ts b/packages/scripts/src/buildAssets/popularTanks.ts index 5dc00f083..c71709f24 100644 --- a/packages/scripts/src/buildAssets/popularTanks.ts +++ b/packages/scripts/src/buildAssets/popularTanks.ts @@ -1,6 +1,5 @@ import { assertSecret, - encodePBBuffer, fetchTankDefinitions, PopularTanks, } from '@blitzkit/core'; @@ -70,7 +69,7 @@ export async function popularTanks() { await commitAssets('popular tanks', [ { path: 'definitions/popular-tanks.pb', - content: encodePBBuffer(PopularTanks, popularTanks), + content: PopularTanks.encode(popularTanks).finish(), }, ]); } diff --git a/packages/scripts/src/buildAssets/tankArmors.ts b/packages/scripts/src/buildAssets/tankArmors.ts index fd733144e..6aaefffce 100644 --- a/packages/scripts/src/buildAssets/tankArmors.ts +++ b/packages/scripts/src/buildAssets/tankArmors.ts @@ -29,10 +29,11 @@ export async function tankArmors() { const id = toUniqueId(nation, tank.id); const model = await extractArmor(DATA, `${nation}-${tankKey}`); + const content = await nodeIO.writeBinary(model); changes.push({ path: `3d/tanks/armor/${id}.glb`, - content: Buffer.from(await nodeIO.writeBinary(model)), + content, }); }), ); diff --git a/packages/scripts/src/buildAssets/tankModels.ts b/packages/scripts/src/buildAssets/tankModels.ts index aeeb0fe70..95901857a 100644 --- a/packages/scripts/src/buildAssets/tankModels.ts +++ b/packages/scripts/src/buildAssets/tankModels.ts @@ -1,7 +1,7 @@ import { toUniqueId } from '@blitzkit/core'; import { NodeIO } from '@gltf-transform/core'; import { ALL_EXTENSIONS } from '@gltf-transform/extensions'; -import { readdir } from 'fs/promises'; +import { readdir, writeFile } from 'fs/promises'; import { extractModel } from '../core/blitz/extractModel'; import { readXMLDVPL } from '../core/blitz/readXMLDVPL'; import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL'; @@ -33,6 +33,8 @@ export async function tankModels() { const id = toUniqueId(nation, tank.id); + if (id !== 21777) continue; + const parameters = await readYAMLDVPL( `${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml`, ); @@ -41,6 +43,8 @@ export async function tankModels() { parameters.resourcesPath.blitzModelPath.replace(/\.sc2$/, ''), ); + await writeFile('../../temp/model.glb', await nodeIO.writeBinary(model)); + changes.push({ path: `3d/tanks/models/${id}.glb`, content: Buffer.from(await nodeIO.writeBinary(model)), diff --git a/packages/scripts/src/buildRatingLeaderboard.ts b/packages/scripts/src/buildRatingLeaderboard.ts index b5b68436a..edadf6762 100644 --- a/packages/scripts/src/buildRatingLeaderboard.ts +++ b/packages/scripts/src/buildRatingLeaderboard.ts @@ -164,7 +164,7 @@ const content = RatingLeaderboard.encode({ const changes: FileChange[] = [ { path: `regions/${normalizedServer}/rating/${info.current_season}.pb`, - content: Buffer.from(content), + content, }, ]; diff --git a/packages/scripts/src/core/blitz/readTexture/index.ts b/packages/scripts/src/core/blitz/readTexture/index.ts index 1a06f6582..73b2495c6 100644 --- a/packages/scripts/src/core/blitz/readTexture/index.ts +++ b/packages/scripts/src/core/blitz/readTexture/index.ts @@ -1,12 +1,15 @@ import { DdsReadStream, PvrReadStream } from '@blitzkit/core'; import { existsSync } from 'fs'; import sharp from 'sharp'; +import { DATA_COMPRESSED } from '../../../buildAssets/constants'; import { readDVPLFile } from '../readDVPLFile'; import { TextureMutation } from './constants'; export async function readTexture(path: string, mutation?: TextureMutation) { const ddsTexturePath = path.replace('.tex', '.dx11.dds'); - const isDds = existsSync(`${ddsTexturePath}.dvpl`); + const isDds = existsSync( + `${ddsTexturePath}${DATA_COMPRESSED ? '.dvpl' : ''}`, + ); const resolvedTexturePath = isDds ? ddsTexturePath : ddsTexturePath.replace('.dds', '.pvr'); @@ -75,5 +78,4 @@ export async function readTexture(path: string, mutation?: TextureMutation) { default: return await sharp(raw.data, { raw }).jpeg().toBuffer(); } - } diff --git a/packages/scripts/src/core/github/commitMultipleFiles.ts b/packages/scripts/src/core/github/commitMultipleFiles.ts index 8b33c4082..2b9804671 100644 --- a/packages/scripts/src/core/github/commitMultipleFiles.ts +++ b/packages/scripts/src/core/github/commitMultipleFiles.ts @@ -1,11 +1,18 @@ import { asset } from '@blitzkit/core'; import ProgressBar from 'progress'; +import { compareUint8Arrays } from './compareUint8Arrays'; import { GithubChangeBlob, createBlob } from './createBlob'; import { octokit } from './octokit'; export interface FileChange { path: string; - content: Buffer; + content: Uint8Array; +} + +enum DiffStatus { + New, + Changed, + Unchanged, } const heuristicFormats = ['glb', 'webp', 'jpg', 'png']; @@ -20,73 +27,60 @@ export async function commitMultipleFiles( await Promise.all( changesRaw.map(async (change) => { - const response = await fetch(asset(change.path)); - let isNew: boolean; - let isDifferent: boolean; - let diff: number; + const response = await fetch(asset(change.path), { + headers: { 'Accept-Encoding': 'identity' }, + }); + let diff: { change: number; status: DiffStatus }; if (response.status === 404) { - isNew = true; - diff = change.content.length; - isDifferent = true; + diff = { change: change.content.length, status: DiffStatus.New }; } else if ( response.headers.has('Content-Length') && - parseInt(response.headers.get('Content-Length')!) !== - change.content.length + Number(response.headers.get('Content-Length')) !== change.content.length ) { - isNew = false; - diff = - change.content.length - - parseInt(response.headers.get('Content-Length')!); - isDifferent = true; - - console.log('diff header content length', 'content:', change.content.length, 'header:', response.headers.get('Content-Length')); + diff = { + change: + change.content.length - + Number(response.headers.get('Content-Length')), + status: DiffStatus.Changed, + }; } else if (response.status === 200) { - isNew = false; - const buffer = Buffer.from(await response.arrayBuffer()); - - if (buffer.equals(change.content)) { - diff = 0; - isDifferent = false; + if ( + heuristicFormats.some((format) => change.path.endsWith(`.${format}`)) + ) { + /** + * Heuristic: if it's a large blob like a .glb, a diff = 0 might end + * up being a false positive with isDifferent = true. So, it's okay + * to assume it's unchanged. + */ + diff = { change: 0, status: DiffStatus.Unchanged }; } else { - diff = change.content.length - buffer.length; - isDifferent = true; - } + const buffer = new Uint8Array(await response.arrayBuffer()); - console.log('diff body buffer', diff); + if (compareUint8Arrays(buffer, change.content)) { + diff = { change: 0, status: DiffStatus.Unchanged }; + } else { + diff = { + change: change.content.length - buffer.length, + status: DiffStatus.Changed, + }; + } + } } else { throw new Error( `Unexpected status code ${response.status} for ${change.path}`, ); } - /** - * Heuristic: if it's a large blob like a .glb, a diff = 0 might end up - * being a false positive with isDifferent = true. So, it's okay to - * assume it's unchanged. - */ - if ( - diff === 0 && - heuristicFormats.some((format) => change.path.endsWith(`.${format}`)) - ) { - isDifferent = false; - isNew = false; - } - - if (isNew) { - console.log( - `🟢 (+${change.content.length.toLocaleString()}B) ${change.path}`, - ); - changes.push(change); - } else if (isDifferent) { - console.log( - `🟡 (${diff > 0 ? '+' : ''}${diff.toLocaleString()}B) ${change.path}`, - ); - changes.push(change); + if (diff.status === DiffStatus.Unchanged) { + console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`); } else { console.log( - `🔵 (${change.content.length.toLocaleString()}B) ${change.path}`, + `${ + diff.status === DiffStatus.New ? '🟢' : '🟡' + } (+${diff.change.toLocaleString()}B) ${change.path}`, ); + changes.push(change); } }), ); diff --git a/packages/scripts/src/core/github/compareUint8Arrays.ts b/packages/scripts/src/core/github/compareUint8Arrays.ts new file mode 100644 index 000000000..fd927cb9c --- /dev/null +++ b/packages/scripts/src/core/github/compareUint8Arrays.ts @@ -0,0 +1,9 @@ +export function compareUint8Arrays(a: Uint8Array, b: Uint8Array) { + if (a.length !== b.length) return false; + + for (let index = 0; index < a.length; index++) { + if (a[index] !== b[index]) return false; + } + + return true; +} diff --git a/packages/scripts/src/core/github/createBlob.ts b/packages/scripts/src/core/github/createBlob.ts index a9f5f94c4..986d86023 100644 --- a/packages/scripts/src/core/github/createBlob.ts +++ b/packages/scripts/src/core/github/createBlob.ts @@ -20,7 +20,7 @@ export const createBlob = throttle( await octokit.git.createBlob({ owner, repo, - content: change.content.toString('base64'), + content: Buffer.from(change.content).toString('base64'), encoding: 'base64', }) ).data; diff --git a/packages/scripts/src/discoverIds.ts b/packages/scripts/src/discoverIds.ts index c57aaa541..70a5f8c07 100644 --- a/packages/scripts/src/discoverIds.ts +++ b/packages/scripts/src/discoverIds.ts @@ -134,14 +134,12 @@ function post() { const files = idsChunked.map((chunk, index) => { const didsWriteStream = new DidsWriteStream().dids(chunk); const buffer = didsWriteStream.uint8Array; - const compressed = compress(buffer); + const content = compress(buffer); console.log( - `Chunk ${index} with ${chunk.length} ids (compressed: ${compressed.length}; uncompressed: ${buffer.length})`, + `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`, ); - const content = Buffer.from(compressed); - return { content, path: `ids/${index}.dids.lz4`, @@ -151,13 +149,12 @@ function post() { commitAssets('discovered ids', [ ...files, { - content: Buffer.from( + content: new TextEncoder().encode( JSON.stringify({ chunks: idsChunked.length, count: ids.length, time: Date.now(), } satisfies DiscoveredIdsDefinitions), - 'utf-8', ), path: 'ids/manifest.json', }, diff --git a/packages/scripts/src/reviews.ts b/packages/scripts/src/reviews.ts index 6fea62cf5..1db57687a 100644 --- a/packages/scripts/src/reviews.ts +++ b/packages/scripts/src/reviews.ts @@ -1,6 +1,5 @@ import { asset, - encodePBBuffer, fetchTankDefinitions, Reviews, Video, @@ -81,7 +80,7 @@ for (const tank of tanksSanitized) { } await commitAssets('reviews', [ { - content: encodePBBuffer(Reviews, reviews), + content: Reviews.encode(reviews).finish(), path: 'definitions/reviews.pb', }, ]);