fix: identity encoding

packages/core/src/protobuf/encodePBBuffer.ts

@@ -1,5 +0,0 @@
-import { MessageFns } from '../protos';
-
-export function encodePBBuffer<Type>(message: MessageFns<Type>, data: Type) {
-  return Buffer.from(message.encode(data).finish());
-}

packages/core/src/protobuf/index.ts

@@ -1,3 +1,2 @@
-export * from './encodePBBuffer';
 export * from './fetchPB';
 export * from './pbMap';

packages/scripts/src/averages.ts

@@ -8,7 +8,6 @@ import {
   Samples,
   averageDefinitionsAllStatsKeys,
   emptySamples,
-  encodePBBuffer,
   getAccountInfo,
   getTankStats,
   idToRegion,
@@ -235,11 +234,11 @@ async function postWork() {
   commitAssets('averages', [
     {
       path: `averages/${latest}.pb`,
-      content: encodePBBuffer(AverageDefinitions, averageDefinitions),
+      content: AverageDefinitions.encode(averageDefinitions).finish(),
     },
     {
       path: 'averages/manifest.json',
-      content: Buffer.from(JSON.stringify(manifest), 'utf-8'),
+      content: new TextEncoder().encode(JSON.stringify(manifest)),
     },
   ]);
 }

packages/scripts/src/buildAssets/definitions.ts

@@ -7,7 +7,6 @@ import {
   ConsumableTankCategoryFilterCategory,
   Crew,
   CrewType,
-  encodePBBuffer,
   EquipmentDefinitions,
   EquipmentSlot,
   GameDefinitions,
@@ -1772,39 +1771,39 @@ export async function definitions() {
 
   await commitAssets('definitions', [
     {
-      content: encodePBBuffer(GameDefinitions, gameDefinitions),
+      content: GameDefinitions.encode(gameDefinitions).finish(),
       path: 'definitions/game.pb',
     },
     {
-      content: encodePBBuffer(TankDefinitions, tankDefinitions),
+      content: TankDefinitions.encode(tankDefinitions).finish(),
       path: 'definitions/tanks.pb',
     },
     {
-      content: encodePBBuffer(ModelDefinitions, modelDefinitions),
+      content: ModelDefinitions.encode(modelDefinitions).finish(),
       path: 'definitions/models.pb',
     },
     {
-      content: encodePBBuffer(EquipmentDefinitions, equipmentDefinitions),
+      content: EquipmentDefinitions.encode(equipmentDefinitions).finish(),
       path: 'definitions/equipment.pb',
     },
     {
-      content: encodePBBuffer(ConsumableDefinitions, consumableDefinitions),
+      content: ConsumableDefinitions.encode(consumableDefinitions).finish(),
       path: 'definitions/consumables.pb',
     },
     {
-      content: encodePBBuffer(ProvisionDefinitions, provisionDefinitions),
+      content: ProvisionDefinitions.encode(provisionDefinitions).finish(),
       path: 'definitions/provisions.pb',
     },
     {
-      content: encodePBBuffer(SkillDefinitions, skillDefinitions),
+      content: SkillDefinitions.encode(skillDefinitions).finish(),
       path: 'definitions/skills.pb',
     },
     {
-      content: encodePBBuffer(MapDefinitions, mapDefinitions),
+      content: MapDefinitions.encode(mapDefinitions).finish(),
       path: 'definitions/maps.pb',
     },
     {
-      content: encodePBBuffer(CamouflageDefinitions, camouflageDefinitions),
+      content: CamouflageDefinitions.encode(camouflageDefinitions).finish(),
       path: 'definitions/camouflages.pb',
     },
   ]);

packages/scripts/src/buildAssets/glossary.ts

@@ -1,4 +1,4 @@
-import { Avatar, encodePBBuffer, fetchGlossary, Gallery } from '@blitzkit/core';
+import { Avatar, fetchGlossary, Gallery } from '@blitzkit/core';
 import locales from '@blitzkit/i18n/locales.json';
 import { extname } from 'path';
 import ProgressBar from 'progress';
@@ -68,13 +68,13 @@ export async function glossary() {
 
       gallery.avatars.push(avatar);
 
-      const content = await fetch(url).then((response) =>
-        response.arrayBuffer(),
-      );
+      const content = await fetch(url)
+        .then((response) => response.arrayBuffer())
+        .then((buffer) => new Uint8Array(buffer));
 
       changes.push({
         path: `gallery/avatars/${key}${avatar.extension}`,
-        content: Buffer.from(content),
+        content,
       });
 
       bar.tick();
@@ -83,7 +83,7 @@ export async function glossary() {
 
   changes.push({
     path: 'definitions/gallery.pb',
-    content: encodePBBuffer(Gallery, gallery),
+    content: Gallery.encode(gallery).finish(),
   });
 
   await commitAssets('glossary', changes);

packages/scripts/src/buildAssets/migration.ts

@@ -32,7 +32,7 @@ export async function migration() {
   await commitAssets('migration', [
     {
       path: 'definitions/migration.json',
-      content: Buffer.from(JSON.stringify(map)),
+      content: new TextEncoder().encode(JSON.stringify(map)),
     },
   ]);
 }

packages/scripts/src/buildAssets/popularTanks.ts

@@ -1,6 +1,5 @@
 import {
   assertSecret,
-  encodePBBuffer,
   fetchTankDefinitions,
   PopularTanks,
 } from '@blitzkit/core';
@@ -70,7 +69,7 @@ export async function popularTanks() {
   await commitAssets('popular tanks', [
     {
       path: 'definitions/popular-tanks.pb',
-      content: encodePBBuffer(PopularTanks, popularTanks),
+      content: PopularTanks.encode(popularTanks).finish(),
     },
   ]);
 }

packages/scripts/src/buildAssets/tankArmors.ts

@@ -29,10 +29,11 @@ export async function tankArmors() {
 
           const id = toUniqueId(nation, tank.id);
           const model = await extractArmor(DATA, `${nation}-${tankKey}`);
+          const content = await nodeIO.writeBinary(model);
 
           changes.push({
             path: `3d/tanks/armor/${id}.glb`,
-            content: Buffer.from(await nodeIO.writeBinary(model)),
+            content,
           });
         }),
       );

packages/scripts/src/buildAssets/tankModels.ts

@@ -1,7 +1,7 @@
 import { toUniqueId } from '@blitzkit/core';
 import { NodeIO } from '@gltf-transform/core';
 import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
-import { readdir } from 'fs/promises';
+import { readdir, writeFile } from 'fs/promises';
 import { extractModel } from '../core/blitz/extractModel';
 import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
 import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
@@ -33,6 +33,8 @@ export async function tankModels() {
 
       const id = toUniqueId(nation, tank.id);
 
+      if (id !== 21777) continue;
+
       const parameters = await readYAMLDVPL<TankParameters>(
         `${DATA}/3d/Tanks/Parameters/${nation}/${tankKey}.yaml`,
       );
@@ -41,6 +43,8 @@ export async function tankModels() {
         parameters.resourcesPath.blitzModelPath.replace(/\.sc2$/, ''),
       );
 
+      await writeFile('../../temp/model.glb', await nodeIO.writeBinary(model));
+
       changes.push({
         path: `3d/tanks/models/${id}.glb`,
         content: Buffer.from(await nodeIO.writeBinary(model)),

packages/scripts/src/buildRatingLeaderboard.ts

@@ -164,7 +164,7 @@ const content = RatingLeaderboard.encode({
 const changes: FileChange[] = [
   {
     path: `regions/${normalizedServer}/rating/${info.current_season}.pb`,
-    content: Buffer.from(content),
+    content,
   },
 ];
 

packages/scripts/src/core/blitz/readTexture/index.ts

@@ -1,12 +1,15 @@
 import { DdsReadStream, PvrReadStream } from '@blitzkit/core';
 import { existsSync } from 'fs';
 import sharp from 'sharp';
+import { DATA_COMPRESSED } from '../../../buildAssets/constants';
 import { readDVPLFile } from '../readDVPLFile';
 import { TextureMutation } from './constants';
 
 export async function readTexture(path: string, mutation?: TextureMutation) {
   const ddsTexturePath = path.replace('.tex', '.dx11.dds');
-  const isDds = existsSync(`${ddsTexturePath}.dvpl`);
+  const isDds = existsSync(
+    `${ddsTexturePath}${DATA_COMPRESSED ? '.dvpl' : ''}`,
+  );
   const resolvedTexturePath = isDds
     ? ddsTexturePath
     : ddsTexturePath.replace('.dds', '.pvr');
@@ -75,5 +78,4 @@ export async function readTexture(path: string, mutation?: TextureMutation) {
     default:
       return await sharp(raw.data, { raw }).jpeg().toBuffer();
   }
-
 }

packages/scripts/src/core/github/commitMultipleFiles.ts

@@ -1,11 +1,18 @@
 import { asset } from '@blitzkit/core';
 import ProgressBar from 'progress';
+import { compareUint8Arrays } from './compareUint8Arrays';
 import { GithubChangeBlob, createBlob } from './createBlob';
 import { octokit } from './octokit';
 
 export interface FileChange {
   path: string;
-  content: Buffer;
+  content: Uint8Array;
+}
+
+enum DiffStatus {
+  New,
+  Changed,
+  Unchanged,
 }
 
 const heuristicFormats = ['glb', 'webp', 'jpg', 'png'];
@@ -20,73 +27,60 @@ export async function commitMultipleFiles(
 
   await Promise.all(
     changesRaw.map(async (change) => {
-      const response = await fetch(asset(change.path));
-      let isNew: boolean;
-      let isDifferent: boolean;
-      let diff: number;
+      const response = await fetch(asset(change.path), {
+        headers: { 'Accept-Encoding': 'identity' },
+      });
+      let diff: { change: number; status: DiffStatus };
 
       if (response.status === 404) {
-        isNew = true;
-        diff = change.content.length;
-        isDifferent = true;
+        diff = { change: change.content.length, status: DiffStatus.New };
       } else if (
         response.headers.has('Content-Length') &&
-        parseInt(response.headers.get('Content-Length')!) !==
-          change.content.length
+        Number(response.headers.get('Content-Length')) !== change.content.length
       ) {
-        isNew = false;
-        diff =
-          change.content.length -
-          parseInt(response.headers.get('Content-Length')!);
-        isDifferent = true;
-
-        console.log('diff header content length', 'content:', change.content.length, 'header:', response.headers.get('Content-Length'));
+        diff = {
+          change:
+            change.content.length -
+            Number(response.headers.get('Content-Length')),
+          status: DiffStatus.Changed,
+        };
       } else if (response.status === 200) {
-        isNew = false;
-        const buffer = Buffer.from(await response.arrayBuffer());
-
-        if (buffer.equals(change.content)) {
-          diff = 0;
-          isDifferent = false;
+        if (
+          heuristicFormats.some((format) => change.path.endsWith(`.${format}`))
+        ) {
+          /**
+           * Heuristic: if it's a large blob like a .glb, a diff = 0 might end
+           * up being a false positive with isDifferent = true. So, it's okay
+           * to assume it's unchanged.
+           */
+          diff = { change: 0, status: DiffStatus.Unchanged };
         } else {
-          diff = change.content.length - buffer.length;
-          isDifferent = true;
-        }
+          const buffer = new Uint8Array(await response.arrayBuffer());
 
-        console.log('diff body buffer', diff);
+          if (compareUint8Arrays(buffer, change.content)) {
+            diff = { change: 0, status: DiffStatus.Unchanged };
+          } else {
+            diff = {
+              change: change.content.length - buffer.length,
+              status: DiffStatus.Changed,
+            };
+          }
+        }
       } else {
         throw new Error(
           `Unexpected status code ${response.status} for ${change.path}`,
         );
       }
 
-      /**
-       * Heuristic: if it's a large blob like a .glb, a diff = 0 might end up
-       * being a false positive with isDifferent = true. So, it's okay to
-       * assume it's unchanged.
-       */
-      if (
-        diff === 0 &&
-        heuristicFormats.some((format) => change.path.endsWith(`.${format}`))
-      ) {
-        isDifferent = false;
-        isNew = false;
-      }
-
-      if (isNew) {
-        console.log(
-          `🟢 (+${change.content.length.toLocaleString()}B) ${change.path}`,
-        );
-        changes.push(change);
-      } else if (isDifferent) {
-        console.log(
-          `🟡 (${diff > 0 ? '+' : ''}${diff.toLocaleString()}B) ${change.path}`,
-        );
-        changes.push(change);
+      if (diff.status === DiffStatus.Unchanged) {
+        console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`);
       } else {
         console.log(
-          `🔵 (${change.content.length.toLocaleString()}B) ${change.path}`,
+          `${
+            diff.status === DiffStatus.New ? '🟢' : '🟡'
+          } (+${diff.change.toLocaleString()}B) ${change.path}`,
         );
+        changes.push(change);
       }
     }),
   );

packages/scripts/src/core/github/compareUint8Arrays.ts

@@ -0,0 +1,9 @@
+export function compareUint8Arrays(a: Uint8Array, b: Uint8Array) {
+  if (a.length !== b.length) return false;
+
+  for (let index = 0; index < a.length; index++) {
+    if (a[index] !== b[index]) return false;
+  }
+
+  return true;
+}

packages/scripts/src/core/github/createBlob.ts

@@ -20,7 +20,7 @@ export const createBlob = throttle(
           await octokit.git.createBlob({
             owner,
             repo,
-            content: change.content.toString('base64'),
+            content: Buffer.from(change.content).toString('base64'),
             encoding: 'base64',
           })
         ).data;

packages/scripts/src/discoverIds.ts

@@ -134,14 +134,12 @@ function post() {
   const files = idsChunked.map((chunk, index) => {
     const didsWriteStream = new DidsWriteStream().dids(chunk);
     const buffer = didsWriteStream.uint8Array;
-    const compressed = compress(buffer);
+    const content = compress(buffer);
 
     console.log(
-      `Chunk ${index} with ${chunk.length} ids (compressed: ${compressed.length}; uncompressed: ${buffer.length})`,
+      `Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`,
     );
 
-    const content = Buffer.from(compressed);
-
     return {
       content,
       path: `ids/${index}.dids.lz4`,
@@ -151,13 +149,12 @@ function post() {
   commitAssets('discovered ids', [
     ...files,
     {
-      content: Buffer.from(
+      content: new TextEncoder().encode(
         JSON.stringify({
           chunks: idsChunked.length,
           count: ids.length,
           time: Date.now(),
         } satisfies DiscoveredIdsDefinitions),
-        'utf-8',
       ),
       path: 'ids/manifest.json',
     },

packages/scripts/src/reviews.ts

@@ -1,6 +1,5 @@
 import {
   asset,
-  encodePBBuffer,
   fetchTankDefinitions,
   Reviews,
   Video,
@@ -81,7 +80,7 @@ for (const tank of tanksSanitized) {
 }
 await commitAssets('reviews', [
   {
-    content: encodePBBuffer(Reviews, reviews),
+    content: Reviews.encode(reviews).finish(),
     path: 'definitions/reviews.pb',
   },
 ]);