fix: types

packages/scripts/package.json

@@ -3,10 +3,15 @@
   "private": true,
   "type": "module",
   "dependencies": {
+    "progress": "^2.0.3",
     "@gltf-transform/core": "^3.10.1",
-    "lodash": "^4.17.21"
+    "decompress": "^4.2.1",
+    "googleapis": "^134.0.0",
+    "fast-xml-parser": "^4.4.1"
   },
   "devDependencies": {
+    "@types/decompress": "^4.2.7",
+    "@types/progress": "^2.0.7",
     "@types/lodash": "^4.17.7"
   }
 }

packages/scripts/src/applyPatches.ts

@@ -1,13 +1,13 @@
+import { assertSecret } from '@blitzkit/core';
 import { mkdir, writeFile } from 'fs/promises';
 import { parse as parsePath } from 'path';
 import ProgressBar from 'progress';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+import { dvp } from '../../../submodules/blitzkit-closed/src/dvp';
+import { writeDVPL } from '../../website/src/core/blitz/writeDVPL';
 import { readStringDVPL } from '../src/core/blitz/readStringDVPL';
-import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL';
-import { writeDVPL } from '../src/core/blitz/writeDVPL';
-import { assertSecret } from '../src/core/blitzkit/secret';
-import { dvp } from '../submodules/blitzkit-closed/src/dvp';
 import { DATA } from './buildAssets/constants';
+import { readYAMLDVPL } from './core/blitz/readYAMLDVPL';
 
 const versionTextFile = await readStringDVPL(`${DATA}/version.txt.dvpl`);
 const currentVersion = versionTextFile
@@ -21,7 +21,9 @@ console.log(`Installing patches for ${currentVersion}...`);
 let patchIndex = 1;
 while (true) {
   const response = await fetch(
-    `${assertSecret(process.env.WOTB_DLC_CDN)}/dlc/s${currentVersion}_${patchIndex}.yaml`,
+    `${assertSecret(
+      process.env.WOTB_DLC_CDN,
+    )}/dlc/s${currentVersion}_${patchIndex}.yaml`,
   );
 
   if (response.status === 200) {
@@ -32,7 +34,10 @@ while (true) {
       `${assertSecret(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11}`,
     ).then((response) => response.arrayBuffer());
     const dvpd = await fetch(
-      `${assertSecret(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11.replace('.dvpm', '.dvpd')}`,
+      `${assertSecret(process.env.WOTB_DLC_CDN)}/dlc/${data.dx11.replace(
+        '.dvpm',
+        '.dvpd',
+      )}`,
     ).then((response) => response.arrayBuffer());
     const files = await dvp(dvpm, dvpd);
     const bar = new ProgressBar(
@@ -58,7 +63,9 @@ while (true) {
       console.log('Found dynamic content localizations; patching...');
 
       const localizationsResponse = await fetch(
-        `${assertSecret(process.env.WOTB_DLC_CDN)}/dlc/${data.dynamicContentLocalizationsDir}/en.yaml`,
+        `${assertSecret(process.env.WOTB_DLC_CDN)}/dlc/${
+          data.dynamicContentLocalizationsDir
+        }/en.yaml`,
       );
       const newStrings = parseYaml(await localizationsResponse.text());
       const oldStrings = await readYAMLDVPL<Record<string, string>>(

packages/scripts/src/averages.ts

@@ -1,23 +1,22 @@
-import { chunk, times } from 'lodash';
-import { Region, REGIONS } from '../src/constants/regions';
-import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
-import getTankStats from '../src/core/blitz/getTankStats';
-import { idToRegion } from '../src/core/blitz/idToRegion';
 import {
   AverageDefinitions,
   AverageDefinitionsEntry,
   AverageDefinitionsEntrySubPartial,
   AverageDefinitionsManifest,
+  IndividualTankStats,
+  REGIONS,
+  Region,
   Samples,
-} from '../src/core/blitzkit/averageDefinitions';
-import {
   averageDefinitionsAllStatsKeys,
   emptySamples,
-} from '../src/core/blitzkit/averageDefinitions/constants';
-import { commitAssets } from '../src/core/blitzkit/commitAssets';
-import { fetchPreDiscoveredIds } from '../src/core/blitzkit/fetchPreDiscoveredIds';
-import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64';
-import { IndividualTankStats } from '../src/types/tanksStats';
+  encodeProtobufToBase64,
+  idToRegion,
+} from '@blitzkit/core';
+import { chunk, times } from 'lodash';
+import { getAccountInfo } from '../../website/src/core/blitz/getAccountInfo';
+import getTankStats from '../../website/src/core/blitz/getTankStats';
+import { commitAssets } from './core/github/commitAssets';
+import { fetchPreDiscoveredIds } from './core/github/fetchPreDiscoveredIds';
 
 interface DataPoint {
   x: number;
@@ -130,7 +129,9 @@ times(THREADS, async () => {
 
 async function postWork() {
   console.log(
-    `Generating statistics based on ${samples.d_120.toLocaleString()} players (${samples.total.toLocaleString()} checked in total) and ${tankIds.length} tanks...`,
+    `Generating statistics based on ${samples.d_120.toLocaleString()} players (${samples.total.toLocaleString()} checked in total) and ${
+      tankIds.length
+    } tanks...`,
   );
 
   const averages: Record<number, AverageDefinitionsEntry> = {};
@@ -224,7 +225,7 @@ async function postWork() {
   commitAssets('averages', [
     {
       path: `averages/${latest}.pb`,
-      content: await encodeToBase64(
+      content: await encodeProtobufToBase64(
         'average_definitions',
         'blitzkit.AverageDefinitions',
         averageDefinitions,

packages/scripts/src/buildAssets/boosterIcons.ts

@@ -2,8 +2,8 @@ import { readdir } from 'fs/promises';
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
 import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 export async function boosterIcons() {

packages/scripts/src/buildAssets/camouflageIcons.ts

@@ -1,6 +1,6 @@
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 export async function camouflageIcons() {

packages/scripts/src/buildAssets/consumableProvisionIcons.ts

@@ -2,12 +2,12 @@ import { existsSync } from 'fs';
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
 import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
 import { DATA } from './constants';
 import { ConsumablesCommon, ProvisionsCommon } from './definitions';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 
 interface Mappings {
   Header: { version: number };

packages/scripts/src/buildAssets/currencies.ts

@@ -1,8 +1,8 @@
+import { assertSecret } from '@blitzkit/core';
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
-import { assertSecret } from '../../src/core/blitzkit/secret';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 const ICONS = [

packages/scripts/src/buildAssets/definitions.ts

@@ -1,47 +1,39 @@
+import {
+  assertSecret,
+  CamouflageDefinitions,
+  ConsumableDefinitions,
+  ConsumableEntry,
+  Crew,
+  CrewMember,
+  EquipmentDefinitions,
+  EquipmentPreset,
+  EquipmentRow,
+  GameDefinitions,
+  GunDefinition,
+  ModelArmor,
+  ModelDefinitions,
+  ProvisionDefinitions,
+  ProvisionEntry,
+  ShellType,
+  superCompress,
+  TankClass,
+  TankDefinitions,
+  TankFilterDefinitionCategory,
+  TankPrice,
+  Tier,
+  Unlock,
+} from '@blitzkit/core';
 import { SkillDefinitions } from '@blitzkit/core/src/blitzkit/skillDefinitions';
 import { readdir } from 'fs/promises';
 import { parse as parsePath } from 'path';
 import { Vector3Tuple } from 'three';
 import { parse as parseYaml } from 'yaml';
-import { TankClass } from '../../src/components/Tanks';
-import { WARGAMING_APPLICATION_ID } from '../../src/constants/wargamingApplicationID';
-import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { toUniqueId } from '../../src/core/blitz/toUniqueId';
-import { CamouflageDefinitions } from '../../src/core/blitzkit/camouflageDefinitions';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import {
-  ConsumableDefinitions,
-  ConsumableEntry,
-  TankFilterDefinitionCategory,
-} from '../../src/core/blitzkit/consumableDefinitions';
-import {
-  EquipmentDefinitions,
-  EquipmentPreset,
-  EquipmentRow,
-} from '../../src/core/blitzkit/equipmentDefinitions';
-import { GameDefinitions } from '../../src/core/blitzkit/gameDefinitions';
-import { MapDefinitions } from '../../src/core/blitzkit/mapDefinitions';
-import {
-  ModelArmor,
-  ModelDefinitions,
-} from '../../src/core/blitzkit/modelDefinitions';
-import {
-  ProvisionDefinitions,
-  ProvisionEntry,
-} from '../../src/core/blitzkit/provisionDefinitions';
-import { superCompress } from '../../src/core/blitzkit/superCompress';
-import {
-  Crew,
-  CrewMember,
-  GunDefinition,
-  ShellType,
-  TankDefinitions,
-  TankPrice,
-  Tier,
-  Unlock,
-} from '../../src/core/blitzkit/tankDefinitions';
+import { toUniqueId } from '../../../website/src/core/blitz/toUniqueId';
+import { MapDefinitions } from '../../../website/src/core/blitzkit/mapDefinitions';
+import { readStringDVPL } from '../core/blitz/readStringDVPL';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
 import { DATA } from './constants';
 import { Avatar } from './skillIcons';
 import { TankParameters } from './tankIcons';
@@ -458,7 +450,9 @@ export async function definitions() {
   console.log('Building definitions...');
 
   const wargamingTankopedia = (await fetch(
-    `https://api.wotblitz.com/wotb/encyclopedia/vehicles/?application_id=${WARGAMING_APPLICATION_ID}&fields=description`,
+    `https://api.wotblitz.com/wotb/encyclopedia/vehicles/?application_id=${assertSecret(
+      process.env.NEXT_PUBLIC_WARGAMING_APPLICATION_ID,
+    )}&fields=description`,
   ).then((response) => response.json())) as {
     data: { [key: number]: { description: null | string } };
   };

packages/scripts/src/buildAssets/equipmentIcons.ts

@@ -1,10 +1,10 @@
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
 import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 import { OptionalDevices } from './definitions';
 

packages/scripts/src/buildAssets/flags.ts

@@ -1,9 +1,9 @@
 import { readdir } from 'fs/promises';
 import sharp from 'sharp';
-import { readBase64DVPL } from '../../src/core/blitz/readBase64DVPL';
-import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { readBase64DVPL } from '../core/blitz/readBase64DVPL';
+import { readDVPLFile } from '../core/blitz/readDVPLFile';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 export async function flags() {
@@ -11,7 +11,9 @@ export async function flags() {
 
   const changes = [
     ...(await Promise.all(
-      (await readdir(`${DATA}/Gfx/Lobby/flags`))
+      (
+        await readdir(`${DATA}/Gfx/Lobby/flags`)
+      )
         .filter(
           (flag) =>
             flag.startsWith('flag_profile-stat_') &&
@@ -34,7 +36,9 @@ export async function flags() {
         }),
     )),
     ...(await Promise.all(
-      (await readdir(`${DATA}/Gfx/Lobby/flags`))
+      (
+        await readdir(`${DATA}/Gfx/Lobby/flags`)
+      )
         .filter(
           (flag) =>
             flag.startsWith('flag_tutor-tank_') &&
@@ -54,7 +58,9 @@ export async function flags() {
         }),
     )),
     ...(await Promise.all(
-      (await readdir(`${DATA}/Gfx/Lobby/flags`))
+      (
+        await readdir(`${DATA}/Gfx/Lobby/flags`)
+      )
         .filter(
           (flag) =>
             flag.startsWith('flag_filter_') &&

packages/scripts/src/buildAssets/gameModeBanners.ts

@@ -1,8 +1,8 @@
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 import { SquadBattleTypeStyles } from './definitions';
 

packages/scripts/src/buildAssets/moduleIcons.ts

@@ -1,15 +1,17 @@
 import { readdir } from 'fs/promises';
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 export async function moduleIcons() {
   console.log('Building module icons...');
 
   const changes = await Promise.all(
-    (await readdir(`${DATA}/Gfx/UI/ModulesTechTree`))
+    (
+      await readdir(`${DATA}/Gfx/UI/ModulesTechTree`)
+    )
       .filter(
         (file) =>
           !file.endsWith('@2x.packed.webp.dvpl') && file.startsWith('vehicle'),

packages/scripts/src/buildAssets/shellIcons.ts

@@ -2,8 +2,8 @@ import { readdir } from 'fs/promises';
 import sharp from 'sharp';
 import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
 import { readStringDVPL } from '../../src/core/blitz/readStringDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 export async function shellIcons() {
@@ -16,7 +16,9 @@ export async function shellIcons() {
   );
 
   const changes = await Promise.all(
-    (await readdir(`${DATA}/Gfx/Shared/tank-supply/ammunition/big`))
+    (
+      await readdir(`${DATA}/Gfx/Shared/tank-supply/ammunition/big`)
+    )
       .filter((file) => file.endsWith('_l.txt.dvpl'))
       .map(async (file) => {
         const name = file.match(/(.+)_l\.txt\.dvpl/)![1];

packages/scripts/src/buildAssets/skillIcons.ts

@@ -1,9 +1,9 @@
+import { TankClass } from '@blitzkit/core';
 import sharp from 'sharp';
-import { TankClass } from '../../src/components/Tanks';
-import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { readDVPLFile } from '../core/blitz/readDVPLFile';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 
 interface SkillIcon {

packages/scripts/src/buildAssets/tankArmors.ts

@@ -1,10 +1,10 @@
 import { NodeIO } from '@gltf-transform/core';
 import { readdir } from 'fs/promises';
-import { extractArmor } from '../../src/core/blitz/extractArmor';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { toUniqueId } from '../../src/core/blitz/toUniqueId';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { extractArmor } from '../../../website/src/core/blitz/extractArmor';
+import { toUniqueId } from '../../../website/src/core/blitz/toUniqueId';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 import { VehicleDefinitionList } from './definitions';
 

packages/scripts/src/buildAssets/tankIcons.ts

@@ -1,12 +1,12 @@
+import { NATION_IDS } from '@blitzkit/core';
 import { readdir } from 'fs/promises';
 import sharp from 'sharp';
 import { Vector3Tuple } from 'three';
-import { NATION_IDS } from '../../src/constants/nations';
-import { readDVPLFile } from '../../src/core/blitz/readDVPLFile';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { readDVPLFile } from '../core/blitz/readDVPLFile';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 import { VehicleDefinitionList } from './definitions';
 

packages/scripts/src/buildAssets/tankModels.ts

@@ -1,11 +1,11 @@
 import { NodeIO } from '@gltf-transform/core';
 import { readdir } from 'fs/promises';
-import { extractModel } from '../../src/core/blitz/extractModel';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { toUniqueId } from '../../src/core/blitz/toUniqueId';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { FileChange } from '../../src/core/blitzkit/commitMultipleFiles';
+import { extractModel } from '../../../website/src/core/blitz/extractModel';
+import { toUniqueId } from '../../../website/src/core/blitz/toUniqueId';
+import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
 import { DATA } from './constants';
 import { VehicleDefinitionList } from './definitions';
 import { TankParameters } from './tankIcons';

packages/scripts/src/buildRatingLeaderboard.ts

@@ -1,24 +1,25 @@
+import {
+  BkrlBaseEntry,
+  BkrlFormat,
+  BkrlSuperset1Entry,
+  BkrlWriteStream,
+  CdonValue,
+  CdonWriteStream,
+  RegionSubdomain,
+  superCompress,
+} from '@blitzkit/core';
 import { chunk } from 'lodash';
 import { argv } from 'process';
 import {
   RatingInfo,
   RatingNeighbors,
   RatingPlayer,
-} from '../src/commands/ratingLeaderboard';
-import { RegionSubdomain } from '../src/constants/regions';
-import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
-import regionSubdomainToRegion from '../src/core/blitz/regionSubdomainToRegion';
-import { commitAssets } from '../src/core/blitzkit/commitAssets';
-import { FileChange } from '../src/core/blitzkit/commitMultipleFiles.js';
-import { patientFetchJSON } from '../src/core/blitzkit/patientFetchJSON';
-import { superCompress } from '../src/core/blitzkit/superCompress';
-import {
-  BkrlBaseEntry,
-  BkrlFormat,
-  BkrlSuperset1Entry,
-  BkrlWriteStream,
-} from '../src/core/streams/bkrl';
-import { CdonValue, CdonWriteStream } from '../src/core/streams/cdon';
+} from '../../bot/src/commands/ratingLeaderboard';
+import { getAccountInfo } from '../../website/src/core/blitz/getAccountInfo';
+import regionSubdomainToRegion from '../../website/src/core/blitz/regionSubdomainToRegion';
+import { patientFetchJSON } from '../../website/src/core/blitzkit/patientFetchJSON';
+import { commitAssets } from './core/github/commitAssets';
+import { FileChange } from './core/github/commitMultipleFiles';
 
 /*
  * Central North American Time (UTC-5): use 0 5 * * *.

packages/scripts/src/core/blitz/readBase64DVPL.ts

@@ -0,0 +1,6 @@
+import { readDVPLFile } from './readDVPLFile';
+
+export async function readBase64DVPL(file: string) {
+  const buffer = await readDVPLFile(file);
+  return buffer.toString('base64');
+}

packages/scripts/src/core/blitz/readDVPL.ts

@@ -0,0 +1,60 @@
+import { crc32 } from 'crc';
+import { decompressBlock } from 'lz4js';
+
+/**
+ * Thanks Maddoxkkm! Modified heavily to be modern.
+ * https://github.com/Maddoxkkm/dvpl_converter/
+ */
+export function readDVPL(buffer: Buffer) {
+  const footerBuffer = buffer.subarray(buffer.length - 20, buffer.length);
+
+  if (
+    footerBuffer.toString('utf8', 16, 20) !== 'DVPL' ||
+    footerBuffer.length !== 20
+  ) {
+    throw new SyntaxError('Invalid DVPL footer');
+  }
+
+  const footerData = {
+    oSize: footerBuffer.readUInt32LE(0),
+    cSize: footerBuffer.readUInt32LE(4),
+    crc32: footerBuffer.readUInt32LE(8),
+    type: footerBuffer.readUInt32LE(12),
+  };
+  const targetBlock = buffer.subarray(0, buffer.length - 20);
+
+  if (targetBlock.length !== footerData.cSize) {
+    throw new RangeError('DVPL size mismatch');
+  }
+  if (crc32(targetBlock) !== footerData.crc32) {
+    throw new TypeError('DVPL CRC32 mismatch');
+  }
+
+  if (footerData.type === 0) {
+    if (!(footerData.oSize === footerData.cSize && footerData.type === 0)) {
+      throw new RangeError('DVPL type and size mismatch');
+    }
+
+    return targetBlock;
+  } else if (footerData.type === 1 || footerData.type === 2) {
+    const source = new Uint8Array(targetBlock);
+    const destination = new Uint8Array(footerData.oSize);
+    const decompressedSize = decompressBlock(
+      source,
+      destination,
+      0,
+      source.length,
+      0,
+    );
+
+    if (decompressedSize !== footerData.oSize) {
+      throw new RangeError(
+        `Decompressed DVPL size mismatch (${decompressedSize} vs ${footerData.oSize})`,
+      );
+    }
+
+    return Buffer.from(destination);
+  } else {
+    throw new SyntaxError('Unknown DVPL format');
+  }
+}

packages/scripts/src/core/blitz/readDVPLFile.ts

@@ -0,0 +1,6 @@
+import { readFile } from 'fs/promises';
+import { readDVPL } from './readDVPL';
+
+export async function readDVPLFile(file: string) {
+  return readDVPL(await readFile(file));
+}

packages/scripts/src/core/blitz/readStringDVPL.ts

@@ -0,0 +1,6 @@
+import { readDVPLFile } from './readDVPLFile';
+
+export async function readStringDVPL(file: string) {
+  const buffer = await readDVPLFile(file);
+  return buffer.toString();
+}

packages/scripts/src/core/blitz/readXMLDVPL.ts

@@ -0,0 +1,9 @@
+import { XMLParser } from 'fast-xml-parser';
+import { readStringDVPL } from './readStringDVPL';
+
+export async function readXMLDVPL<Type>(file: string) {
+  const parser = new XMLParser();
+  const xml = await readStringDVPL(file);
+
+  return parser.parse(xml) as Type;
+}

packages/scripts/src/core/blitz/readYAML.ts

@@ -0,0 +1,7 @@
+import { readFile } from 'fs/promises';
+import { parse } from 'yaml';
+
+export async function readYAML<Type>(file: string) {
+  const yaml = (await readFile(file)).toString();
+  return parse(yaml) as Type;
+}

packages/scripts/src/core/blitz/readYAMLDVPL.ts

@@ -0,0 +1,8 @@
+import { parse } from 'yaml';
+import { readStringDVPL } from './readStringDVPL';
+
+export async function readYAMLDVPL<Type>(file: string) {
+  const yaml = await readStringDVPL(file);
+
+  return parse(yaml) as Type;
+}

packages/scripts/src/core/github/commitAssets.ts

@@ -0,0 +1,16 @@
+import { assertSecret } from '@blitzkit/core';
+import commitMultipleFiles, { FileChange } from './commitMultipleFiles';
+
+export async function commitAssets(message: string, changes: FileChange[]) {
+  console.log(`Committing ${message}...`);
+
+  if (changes.length === 0) return;
+
+  await commitMultipleFiles(
+    'tresabhi',
+    assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+    assertSecret(process.env.NEXT_PUBLIC_ASSET_BRANCH),
+    `${message} - ${new Date().toDateString()}`,
+    changes,
+  );
+}

packages/scripts/src/core/github/commitMultipleFiles.ts

@@ -0,0 +1,66 @@
+import ProgressBar from 'progress';
+import {
+  GithubChangeBlob,
+  createBlob,
+} from '../../../../website/src/core/blitzkit/createBlob';
+import { octokit } from '../../../../website/src/core/blitzkit/octokit';
+
+export interface FileChange {
+  path: string;
+  content: string;
+  encoding: 'utf-8' | 'base64';
+}
+
+export default async function commitMultipleFiles(
+  owner: string,
+  repo: string,
+  branch: string,
+  message: string,
+  changes: FileChange[],
+) {
+  const latestCommitSha = (
+    await octokit.git.getRef({
+      owner,
+      repo,
+      ref: `heads/${branch}`,
+    })
+  ).data.object.sha;
+  const treeSha = (
+    await octokit.git.getCommit({
+      owner,
+      repo,
+      commit_sha: latestCommitSha,
+    })
+  ).data.tree.sha;
+  const blobs: GithubChangeBlob[] = [];
+  const bar = new ProgressBar('Blobs :bar', changes.length);
+
+  for (const changeIndexString in changes) {
+    const changeIndex = parseInt(changeIndexString);
+    const change = changes[changeIndex];
+    const createdBlob = await createBlob(owner, repo, change);
+    blobs.push(createdBlob!);
+    bar.tick();
+  }
+
+  const { data: treeData } = await octokit.git.createTree({
+    owner,
+    repo,
+    base_tree: treeSha,
+    tree: blobs,
+  });
+  const { data: newCommitData } = await octokit.git.createCommit({
+    owner,
+    repo,
+    message,
+    tree: treeData.sha,
+    parents: [latestCommitSha],
+  });
+  await octokit.git.updateRef({
+    owner,
+    repo,
+    ref: `heads/${branch}`,
+    sha: newCommitData.sha,
+    force: true,
+  });
+}

packages/scripts/src/core/github/fetchPreDiscoveredIds.ts

@@ -0,0 +1,36 @@
+import { asset, DidsReadStream } from '@blitzkit/core';
+import { times } from 'lodash';
+import { decompress } from 'lz4js';
+import ProgressBar from 'progress';
+import { DiscoveredIdsDefinitions } from '../../../../website/src/core/blitzkit/discoveredIdDefinitions';
+
+export async function fetchPreDiscoveredIds() {
+  const idChunks: number[][] = [];
+  const manifest = (await fetch(asset('ids/manifest.json')).then((response) =>
+    response.json(),
+  )) as DiscoveredIdsDefinitions;
+  const bar = new ProgressBar(
+    `Fetching ${manifest.count.toLocaleString()} ids in ${
+      manifest.chunks
+    } chunks :bar`,
+    manifest.chunks,
+  );
+
+  bar.render();
+  await Promise.all(
+    times(manifest.chunks, async (chunkIndex) => {
+      const preDiscovered = await fetch(
+        asset(`ids/${chunkIndex}.dids.lz4`),
+      ).then(async (response) => {
+        const buffer = await response.arrayBuffer();
+        const decompressed = decompress(new Uint8Array(buffer)).buffer;
+        return new DidsReadStream(decompressed as ArrayBuffer).dids();
+      });
+
+      idChunks.push(preDiscovered);
+      bar.tick();
+    }),
+  );
+
+  return idChunks.sort((a, b) => a[0] - b[0]).flat();
+}

packages/scripts/src/discoverIds.ts

@@ -1,13 +1,18 @@
+import {
+  DidsReadStream,
+  DidsWriteStream,
+  MIN_REGION_IDS,
+  REGIONS,
+  Region,
+  asset,
+  idToRegion,
+} from '@blitzkit/core';
 import { chunk, times, uniq } from 'lodash';
 import { compress, decompress } from 'lz4js';
-import { REGIONS, Region } from '../src/constants/regions';
-import { retryAbleBlitzFetchEvent } from '../src/core/blitz/fetchBlitz';
-import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
-import { MIN_IDS, idToRegion } from '../src/core/blitz/idToRegion';
-import { asset } from '../src/core/blitzkit/asset';
-import { commitAssets } from '../src/core/blitzkit/commitAssets';
-import { DiscoveredIdsDefinitions } from '../src/core/blitzkit/discoveredIdDefinitions';
-import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
+import { retryAbleBlitzFetchEvent } from '../../website/src/core/blitz/fetchBlitz';
+import { getAccountInfo } from '../../website/src/core/blitz/getAccountInfo';
+import { DiscoveredIdsDefinitions } from '../../website/src/core/blitzkit/discoveredIdDefinitions';
+import { commitAssets } from './core/github/commitAssets';
 
 const CHUNK_SIZE = 2 ** 21;
 const RUN_TIME = 1000 * 60 * 60 * 5.5;
@@ -47,9 +52,9 @@ while (chunkIndex < preDiscoveredManifest.chunks) {
 }
 
 const regionalIdIndex: Record<Region, number> = {
-  asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
-  com: ids.findLast((id) => idToRegion(id) === 'com') ?? MIN_IDS.com,
-  eu: ids.findLast((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu,
+  asia: ids.findLast((id) => idToRegion(id) === 'asia') ?? MIN_REGION_IDS.asia,
+  com: ids.findLast((id) => idToRegion(id) === 'com') ?? MIN_REGION_IDS.com,
+  eu: ids.findLast((id) => idToRegion(id) === 'eu') ?? MIN_REGION_IDS.eu,
 };
 const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
 let regionIndex = 0;

packages/scripts/src/downloadZipClient.ts

@@ -1,3 +0,0 @@
-import { assertSecret } from '../src/core/blitzkit/secret';
-
-assertSecret(process.env.ZIP_URL);

packages/scripts/src/reviews.ts

@@ -1,22 +1,25 @@
+import {
+  asset,
+  decodeProtobuf,
+  encodeProtobufToBase64,
+  Reviews,
+  Video,
+  youtubers,
+} from '@blitzkit/core';
 import { readdir } from 'fs/promises';
 import { google } from 'googleapis';
 import { cloneDeep, uniqBy } from 'lodash';
 import { parse as parseYaml } from 'yaml';
-import { youtubers } from '../src/constants/youtubers';
-import { readXMLDVPL } from '../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL';
-import { toUniqueId } from '../src/core/blitz/toUniqueId';
-import { asset } from '../src/core/blitzkit/asset';
-import { commitAssets } from '../src/core/blitzkit/commitAssets';
-import { Reviews, Video } from '../src/core/blitzkit/reviews';
-import { decode } from '../src/core/protobuf/decode';
-import { encodeToBase64 } from '../src/core/protobuf/encodeToBase64';
+import { toUniqueId } from '../../website/src/core/blitz/toUniqueId';
 import { DATA } from './buildAssets/constants';
 import {
   BlitzStrings,
   botPattern,
   VehicleDefinitionList,
 } from './buildAssets/definitions';
+import { readXMLDVPL } from './core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from './core/blitz/readYAMLDVPL';
+import { commitAssets } from './core/github/commitAssets';
 
 const MAX_QUERIES = 128;
 
@@ -25,7 +28,11 @@ console.log('Finding reviews...');
 const currentReviews = await fetch(asset('definitions/reviews.pb'))
   .then((response) => response.arrayBuffer())
   .then((buffer) =>
-    decode<Reviews>('reviews', 'blitzkit.Reviews', new Uint8Array(buffer)),
+    decodeProtobuf<Reviews>(
+      'reviews',
+      'blitzkit.Reviews',
+      new Uint8Array(buffer),
+    ),
   );
 const auth = await google.auth.getClient({
   scopes: ['https://www.googleapis.com/auth/youtube.readonly'],
@@ -111,7 +118,7 @@ for (const tank of tanksSanitized) {
           ({
             id: item.id!.videoId!,
             author: item.snippet!.channelId!,
-          }) satisfies Video,
+          } satisfies Video),
       ),
     };
 
@@ -124,7 +131,11 @@ for (const tank of tanksSanitized) {
   }
 }
 
-const content = await encodeToBase64('reviews', 'blitzkit.Reviews', reviews);
+const content = await encodeProtobufToBase64(
+  'reviews',
+  'blitzkit.Reviews',
+  reviews,
+);
 
 await commitAssets('reviews', [
   { content, encoding: 'base64', path: 'definitions/reviews.pb' },

packages/scripts/src/sitemap.ts

@@ -1,4 +1,4 @@
-import { tankDefinitions } from '../src/core/blitzkit/tankDefinitions';
+import { tankDefinitions } from '@blitzkit/core';
 
 // random vercel bug forces require import
 const { writeFileSync } = require('fs');