Merge branch 'main' into ue

packages/scripts/src/buildAssets/boosterIcons.ts

@@ -5,10 +5,12 @@ import { vfs } from "./constants";
 export async function boosterIcons() {
   console.log("Building booster icons...");
 
-  using uploader = new AssetUploader("booster icons");
-  const boosterFiles = (vfs.dir(`Data/Gfx/Shared/boosters`)).filter(
-    (file) => !file.includes("@2x.txt") && !file.startsWith("texture0")
-  );
+  const uploader = new AssetUploader("booster icons");
+  const boosterFiles = vfs
+    .dir("Data/Gfx/Shared/boosters")
+    .filter(
+      (file) => !file.includes("@2x.txt") && !file.startsWith("texture0")
+    );
   const image = sharp(
     await vfs.file(`Data/Gfx/Shared/boosters/texture0.packed.webp`)
   );

packages/scripts/src/buildAssets/constants.ts

@@ -1,12 +1,33 @@
 import { assertSecret } from "@blitzkit/core";
-import { SteamVFS } from "../core/steam/vfs";
+import { AbstractVFS } from "../core/vfs/abstract";
+import { SteamVFS } from "../core/vfs/steam";
+import { ZipVFS } from "../core/vfs/zip";
 
-const APP = 444200;
-const DEPOT = 444202;
+const steamPattern = /steam:(\d+)\/(\d+)/;
+const zipPattern = /zip:(.+)/;
 
-export const vfs = await new SteamVFS(
-  assertSecret(import.meta.env.STEAM_USERNAME),
-  assertSecret(import.meta.env.STEAM_PASSWORD),
-  APP,
-  DEPOT
-).init();
+const source = Bun.argv
+  .find((arg) => arg.startsWith("--source="))
+  ?.split("=")[1];
+
+let resolved: AbstractVFS;
+
+if (!source) {
+  throw new Error("No source specified");
+} else if (steamPattern.test(source)) {
+  const [, app, depot] = source.match(steamPattern)!.map(Number);
+
+  resolved = await new SteamVFS(
+    assertSecret(import.meta.env.STEAM_USERNAME),
+    assertSecret(import.meta.env.STEAM_PASSWORD),
+    app,
+    depot
+  ).init();
+} else if (zipPattern.test(source)) {
+  const [, url] = source.match(zipPattern)!;
+  resolved = await new ZipVFS(url).init();
+} else {
+  throw new Error("No valid source specified");
+}
+
+export const vfs = resolved;

packages/scripts/src/buildAssets/definitions.ts

@@ -534,11 +534,11 @@ const blitzModuleTypeToBlitzkit: Record<keyof BlitzModuleType, ModuleType> = {
 export async function definitions() {
   console.log("Building definitions...");
 
-  using uploader = new AssetUploader("definitions");
+  const uploader = new AssetUploader("definitions");
   const gameDefinitions: GameDefinitions = {
-    version: (await vfs.text('Data/version.txt')).split(" ")[0],
+    version: (await vfs.text("Data/version.txt")).split(" ")[0],
     nations: (
-      await vfs.yaml<AvailableNationsYaml>('Data/available_nations.yaml')
+      await vfs.yaml<AvailableNationsYaml>("Data/available_nations.yaml")
     ).available_nations,
     gameModes: {},
     roles: {},
@@ -561,10 +561,12 @@ export async function definitions() {
       [TankClass.TANK_DESTROYER]: { skills: [] },
     },
   };
-  const nations = vfs.dir('Data/XML/item_defs/vehicles').filter((nation) => nation !== "common")
+  const nations = vfs
+    .dir("Data/XML/item_defs/vehicles")
+    .filter((nation) => nation !== "common");
   const tankStringIdMap: Record<string, number> = {};
   const optionalDevices = await vfs.xml<{ root: OptionalDevices }>(
-    'Data/XML/item_defs/vehicles/common/optional_devices.xml'
+    "Data/XML/item_defs/vehicles/common/optional_devices.xml"
   );
   const stringsI18n: Record<string, Record<string, string>> = {};
 
@@ -617,9 +619,7 @@ export async function definitions() {
   const provisions: ProvisionsCommon = {};
 
   for (const match of (
-    await vfs.text(
-      `Data/XML/item_defs/vehicles/common/consumables/list.xml`
-    )
+    await vfs.text(`Data/XML/item_defs/vehicles/common/consumables/list.xml`)
   ).matchAll(/<items path="(.+)\.xml"\/>/g)) {
     if (match[1] === "prototypes") continue;
 
@@ -634,9 +634,7 @@ export async function definitions() {
   }
 
   for (const match of (
-    await vfs.text(
-      `Data/XML/item_defs/vehicles/common/provisions/list.xml`
-    )
+    await vfs.text(`Data/XML/item_defs/vehicles/common/provisions/list.xml`)
   ).matchAll(/<items path="(.+)\.xml"\/>/g)) {
     if (match[1] === "prototypes") continue;
 

packages/scripts/src/buildAssets/skillIcons.ts

@@ -28,7 +28,7 @@ export async function skillIcons() {
   const avatar = await vfs.xml<{ root: Avatar }>(
     `Data/XML/item_defs/tankmen/avatar.xml`
   );
-  using uploader = new AssetUploader("skill icons");
+  const uploader = new AssetUploader("skill icons");
 
   for (const key in avatar.root.skills) {
     const skill = avatar.root.skills[key];

packages/scripts/src/core/blitz/extractArmor.ts

@@ -6,7 +6,7 @@ import {
 } from "@blitzkit/core";
 import { Accessor, Document, Node, Scene } from "@gltf-transform/core";
 import { times } from "lodash-es";
-import { SteamVFS } from "../steam/vfs";
+import { SteamVFS } from "../vfs/vfs";
 import {
   vertexAttributeGLTFName,
   vertexAttributeGltfVectorSizes,

packages/scripts/src/core/blitz/extractModel/index.ts

@@ -9,7 +9,7 @@ import { Document, Material, Node, Scene } from "@gltf-transform/core";
 import { dedup, prune } from "@gltf-transform/functions";
 import { times } from "lodash-es";
 import { dirname } from "path";
-import { SteamVFS } from "../../steam/vfs";
+import { SteamVFS } from "../../vfs/vfs";
 import { readBaseColor } from "../readBaseColor";
 import { readNormal } from "../readNormal";
 import { readRoughnessMetallic } from "../readRoughnessMetallic";

packages/scripts/src/core/github/assetUploader.ts

@@ -99,7 +99,7 @@ export class AssetUploader {
     }
 
     if (diff.status === DiffStatus.Unchanged) {
-      // console.log(`🔵 ${change.path}`);
+      console.log(`🔵 ${change.path}`);
     } else {
       console.log(
         `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${

packages/scripts/src/core/vfs/abstract.ts

@@ -0,0 +1,84 @@
+import { readDVPL } from "@blitzkit/core";
+import { XMLParser } from "fast-xml-parser";
+import { normalize } from "path/posix";
+import { parse } from "yaml";
+
+export abstract class AbstractVFS {
+  textDecoder = new TextDecoder();
+  private xmlParser = new XMLParser();
+
+  abstract init(): Promise<typeof this>;
+
+  abstract dispose(): void;
+
+  abstract has(path: string): boolean;
+
+  abstract raw(path: string): Promise<Uint8Array>;
+
+  abstract paths(): string[];
+
+  resolve(path: string) {
+    const normalized = normalize(path);
+    if (this.has(normalized)) return normalized;
+
+    const dvplPath = `${path}.dvpl`;
+    if (this.has(dvplPath)) return dvplPath;
+
+    return null;
+  }
+
+  assert(path: string) {
+    const resolved = this.resolve(path);
+
+    if (resolved === null) throw new Error(`File not found: ${path}`);
+
+    return resolved;
+  }
+
+  async file(file: string) {
+    const resolved = this.assert(file);
+    const raw = await this.raw(resolved);
+    let buffer = raw;
+
+    if (resolved.endsWith(".dvpl")) {
+      buffer = new Uint8Array(readDVPL(Buffer.from(raw)));
+    }
+
+    return buffer;
+  }
+
+  dir(path: string) {
+    const parentSegments = path.split("/").length;
+    const children = new Set<string>();
+
+    for (const child of this.paths()) {
+      if (!child.startsWith(path) || child === path) continue;
+
+      const childSegments = child.split("/");
+      const nextSegment = childSegments[parentSegments];
+
+      children.add(nextSegment);
+    }
+
+    return Array.from(children);
+  }
+
+  async text(path: string) {
+    const file = await this.file(path);
+    return this.textDecoder.decode(file);
+  }
+
+  async yaml<Type>(path: string) {
+    const file = await this.text(path);
+    return parse(file) as Type;
+  }
+
+  async xml<Type>(path: string) {
+    const file = await this.text(path);
+    return this.xmlParser.parse(file) as Type;
+  }
+
+  [Symbol.dispose]() {
+    this.dispose();
+  }
+}

packages/scripts/src/core/vfs/steam.ts

@@ -1,8 +1,5 @@
-import { readDVPL } from "@blitzkit/core/src/blitz";
-import { XMLParser } from "fast-xml-parser";
-import { normalize } from "path/posix";
 import SteamUser, { EConnectionProtocol } from "steam-user";
-import { parse } from "yaml";
+import { AbstractVFS } from "./abstract";
 
 interface SteamManifestFile {
   chunks: {
@@ -33,11 +30,8 @@ interface SteamManifest {
   crc_clear: number;
 }
 
-export class SteamVFS {
+export class SteamVFS extends AbstractVFS {
   private steam = new SteamUser({ protocol: EConnectionProtocol.TCP });
-  private textDecoder = new TextDecoder();
-  private xmlParser = new XMLParser();
-
   private manifest: Map<string, SteamManifestFile> = new Map();
 
   constructor(
@@ -45,7 +39,9 @@ export class SteamVFS {
     private password: string,
     private app: number,
     private depot: number
-  ) {}
+  ) {
+    super();
+  }
 
   async init() {
     this.steam.logOn({
@@ -71,10 +67,8 @@ export class SteamVFS {
       );
     });
 
-    const paths: string[] = [];
     for (const file of manifest.files) {
       const path = file.filename.replaceAll("\\", "/");
-      paths.push(path);
       this.manifest.set(path, file);
     }
 
@@ -82,74 +76,23 @@ export class SteamVFS {
   }
 
   has(path: string) {
-    const normalized = normalize(path);
-    const dvplPath = `${normalized}.dvpl`;
-
-    return (
-      (this.manifest.has(dvplPath) && dvplPath) ||
-      (this.manifest.has(normalized) && normalized)
-    );
+    return this.manifest.has(path);
   }
 
-  assert(requested: string) {
-    const path = this.has(requested);
-
-    if (!path) throw new Error(`File not found: ${requested}`);
-
-    return path;
-  }
-
-  dir(path: string) {
-    const parentSegments = path.split("/").length;
-    const children: string[] = [];
-
-    for (const child of this.manifest.keys()) {
-      const childSegments = child.split("/");
-
-      if (
-        child.startsWith(path) &&
-        parentSegments + 1 === childSegments.length
-      ) {
-        children.push(childSegments.at(-1)!);
-      }
-    }
-
-    return children;
-  }
-
-  async file(requested: string) {
-    const path = this.assert(requested);
+  async raw(path: string) {
     const fileManifest = this.manifest.get(path)!;
     const downloaded: { type: "complete"; file: Buffer } =
       // @ts-expect-error
       await this.steam.downloadFile(this.app, this.depot, fileManifest);
-    let buffer = downloaded.file;
 
-    if (path.endsWith(".dvpl")) buffer = readDVPL(downloaded.file);
-
-    return new Uint8Array(buffer);
+    return new Uint8Array(downloaded.file);
   }
 
-  async text(path: string) {
-    const file = await this.file(path);
-    return this.textDecoder.decode(file);
-  }
-
-  async yaml<Type>(path: string) {
-    const file = await this.text(path);
-    return parse(file) as Type;
-  }
-
-  async xml<Type>(path: string) {
-    const file = await this.text(path);
-    return this.xmlParser.parse(file) as Type;
+  paths() {
+    return Array.from(this.manifest.keys());
   }
 
   dispose() {
     this.steam.logOff();
   }
-
-  [Symbol.dispose]() {
-    this.dispose();
-  }
 }

packages/scripts/src/core/vfs/zip.ts

@@ -0,0 +1,38 @@
+import { normalize } from "path/posix";
+import { HTTPRangeReader, unzip, ZipEntry } from "unzipit";
+import { AbstractVFS } from "./abstract";
+
+export class ZipVFS extends AbstractVFS {
+  entries = new Map<string, ZipEntry>();
+
+  constructor(private url: string, private root = "app/") {
+    super();
+  }
+
+  async init() {
+    const reader = new HTTPRangeReader(this.url);
+    const { entries } = await unzip(reader);
+
+    for (const entry of Object.values(entries)) {
+      const normalized = normalize(entry.name.slice(this.root.length));
+      this.entries.set(normalized, entry);
+    }
+
+    return this;
+  }
+
+  has(path: string) {
+    return this.entries.has(path);
+  }
+
+  async raw(path: string) {
+    const arrayBuffer = await this.entries.get(path)!.arrayBuffer();
+    return new Uint8Array(arrayBuffer);
+  }
+
+  paths() {
+    return Array.from(this.entries.keys());
+  }
+
+  dispose() {}
+}