Merge branch 'main' into ue

.github/workflows/assets.yml

@@ -16,10 +16,11 @@ on:
           - blitzkit/assets-dev
         default: blitzkit/assets
 
-      archive_url:
+      asset_source:
         type: string
         required: false
-        description: Archive URL to download instead of Steam client
+        description: "Archive URL or Steam to download (example: steam:APP_ID/DEPOT_ID or zip:URL_TO_ZIP)"
+        default: "steam:444200/444202"
 
   schedule:
     - cron: "0 2 * * *"
@@ -78,4 +79,4 @@ jobs:
         run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
 
       - name: Build assets
-        run: bun run build:assets --all-targets --manual
+        run: bun run build:assets --all-targets --source=${{ github.event.inputs.asset_source || 'steam:444200/444202' }}

bun.lock

@@ -122,6 +122,7 @@
         "steam-user": "^5.2.3",
         "three": "^0.180.0",
         "ts-proto": "^2.7.7",
+        "unzipit": "^1.4.3",
         "yaml": "^2.8.1",
       },
     },
@@ -2452,6 +2453,8 @@
 
     "unstorage": ["[email protected]", "", { "dependencies": { "anymatch": "^3.1.3", "chokidar": "^4.0.3", "destr": "^2.0.5", "h3": "^1.15.4", "lru-cache": "^10.4.3", "node-fetch-native": "^1.6.7", "ofetch": "^1.4.1", "ufo": "^1.6.1" }, "peerDependencies": { "@azure/app-configuration": "^1.8.0", "@azure/cosmos": "^4.2.0", "@azure/data-tables": "^13.3.0", "@azure/identity": "^4.6.0", "@azure/keyvault-secrets": "^4.9.0", "@azure/storage-blob": "^12.26.0", "@capacitor/preferences": "^6.0.3 || ^7.0.0", "@deno/kv": ">=0.9.0", "@netlify/blobs": "^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0", "@planetscale/database": "^1.19.0", "@upstash/redis": "^1.34.3", "@vercel/blob": ">=0.27.1", "@vercel/functions": "^2.2.12 || ^3.0.0", "@vercel/kv": "^1.0.1", "aws4fetch": "^1.0.20", "db0": ">=0.2.1", "idb-keyval": "^6.2.1", "ioredis": "^5.4.2", "uploadthing": "^7.4.4" }, "optionalPeers": ["@azure/app-configuration", "@azure/cosmos", "@azure/data-tables", "@azure/identity", "@azure/keyvault-secrets", "@azure/storage-blob", "@capacitor/preferences", "@deno/kv", "@netlify/blobs", "@planetscale/database", "@upstash/redis", "@vercel/blob", "@vercel/functions", "@vercel/kv", "aws4fetch", "db0", "idb-keyval", "ioredis", "uploadthing"] }, "sha512-KKGwRTT0iVBCErKemkJCLs7JdxNVfqTPc/85ae1XES0+bsHbc/sFBfVi5kJp156cc51BHinIH2l3k0EZ24vOBQ=="],
 
+    "unzipit": ["[email protected]", "", { "dependencies": { "uzip-module": "^1.0.2" } }, "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg=="],
+
     "upath": ["[email protected]", "", {}, "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg=="],
 
     "update-browserslist-db": ["[email protected]", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw=="],
@@ -2470,6 +2473,8 @@
 
     "utility-types": ["[email protected]", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="],
 
+    "uzip-module": ["[email protected]", "", {}, "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA=="],
+
     "varuna": ["varuna@workspace:packages/varuna"],
 
     "vfile": ["[email protected]", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="],

docs/changelogs/1.29.7.md

@@ -0,0 +1,5 @@
+# BlitzKit 1.29.7
+
+## Features
+
+- Re-introduced the ability to deploy BlitzKit OpenTests

package.json

@@ -1,6 +1,6 @@
 {
   "name": "blitzkit",
-  "version": "1.29.6",
+  "version": "1.29.7",
   "license": "BlitzKit License",
   "private": true,
   "scripts": {

packages/core/src/blitz/blitzVFS.ts

@@ -1,25 +0,0 @@
-export enum BlitzVFSSourceType {
-  Steam,
-}
-
-type BlitzVFSSource = {
-  type: BlitzVFSSourceType;
-
-  username: string;
-  password: string;
-
-  app: number;
-  os: "windows";
-};
-
-export class BlitzVFS {
-  constructor(private source: BlitzVFSSource) {}
-
-  async init() {
-    return this;
-  }
-
-  async file(path: string) {
-    return new Uint8Array();
-  }
-}

packages/scripts/package.json

@@ -31,6 +31,7 @@
     "steam-user": "^5.2.3",
     "three": "^0.180.0",
     "ts-proto": "^2.7.7",
+    "unzipit": "^1.4.3",
     "yaml": "^2.8.1"
   },
   "private": true,

packages/scripts/src/buildAssets/boosterIcons.ts

@@ -5,10 +5,12 @@ import { vfs } from "./constants";
 export async function boosterIcons() {
   console.log("Building booster icons...");
 
-  using uploader = new AssetUploader("booster icons");
-  const boosterFiles = (vfs.dir(`Data/Gfx/Shared/boosters`)).filter(
-    (file) => !file.includes("@2x.txt") && !file.startsWith("texture0")
-  );
+  const uploader = new AssetUploader("booster icons");
+  const boosterFiles = vfs
+    .dir("Data/Gfx/Shared/boosters")
+    .filter(
+      (file) => !file.includes("@2x.txt") && !file.startsWith("texture0")
+    );
   const image = sharp(
     await vfs.file(`Data/Gfx/Shared/boosters/texture0.packed.webp`)
   );

packages/scripts/src/buildAssets/constants.ts

@@ -1,12 +1,33 @@
 import { assertSecret } from "@blitzkit/core";
-import { SteamVFS } from "../core/steam/vfs";
+import { AbstractVFS } from "../core/vfs/abstract";
+import { SteamVFS } from "../core/vfs/steam";
+import { ZipVFS } from "../core/vfs/zip";
 
-const APP = 444200;
-const DEPOT = 444202;
+const steamPattern = /steam:(\d+)\/(\d+)/;
+const zipPattern = /zip:(.+)/;
 
-export const vfs = await new SteamVFS(
-  assertSecret(import.meta.env.STEAM_USERNAME),
-  assertSecret(import.meta.env.STEAM_PASSWORD),
-  APP,
-  DEPOT
-).init();
+const source = Bun.argv
+  .find((arg) => arg.startsWith("--source="))
+  ?.split("=")[1];
+
+let resolved: AbstractVFS;
+
+if (!source) {
+  throw new Error("No source specified");
+} else if (steamPattern.test(source)) {
+  const [, app, depot] = source.match(steamPattern)!.map(Number);
+
+  resolved = await new SteamVFS(
+    assertSecret(import.meta.env.STEAM_USERNAME),
+    assertSecret(import.meta.env.STEAM_PASSWORD),
+    app,
+    depot
+  ).init();
+} else if (zipPattern.test(source)) {
+  const [, url] = source.match(zipPattern)!;
+  resolved = await new ZipVFS(url).init();
+} else {
+  throw new Error("No valid source specified");
+}
+
+export const vfs = resolved;

packages/scripts/src/buildAssets/definitions.ts

@@ -534,11 +534,11 @@ const blitzModuleTypeToBlitzkit: Record<keyof BlitzModuleType, ModuleType> = {
 export async function definitions() {
   console.log("Building definitions...");
 
-  using uploader = new AssetUploader("definitions");
+  const uploader = new AssetUploader("definitions");
   const gameDefinitions: GameDefinitions = {
-    version: (await vfs.text('Data/version.txt')).split(" ")[0],
+    version: (await vfs.text("Data/version.txt")).split(" ")[0],
     nations: (
-      await vfs.yaml<AvailableNationsYaml>('Data/available_nations.yaml')
+      await vfs.yaml<AvailableNationsYaml>("Data/available_nations.yaml")
     ).available_nations,
     gameModes: {},
     roles: {},
@@ -561,10 +561,12 @@ export async function definitions() {
       [TankClass.TANK_DESTROYER]: { skills: [] },
     },
   };
-  const nations = vfs.dir('Data/XML/item_defs/vehicles').filter((nation) => nation !== "common")
+  const nations = vfs
+    .dir("Data/XML/item_defs/vehicles")
+    .filter((nation) => nation !== "common");
   const tankStringIdMap: Record<string, number> = {};
   const optionalDevices = await vfs.xml<{ root: OptionalDevices }>(
-    'Data/XML/item_defs/vehicles/common/optional_devices.xml'
+    "Data/XML/item_defs/vehicles/common/optional_devices.xml"
   );
   const stringsI18n: Record<string, Record<string, string>> = {};
 
@@ -617,9 +619,7 @@ export async function definitions() {
   const provisions: ProvisionsCommon = {};
 
   for (const match of (
-    await vfs.text(
-      `Data/XML/item_defs/vehicles/common/consumables/list.xml`
-    )
+    await vfs.text(`Data/XML/item_defs/vehicles/common/consumables/list.xml`)
   ).matchAll(/<items path="(.+)\.xml"\/>/g)) {
     if (match[1] === "prototypes") continue;
 
@@ -634,9 +634,7 @@ export async function definitions() {
   }
 
   for (const match of (
-    await vfs.text(
-      `Data/XML/item_defs/vehicles/common/provisions/list.xml`
-    )
+    await vfs.text(`Data/XML/item_defs/vehicles/common/provisions/list.xml`)
   ).matchAll(/<items path="(.+)\.xml"\/>/g)) {
     if (match[1] === "prototypes") continue;
 

packages/scripts/src/buildAssets/skillIcons.ts

@@ -28,7 +28,7 @@ export async function skillIcons() {
   const avatar = await vfs.xml<{ root: Avatar }>(
     `Data/XML/item_defs/tankmen/avatar.xml`
   );
-  using uploader = new AssetUploader("skill icons");
+  const uploader = new AssetUploader("skill icons");
 
   for (const key in avatar.root.skills) {
     const skill = avatar.root.skills[key];

packages/scripts/src/core/blitz/extractArmor.ts

@@ -6,7 +6,7 @@ import {
 } from "@blitzkit/core";
 import { Accessor, Document, Node, Scene } from "@gltf-transform/core";
 import { times } from "lodash-es";
-import { SteamVFS } from "../steam/vfs";
+import { SteamVFS } from "../vfs/vfs";
 import {
   vertexAttributeGLTFName,
   vertexAttributeGltfVectorSizes,

packages/scripts/src/core/blitz/extractModel/index.ts

@@ -9,7 +9,7 @@ import { Document, Material, Node, Scene } from "@gltf-transform/core";
 import { dedup, prune } from "@gltf-transform/functions";
 import { times } from "lodash-es";
 import { dirname } from "path";
-import { SteamVFS } from "../../steam/vfs";
+import { SteamVFS } from "../../vfs/vfs";
 import { readBaseColor } from "../readBaseColor";
 import { readNormal } from "../readNormal";
 import { readRoughnessMetallic } from "../readRoughnessMetallic";

packages/scripts/src/core/github/assetUploader.ts

@@ -99,7 +99,7 @@ export class AssetUploader {
     }
 
     if (diff.status === DiffStatus.Unchanged) {
-      // console.log(`🔵 ${change.path}`);
+      console.log(`🔵 ${change.path}`);
     } else {
       console.log(
         `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${

packages/scripts/src/core/vfs/abstract.ts

@@ -0,0 +1,84 @@
+import { readDVPL } from "@blitzkit/core";
+import { XMLParser } from "fast-xml-parser";
+import { normalize } from "path/posix";
+import { parse } from "yaml";
+
+export abstract class AbstractVFS {
+  textDecoder = new TextDecoder();
+  private xmlParser = new XMLParser();
+
+  abstract init(): Promise<typeof this>;
+
+  abstract dispose(): void;
+
+  abstract has(path: string): boolean;
+
+  abstract raw(path: string): Promise<Uint8Array>;
+
+  abstract paths(): string[];
+
+  resolve(path: string) {
+    const normalized = normalize(path);
+    if (this.has(normalized)) return normalized;
+
+    const dvplPath = `${path}.dvpl`;
+    if (this.has(dvplPath)) return dvplPath;
+
+    return null;
+  }
+
+  assert(path: string) {
+    const resolved = this.resolve(path);
+
+    if (resolved === null) throw new Error(`File not found: ${path}`);
+
+    return resolved;
+  }
+
+  async file(file: string) {
+    const resolved = this.assert(file);
+    const raw = await this.raw(resolved);
+    let buffer = raw;
+
+    if (resolved.endsWith(".dvpl")) {
+      buffer = new Uint8Array(readDVPL(Buffer.from(raw)));
+    }
+
+    return buffer;
+  }
+
+  dir(path: string) {
+    const parentSegments = path.split("/").length;
+    const children = new Set<string>();
+
+    for (const child of this.paths()) {
+      if (!child.startsWith(path) || child === path) continue;
+
+      const childSegments = child.split("/");
+      const nextSegment = childSegments[parentSegments];
+
+      children.add(nextSegment);
+    }
+
+    return Array.from(children);
+  }
+
+  async text(path: string) {
+    const file = await this.file(path);
+    return this.textDecoder.decode(file);
+  }
+
+  async yaml<Type>(path: string) {
+    const file = await this.text(path);
+    return parse(file) as Type;
+  }
+
+  async xml<Type>(path: string) {
+    const file = await this.text(path);
+    return this.xmlParser.parse(file) as Type;
+  }
+
+  [Symbol.dispose]() {
+    this.dispose();
+  }
+}

packages/scripts/src/core/vfs/steam.ts

@@ -1,8 +1,5 @@
-import { readDVPL } from "@blitzkit/core/src/blitz";
-import { XMLParser } from "fast-xml-parser";
-import { normalize } from "path/posix";
 import SteamUser, { EConnectionProtocol } from "steam-user";
-import { parse } from "yaml";
+import { AbstractVFS } from "./abstract";
 
 interface SteamManifestFile {
   chunks: {
@@ -33,11 +30,8 @@ interface SteamManifest {
   crc_clear: number;
 }
 
-export class SteamVFS {
+export class SteamVFS extends AbstractVFS {
   private steam = new SteamUser({ protocol: EConnectionProtocol.TCP });
-  private textDecoder = new TextDecoder();
-  private xmlParser = new XMLParser();
-
   private manifest: Map<string, SteamManifestFile> = new Map();
 
   constructor(
@@ -45,7 +39,9 @@ export class SteamVFS {
     private password: string,
     private app: number,
     private depot: number
-  ) {}
+  ) {
+    super();
+  }
 
   async init() {
     this.steam.logOn({
@@ -71,10 +67,8 @@ export class SteamVFS {
       );
     });
 
-    const paths: string[] = [];
     for (const file of manifest.files) {
       const path = file.filename.replaceAll("\\", "/");
-      paths.push(path);
       this.manifest.set(path, file);
     }
 
@@ -82,74 +76,23 @@ export class SteamVFS {
   }
 
   has(path: string) {
-    const normalized = normalize(path);
-    const dvplPath = `${normalized}.dvpl`;
-
-    return (
-      (this.manifest.has(dvplPath) && dvplPath) ||
-      (this.manifest.has(normalized) && normalized)
-    );
+    return this.manifest.has(path);
   }
 
-  assert(requested: string) {
-    const path = this.has(requested);
-
-    if (!path) throw new Error(`File not found: ${requested}`);
-
-    return path;
-  }
-
-  dir(path: string) {
-    const parentSegments = path.split("/").length;
-    const children: string[] = [];
-
-    for (const child of this.manifest.keys()) {
-      const childSegments = child.split("/");
-
-      if (
-        child.startsWith(path) &&
-        parentSegments + 1 === childSegments.length
-      ) {
-        children.push(childSegments.at(-1)!);
-      }
-    }
-
-    return children;
-  }
-
-  async file(requested: string) {
-    const path = this.assert(requested);
+  async raw(path: string) {
     const fileManifest = this.manifest.get(path)!;
     const downloaded: { type: "complete"; file: Buffer } =
       // @ts-expect-error
       await this.steam.downloadFile(this.app, this.depot, fileManifest);
-    let buffer = downloaded.file;
 
-    if (path.endsWith(".dvpl")) buffer = readDVPL(downloaded.file);
-
-    return new Uint8Array(buffer);
+    return new Uint8Array(downloaded.file);
   }
 
-  async text(path: string) {
-    const file = await this.file(path);
-    return this.textDecoder.decode(file);
-  }
-
-  async yaml<Type>(path: string) {
-    const file = await this.text(path);
-    return parse(file) as Type;
-  }
-
-  async xml<Type>(path: string) {
-    const file = await this.text(path);
-    return this.xmlParser.parse(file) as Type;
+  paths() {
+    return Array.from(this.manifest.keys());
   }
 
   dispose() {
     this.steam.logOff();
   }
-
-  [Symbol.dispose]() {
-    this.dispose();
-  }
 }

packages/scripts/src/core/vfs/zip.ts

@@ -0,0 +1,38 @@
+import { normalize } from "path/posix";
+import { HTTPRangeReader, unzip, ZipEntry } from "unzipit";
+import { AbstractVFS } from "./abstract";
+
+export class ZipVFS extends AbstractVFS {
+  entries = new Map<string, ZipEntry>();
+
+  constructor(private url: string, private root = "app/") {
+    super();
+  }
+
+  async init() {
+    const reader = new HTTPRangeReader(this.url);
+    const { entries } = await unzip(reader);
+
+    for (const entry of Object.values(entries)) {
+      const normalized = normalize(entry.name.slice(this.root.length));
+      this.entries.set(normalized, entry);
+    }
+
+    return this;
+  }
+
+  has(path: string) {
+    return this.entries.has(path);
+  }
+
+  async raw(path: string) {
+    const arrayBuffer = await this.entries.get(path)!.arrayBuffer();
+    return new Uint8Array(arrayBuffer);
+  }
+
+  paths() {
+    return Array.from(this.entries.keys());
+  }
+
+  dispose() {}
+}