diff --git a/.github/workflows/assets.yml b/.github/workflows/assets.yml index 1c197cf22..222b45e9d 100644 --- a/.github/workflows/assets.yml +++ b/.github/workflows/assets.yml @@ -16,10 +16,11 @@ on: - blitzkit/assets-dev default: blitzkit/assets - archive_url: + asset_source: type: string required: false - description: Archive URL to download instead of Steam client + description: "Archive URL or Steam to download (example: steam:APP_ID/DEPOT_ID or zip:URL_TO_ZIP)" + default: "steam:444200/444202" schedule: - cron: "0 2 * * *" @@ -78,4 +79,4 @@ jobs: run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - name: Build assets - run: bun run build:assets --all-targets --manual + run: bun run build:assets --all-targets --source=${{ github.event.inputs.asset_source || 'steam:444200/444202' }} diff --git a/bun.lock b/bun.lock index 29085f808..b3b55bd47 100644 --- a/bun.lock +++ b/bun.lock @@ -122,6 +122,7 @@ "steam-user": "^5.2.3", "three": "^0.180.0", "ts-proto": "^2.7.7", + "unzipit": "^1.4.3", "yaml": "^2.8.1", }, }, @@ -2452,6 +2453,8 @@ "unstorage": ["unstorage@1.17.1", "", { "dependencies": { "anymatch": "^3.1.3", "chokidar": "^4.0.3", "destr": "^2.0.5", "h3": "^1.15.4", "lru-cache": "^10.4.3", "node-fetch-native": "^1.6.7", "ofetch": "^1.4.1", "ufo": "^1.6.1" }, "peerDependencies": { "@azure/app-configuration": "^1.8.0", "@azure/cosmos": "^4.2.0", "@azure/data-tables": "^13.3.0", "@azure/identity": "^4.6.0", "@azure/keyvault-secrets": "^4.9.0", "@azure/storage-blob": "^12.26.0", "@capacitor/preferences": "^6.0.3 || ^7.0.0", "@deno/kv": ">=0.9.0", "@netlify/blobs": "^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0", "@planetscale/database": "^1.19.0", "@upstash/redis": "^1.34.3", "@vercel/blob": ">=0.27.1", "@vercel/functions": "^2.2.12 || ^3.0.0", "@vercel/kv": "^1.0.1", "aws4fetch": "^1.0.20", "db0": ">=0.2.1", "idb-keyval": "^6.2.1", "ioredis": "^5.4.2", "uploadthing": "^7.4.4" }, "optionalPeers": ["@azure/app-configuration", "@azure/cosmos", "@azure/data-tables", "@azure/identity", "@azure/keyvault-secrets", "@azure/storage-blob", "@capacitor/preferences", "@deno/kv", "@netlify/blobs", "@planetscale/database", "@upstash/redis", "@vercel/blob", "@vercel/functions", "@vercel/kv", "aws4fetch", "db0", "idb-keyval", "ioredis", "uploadthing"] }, "sha512-KKGwRTT0iVBCErKemkJCLs7JdxNVfqTPc/85ae1XES0+bsHbc/sFBfVi5kJp156cc51BHinIH2l3k0EZ24vOBQ=="], + "unzipit": ["unzipit@1.4.3", "", { "dependencies": { "uzip-module": "^1.0.2" } }, "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg=="], + "upath": ["upath@1.2.0", "", {}, "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg=="], "update-browserslist-db": ["update-browserslist-db@1.1.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw=="], @@ -2470,6 +2473,8 @@ "utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="], + "uzip-module": ["uzip-module@1.0.3", "", {}, "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA=="], + "varuna": ["varuna@workspace:packages/varuna"], "vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], diff --git a/docs/changelogs/1.29.7.md b/docs/changelogs/1.29.7.md new file mode 100644 index 000000000..e809d92e2 --- /dev/null +++ b/docs/changelogs/1.29.7.md @@ -0,0 +1,5 @@ +# BlitzKit 1.29.7 + +## Features + +- Re-introduced the ability to deploy BlitzKit OpenTests diff --git a/package.json b/package.json index e048da45c..a2f7fb883 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "blitzkit", - "version": "1.29.6", + "version": "1.29.7", "license": "BlitzKit License", "private": true, "scripts": { diff --git a/packages/core/src/blitz/blitzVFS.ts b/packages/core/src/blitz/blitzVFS.ts deleted file mode 100644 index 776381056..000000000 --- a/packages/core/src/blitz/blitzVFS.ts +++ /dev/null @@ -1,25 +0,0 @@ -export enum BlitzVFSSourceType { - Steam, -} - -type BlitzVFSSource = { - type: BlitzVFSSourceType; - - username: string; - password: string; - - app: number; - os: "windows"; -}; - -export class BlitzVFS { - constructor(private source: BlitzVFSSource) {} - - async init() { - return this; - } - - async file(path: string) { - return new Uint8Array(); - } -} diff --git a/packages/scripts/package.json b/packages/scripts/package.json index fd7d98399..3e3d830c1 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -31,6 +31,7 @@ "steam-user": "^5.2.3", "three": "^0.180.0", "ts-proto": "^2.7.7", + "unzipit": "^1.4.3", "yaml": "^2.8.1" }, "private": true, diff --git a/packages/scripts/src/buildAssets/boosterIcons.ts b/packages/scripts/src/buildAssets/boosterIcons.ts index 486a732b2..4046ba88b 100644 --- a/packages/scripts/src/buildAssets/boosterIcons.ts +++ b/packages/scripts/src/buildAssets/boosterIcons.ts @@ -5,10 +5,12 @@ import { vfs } from "./constants"; export async function boosterIcons() { console.log("Building booster icons..."); - using uploader = new AssetUploader("booster icons"); - const boosterFiles = (vfs.dir(`Data/Gfx/Shared/boosters`)).filter( - (file) => !file.includes("@2x.txt") && !file.startsWith("texture0") - ); + const uploader = new AssetUploader("booster icons"); + const boosterFiles = vfs + .dir("Data/Gfx/Shared/boosters") + .filter( + (file) => !file.includes("@2x.txt") && !file.startsWith("texture0") + ); const image = sharp( await vfs.file(`Data/Gfx/Shared/boosters/texture0.packed.webp`) ); diff --git a/packages/scripts/src/buildAssets/constants.ts b/packages/scripts/src/buildAssets/constants.ts index e6814d55b..d189bbd3b 100644 --- a/packages/scripts/src/buildAssets/constants.ts +++ b/packages/scripts/src/buildAssets/constants.ts @@ -1,12 +1,33 @@ import { assertSecret } from "@blitzkit/core"; -import { SteamVFS } from "../core/steam/vfs"; +import { AbstractVFS } from "../core/vfs/abstract"; +import { SteamVFS } from "../core/vfs/steam"; +import { ZipVFS } from "../core/vfs/zip"; -const APP = 444200; -const DEPOT = 444202; +const steamPattern = /steam:(\d+)\/(\d+)/; +const zipPattern = /zip:(.+)/; -export const vfs = await new SteamVFS( - assertSecret(import.meta.env.STEAM_USERNAME), - assertSecret(import.meta.env.STEAM_PASSWORD), - APP, - DEPOT -).init(); +const source = Bun.argv + .find((arg) => arg.startsWith("--source=")) + ?.split("=")[1]; + +let resolved: AbstractVFS; + +if (!source) { + throw new Error("No source specified"); +} else if (steamPattern.test(source)) { + const [, app, depot] = source.match(steamPattern)!.map(Number); + + resolved = await new SteamVFS( + assertSecret(import.meta.env.STEAM_USERNAME), + assertSecret(import.meta.env.STEAM_PASSWORD), + app, + depot + ).init(); +} else if (zipPattern.test(source)) { + const [, url] = source.match(zipPattern)!; + resolved = await new ZipVFS(url).init(); +} else { + throw new Error("No valid source specified"); +} + +export const vfs = resolved; diff --git a/packages/scripts/src/buildAssets/definitions.ts b/packages/scripts/src/buildAssets/definitions.ts index fc669459f..94608ef77 100644 --- a/packages/scripts/src/buildAssets/definitions.ts +++ b/packages/scripts/src/buildAssets/definitions.ts @@ -534,11 +534,11 @@ const blitzModuleTypeToBlitzkit: Record = { export async function definitions() { console.log("Building definitions..."); - using uploader = new AssetUploader("definitions"); + const uploader = new AssetUploader("definitions"); const gameDefinitions: GameDefinitions = { - version: (await vfs.text('Data/version.txt')).split(" ")[0], + version: (await vfs.text("Data/version.txt")).split(" ")[0], nations: ( - await vfs.yaml('Data/available_nations.yaml') + await vfs.yaml("Data/available_nations.yaml") ).available_nations, gameModes: {}, roles: {}, @@ -561,10 +561,12 @@ export async function definitions() { [TankClass.TANK_DESTROYER]: { skills: [] }, }, }; - const nations = vfs.dir('Data/XML/item_defs/vehicles').filter((nation) => nation !== "common") + const nations = vfs + .dir("Data/XML/item_defs/vehicles") + .filter((nation) => nation !== "common"); const tankStringIdMap: Record = {}; const optionalDevices = await vfs.xml<{ root: OptionalDevices }>( - 'Data/XML/item_defs/vehicles/common/optional_devices.xml' + "Data/XML/item_defs/vehicles/common/optional_devices.xml" ); const stringsI18n: Record> = {}; @@ -617,9 +619,7 @@ export async function definitions() { const provisions: ProvisionsCommon = {}; for (const match of ( - await vfs.text( - `Data/XML/item_defs/vehicles/common/consumables/list.xml` - ) + await vfs.text(`Data/XML/item_defs/vehicles/common/consumables/list.xml`) ).matchAll(//g)) { if (match[1] === "prototypes") continue; @@ -634,9 +634,7 @@ export async function definitions() { } for (const match of ( - await vfs.text( - `Data/XML/item_defs/vehicles/common/provisions/list.xml` - ) + await vfs.text(`Data/XML/item_defs/vehicles/common/provisions/list.xml`) ).matchAll(//g)) { if (match[1] === "prototypes") continue; diff --git a/packages/scripts/src/buildAssets/skillIcons.ts b/packages/scripts/src/buildAssets/skillIcons.ts index 4c6b67a4c..60dec1f5d 100644 --- a/packages/scripts/src/buildAssets/skillIcons.ts +++ b/packages/scripts/src/buildAssets/skillIcons.ts @@ -28,7 +28,7 @@ export async function skillIcons() { const avatar = await vfs.xml<{ root: Avatar }>( `Data/XML/item_defs/tankmen/avatar.xml` ); - using uploader = new AssetUploader("skill icons"); + const uploader = new AssetUploader("skill icons"); for (const key in avatar.root.skills) { const skill = avatar.root.skills[key]; diff --git a/packages/scripts/src/core/blitz/extractArmor.ts b/packages/scripts/src/core/blitz/extractArmor.ts index ddfc3c21b..33dd87095 100644 --- a/packages/scripts/src/core/blitz/extractArmor.ts +++ b/packages/scripts/src/core/blitz/extractArmor.ts @@ -6,7 +6,7 @@ import { } from "@blitzkit/core"; import { Accessor, Document, Node, Scene } from "@gltf-transform/core"; import { times } from "lodash-es"; -import { SteamVFS } from "../steam/vfs"; +import { SteamVFS } from "../vfs/vfs"; import { vertexAttributeGLTFName, vertexAttributeGltfVectorSizes, diff --git a/packages/scripts/src/core/blitz/extractModel/index.ts b/packages/scripts/src/core/blitz/extractModel/index.ts index 39340bf2a..14a14e4f2 100644 --- a/packages/scripts/src/core/blitz/extractModel/index.ts +++ b/packages/scripts/src/core/blitz/extractModel/index.ts @@ -9,7 +9,7 @@ import { Document, Material, Node, Scene } from "@gltf-transform/core"; import { dedup, prune } from "@gltf-transform/functions"; import { times } from "lodash-es"; import { dirname } from "path"; -import { SteamVFS } from "../../steam/vfs"; +import { SteamVFS } from "../../vfs/vfs"; import { readBaseColor } from "../readBaseColor"; import { readNormal } from "../readNormal"; import { readRoughnessMetallic } from "../readRoughnessMetallic"; diff --git a/packages/scripts/src/core/github/assetUploader.ts b/packages/scripts/src/core/github/assetUploader.ts index ce24de2e3..0a064e25e 100644 --- a/packages/scripts/src/core/github/assetUploader.ts +++ b/packages/scripts/src/core/github/assetUploader.ts @@ -99,7 +99,7 @@ export class AssetUploader { } if (diff.status === DiffStatus.Unchanged) { - // console.log(`🔵 ${change.path}`); + console.log(`🔵 ${change.path}`); } else { console.log( `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${ diff --git a/packages/scripts/src/core/vfs/abstract.ts b/packages/scripts/src/core/vfs/abstract.ts new file mode 100644 index 000000000..122c33f15 --- /dev/null +++ b/packages/scripts/src/core/vfs/abstract.ts @@ -0,0 +1,84 @@ +import { readDVPL } from "@blitzkit/core"; +import { XMLParser } from "fast-xml-parser"; +import { normalize } from "path/posix"; +import { parse } from "yaml"; + +export abstract class AbstractVFS { + textDecoder = new TextDecoder(); + private xmlParser = new XMLParser(); + + abstract init(): Promise; + + abstract dispose(): void; + + abstract has(path: string): boolean; + + abstract raw(path: string): Promise; + + abstract paths(): string[]; + + resolve(path: string) { + const normalized = normalize(path); + if (this.has(normalized)) return normalized; + + const dvplPath = `${path}.dvpl`; + if (this.has(dvplPath)) return dvplPath; + + return null; + } + + assert(path: string) { + const resolved = this.resolve(path); + + if (resolved === null) throw new Error(`File not found: ${path}`); + + return resolved; + } + + async file(file: string) { + const resolved = this.assert(file); + const raw = await this.raw(resolved); + let buffer = raw; + + if (resolved.endsWith(".dvpl")) { + buffer = new Uint8Array(readDVPL(Buffer.from(raw))); + } + + return buffer; + } + + dir(path: string) { + const parentSegments = path.split("/").length; + const children = new Set(); + + for (const child of this.paths()) { + if (!child.startsWith(path) || child === path) continue; + + const childSegments = child.split("/"); + const nextSegment = childSegments[parentSegments]; + + children.add(nextSegment); + } + + return Array.from(children); + } + + async text(path: string) { + const file = await this.file(path); + return this.textDecoder.decode(file); + } + + async yaml(path: string) { + const file = await this.text(path); + return parse(file) as Type; + } + + async xml(path: string) { + const file = await this.text(path); + return this.xmlParser.parse(file) as Type; + } + + [Symbol.dispose]() { + this.dispose(); + } +} diff --git a/packages/scripts/src/core/steam/vfs.ts b/packages/scripts/src/core/vfs/steam.ts similarity index 54% rename from packages/scripts/src/core/steam/vfs.ts rename to packages/scripts/src/core/vfs/steam.ts index 13318d883..a1432d382 100644 --- a/packages/scripts/src/core/steam/vfs.ts +++ b/packages/scripts/src/core/vfs/steam.ts @@ -1,8 +1,5 @@ -import { readDVPL } from "@blitzkit/core/src/blitz"; -import { XMLParser } from "fast-xml-parser"; -import { normalize } from "path/posix"; import SteamUser, { EConnectionProtocol } from "steam-user"; -import { parse } from "yaml"; +import { AbstractVFS } from "./abstract"; interface SteamManifestFile { chunks: { @@ -33,11 +30,8 @@ interface SteamManifest { crc_clear: number; } -export class SteamVFS { +export class SteamVFS extends AbstractVFS { private steam = new SteamUser({ protocol: EConnectionProtocol.TCP }); - private textDecoder = new TextDecoder(); - private xmlParser = new XMLParser(); - private manifest: Map = new Map(); constructor( @@ -45,7 +39,9 @@ export class SteamVFS { private password: string, private app: number, private depot: number - ) {} + ) { + super(); + } async init() { this.steam.logOn({ @@ -71,10 +67,8 @@ export class SteamVFS { ); }); - const paths: string[] = []; for (const file of manifest.files) { const path = file.filename.replaceAll("\\", "/"); - paths.push(path); this.manifest.set(path, file); } @@ -82,74 +76,23 @@ export class SteamVFS { } has(path: string) { - const normalized = normalize(path); - const dvplPath = `${normalized}.dvpl`; - - return ( - (this.manifest.has(dvplPath) && dvplPath) || - (this.manifest.has(normalized) && normalized) - ); + return this.manifest.has(path); } - assert(requested: string) { - const path = this.has(requested); - - if (!path) throw new Error(`File not found: ${requested}`); - - return path; - } - - dir(path: string) { - const parentSegments = path.split("/").length; - const children: string[] = []; - - for (const child of this.manifest.keys()) { - const childSegments = child.split("/"); - - if ( - child.startsWith(path) && - parentSegments + 1 === childSegments.length - ) { - children.push(childSegments.at(-1)!); - } - } - - return children; - } - - async file(requested: string) { - const path = this.assert(requested); + async raw(path: string) { const fileManifest = this.manifest.get(path)!; const downloaded: { type: "complete"; file: Buffer } = // @ts-expect-error await this.steam.downloadFile(this.app, this.depot, fileManifest); - let buffer = downloaded.file; - if (path.endsWith(".dvpl")) buffer = readDVPL(downloaded.file); - - return new Uint8Array(buffer); + return new Uint8Array(downloaded.file); } - async text(path: string) { - const file = await this.file(path); - return this.textDecoder.decode(file); - } - - async yaml(path: string) { - const file = await this.text(path); - return parse(file) as Type; - } - - async xml(path: string) { - const file = await this.text(path); - return this.xmlParser.parse(file) as Type; + paths() { + return Array.from(this.manifest.keys()); } dispose() { this.steam.logOff(); } - - [Symbol.dispose]() { - this.dispose(); - } } diff --git a/packages/scripts/src/core/vfs/zip.ts b/packages/scripts/src/core/vfs/zip.ts new file mode 100644 index 000000000..80169182e --- /dev/null +++ b/packages/scripts/src/core/vfs/zip.ts @@ -0,0 +1,38 @@ +import { normalize } from "path/posix"; +import { HTTPRangeReader, unzip, ZipEntry } from "unzipit"; +import { AbstractVFS } from "./abstract"; + +export class ZipVFS extends AbstractVFS { + entries = new Map(); + + constructor(private url: string, private root = "app/") { + super(); + } + + async init() { + const reader = new HTTPRangeReader(this.url); + const { entries } = await unzip(reader); + + for (const entry of Object.values(entries)) { + const normalized = normalize(entry.name.slice(this.root.length)); + this.entries.set(normalized, entry); + } + + return this; + } + + has(path: string) { + return this.entries.has(path); + } + + async raw(path: string) { + const arrayBuffer = await this.entries.get(path)!.arrayBuffer(); + return new Uint8Array(arrayBuffer); + } + + paths() { + return Array.from(this.entries.keys()); + } + + dispose() {} +}