TresAbhi/blitzkit
1
0 Fork 0
mirror of https://github.com/blitzkit/blitzkit.git synced 2026-10-24 03:59:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: code split streams

Browse Source
This commit is contained in:
TrèsAbhi committed 2023-12-25 12:37:37 -06:00
1 parent 510eebd2f4
commit cb87a3067c
9 files changed
+856 -832
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/buildAssets.ts
+3 -2
View File
@@ -6,7 +6,7 @@ import { argv } from 'process';
import sharp from 'sharp'; import sharp from 'sharp';
import { TankType } from '../src/components/Tanks'; import { TankType } from '../src/components/Tanks';
import { NATION_IDS } from '../src/constants/nations'; import { NATION_IDS } from '../src/constants/nations';
import { SCPGStream, VertexAttribute } from '../src/core/blitz/SCPGStream'; import { extractModel } from '../src/core/blitz/extractModel';
import { readBase64DVPL } from '../src/core/blitz/readBase64DVPL'; import { readBase64DVPL } from '../src/core/blitz/readBase64DVPL';
import { readDVPLFile } from '../src/core/blitz/readDVPLFile'; import { readDVPLFile } from '../src/core/blitz/readDVPLFile';
import { readStringDVPL } from '../src/core/blitz/readStringDVPL'; import { readStringDVPL } from '../src/core/blitz/readStringDVPL';
@@ -25,6 +25,7 @@ import {
Tier, Tier,
TurretDefinition, TurretDefinition,
} from '../src/core/blitzkrieg/tankDefinitions'; } from '../src/core/blitzkrieg/tankDefinitions';
import { VertexAttribute } from '../src/core/streams/scpg';
const vertexAttributeGLTFName: Partial<Record<VertexAttribute, string>> = { const vertexAttributeGLTFName: Partial<Record<VertexAttribute, string>> = {
[VertexAttribute.VERTEX]: 'POSITION', [VertexAttribute.VERTEX]: 'POSITION',
@@ -526,7 +527,7 @@ if (allTargets || targets?.includes('tankModels')) {
const parameters = await readYAMLDVPL<TankParameters>( const parameters = await readYAMLDVPL<TankParameters>(
`${DATA}/${DOI.tankParameters}/${nation}/${tankIndex}.yaml.dvpl`, `${DATA}/${DOI.tankParameters}/${nation}/${tankIndex}.yaml.dvpl`,
); );
const model = await SCPGStream.extractModel( const model = await extractModel(
DATA, DATA,
parameters.resourcesPath.blitzModelPath.replace(/\.sc2$/, ''), parameters.resourcesPath.blitzModelPath.replace(/\.sc2$/, ''),
); );
src/core/blitz/SCPGStream.ts
-830
View File
@@ -1,830 +0,0 @@
import { Document, Material, Node, Scene } from '@gltf-transform/core';
import { writeFileSync } from 'fs';
import { range, times } from 'lodash';
import { dirname } from 'path';
import sharp from 'sharp';
import { Matrix4, Quaternion, Vector3, Vector4Tuple } from 'three';
import { Hierarchy, SC2, Textures } from '../../types/sc2';
import { readTexture } from '../blitzkrieg/readTexture';
import { readDVPL } from './readDVPL';
import { readDVPLFile } from './readDVPLFile';
const MAX_FLOAT32 = 2 ** 127 * (2 - 2 ** -23);
enum KAType {
NONE = 0,
BOOLEAN = 1,
INT32 = 2,
FLOAT = 3,
STRING = 4,
WIDE_STRING = 5,
BYTE_ARRAY = 6,
UINT32 = 7,
KEYED_ARCHIVE = 8,
INT64 = 9,
UINT64 = 10,
VECTOR2 = 11,
VECTOR3 = 12,
VECTOR4 = 13,
MATRIX2 = 14,
MATRIX3 = 15,
MATRIX4 = 16,
COLOR = 17,
FASTNAME = 18,
AABBOX3 = 19,
FILEPATH = 20,
FLOAT64 = 21,
INT8 = 22,
UINT8 = 23,
INT16 = 24,
UINT16 = 25,
ARRAY = 27,
TRANSFORM = 29,
}
export enum VertexAttribute {
VERTEX = 0,
NORMAL = 1,
COLOR = 2,
TEXCOORD0 = 3,
TEXCOORD1 = 4,
TEXCOORD2 = 5,
TEXCOORD3 = 6,
TANGENT = 7,
BINORMAL = 8,
HARD_JOINTINDEX = 9,
PIVOT4 = 10,
PIVOT_DEPRECATED = 11,
FLEXIBILITY = 12,
ANGLE_SIN_COS = 13,
JOINTINDEX = 14,
JOINTWEIGHT = 15,
CUBETEXCOORD0 = 16,
CUBETEXCOORD1 = 17,
CUBETEXCOORD2 = 18,
CUBETEXCOORD3 = 19,
}
export const vertexAttributeVectorSizes = {
[VertexAttribute.VERTEX]: 3,
[VertexAttribute.NORMAL]: 3,
[VertexAttribute.COLOR]: 1,
[VertexAttribute.TEXCOORD0]: 2,
[VertexAttribute.TEXCOORD1]: 2,
[VertexAttribute.TEXCOORD2]: 2,
[VertexAttribute.TEXCOORD3]: 2,
[VertexAttribute.TANGENT]: 3,
[VertexAttribute.BINORMAL]: 3,
[VertexAttribute.HARD_JOINTINDEX]: 1,
[VertexAttribute.CUBETEXCOORD0]: 3,
[VertexAttribute.CUBETEXCOORD1]: 3,
[VertexAttribute.CUBETEXCOORD2]: 3,
[VertexAttribute.CUBETEXCOORD3]: 3,
[VertexAttribute.PIVOT4]: 4,
[VertexAttribute.PIVOT_DEPRECATED]: 3,
[VertexAttribute.FLEXIBILITY]: 1,
[VertexAttribute.ANGLE_SIN_COS]: 2,
[VertexAttribute.JOINTINDEX]: 4,
[VertexAttribute.JOINTWEIGHT]: 4,
} as const;
const vertexAttributeGLTFName: Partial<Record<VertexAttribute, string>> = {
[VertexAttribute.VERTEX]: 'POSITION',
[VertexAttribute.NORMAL]: 'NORMAL',
[VertexAttribute.COLOR]: 'COLOR_0',
[VertexAttribute.TEXCOORD0]: 'TEXCOORD_0',
[VertexAttribute.TEXCOORD1]: 'TEXCOORD_0',
[VertexAttribute.TEXCOORD2]: 'TEXCOORD_0',
[VertexAttribute.TEXCOORD3]: 'TEXCOORD_0',
[VertexAttribute.TANGENT]: 'TANGENT',
[VertexAttribute.JOINTINDEX]: 'JOINT_0',
[VertexAttribute.JOINTWEIGHT]: 'WEIGHT_0',
};
interface PolygonGroupRaw {
'##name': 'PolygonGroup';
'#id': Buffer;
cubeTextureCoordCount: number;
indexCount: number;
indexFormat: 0 | 1;
indices: Buffer;
packing: 0 | 1;
primitiveCount: number;
rhi_primitiveType: 1 | 2 | 10;
textureCoordCount: number;
vertexCount: number;
vertexFormat: number;
vertices: Buffer;
}
type BlitzkriegVertex = { attribute: VertexAttribute; value: number[] }[];
interface BlitzkriegPolygonGroup {
vertices: BlitzkriegVertex[];
indices: number[];
}
export class SCPGStream {
public index = 0;
constructor(public buffer: Buffer) {}
skip(size: number) {
this.index += size;
}
back(size: number) {
this.skip(-size);
}
read(size: number) {
return this.buffer.subarray(this.index, this.index + size);
}
consume(size: number) {
const subarray = this.read(size);
this.skip(size);
return subarray;
}
readRemaining() {
return this.read(Number.POSITIVE_INFINITY);
}
readRemainingLength() {
return this.buffer.length - this.index;
}
consumeRemaining() {
return this.consume(Number.POSITIVE_INFINITY);
}
consumeByteArray(size: number) {
return this.consume(size);
}
consumeAscii(size: number) {
return this.consume(size).toString('ascii');
}
consumeInt8() {
return this.consume(1).readInt8();
}
consumeInt16() {
return this.consume(2).readInt16LE();
}
consumeInt32() {
return this.consume(4).readInt32LE();
}
consumeInt64() {
return this.consume(8).readBigInt64LE();
}
consumeUInt8() {
return this.consume(1).readUInt8();
}
consumeUInt16() {
return this.consume(2).readUInt16LE();
}
consumeUInt32() {
return this.consume(4).readUInt32LE();
}
consumeUInt64() {
return this.consume(8).readBigUInt64LE();
}
consumeFloat() {
return this.consume(4).readFloatLE();
}
consumeBoolean() {
return this.consume(1).readUInt8() === 1;
}
consumeVectorN(size: number) {
return times(size, () => this.consumeFloat());
}
consumeVector2() {
return this.consumeVectorN(2);
}
consumeVector3() {
return this.consumeVectorN(3);
}
consumeVector4() {
return this.consumeVectorN(4);
}
consumeMatrixN(size: number) {
return times(size, () => this.consumeVectorN(size));
}
consumeMatrix2() {
return this.consumeMatrixN(2);
}
consumeMatrix3() {
return this.consumeMatrixN(3);
}
consumeMatrix4() {
return this.consumeMatrixN(4);
}
static getVertexStride(vertexFormat: number) {
const flags = vertexFormat.toString(2);
return Object.values(VertexAttribute)
.filter((value) => typeof value === 'number')
.reduce(
(accumulator, attribute) =>
flags[attribute as VertexAttribute] === '1'
? accumulator +
vertexAttributeVectorSizes[attribute as VertexAttribute] * 4
: accumulator,
0,
);
}
consumeSCGHeader() {
return {
name: this.consumeAscii(4),
version: this.consumeUInt32(),
nodeCount: this.consumeUInt32(),
nodeCount2: this.consumeUInt32(),
};
}
consumeSCG() {
const header = this.consumeSCGHeader();
const polygonGroupsRaw: PolygonGroupRaw[] = [];
const polygonGroups = new Map<bigint, BlitzkriegPolygonGroup>();
for (let index = 0; index < header.nodeCount; index++) {
const polygonGroupRaw = this.consumeKA() as PolygonGroupRaw;
polygonGroupsRaw.push(polygonGroupRaw);
}
polygonGroupsRaw.forEach((polygonGroupRaw) => {
const indicesStream = new SCPGStream(polygonGroupRaw.indices);
const indices: number[] = [];
for (let index = 0; index < polygonGroupRaw.indexCount; index++) {
indices.push(
polygonGroupRaw.indexFormat === 0
? indicesStream.consumeUInt16()
: indicesStream.consumeUInt32(),
);
}
const verticesStream = new SCPGStream(polygonGroupRaw.vertices);
const vertices: BlitzkriegVertex[] = [];
const stride =
SCPGStream.getVertexStride(polygonGroupRaw.vertexFormat) *
polygonGroupRaw.vertexCount;
if (verticesStream.buffer.length !== stride) {
console.warn(
`Vertex stride mismatch; expected ${stride}, got ${verticesStream.buffer.length}`,
);
return;
}
const vertexFormat = (
polygonGroupRaw.vertexFormat
.toString(2)
.split('')
.map((bitString, index) =>
bitString === '1' ? index : null,
) as VertexAttribute[]
).filter((type) => type !== null);
for (let index = 0; index < polygonGroupRaw.vertexCount; index++) {
vertices[index] = vertexFormat.map((attribute) => {
return {
attribute,
value: verticesStream.consumeVectorN(
vertexAttributeVectorSizes[attribute],
),
};
});
}
if (verticesStream.readRemainingLength() !== 0) {
throw new RangeError('Vertices stream was not fully consumed');
}
polygonGroups.set(polygonGroupRaw['#id'].readBigUInt64LE(), {
vertices,
indices,
});
});
return polygonGroups;
}
consumeSC2Header() {
const header = {
name: this.consumeAscii(4),
version: this.consumeUInt32(),
nodeCount: this.consumeUInt32(),
};
const versionTags = this.consumeKA();
const descriptor = this.consumeSC2Descriptor();
return { header, versionTags, descriptor };
}
consumeSC2Descriptor() {
const size = this.consumeUInt32();
const fileType = this.consumeUInt32();
// other felids that we don't care about for some reason lol
this.skip(size - 4);
return { size, fileType };
}
consumeSC2() {
this.consumeSC2Header();
return this.consumeKA() as SC2;
}
consumeKAValue(
type: KAType = this.consumeUInt8(),
stringTable?: Record<number, string>,
): unknown {
switch (type) {
case KAType.NONE:
return undefined;
case KAType.FILEPATH:
case KAType.STRING: {
if (stringTable) {
const id = this.consumeUInt32();
return stringTable[id];
} else {
const length = this.consumeUInt32();
return this.consumeAscii(length);
}
}
case KAType.BYTE_ARRAY: {
const length = this.consumeUInt32();
return this.consumeByteArray(length);
}
case KAType.FLOAT:
return this.consumeFloat();
case KAType.ARRAY: {
const length = this.consumeUInt32();
const value = [];
for (let index = 0; index < length; index++) {
value.push(this.consumeKAValue(undefined, stringTable));
}
return value;
}
case KAType.KEYED_ARCHIVE: {
this.skip(4); // UInt32 length of the nested archive in bytes
return this.consumeKA(stringTable);
}
case KAType.INT8:
return this.consumeInt8();
case KAType.INT16:
return this.consumeInt16();
case KAType.INT32:
return this.consumeInt32();
case KAType.INT64:
return this.consumeInt64();
case KAType.UINT8:
return this.consumeUInt8();
case KAType.UINT16:
return this.consumeUInt16();
case KAType.UINT32:
return this.consumeUInt32();
case KAType.UINT64:
return this.consumeUInt64();
case KAType.BOOLEAN:
return this.consumeBoolean();
case KAType.VECTOR2:
return this.consumeVector2();
case KAType.VECTOR3:
return this.consumeVector3();
case KAType.VECTOR4:
return this.consumeVector4();
case KAType.COLOR:
return this.consumeVector4(); // as RGBA
case KAType.MATRIX2:
return this.consumeMatrix2();
case KAType.MATRIX3:
return this.consumeMatrix3();
case KAType.MATRIX4:
return this.consumeMatrix4();
case KAType.AABBOX3: {
const minimum = this.consumeVector3();
const maximum = this.consumeVector3();
return { minimum, maximum };
}
case KAType.FASTNAME: {
if (!stringTable) throw new Error('No string table provided');
const index = this.consumeUInt32();
return stringTable[index];
}
case KAType.TRANSFORM: {
const position = this.consumeVector3();
const scale = this.consumeVector3();
const rotation = this.consumeVector4();
return {
position,
scale,
rotation,
};
}
default:
throw new TypeError(
`Unhandled KA type: ${type} (${KAType[type] ?? 'unknown type'})`,
);
}
}
consumeKA(stringTable?: Record<number, string>) {
this.skip(2); // "KA"
const version = this.consumeUInt16();
const pairs: Record<string, any> = {};
if (version === 0x0001) {
const count = this.consumeUInt32();
for (let index = 0; index < count; index++) {
const name = this.consumeKAValue(undefined, stringTable) as string;
const value = this.consumeKAValue(undefined, stringTable);
pairs[name] = value;
}
} else if (version === 0x0002) {
const count = this.consumeUInt32();
const strings: string[] = [];
const stringTable: Record<number, string> = {};
for (let index = 0; index < count; index++) {
const length = this.consumeUInt16();
const string = this.consumeAscii(length);
strings.push(string);
}
for (let index = 0; index < count; index++) {
const id = this.consumeUInt32();
stringTable[id] = strings[index];
}
const nodeCount = this.consumeUInt32();
for (let index = 0; index < nodeCount; index++) {
const keyId = this.consumeUInt32();
const value = this.consumeKAValue(undefined, stringTable);
const key = stringTable[keyId];
pairs[key] = value;
}
} else if (version === 0x0102) {
if (!stringTable) throw new Error('No string table provided');
const count = this.consumeUInt32();
for (let index = 0; index < count; index++) {
const keyIndex = this.consumeUInt32();
const key = stringTable[keyIndex];
const valueType = this.consumeUInt8();
if (valueType === KAType.STRING) {
const valueIndex = this.consumeUInt32();
const value = stringTable[valueIndex];
pairs[key] = value;
} else {
const value = this.consumeKAValue(valueType, stringTable);
pairs[key] = value;
}
}
} else if (version === 0xff02) {
// same as 0x0102 but empty and with no count
return {};
} else throw new RangeError(`Unhandled KA version: ${version}`);
return pairs;
}
static fromDVPL(buffer: Buffer) {
return new SCPGStream(readDVPL(buffer));
}
static async fromDVPLFile(file: string) {
return new SCPGStream(await readDVPLFile(file));
}
static async extractModel(data: string, path: string) {
const sc2Path = `${data}/3d/${path}.sc2.dvpl`;
const scgPath = `${data}/3d/${path}.scg.dvpl`;
const sc2 = (await SCPGStream.fromDVPLFile(sc2Path)).consumeSC2();
const scg = (await SCPGStream.fromDVPLFile(scgPath)).consumeSCG();
const document = new Document();
const scene = document.createScene();
const buffer = document.createBuffer();
const materials = new Map<bigint, Material | bigint>();
writeFileSync('test.sc2.json', JSON.stringify(sc2, null, 2));
// create materials
await Promise.all(
sc2['#dataNodes'].map(async (node) => {
const id = node['#id'].readBigUInt64LE();
if (node.parentMaterialKey !== undefined) {
/**
* material depends on a parent and doesn't seem to have any
* properties so we just point it to the parent
*/
materials.set(id, node.parentMaterialKey);
return;
}
const material = document.createMaterial(node.materialName);
let textures: Textures | undefined = undefined;
if (node.textures) {
textures = node.textures;
} else if (typeof node.configCount === 'number') {
textures = node.configArchive_0.textures;
}
if (textures) {
material.setBaseColorTexture(
document
.createTexture(node.materialName)
.setMimeType('image/jpg')
.setImage(
await readTexture(
`${data}/3d/${dirname(path)}/${textures.albedo}`,
),
),
);
if (textures.baseRMMap) {
material.setMetallicRoughnessTexture(
document
.createTexture(node.materialName)
.setMimeType('image/jpg')
.setImage(
await sharp(
await readTexture(
`${data}/3d/${dirname(path)}/${textures.baseRMMap}`,
),
)
.raw()
.toBuffer({ resolveWithObject: true })
.then(({ data, info }) => {
// fake it till you make it
for (let index = 0; index < data.length; index++) {
data[index] = 255 - data[index];
}
return sharp(data, { raw: info }).jpeg().toBuffer();
}),
),
);
}
if (textures.baseNormalMap) {
material.setNormalTexture(
document
.createTexture(node.materialName)
.setMimeType('image/jpg')
.setImage(
await sharp(
await readTexture(
`${data}/3d/${dirname(path)}/${textures.baseNormalMap}`,
),
)
.raw()
.toBuffer({ resolveWithObject: true })
.then(({ data, info }) => {
// https://www.youtube.com/watch?v=CGys6CnnYSg
for (let index = 0; index < data.length; index += 3) {
data[index + 2] = 255 - data[index + 2];
}
return sharp(data, { raw: info }).jpeg().toBuffer();
}),
),
);
}
}
materials.set(node['#id'].readBigUInt64LE(), material);
}),
);
// replace children materials with parents
materials.forEach((material, id) => {
if (typeof material !== 'bigint') return;
let resolvedMaterial: Material | bigint = material;
while (typeof resolvedMaterial === 'bigint') {
const linkedParentMaterial = materials.get(resolvedMaterial);
if (linkedParentMaterial === undefined) {
throw new Error('Could not resolve material');
}
resolvedMaterial = linkedParentMaterial;
}
materials.set(id, resolvedMaterial);
});
writeFileSync('test.sc2.json', JSON.stringify(sc2, null, 2));
function parseHierarchies(hierarchies: Hierarchy[], parent: Scene | Node) {
hierarchies.forEach((hierarchy) => {
const node = document.createNode(hierarchy.name);
const components = times(
hierarchy.components.count,
(index) => hierarchy.components[index.toString().padStart(4, '0')],
);
components.forEach((component) => {
switch (component['comp.typename']) {
case 'LodComponent':
case 'DecorItemComponent':
case 'NewSlotComponent':
case 'SlotComponent':
break;
case 'TransformComponent': {
const translation = new Vector3();
const rotation = new Quaternion();
const scale = new Vector3();
new Matrix4()
.multiplyMatrices(
new Matrix4().compose(
new Vector3().fromArray(component['tc.worldTranslation']),
new Quaternion().fromArray(component['tc.worldRotation']),
new Vector3().fromArray(component['tc.worldScale']),
),
new Matrix4().compose(
new Vector3().fromArray(component['tc.localTranslation']),
new Quaternion().fromArray(component['tc.localRotation']),
new Vector3().fromArray(component['tc.localScale']),
),
)
.decompose(translation, rotation, scale);
node.setTranslation(translation.toArray());
node.setRotation(rotation.toArray() as Vector4Tuple);
node.setScale(scale.toArray());
break;
}
case 'RenderComponent': {
let minLODDistanceBatchId: undefined | number = undefined;
if (component['rc.renderObj']['rb0.lodIndex'] === -1) {
minLODDistanceBatchId = 0;
} else {
const lodList = components.find(
(component) => component['comp.typename'] === 'LodComponent',
);
if (
lodList === undefined ||
lodList['comp.typename'] !== 'LodComponent' // type annotation hack
) {
throw new SyntaxError('Missing LodComponent');
}
let minLODDistance = Infinity;
range(component['rc.renderObj']['ro.batchCount']).forEach(
(id) => {
const lodIndex =
component['rc.renderObj'][`rb${id}.lodIndex`];
const lodDistance =
lodList['lc.loddist'][`distance${lodIndex}`];
if (
lodDistance <= minLODDistance ||
lodDistance === MAX_FLOAT32
) {
minLODDistance = lodDistance;
minLODDistanceBatchId = id;
}
},
);
}
const batch =
component['rc.renderObj']['ro.batches'][
minLODDistanceBatchId!.toString().padStart(4, '0')
];
const polygonGroup = scg.get(batch['rb.datasource']);
if (!polygonGroup) {
console.warn(
`Missing polygon group ${batch['rb.datasource']}; skipping...`,
);
break;
}
const indexedVertexAttributes: VertexAttribute[] = [];
const unpackedVertices: Partial<
Record<VertexAttribute, number[]>
> = {};
polygonGroup.vertices.forEach((vertex) => {
vertex.map((vertexItem) => {
if (!(vertexItem.attribute in unpackedVertices)) {
unpackedVertices[vertexItem.attribute] = [];
indexedVertexAttributes.push(vertexItem.attribute);
}
unpackedVertices[vertexItem.attribute]!.push(
...vertexItem.value,
);
});
});
const indexAccessor = document
.createAccessor()
.setType('SCALAR')
.setArray(new Uint16Array(polygonGroup.indices))
.setBuffer(buffer);
const material = materials.get(batch['rb.nmatname']);
if (!(material instanceof Material)) {
throw new Error(
`Material ${batch['rb.nmatname']} is unresolved`,
);
}
const primitive = document
.createPrimitive()
.setIndices(indexAccessor)
.setMaterial(material);
indexedVertexAttributes.forEach((attribute) => {
if (!(attribute in vertexAttributeGLTFName)) return;
const vertexSize = vertexAttributeVectorSizes[attribute];
if (
!primitive.getAttribute(vertexAttributeGLTFName[attribute]!)
) {
primitive.setAttribute(
vertexAttributeGLTFName[attribute]!,
document
.createAccessor()
.setType(vertexSize === 1 ? 'SCALAR' : `VEC${vertexSize}`)
.setArray(new Float32Array(unpackedVertices[attribute]!))
.setBuffer(buffer),
);
}
});
const mesh = document
.createMesh(batch['##name'])
.addPrimitive(primitive);
node.setMesh(mesh);
break;
}
default:
throw new TypeError(
`Unhandled component type: ${component['comp.typename']}`,
);
}
});
if (hierarchy['#hierarchy']) {
parseHierarchies(hierarchy['#hierarchy'], node);
}
parent.addChild(node);
});
}
// rotate 90 degrees on the x axis
const rootNode = document
.createNode()
.setRotation([Math.cos(Math.PI / 4), 0, 0, -Math.sin(Math.PI / 4)]);
scene.addChild(rootNode);
parseHierarchies(sc2['#hierarchy'], rootNode);
return document;
}
}
src/core/blitz/extractModel.ts
+330
View File
@@ -0,0 +1,330 @@
import { Document, Material, Node, Scene } from '@gltf-transform/core';
import { writeFileSync } from 'fs';
import { range, times } from 'lodash';
import { dirname } from 'path';
import sharp from 'sharp';
import { Matrix4, Quaternion, Vector3, Vector4Tuple } from 'three';
import { readTexture } from '../blitzkrieg/readTexture';
import { Hierarchy, Sc2Stream, Textures } from '../streams/sc2';
import { ScgStream, vertexAttributeVectorSizes } from '../streams/scg';
import { VertexAttribute } from '../streams/scpg';
import { readDVPLFile } from './readDVPLFile';
const MAX_FLOAT32 = 2 ** 127 * (2 - 2 ** -23);
const vertexAttributeGLTFName: Partial<Record<VertexAttribute, string>> = {
[VertexAttribute.VERTEX]: 'POSITION',
[VertexAttribute.NORMAL]: 'NORMAL',
[VertexAttribute.COLOR]: 'COLOR_0',
[VertexAttribute.TEXCOORD0]: 'TEXCOORD_0',
[VertexAttribute.TEXCOORD1]: 'TEXCOORD_0',
[VertexAttribute.TEXCOORD2]: 'TEXCOORD_0',
[VertexAttribute.TEXCOORD3]: 'TEXCOORD_0',
[VertexAttribute.TANGENT]: 'TANGENT',
[VertexAttribute.JOINTINDEX]: 'JOINT_0',
[VertexAttribute.JOINTWEIGHT]: 'WEIGHT_0',
};
export async function extractModel(data: string, path: string) {
const sc2Path = `${data}/3d/${path}.sc2.dvpl`;
const scgPath = `${data}/3d/${path}.scg.dvpl`;
const sc2 = new Sc2Stream(await readDVPLFile(sc2Path)).sc2();
const scg = new ScgStream(await readDVPLFile(scgPath)).scg();
const document = new Document();
const scene = document.createScene();
const buffer = document.createBuffer();
const materials = new Map<bigint, Material | bigint>();
writeFileSync('test.sc2.json', JSON.stringify(sc2, null, 2));
// create materials
await Promise.all(
sc2['#dataNodes'].map(async (node) => {
const id = node['#id'].readBigUInt64LE();
if (node.parentMaterialKey !== undefined) {
/**
* material depends on a parent and doesn't seem to have any
* properties so we just point it to the parent
*/
materials.set(id, node.parentMaterialKey);
return;
}
const material = document.createMaterial(node.materialName);
let textures: Textures | undefined = undefined;
if (node.textures) {
textures = node.textures;
} else if (typeof node.configCount === 'number') {
textures = node.configArchive_0.textures;
}
if (textures) {
material.setBaseColorTexture(
document
.createTexture(node.materialName)
.setMimeType('image/jpg')
.setImage(
await readTexture(
`${data}/3d/${dirname(path)}/${textures.albedo}`,
),
),
);
if (textures.baseRMMap) {
material.setMetallicRoughnessTexture(
document
.createTexture(node.materialName)
.setMimeType('image/jpg')
.setImage(
await sharp(
await readTexture(
`${data}/3d/${dirname(path)}/${textures.baseRMMap}`,
),
)
.raw()
.toBuffer({ resolveWithObject: true })
.then(({ data, info }) => {
// fake it till you make it
for (let index = 0; index < data.length; index++) {
data[index] = 255 - data[index];
}
return sharp(data, { raw: info }).jpeg().toBuffer();
}),
),
);
}
if (textures.baseNormalMap) {
material.setNormalTexture(
document
.createTexture(node.materialName)
.setMimeType('image/jpg')
.setImage(
await sharp(
await readTexture(
`${data}/3d/${dirname(path)}/${textures.baseNormalMap}`,
),
)
.raw()
.toBuffer({ resolveWithObject: true })
.then(({ data, info }) => {
// https://www.youtube.com/watch?v=CGys6CnnYSg
for (let index = 0; index < data.length; index += 3) {
data[index + 2] = 255 - data[index + 2];
}
return sharp(data, { raw: info }).jpeg().toBuffer();
}),
),
);
}
}
materials.set(node['#id'].readBigUInt64LE(), material);
}),
);
// replace children materials with parents
materials.forEach((material, id) => {
if (typeof material !== 'bigint') return;
let resolvedMaterial: Material | bigint = material;
while (typeof resolvedMaterial === 'bigint') {
const linkedParentMaterial = materials.get(resolvedMaterial);
if (linkedParentMaterial === undefined) {
throw new Error('Could not resolve material');
}
resolvedMaterial = linkedParentMaterial;
}
materials.set(id, resolvedMaterial);
});
writeFileSync('test.sc2.json', JSON.stringify(sc2, null, 2));
function parseHierarchies(hierarchies: Hierarchy[], parent: Scene | Node) {
hierarchies.forEach((hierarchy) => {
const node = document.createNode(hierarchy.name);
const components = times(
hierarchy.components.count,
(index) => hierarchy.components[index.toString().padStart(4, '0')],
);
components.forEach((component) => {
switch (component['comp.typename']) {
case 'LodComponent':
case 'DecorItemComponent':
case 'NewSlotComponent':
case 'SlotComponent':
break;
case 'TransformComponent': {
const translation = new Vector3();
const rotation = new Quaternion();
const scale = new Vector3();
new Matrix4()
.multiplyMatrices(
new Matrix4().compose(
new Vector3().fromArray(component['tc.worldTranslation']),
new Quaternion().fromArray(component['tc.worldRotation']),
new Vector3().fromArray(component['tc.worldScale']),
),
new Matrix4().compose(
new Vector3().fromArray(component['tc.localTranslation']),
new Quaternion().fromArray(component['tc.localRotation']),
new Vector3().fromArray(component['tc.localScale']),
),
)
.decompose(translation, rotation, scale);
node.setTranslation(translation.toArray());
node.setRotation(rotation.toArray() as Vector4Tuple);
node.setScale(scale.toArray());
break;
}
case 'RenderComponent': {
let minLODDistanceBatchId: undefined | number = undefined;
if (component['rc.renderObj']['rb0.lodIndex'] === -1) {
minLODDistanceBatchId = 0;
} else {
const lodList = components.find(
(component) => component['comp.typename'] === 'LodComponent',
);
if (
lodList === undefined ||
lodList['comp.typename'] !== 'LodComponent' // type annotation hack
) {
throw new SyntaxError('Missing LodComponent');
}
let minLODDistance = Infinity;
range(component['rc.renderObj']['ro.batchCount']).forEach(
(id) => {
const lodIndex =
component['rc.renderObj'][`rb${id}.lodIndex`];
const lodDistance =
lodList['lc.loddist'][`distance${lodIndex}`];
if (
lodDistance <= minLODDistance ||
lodDistance === MAX_FLOAT32
) {
minLODDistance = lodDistance;
minLODDistanceBatchId = id;
}
},
);
}
const batch =
component['rc.renderObj']['ro.batches'][
minLODDistanceBatchId!.toString().padStart(4, '0')
];
const polygonGroup = scg.get(batch['rb.datasource']);
if (!polygonGroup) {
console.warn(
`Missing polygon group ${batch['rb.datasource']}; skipping...`,
);
break;
}
const indexedVertexAttributes: VertexAttribute[] = [];
const unpackedVertices: Partial<Record<VertexAttribute, number[]>> =
{};
polygonGroup.vertices.forEach((vertex) => {
vertex.map((vertexItem) => {
if (!(vertexItem.attribute in unpackedVertices)) {
unpackedVertices[vertexItem.attribute] = [];
indexedVertexAttributes.push(vertexItem.attribute);
}
unpackedVertices[vertexItem.attribute]!.push(
...vertexItem.value,
);
});
});
const indexAccessor = document
.createAccessor()
.setType('SCALAR')
.setArray(new Uint16Array(polygonGroup.indices))
.setBuffer(buffer);
const material = materials.get(batch['rb.nmatname']);
if (!(material instanceof Material)) {
throw new Error(`Material ${batch['rb.nmatname']} is unresolved`);
}
const primitive = document
.createPrimitive()
.setIndices(indexAccessor)
.setMaterial(material);
indexedVertexAttributes.forEach((attribute) => {
if (!(attribute in vertexAttributeGLTFName)) return;
const vertexSize = vertexAttributeVectorSizes[attribute];
if (
!primitive.getAttribute(vertexAttributeGLTFName[attribute]!)
) {
primitive.setAttribute(
vertexAttributeGLTFName[attribute]!,
document
.createAccessor()
.setType(vertexSize === 1 ? 'SCALAR' : `VEC${vertexSize}`)
.setArray(new Float32Array(unpackedVertices[attribute]!))
.setBuffer(buffer),
);
}
});
const mesh = document
.createMesh(batch['##name'])
.addPrimitive(primitive);
node.setMesh(mesh);
break;
}
default:
throw new TypeError(
`Unhandled component type: ${component['comp.typename']}`,
);
}
});
if (hierarchy['#hierarchy']) {
parseHierarchies(hierarchy['#hierarchy'], node);
}
parent.addChild(node);
});
}
// rotate 90 degrees on the x axis
const rootNode = document
.createNode()
.setRotation([Math.cos(Math.PI / 4), 0, 0, -Math.sin(Math.PI / 4)]);
scene.addChild(rootNode);
parseHierarchies(sc2['#hierarchy'], rootNode);
return document;
}
src/core/blitzkrieg/readDds.ts
+15
View File
@@ -0,0 +1,15 @@
import { readDVPLFile } from '../blitz/readDVPLFile';
import { PrimitiveStream } from '../streams/primitive';
export function readDds(buffer: Buffer) {
const stream = new PrimitiveStream(buffer);
const name = stream.consumeAscii(4);
if (name !== 'DDS ') throw new TypeError('Not a DDS file');
}
readDds(
await readDVPLFile(
'C:/Program Files (x86)/Steam/steamapps/common/World of Tanks Blitz/Data/3d/Tanks/German/images_pbr/E_100_BC.dx11.dds.dvpl',
),
);
src/core/streams/primitive.ts
+67
View File
@@ -0,0 +1,67 @@
export class PrimitiveStream {
public index = 0;
constructor(public buffer: Buffer) {}
skip(size: number) {
this.index += size;
}
read(size: number) {
return this.buffer.subarray(this.index, this.index + size);
}
consume(size: number) {
const subarray = this.read(size);
this.skip(size);
return subarray;
}
readRemaining() {
return this.read(Number.POSITIVE_INFINITY);
}
readRemainingLength() {
return this.buffer.length - this.index;
}
consumeRemaining() {
return this.consume(Number.POSITIVE_INFINITY);
}
consumeByteArray(size: number) {
return this.consume(size);
}
consumeAscii(size: number) {
return this.consume(size).toString('ascii');
}
int8() {
return this.consume(1).readInt8();
}
int16() {
return this.consume(2).readInt16LE();
}
int32() {
return this.consume(4).readInt32LE();
}
int64() {
return this.consume(8).readBigInt64LE();
}
uint8() {
return this.consume(1).readUInt8();
}
uint16() {
return this.consume(2).readUInt16LE();
}
uint32() {
return this.consume(4).readUInt32LE();
}
uint64() {
return this.consume(8).readBigUInt64LE();
}
float() {
return this.consume(4).readFloatLE();
}
boolean() {
return this.consume(1).readUInt8() === 1;
}
}
src/types/sc2.ts → src/core/streams/sc2.ts
+33
View File
@@ -1,4 +1,5 @@
import { Vector3Tuple, Vector4Tuple } from 'three'; import { Vector3Tuple, Vector4Tuple } from 'three';
import { ScpgStream } from './scpg';
export interface SC2 { export interface SC2 {
'#dataNodes': DataNode[]; '#dataNodes': DataNode[];
@@ -172,3 +173,35 @@ export type Component =
interface SceneComponents { interface SceneComponents {
count: number; count: number;
} }
export class Sc2Stream extends ScpgStream {
header() {
return {
name: this.consumeAscii(4),
version: this.uint32(),
nodeCount: this.uint32(),
};
}
versionTags() {
return this.ka();
}
descriptor() {
const size = this.uint32();
const fileType = this.uint32();
// other felids that we don't have any info about
this.skip(size - 4);
return { size, fileType };
}
sc2() {
this.header();
this.versionTags();
this.descriptor();
return this.ka() as SC2;
}
}
src/core/streams/scg.ts
+142
View File
@@ -0,0 +1,142 @@
import { ScpgStream, VertexAttribute } from './scpg';
interface PolygonGroupRaw {
'##name': 'PolygonGroup';
'#id': Buffer;
cubeTextureCoordCount: number;
indexCount: number;
indexFormat: 0 | 1;
indices: Buffer;
packing: 0 | 1;
primitiveCount: number;
rhi_primitiveType: 1 | 2 | 10;
textureCoordCount: number;
vertexCount: number;
vertexFormat: number;
vertices: Buffer;
}
type BlitzkriegVertex = { attribute: VertexAttribute; value: number[] }[];
interface BlitzkriegPolygonGroup {
vertices: BlitzkriegVertex[];
indices: number[];
}
export const vertexAttributeVectorSizes = {
[VertexAttribute.VERTEX]: 3,
[VertexAttribute.NORMAL]: 3,
[VertexAttribute.COLOR]: 1,
[VertexAttribute.TEXCOORD0]: 2,
[VertexAttribute.TEXCOORD1]: 2,
[VertexAttribute.TEXCOORD2]: 2,
[VertexAttribute.TEXCOORD3]: 2,
[VertexAttribute.TANGENT]: 3,
[VertexAttribute.BINORMAL]: 3,
[VertexAttribute.HARD_JOINTINDEX]: 1,
[VertexAttribute.CUBETEXCOORD0]: 3,
[VertexAttribute.CUBETEXCOORD1]: 3,
[VertexAttribute.CUBETEXCOORD2]: 3,
[VertexAttribute.CUBETEXCOORD3]: 3,
[VertexAttribute.PIVOT4]: 4,
[VertexAttribute.PIVOT_DEPRECATED]: 3,
[VertexAttribute.FLEXIBILITY]: 1,
[VertexAttribute.ANGLE_SIN_COS]: 2,
[VertexAttribute.JOINTINDEX]: 4,
[VertexAttribute.JOINTWEIGHT]: 4,
} as const;
export class ScgStream extends ScpgStream {
header() {
return {
name: this.consumeAscii(4),
version: this.uint32(),
nodeCount: this.uint32(),
nodeCount2: this.uint32(),
};
}
scg() {
const header = this.header();
const polygonGroupsRaw: PolygonGroupRaw[] = [];
const polygonGroups = new Map<bigint, BlitzkriegPolygonGroup>();
for (let index = 0; index < header.nodeCount; index++) {
const polygonGroupRaw = this.ka() as PolygonGroupRaw;
polygonGroupsRaw.push(polygonGroupRaw);
}
polygonGroupsRaw.forEach((polygonGroupRaw) => {
const indicesStream = new ScpgStream(polygonGroupRaw.indices);
const indices: number[] = [];
for (let index = 0; index < polygonGroupRaw.indexCount; index++) {
indices.push(
polygonGroupRaw.indexFormat === 0
? indicesStream.uint16()
: indicesStream.uint32(),
);
}
const verticesStream = new ScpgStream(polygonGroupRaw.vertices);
const vertices: BlitzkriegVertex[] = [];
const stride =
ScgStream.vertexStride(polygonGroupRaw.vertexFormat) *
polygonGroupRaw.vertexCount;
if (verticesStream.buffer.length !== stride) {
console.warn(
`Vertex stride mismatch; expected ${stride}, got ${verticesStream.buffer.length}`,
);
return;
}
const vertexFormat = (
polygonGroupRaw.vertexFormat
.toString(2)
.split('')
.map((bitString, index) =>
bitString === '1' ? index : null,
) as VertexAttribute[]
).filter((type) => type !== null);
for (let index = 0; index < polygonGroupRaw.vertexCount; index++) {
vertices[index] = vertexFormat.map((attribute) => {
return {
attribute,
value: verticesStream.vectorN(
vertexAttributeVectorSizes[attribute],
),
};
});
}
if (verticesStream.readRemainingLength() !== 0) {
throw new RangeError('Vertices stream was not fully consumed');
}
polygonGroups.set(polygonGroupRaw['#id'].readBigUInt64LE(), {
vertices,
indices,
});
});
return polygonGroups;
}
static vertexStride(vertexFormat: number) {
const flags = vertexFormat.toString(2);
return Object.values(VertexAttribute)
.filter((value) => typeof value === 'number')
.reduce(
(accumulator, attribute) =>
flags[attribute as VertexAttribute] === '1'
? accumulator +
vertexAttributeVectorSizes[attribute as VertexAttribute] * 4
: accumulator,
0,
);
}
}
src/core/streams/scpg.ts
+263
View File
@@ -0,0 +1,263 @@
import { times } from 'lodash';
import { PrimitiveStream } from './primitive';
enum KAType {
NONE = 0,
BOOLEAN = 1,
INT32 = 2,
FLOAT = 3,
STRING = 4,
WIDE_STRING = 5,
BYTE_ARRAY = 6,
UINT32 = 7,
KEYED_ARCHIVE = 8,
INT64 = 9,
UINT64 = 10,
VECTOR2 = 11,
VECTOR3 = 12,
VECTOR4 = 13,
MATRIX2 = 14,
MATRIX3 = 15,
MATRIX4 = 16,
COLOR = 17,
FASTNAME = 18,
AABBOX3 = 19,
FILEPATH = 20,
FLOAT64 = 21,
INT8 = 22,
UINT8 = 23,
INT16 = 24,
UINT16 = 25,
ARRAY = 27,
TRANSFORM = 29,
}
export enum VertexAttribute {
VERTEX = 0,
NORMAL = 1,
COLOR = 2,
TEXCOORD0 = 3,
TEXCOORD1 = 4,
TEXCOORD2 = 5,
TEXCOORD3 = 6,
TANGENT = 7,
BINORMAL = 8,
HARD_JOINTINDEX = 9,
PIVOT4 = 10,
PIVOT_DEPRECATED = 11,
FLEXIBILITY = 12,
ANGLE_SIN_COS = 13,
JOINTINDEX = 14,
JOINTWEIGHT = 15,
CUBETEXCOORD0 = 16,
CUBETEXCOORD1 = 17,
CUBETEXCOORD2 = 18,
CUBETEXCOORD3 = 19,
}
export class ScpgStream extends PrimitiveStream {
vectorN(size: number) {
return times(size, () => this.float());
}
vector2() {
return this.vectorN(2);
}
vector3() {
return this.vectorN(3);
}
vector4() {
return this.vectorN(4);
}
matrixN(size: number) {
return times(size, () => this.vectorN(size));
}
matrix2() {
return this.matrixN(2);
}
matrix3() {
return this.matrixN(3);
}
matrix4() {
return this.matrixN(4);
}
kaValue(
type: KAType = this.uint8(),
stringTable?: Record<number, string>,
): unknown {
switch (type) {
case KAType.NONE:
return undefined;
case KAType.FILEPATH:
case KAType.STRING: {
if (stringTable) {
const id = this.uint32();
return stringTable[id];
} else {
const length = this.uint32();
return this.consumeAscii(length);
}
}
case KAType.BYTE_ARRAY: {
const length = this.uint32();
return this.consumeByteArray(length);
}
case KAType.FLOAT:
return this.float();
case KAType.ARRAY: {
const length = this.uint32();
const value = [];
for (let index = 0; index < length; index++) {
value.push(this.kaValue(undefined, stringTable));
}
return value;
}
case KAType.KEYED_ARCHIVE: {
this.skip(4); // UInt32 length of the nested archive in bytes
return this.ka(stringTable);
}
case KAType.INT8:
return this.int8();
case KAType.INT16:
return this.int16();
case KAType.INT32:
return this.int32();
case KAType.INT64:
return this.int64();
case KAType.UINT8:
return this.uint8();
case KAType.UINT16:
return this.uint16();
case KAType.UINT32:
return this.uint32();
case KAType.UINT64:
return this.uint64();
case KAType.BOOLEAN:
return this.boolean();
case KAType.VECTOR2:
return this.vector2();
case KAType.VECTOR3:
return this.vector3();
case KAType.VECTOR4:
return this.vector4();
case KAType.COLOR:
return this.vector4(); // as RGBA
case KAType.MATRIX2:
return this.matrix2();
case KAType.MATRIX3:
return this.matrix3();
case KAType.MATRIX4:
return this.matrix4();
case KAType.AABBOX3: {
const minimum = this.vector3();
const maximum = this.vector3();
return { minimum, maximum };
}
case KAType.FASTNAME: {
if (!stringTable) throw new Error('No string table provided');
const index = this.uint32();
return stringTable[index];
}
case KAType.TRANSFORM: {
const position = this.vector3();
const scale = this.vector3();
const rotation = this.vector4();
return {
position,
scale,
rotation,
};
}
default:
throw new TypeError(
`Unhandled KA type: ${type} (${KAType[type] ?? 'unknown type'})`,
);
}
}
ka(stringTable?: Record<number, string>) {
this.skip(2); // "KA"
const version = this.uint16();
const pairs: Record<string, any> = {};
if (version === 0x0001) {
const count = this.uint32();
for (let index = 0; index < count; index++) {
const name = this.kaValue(undefined, stringTable) as string;
const value = this.kaValue(undefined, stringTable);
pairs[name] = value;
}
} else if (version === 0x0002) {
const count = this.uint32();
const strings: string[] = [];
const stringTable: Record<number, string> = {};
for (let index = 0; index < count; index++) {
const length = this.uint16();
const string = this.consumeAscii(length);
strings.push(string);
}
for (let index = 0; index < count; index++) {
const id = this.uint32();
stringTable[id] = strings[index];
}
const nodeCount = this.uint32();
for (let index = 0; index < nodeCount; index++) {
const keyId = this.uint32();
const value = this.kaValue(undefined, stringTable);
const key = stringTable[keyId];
pairs[key] = value;
}
} else if (version === 0x0102) {
if (!stringTable) throw new Error('No string table provided');
const count = this.uint32();
for (let index = 0; index < count; index++) {
const keyIndex = this.uint32();
const key = stringTable[keyIndex];
const valueType = this.uint8();
if (valueType === KAType.STRING) {
const valueIndex = this.uint32();
const value = stringTable[valueIndex];
pairs[key] = value;
} else {
const value = this.kaValue(valueType, stringTable);
pairs[key] = value;
}
}
} else if (version === 0xff02) {
// same as 0x0102 but empty and with no count
return {};
} else throw new RangeError(`Unhandled KA version: ${version}`);
return pairs;
}
}
src/core/streams/windows.ts
+3
View File
@@ -0,0 +1,3 @@
import { PrimitiveStream } from './primitive';
export class WindowsStream extends PrimitiveStream {}