fix: more imports 4

packages/scripts/src/buildAssets/tankArmors.ts

@@ -1,7 +1,7 @@
 import { NodeIO } from '@gltf-transform/core';
 import { readdir } from 'fs/promises';
-import { extractArmor } from '../../../website/src/core/blitz/extractArmor';
 import { toUniqueId } from '../../../website/src/core/blitz/toUniqueId';
+import { extractArmor } from '../core/blitz/extractArmor';
 import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
 import { commitAssets } from '../core/github/commitAssets';
 import { FileChange } from '../core/github/commitMultipleFiles';

packages/scripts/src/buildAssets/tankModels.ts

@@ -1,7 +1,7 @@
 import { NodeIO } from '@gltf-transform/core';
 import { readdir } from 'fs/promises';
-import { extractModel } from '../../../website/src/core/blitz/extractModel';
 import { toUniqueId } from '../../../website/src/core/blitz/toUniqueId';
+import { extractModel } from '../core/blitz/extractModel';
 import { readXMLDVPL } from '../core/blitz/readXMLDVPL';
 import { readYAMLDVPL } from '../core/blitz/readYAMLDVPL';
 import { commitAssets } from '../core/github/commitAssets';

packages/scripts/src/core/blitz/extractArmor.ts

@@ -0,0 +1,144 @@
+import {
+  Hierarchy,
+  Sc2ReadStream,
+  ScgReadStream,
+  VertexAttribute,
+} from '@blitzkit/core';
+import { Accessor, Document, Node, Scene } from '@gltf-transform/core';
+import { times } from 'lodash';
+import {
+  vertexAttributeGLTFName,
+  vertexAttributeGltfVectorSizes,
+} from './extractModel/constants';
+import { readDVPLFile } from './readDVPLFile';
+
+export async function extractArmor(data: string, fileName: string) {
+  const sc2Path = `${data}/3d/Tanks/CollisionMeshes/${fileName}.sc2.dvpl`;
+  const scgPath = `${data}/3d/Tanks/CollisionMeshes/${fileName}.scg.dvpl`;
+  const sc2 = new Sc2ReadStream(
+    (await readDVPLFile(sc2Path)).buffer as ArrayBuffer,
+  ).sc2();
+  const scg = new ScgReadStream(
+    (await readDVPLFile(scgPath)).buffer as ArrayBuffer,
+  ).scg();
+  const document = new Document();
+  const scene = document.createScene();
+  const buffer = document.createBuffer();
+
+  function parseHierarchies(hierarchies: Hierarchy[], parent: Scene | Node) {
+    hierarchies.forEach((hierarchy) => {
+      const components = times(
+        hierarchy.components.count,
+        (index) => hierarchy.components[index.toString().padStart(4, '0')],
+      );
+
+      components.forEach((component) => {
+        switch (component['comp.typename']) {
+          case 'TransformComponent':
+            break;
+
+          case 'RenderComponent': {
+            const batch = component['rc.renderObj']['ro.batches']['0000'];
+            const polygonGroup = scg.get(batch['rb.datasource']);
+
+            if (!polygonGroup) {
+              console.warn(
+                `Missing polygon group ${batch['rb.datasource']} (${hierarchy.name}); skipping...`,
+              );
+
+              break;
+            }
+
+            const hardJointIndices = new Set<number>();
+            const vertexHardJointIndices = new Map<number, number>();
+            const attributes = new Map<VertexAttribute, number[][]>();
+
+            polygonGroup.vertices.forEach((vertex, index) => {
+              vertex.forEach(({ attribute, value }) => {
+                if (!attributes.has(attribute)) {
+                  attributes.set(attribute, []);
+                }
+
+                attributes.get(attribute)!.push(value);
+
+                if (attribute === VertexAttribute.HARD_JOINTINDEX) {
+                  hardJointIndices.add(value[0]);
+                  vertexHardJointIndices.set(index, value[0]);
+                }
+              });
+            });
+
+            if (attributes.has(VertexAttribute.TANGENT)) {
+              attributes.set(
+                VertexAttribute.TANGENT,
+                attributes
+                  .get(VertexAttribute.TANGENT)!
+                  .map((tangent) => [...tangent, 1]),
+              );
+            }
+
+            const accessors = new Map<string, Accessor>();
+
+            attributes.forEach((value, attribute) => {
+              const name = vertexAttributeGLTFName[attribute];
+              if (!name || accessors.has(name)) return;
+              const vertexSize = vertexAttributeGltfVectorSizes[attribute];
+              const accessor = document
+                .createAccessor(name)
+                .setType(vertexSize === 1 ? 'SCALAR' : `VEC${vertexSize}`)
+                .setArray(new Float32Array(value.flat()))
+                .setBuffer(buffer);
+
+              accessors.set(name, accessor);
+            });
+
+            hardJointIndices.forEach((hardJointIndex) => {
+              const node = document.createNode(
+                `${hierarchy.name}_armor_${hardJointIndex}`,
+              );
+              const mesh = document.createMesh(batch['##name']);
+              const indicesAccessor = document
+                .createAccessor()
+                .setType('SCALAR')
+                .setArray(
+                  new Uint16Array(
+                    polygonGroup.indices.filter(
+                      (index) =>
+                        vertexHardJointIndices.get(index) === hardJointIndex,
+                    ),
+                  ),
+                )
+                .setBuffer(buffer);
+              const primitive = document
+                .createPrimitive()
+                .setIndices(indicesAccessor);
+
+              accessors.forEach((accessor, name) => {
+                primitive.setAttribute(name, accessor);
+              });
+
+              mesh.addPrimitive(primitive);
+              node.setMesh(mesh);
+              parent.addChild(node);
+            });
+
+            break;
+          }
+
+          default:
+            throw new TypeError(
+              `Unhandled component type: ${component['comp.typename']}`,
+            );
+        }
+      });
+
+      if (hierarchy['#hierarchy']) {
+        parseHierarchies(hierarchy['#hierarchy'], parent);
+      }
+    });
+  }
+
+  parseHierarchies(sc2['#hierarchy'], scene);
+
+  return document;
+}

packages/scripts/src/core/blitz/extractModel/constants.ts

@@ -0,0 +1,18 @@
+import { VertexAttribute, vertexAttributeVectorSizes } from '@blitzkit/core';
+
+export const vertexAttributeGLTFName: Partial<Record<VertexAttribute, string>> =
+  {
+    [VertexAttribute.VERTEX]: 'POSITION',
+    [VertexAttribute.NORMAL]: 'NORMAL',
+    [VertexAttribute.TEXCOORD0]: 'TEXCOORD_0',
+    [VertexAttribute.TEXCOORD1]: 'TEXCOORD_0',
+    [VertexAttribute.TEXCOORD2]: 'TEXCOORD_0',
+    [VertexAttribute.TEXCOORD3]: 'TEXCOORD_0',
+    [VertexAttribute.TANGENT]: 'TANGENT',
+    [VertexAttribute.JOINTINDEX]: 'JOINT_0',
+    [VertexAttribute.JOINTWEIGHT]: 'WEIGHT_0',
+  };
+export const vertexAttributeGltfVectorSizes = {
+  ...vertexAttributeVectorSizes,
+  [VertexAttribute.TANGENT]: 4,
+} as const;

packages/scripts/src/core/blitz/extractModel/index.ts

@@ -0,0 +1,292 @@
+import {
+  Hierarchy,
+  Sc2ReadStream,
+  ScgReadStream,
+  Textures,
+  VertexAttribute,
+} from '@blitzkit/core';
+import { Document, Material, Node, Scene } from '@gltf-transform/core';
+import { times } from 'lodash';
+import { dirname } from 'path';
+import { Matrix4, Quaternion, Vector3, Vector4Tuple } from 'three';
+import { readDVPLFile } from '../readDVPLFile';
+import { readTexture } from '../readTexture';
+import { TextureMutation } from '../readTexture/constants';
+import {
+  vertexAttributeGLTFName,
+  vertexAttributeGltfVectorSizes,
+} from './constants';
+
+const ERROR_ON_UNKNOWN_COMPONENT = false;
+
+const omitMeshNames = {
+  start: ['chassis_chassis_', 'chassis_track_crash_', 'HP_'],
+  end: ['_POINT'],
+};
+
+export async function extractModel(data: string, path: string) {
+  const sc2Path = `${data}/3d/${path}.sc2.dvpl`;
+  const scgPath = `${data}/3d/${path}.scg.dvpl`;
+  const sc2 = new Sc2ReadStream(
+    (await readDVPLFile(sc2Path)).buffer as ArrayBuffer,
+  ).sc2();
+  const scg = new ScgReadStream(
+    (await readDVPLFile(scgPath)).buffer as ArrayBuffer,
+  ).scg();
+  const document = new Document();
+  const scene = document.createScene();
+  const buffer = document.createBuffer();
+  const materials = new Map<bigint, Material | bigint | undefined>();
+
+  // create materials
+  await Promise.all(
+    sc2['#dataNodes'].map(async (node) => {
+      const id = new DataView(node['#id']).getBigUint64(0, true);
+
+      if (node.parentMaterialKey !== undefined) {
+        /**
+         * material depends on a parent and doesn't seem to have any
+         * properties so we just point it to the parent
+         */
+        materials.set(id, node.parentMaterialKey);
+        return;
+      }
+
+      const material = document.createMaterial(node.materialName);
+      let textures: Textures | undefined = undefined;
+
+      if (node.textures) {
+        textures = node.textures;
+      } else if (typeof node.configCount === 'number') {
+        textures = node.configArchive_0.textures;
+      }
+
+      if (textures) {
+        material.setBaseColorTexture(
+          document
+            .createTexture(node.materialName)
+            .setMimeType('image/jpeg')
+            .setImage(
+              await readTexture(
+                `${data}/3d/${dirname(path)}/${textures.albedo}`,
+                TextureMutation.Albedo,
+              ),
+            ),
+        );
+
+        if (textures.baseRMMap) {
+          material.setMetallicRoughnessTexture(
+            document
+              .createTexture(node.materialName)
+              .setMimeType('image/jpeg')
+              .setImage(
+                await readTexture(
+                  `${data}/3d/${dirname(path)}/${textures.baseRMMap}`,
+                  TextureMutation.RoughnessMetallicness,
+                ),
+              ),
+          );
+        }
+
+        if (textures.baseNormalMap ?? textures.normalmap) {
+          const isBase = textures.baseNormalMap !== undefined;
+
+          material.setNormalTexture(
+            document
+              .createTexture(node.materialName)
+              .setMimeType('image/jpeg')
+              .setImage(
+                await readTexture(
+                  `${data}/3d/${dirname(path)}/${
+                    textures.baseNormalMap ?? textures.normalmap
+                  }`,
+                  isBase ? TextureMutation.Normal : undefined,
+                ),
+              ),
+          );
+        }
+
+        if (textures.miscMap) {
+          material.setOcclusionTexture(
+            document
+              .createTexture(node.materialName)
+              .setMimeType('image/jpeg')
+              .setImage(
+                await readTexture(
+                  `${data}/3d/${dirname(path)}/${textures.miscMap}`,
+                  TextureMutation.Miscellaneous,
+                ),
+              ),
+          );
+        }
+
+        materials.set(
+          new DataView(node['#id']).getBigUint64(0, true),
+          material,
+        );
+      }
+    }),
+  );
+
+  // replace children materials with parents
+  materials.forEach((material, id) => {
+    if (typeof material !== 'bigint') return;
+
+    let resolvedMaterial: Material | bigint | undefined = material;
+
+    while (typeof resolvedMaterial === 'bigint') {
+      const linkedParentMaterial = materials.get(resolvedMaterial);
+
+      resolvedMaterial = linkedParentMaterial;
+    }
+
+    materials.set(id, resolvedMaterial);
+  });
+
+  function parseHierarchies(hierarchies: Hierarchy[], parent: Scene | Node) {
+    hierarchies.forEach((hierarchy) => {
+      if (
+        omitMeshNames.start.some((omit) => hierarchy.name.startsWith(omit)) ||
+        omitMeshNames.end.some((omit) => hierarchy.name.endsWith(omit))
+      )
+        return;
+
+      const node = document.createNode(hierarchy.name);
+      const components = times(
+        hierarchy.components.count,
+        (index) => hierarchy.components[index.toString().padStart(4, '0')],
+      );
+
+      components.forEach((component) => {
+        switch (component['comp.typename']) {
+          case 'LodComponent':
+            // found and used later by transform component
+            // TODO: lod component always shows up before transform component; cache it before parsing transform
+            break;
+
+          case 'TransformComponent': {
+            const translation = new Vector3();
+            const rotation = new Quaternion();
+            const scale = new Vector3();
+
+            new Matrix4()
+              .multiplyMatrices(
+                new Matrix4().compose(
+                  new Vector3().fromArray(component['tc.worldTranslation']),
+                  new Quaternion().fromArray(component['tc.worldRotation']),
+                  new Vector3().fromArray(component['tc.worldScale']),
+                ),
+                new Matrix4().compose(
+                  new Vector3().fromArray(component['tc.localTranslation']),
+                  new Quaternion().fromArray(component['tc.localRotation']),
+                  new Vector3().fromArray(component['tc.localScale']),
+                ),
+              )
+              .decompose(translation, rotation, scale);
+
+            node.setTranslation(translation.toArray());
+            node.setRotation(rotation.toArray() as Vector4Tuple);
+            node.setScale(scale.toArray());
+
+            break;
+          }
+
+          case 'RenderComponent': {
+            const renderObject = component['rc.renderObj'];
+
+            times(renderObject['ro.batchCount'], (batchIndex): void => {
+              const lodIndex = renderObject[`rb${batchIndex}.lodIndex`];
+
+              if (lodIndex !== 0) return;
+
+              const batchKey = batchIndex.toString().padStart(4, '0');
+              const batch = renderObject['ro.batches'][batchKey];
+              const material = materials.get(batch['rb.nmatname']);
+              const polygonGroup = scg.get(batch['rb.datasource']);
+
+              if (!(material instanceof Material)) {
+                // probably shadow material
+                return;
+              }
+              if (polygonGroup === undefined) {
+                throw new Error(
+                  `Missing polygon group ${batch['rb.datasource']}`,
+                );
+              }
+
+              const lodNode = document.createNode(batchKey);
+              const indicesAccessor = document
+                .createAccessor()
+                .setType('SCALAR')
+                .setArray(new Uint16Array(polygonGroup.indices))
+                .setBuffer(buffer);
+              const primitive = document
+                .createPrimitive()
+                .setIndices(indicesAccessor)
+                .setMaterial(material);
+              const attributes = new Map<VertexAttribute, number[][]>();
+
+              polygonGroup.vertices.forEach((vertex) => {
+                vertex.forEach(({ attribute, value }) => {
+                  if (!attributes.has(attribute)) {
+                    attributes.set(attribute, []);
+                  }
+
+                  attributes.get(attribute)!.push(value);
+                });
+              });
+
+              if (attributes.has(VertexAttribute.TANGENT)) {
+                attributes.set(
+                  VertexAttribute.TANGENT,
+                  attributes
+                    .get(VertexAttribute.TANGENT)!
+                    .map((tangent) => [...tangent, 1]),
+                );
+              }
+
+              attributes.forEach((value, attribute) => {
+                const name = vertexAttributeGLTFName[attribute];
+                if (!name || primitive.getAttribute(name)) return;
+                const vertexSize = vertexAttributeGltfVectorSizes[attribute];
+
+                const attributeAccessor = document
+                  .createAccessor(name)
+                  .setType(vertexSize === 1 ? 'SCALAR' : `VEC${vertexSize}`)
+                  .setArray(new Float32Array(value.flat()))
+                  .setBuffer(buffer);
+                primitive.setAttribute(name, attributeAccessor);
+              });
+
+              const mesh = document
+                .createMesh(batch['##name'])
+                .addPrimitive(primitive);
+              lodNode.setMesh(mesh);
+              node.addChild(lodNode);
+            });
+
+            break;
+          }
+
+          default: {
+            if (ERROR_ON_UNKNOWN_COMPONENT) {
+              throw new Error(
+                `Unhandled component type: ${component['comp.typename']}`,
+              );
+            }
+          }
+        }
+      });
+
+      if (hierarchy['#hierarchy']) {
+        parseHierarchies(hierarchy['#hierarchy'], node);
+      }
+
+      parent.addChild(node);
+    });
+  }
+
+  parseHierarchies(sc2['#hierarchy'], scene);
+
+  return document;
+}

packages/scripts/src/core/blitz/readTexture/constants.ts

@@ -0,0 +1,6 @@
+export enum TextureMutation {
+  Normal,
+  RoughnessMetallicness,
+  Albedo,
+  Miscellaneous,
+}

packages/scripts/src/core/blitz/readTexture/index.ts

@@ -0,0 +1,78 @@
+import { DdsReadStream, PvrReadStream } from '@blitzkit/core';
+import { existsSync } from 'fs';
+import sharp from 'sharp';
+import { readDVPLFile } from '../readDVPLFile';
+import { TextureMutation } from './constants';
+
+export async function readTexture(path: string, mutation?: TextureMutation) {
+  const ddsTexturePath = path.replace('.tex', '.dx11.dds.dvpl');
+  const isDds = existsSync(ddsTexturePath);
+  const resolvedTexturePath = isDds
+    ? ddsTexturePath
+    : ddsTexturePath.replace('.dds', '.pvr');
+  const decompressedDvpl = await readDVPLFile(resolvedTexturePath);
+
+  const raw = isDds
+    ? await new DdsReadStream(decompressedDvpl.buffer as ArrayBuffer).dds()
+    : new PvrReadStream(decompressedDvpl.buffer as ArrayBuffer).pvr();
+
+  switch (mutation) {
+    case TextureMutation.Normal: {
+      const bytes = 4 * raw.width * raw.height;
+      for (let index = 0; index < bytes; index += 4) {
+        /**
+         * Red is always 255 and blue is always 0. Only alpha and green contain any
+         * sort of information.
+         */
+        let x = raw.data[index + 3] * (2 / 255) - 1;
+        let y = raw.data[index + 1] * (2 / 255) - 1;
+        const underSqrt = 1 - x ** 2 - y ** 2;
+        let z = underSqrt > 0 ? Math.sqrt(underSqrt) : -Math.sqrt(-underSqrt);
+
+        raw.data[index] = Math.round((x + 1) * (255 / 2));
+        raw.data[index + 1] = Math.round((y + 1) * (255 / 2));
+        raw.data[index + 2] = Math.round((z + 1) * (255 / 2));
+        raw.data[index + 3] = 255;
+      }
+
+      return await sharp(raw.data, { raw }).jpeg().toBuffer();
+    }
+
+    case TextureMutation.RoughnessMetallicness: {
+      const newImage = sharp(Buffer.alloc(raw.width * raw.height), {
+        raw: { ...raw, channels: 1 },
+      });
+      const metallicness = sharp(raw.data, { raw })
+        .extractChannel('green')
+        .jpeg()
+        .toBuffer();
+      const roughness = sharp(raw.data, { raw })
+        .extractChannel('alpha')
+        .jpeg()
+        .toBuffer();
+
+      return await newImage
+        .joinChannel(await Promise.all([roughness, metallicness]))
+        .jpeg()
+        .toBuffer();
+    }
+
+    case TextureMutation.Miscellaneous: {
+      /**
+       * Alpha is ambient occlusion and green is emissive. But very few tanks
+       * use emissive so I am ignoring it for now.
+       */
+      return await sharp(raw.data, { raw })
+        .extractChannel('alpha')
+        .jpeg()
+        .toBuffer();
+    }
+
+    case TextureMutation.Albedo: {
+      return await sharp(raw.data, { raw }).removeAlpha().jpeg().toBuffer();
+    }
+
+    default:
+      return await sharp(raw.data, { raw }).jpeg().toBuffer();
+  }
+}

packages/scripts/src/core/github/assetCommit.ts

@@ -0,0 +1,64 @@
+import { assertSecret } from '@blitzkit/core';
+import { octokit } from '../../../../website/src/core/blitzkit/octokit';
+import { GithubChangeBlob, createBlob } from './createBlob';
+
+export class AssetCommit {
+  constructor(public message: string) {}
+  private blobs: GithubChangeBlob[] = [];
+
+  async add(path: string, content: string, encoding: 'utf-8' | 'base64') {
+    console.log(`Blob: adding ${path}...`);
+
+    const blob = await createBlob(
+      'tresabhi',
+      assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+      {
+        content,
+        encoding,
+        path,
+      },
+    );
+
+    this.blobs.push(blob!);
+  }
+
+  async push() {
+    if (this.blobs.length === 0) return;
+
+    const latestCommitSha = (
+      await octokit.git.getRef({
+        owner: 'tresabhi',
+        repo: assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+        ref: `heads/${assertSecret(process.env.NEXT_PUBLIC_ASSET_BRANCH)}`,
+      })
+    ).data.object.sha;
+    const treeSha = (
+      await octokit.git.getCommit({
+        owner: 'tresabhi',
+        repo: assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+        commit_sha: latestCommitSha,
+      })
+    ).data.tree.sha;
+    const blobs: GithubChangeBlob[] = [];
+
+    const { data: treeData } = await octokit.git.createTree({
+      owner: 'tresabhi',
+      repo: assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+      base_tree: treeSha,
+      tree: blobs,
+    });
+    const { data: newCommitData } = await octokit.git.createCommit({
+      owner: 'tresabhi',
+      repo: assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+      message: this.message,
+      tree: treeData.sha,
+      parents: [latestCommitSha],
+    });
+    await octokit.git.updateRef({
+      owner: 'tresabhi',
+      repo: assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+      ref: `heads/${assertSecret(process.env.NEXT_PUBLIC_ASSET_BRANCH)}`,
+      sha: newCommitData.sha,
+    });
+  }
+}

packages/scripts/src/core/github/commitMultipleFiles.ts

@@ -1,9 +1,6 @@
 import ProgressBar from 'progress';
-import {
-  GithubChangeBlob,
-  createBlob,
-} from '../../../../website/src/core/blitzkit/createBlob';
 import { octokit } from '../../../../website/src/core/blitzkit/octokit';
+import { GithubChangeBlob, createBlob } from './createBlob';
 
 export interface FileChange {
   path: string;

packages/scripts/src/core/github/createBlob.ts

@@ -0,0 +1,46 @@
+import { throttle } from 'lodash';
+import { octokit } from '../../../../website/src/core/blitzkit/octokit';
+import { FileChange } from './commitMultipleFiles';
+
+export interface GithubChangeBlob {
+  sha: string;
+  path: string;
+  mode: '100644';
+  type: 'blob';
+}
+
+const TIME_PER_BLOB = 2 ** 4 * 1000;
+
+// github rate limit: 5000 requests per hour
+export const createBlob = throttle(
+  async (owner: string, repo: string, change: FileChange) => {
+    while (true) {
+      try {
+        const { sha } = (
+          await octokit.git.createBlob({
+            owner,
+            repo,
+            content: change.content,
+            encoding: change.encoding,
+          })
+        ).data;
+
+        return {
+          sha,
+          path: change.path,
+          mode: '100644',
+          type: 'blob',
+        } satisfies GithubChangeBlob;
+      } catch (error) {
+        console.warn(
+          `Failed blob ${change.path}; retrying in ${TIME_PER_BLOB}ms...`,
+          // error,
+        );
+
+        await new Promise((resolve) => setTimeout(resolve, TIME_PER_BLOB));
+      }
+    }
+  },
+  (5000 / (60 * 60 * 1000)) * 0.9,
+  { trailing: true },
+);