feat: more compression

docs/guide/bkon.md

@@ -22,6 +22,7 @@ struct String {
 
 - Everything is little endian
 - Magic is always `"BKON"`
+- The index of fast strings must be read as mentioned by the fast string format
 
 ```cpp
 struct BlitzkriegObjectNotation {
@@ -38,11 +39,19 @@ struct BlitzkriegObjectNotation {
 ```cpp
 struct Header {
   uint16 version;
+  FastStringFormat fastStringFormat;
   StringTable stringTable;
 }
 
+// read as uint8
+enum FastStringFormat {
+  Format8,
+  Format16,
+  Format32,
+}
+
 struct StringTable {
-  uint16 count;
+  (uint8 | uint16 | uint32) count;
   String[count] strings;
 }
 ```
@@ -142,7 +151,7 @@ struct ValueString {
 
 struct ValueFastString {
   ValueType.FastString type;
-  uint16 index;
+  (uint8 | uint16 | uint32) index;
 }
 
 struct ValueArray {

scripts/buildAssets/definitions.ts

@@ -21,6 +21,7 @@ import {
   ModelDefinitions,
 } from '../../src/core/blitzkrieg/modelDefinitions';
 import { ProvisionDefinitions } from '../../src/core/blitzkrieg/provisionDefinitions';
+import { superCompress } from '../../src/core/blitzkrieg/superCompress';
 import {
   GunDefinition,
   ShellType,
@@ -912,29 +913,29 @@ export async function definitions(production: boolean) {
     'definitions',
     [
       {
-        content: JSON.stringify(tankDefinitions),
-        encoding: 'utf-8',
-        path: 'definitions/tanks.json',
+        content: superCompress(tankDefinitions),
+        encoding: 'base64',
+        path: 'definitions/tanks.bkon.lz4',
       },
       {
-        content: JSON.stringify(modelDefinitions),
-        encoding: 'utf-8',
-        path: 'definitions/models.json',
+        content: superCompress(modelDefinitions),
+        encoding: 'base64',
+        path: 'definitions/models.bkon.lz4',
       },
       {
-        content: JSON.stringify(equipmentDefinitions),
-        encoding: 'utf-8',
-        path: 'definitions/equipment.json',
+        content: superCompress(equipmentDefinitions),
+        encoding: 'base64',
+        path: 'definitions/equipment.bkon.lz4',
       },
       {
-        content: JSON.stringify(consumableDefinitions),
-        encoding: 'utf-8',
-        path: 'definitions/consumables.json',
+        content: superCompress(consumableDefinitions),
+        encoding: 'base64',
+        path: 'definitions/consumables.bkon.lz4',
       },
       {
-        content: JSON.stringify(provisionDefinitions),
-        encoding: 'utf-8',
-        path: 'definitions/provisions.json',
+        content: superCompress(provisionDefinitions),
+        encoding: 'base64',
+        path: 'definitions/provisions.bkon.lz4',
       },
     ],
     production,

src/core/blitzkrieg/consumableDefinitions.ts

@@ -1,4 +1,5 @@
 import { asset } from './asset';
+import { fetchBkonLz4 } from './fetchBkonLz4';
 
 export type TankFilterDefinitionCategory = 'clip';
 
@@ -33,9 +34,6 @@ export interface ConsumableDefinitions {
   [key: string]: ConsumableEntry;
 }
 
-export const consumableDefinitions = fetch(
-  asset('definitions/consumables.json'),
-  {
-    cache: 'no-cache',
-  },
-).then(async (response) => response.json() as Promise<ConsumableDefinitions>);
+export const consumableDefinitions = fetchBkonLz4<ConsumableDefinitions>(
+  asset('definitions/consumables.bkon.lz4'),
+);

src/core/blitzkrieg/equipmentDefinitions.ts

@@ -1,4 +1,5 @@
 import { asset } from './asset';
+import { fetchBkonLz4 } from './fetchBkonLz4';
 
 type EquipmentOptions = [number, number];
 
@@ -21,6 +22,6 @@ export interface EquipmentDefinitions {
   };
 }
 
-export const equipmentDefinitions = fetch(asset('definitions/equipment.json'), {
-  cache: 'no-cache',
-}).then(async (response) => response.json() as Promise<EquipmentDefinitions>);
+export const equipmentDefinitions = fetchBkonLz4<EquipmentDefinitions>(
+  asset('definitions/equipment.bkon.lz4'),
+);

src/core/blitzkrieg/fetchBkonLz4.ts

@@ -0,0 +1,9 @@
+import { superDecompress } from './superDecompress';
+
+export async function fetchBkonLz4<Type>(url: string) {
+  return await fetch(url, {
+    cache: 'no-cache',
+  }).then(async (response) => {
+    return superDecompress(Buffer.from(await response.arrayBuffer())) as Type;
+  });
+}

src/core/blitzkrieg/modelDefinitions.ts

@@ -1,5 +1,6 @@
 import { Vector3Tuple } from 'three';
 import { asset } from './asset';
+import { fetchBkonLz4 } from './fetchBkonLz4';
 
 export type ModelDefinitions = Record<number, ModelDefinition>;
 
@@ -53,6 +54,6 @@ export interface PitchLimits {
   transition?: number;
 }
 
-export const modelDefinitions = fetch(asset('definitions/models.json'), {
-  cache: 'no-cache',
-}).then(async (response) => response.json() as Promise<ModelDefinitions>);
+export const modelDefinitions = fetchBkonLz4<ModelDefinitions>(
+  asset('definitions/models.bkon.lz4'),
+);

src/core/blitzkrieg/provisionDefinitions.ts

@@ -1,5 +1,6 @@
 import { asset } from './asset';
 import { TankFilterDefinition } from './consumableDefinitions';
+import { fetchBkonLz4 } from './fetchBkonLz4';
 
 export interface ProvisionEntry {
   id: number;
@@ -13,9 +14,6 @@ export interface ProvisionDefinitions {
   [key: string]: ProvisionEntry;
 }
 
-export const provisionDefinitions = fetch(
-  asset('definitions/provisions.json'),
-  {
-    cache: 'no-cache',
-  },
-).then(async (response) => response.json() as Promise<ProvisionDefinitions>);
+export const provisionDefinitions = fetchBkonLz4<ProvisionDefinitions>(
+  asset('definitions/provisions.bkon.lz4'),
+);

src/core/blitzkrieg/superCompress.ts

@@ -0,0 +1,8 @@
+import { encode } from 'lz4';
+import { BkonWriteStream } from '../streams/bkon';
+
+export function superCompress(object: any) {
+  const write = new BkonWriteStream().bkon(object);
+  const compressed = encode(write.buffer);
+  return compressed.toString('base64');
+}

src/core/blitzkrieg/superDecompress.ts

@@ -0,0 +1,8 @@
+import { decode } from 'lz4';
+import { BkonReadStream } from '../streams/bkon';
+
+export function superDecompress<Type>(buffer: Buffer) {
+  const decompressed = decode(buffer);
+  const read = new BkonReadStream(decompressed);
+  return read.bkon() as Type;
+}

src/core/blitzkrieg/tankDefinitions.ts

@@ -2,6 +2,7 @@ import { deburr } from 'lodash';
 import { Vector2Tuple } from 'three';
 import { TankType, TreeType } from '../../components/Tanks';
 import { asset } from './asset';
+import { fetchBkonLz4 } from './fetchBkonLz4';
 
 export type ShellType = 'ap' | 'ap_cr' | 'hc' | 'he';
 
@@ -117,9 +118,9 @@ export const GUN_TYPE_NAMES: Record<GunDefinition['type'], string> = {
   autoReloader: 'Auto reloader',
 };
 
-export const tankDefinitions = fetch(asset('definitions/tanks.json'), {
-  cache: 'no-cache',
-}).then(async (response) => response.json() as Promise<TankDefinitions>);
+export const tankDefinitions = fetchBkonLz4<TankDefinitions>(
+  asset('definitions/tanks.bkon.lz4'),
+);
 
 const entries = new Promise<TankDefinition[]>(async (resolve) => {
   resolve(Object.entries(await tankDefinitions).map(([, entry]) => entry));

src/core/streams/bkon.ts

@@ -20,20 +20,26 @@ enum ValueType {
   Object,
 }
 
-type Value =
+enum FastStringFormat {
+  Format8,
+  Format16,
+  Format32,
+}
+
+export type BkonValue =
   | null
   | boolean
   | number
   | bigint
   | string
-  | Value[]
-  | { [key: string]: Value };
+  | BkonValue[]
+  | { [key: string]: BkonValue };
 
 export class BkonReadStream extends ReadStream {
   bkon() {
     this.magic();
     const header = this.header();
-    const body = this.body(header.stringTable);
+    const body = this.body(header.fastStringFormat, header.stringTable);
 
     return body;
   }
@@ -47,12 +53,16 @@ export class BkonReadStream extends ReadStream {
   header() {
     return {
       version: this.uint16(),
+      fastStringFormat: this.uint8() as FastStringFormat,
       stringTable: this.stringTable(),
     };
   }
 
-  body(stringTable: Record<number, string>) {
-    return this.value(stringTable);
+  body(
+    fastStringFormat: FastStringFormat,
+    stringTable: Record<number, string>,
+  ) {
+    return this.value(fastStringFormat, stringTable);
   }
 
   stringTable() {
@@ -75,7 +85,10 @@ export class BkonReadStream extends ReadStream {
     return string;
   }
 
-  value(stringTable: Record<number, string>): Value {
+  value(
+    fastStringFormat: FastStringFormat,
+    stringTable: Record<number, string>,
+  ): BkonValue {
     const type = this.uint8();
 
     switch (type) {
@@ -118,17 +131,34 @@ export class BkonReadStream extends ReadStream {
       case ValueType.String:
         return this.string();
 
-      case ValueType.FastString:
-        return stringTable[this.uint16()];
+      case ValueType.FastString: {
+        switch (fastStringFormat) {
+          case FastStringFormat.Format8:
+            return stringTable[this.uint8()];
+
+          case FastStringFormat.Format16:
+            return stringTable[this.uint16()];
+
+          case FastStringFormat.Format32:
+            return stringTable[this.uint32()];
+        }
+      }
 
       case ValueType.Array:
-        return times(this.uint32(), () => this.value(stringTable));
+        return times(this.uint32(), () =>
+          this.value(fastStringFormat, stringTable),
+        );
 
       case ValueType.Object: {
         const count = this.uint32();
-        const keys = times(count, () => this.value(stringTable) as string);
-        const values = times(count, () => this.value(stringTable));
-        const object: Record<string, Value> = {};
+        const keys = times(
+          count,
+          () => this.value(fastStringFormat, stringTable) as string,
+        );
+        const values = times(count, () =>
+          this.value(fastStringFormat, stringTable),
+        );
+        const object: Record<string, BkonValue> = {};
 
         keys.forEach((string, index) => (object[string] = values[index]));
 
@@ -142,18 +172,20 @@ export class BkonReadStream extends ReadStream {
 }
 
 export class BkonWriteStream extends WriteStream {
-  bkon(object: Value) {
+  bkon(object: BkonValue) {
     this.magic();
     const stringTable = this.header(object);
     this.body(object, stringTable);
+    return this;
   }
 
-  body(object: Value, stringTable: Map<string, number>) {
+  body(object: BkonValue, stringTable: Map<string, number>) {
     this.value(object, stringTable);
+    return this;
   }
 
-  value(object: Value, stringTable: Map<string, number>) {
-    if (object === null) {
+  value(object: BkonValue, stringTable: Map<string, number>) {
+    if (object === null || object === undefined) {
       this.uint8(ValueType.Null);
     } else if (typeof object === 'boolean') {
       this.uint8(ValueType.Boolean);
@@ -226,17 +258,27 @@ export class BkonWriteStream extends WriteStream {
     }
   }
 
-  header(object: Value) {
+  header(object: BkonValue) {
     this.uint16(1);
     return this.stringTable(object);
   }
 
-  stringTable(object: Value) {
+  stringTable(object: BkonValue) {
     const stringTable = this.buildStringTable(
       this.collectStringCounts(object, new Map()),
     );
 
-    this.uint16(stringTable.size);
+    if (stringTable.size <= 0xff) {
+      this.uint8(FastStringFormat.Format8);
+      this.uint8(stringTable.size);
+    } else if (stringTable.size <= 0xffff) {
+      this.uint8(FastStringFormat.Format16);
+      this.uint16(stringTable.size);
+    } else {
+      this.uint8(FastStringFormat.Format32);
+      this.uint32(stringTable.size);
+    }
+
     stringTable.forEach((_index, string) => this.string(string));
 
     return stringTable;
@@ -252,7 +294,7 @@ export class BkonWriteStream extends WriteStream {
     return stringTable;
   }
 
-  collectStringCounts(object: Value, stringCounts: Map<string, number>) {
+  collectStringCounts(object: BkonValue, stringCounts: Map<string, number>) {
     if (typeof object === 'string') {
       if (stringCounts.has(object)) {
         stringCounts.set(object, stringCounts.get(object)! + 1);