fix: types

packages/scripts/src/core/blitz/readBase64DVPL.ts

@@ -0,0 +1,6 @@
+import { readDVPLFile } from './readDVPLFile';
+
+export async function readBase64DVPL(file: string) {
+  const buffer = await readDVPLFile(file);
+  return buffer.toString('base64');
+}

packages/scripts/src/core/blitz/readDVPL.ts

@@ -0,0 +1,60 @@
+import { crc32 } from 'crc';
+import { decompressBlock } from 'lz4js';
+
+/**
+ * Thanks Maddoxkkm! Modified heavily to be modern.
+ * https://github.com/Maddoxkkm/dvpl_converter/
+ */
+export function readDVPL(buffer: Buffer) {
+  const footerBuffer = buffer.subarray(buffer.length - 20, buffer.length);
+
+  if (
+    footerBuffer.toString('utf8', 16, 20) !== 'DVPL' ||
+    footerBuffer.length !== 20
+  ) {
+    throw new SyntaxError('Invalid DVPL footer');
+  }
+
+  const footerData = {
+    oSize: footerBuffer.readUInt32LE(0),
+    cSize: footerBuffer.readUInt32LE(4),
+    crc32: footerBuffer.readUInt32LE(8),
+    type: footerBuffer.readUInt32LE(12),
+  };
+  const targetBlock = buffer.subarray(0, buffer.length - 20);
+
+  if (targetBlock.length !== footerData.cSize) {
+    throw new RangeError('DVPL size mismatch');
+  }
+  if (crc32(targetBlock) !== footerData.crc32) {
+    throw new TypeError('DVPL CRC32 mismatch');
+  }
+
+  if (footerData.type === 0) {
+    if (!(footerData.oSize === footerData.cSize && footerData.type === 0)) {
+      throw new RangeError('DVPL type and size mismatch');
+    }
+
+    return targetBlock;
+  } else if (footerData.type === 1 || footerData.type === 2) {
+    const source = new Uint8Array(targetBlock);
+    const destination = new Uint8Array(footerData.oSize);
+    const decompressedSize = decompressBlock(
+      source,
+      destination,
+      0,
+      source.length,
+      0,
+    );
+
+    if (decompressedSize !== footerData.oSize) {
+      throw new RangeError(
+        `Decompressed DVPL size mismatch (${decompressedSize} vs ${footerData.oSize})`,
+      );
+    }
+
+    return Buffer.from(destination);
+  } else {
+    throw new SyntaxError('Unknown DVPL format');
+  }
+}

packages/scripts/src/core/blitz/readDVPLFile.ts

@@ -0,0 +1,6 @@
+import { readFile } from 'fs/promises';
+import { readDVPL } from './readDVPL';
+
+export async function readDVPLFile(file: string) {
+  return readDVPL(await readFile(file));
+}

packages/scripts/src/core/blitz/readStringDVPL.ts

@@ -0,0 +1,6 @@
+import { readDVPLFile } from './readDVPLFile';
+
+export async function readStringDVPL(file: string) {
+  const buffer = await readDVPLFile(file);
+  return buffer.toString();
+}

packages/scripts/src/core/blitz/readXMLDVPL.ts

@@ -0,0 +1,9 @@
+import { XMLParser } from 'fast-xml-parser';
+import { readStringDVPL } from './readStringDVPL';
+
+export async function readXMLDVPL<Type>(file: string) {
+  const parser = new XMLParser();
+  const xml = await readStringDVPL(file);
+
+  return parser.parse(xml) as Type;
+}

packages/scripts/src/core/blitz/readYAML.ts

@@ -0,0 +1,7 @@
+import { readFile } from 'fs/promises';
+import { parse } from 'yaml';
+
+export async function readYAML<Type>(file: string) {
+  const yaml = (await readFile(file)).toString();
+  return parse(yaml) as Type;
+}

packages/scripts/src/core/blitz/readYAMLDVPL.ts

@@ -0,0 +1,8 @@
+import { parse } from 'yaml';
+import { readStringDVPL } from './readStringDVPL';
+
+export async function readYAMLDVPL<Type>(file: string) {
+  const yaml = await readStringDVPL(file);
+
+  return parse(yaml) as Type;
+}

packages/scripts/src/core/github/commitAssets.ts

@@ -0,0 +1,16 @@
+import { assertSecret } from '@blitzkit/core';
+import commitMultipleFiles, { FileChange } from './commitMultipleFiles';
+
+export async function commitAssets(message: string, changes: FileChange[]) {
+  console.log(`Committing ${message}...`);
+
+  if (changes.length === 0) return;
+
+  await commitMultipleFiles(
+    'tresabhi',
+    assertSecret(process.env.NEXT_PUBLIC_ASSET_REPO),
+    assertSecret(process.env.NEXT_PUBLIC_ASSET_BRANCH),
+    `${message} - ${new Date().toDateString()}`,
+    changes,
+  );
+}

packages/scripts/src/core/github/commitMultipleFiles.ts

@@ -0,0 +1,66 @@
+import ProgressBar from 'progress';
+import {
+  GithubChangeBlob,
+  createBlob,
+} from '../../../../website/src/core/blitzkit/createBlob';
+import { octokit } from '../../../../website/src/core/blitzkit/octokit';
+
+export interface FileChange {
+  path: string;
+  content: string;
+  encoding: 'utf-8' | 'base64';
+}
+
+export default async function commitMultipleFiles(
+  owner: string,
+  repo: string,
+  branch: string,
+  message: string,
+  changes: FileChange[],
+) {
+  const latestCommitSha = (
+    await octokit.git.getRef({
+      owner,
+      repo,
+      ref: `heads/${branch}`,
+    })
+  ).data.object.sha;
+  const treeSha = (
+    await octokit.git.getCommit({
+      owner,
+      repo,
+      commit_sha: latestCommitSha,
+    })
+  ).data.tree.sha;
+  const blobs: GithubChangeBlob[] = [];
+  const bar = new ProgressBar('Blobs :bar', changes.length);
+
+  for (const changeIndexString in changes) {
+    const changeIndex = parseInt(changeIndexString);
+    const change = changes[changeIndex];
+    const createdBlob = await createBlob(owner, repo, change);
+    blobs.push(createdBlob!);
+    bar.tick();
+  }
+
+  const { data: treeData } = await octokit.git.createTree({
+    owner,
+    repo,
+    base_tree: treeSha,
+    tree: blobs,
+  });
+  const { data: newCommitData } = await octokit.git.createCommit({
+    owner,
+    repo,
+    message,
+    tree: treeData.sha,
+    parents: [latestCommitSha],
+  });
+  await octokit.git.updateRef({
+    owner,
+    repo,
+    ref: `heads/${branch}`,
+    sha: newCommitData.sha,
+    force: true,
+  });
+}

packages/scripts/src/core/github/fetchPreDiscoveredIds.ts

@@ -0,0 +1,36 @@
+import { asset, DidsReadStream } from '@blitzkit/core';
+import { times } from 'lodash';
+import { decompress } from 'lz4js';
+import ProgressBar from 'progress';
+import { DiscoveredIdsDefinitions } from '../../../../website/src/core/blitzkit/discoveredIdDefinitions';
+
+export async function fetchPreDiscoveredIds() {
+  const idChunks: number[][] = [];
+  const manifest = (await fetch(asset('ids/manifest.json')).then((response) =>
+    response.json(),
+  )) as DiscoveredIdsDefinitions;
+  const bar = new ProgressBar(
+    `Fetching ${manifest.count.toLocaleString()} ids in ${
+      manifest.chunks
+    } chunks :bar`,
+    manifest.chunks,
+  );
+
+  bar.render();
+  await Promise.all(
+    times(manifest.chunks, async (chunkIndex) => {
+      const preDiscovered = await fetch(
+        asset(`ids/${chunkIndex}.dids.lz4`),
+      ).then(async (response) => {
+        const buffer = await response.arrayBuffer();
+        const decompressed = decompress(new Uint8Array(buffer)).buffer;
+        return new DidsReadStream(decompressed as ArrayBuffer).dids();
+      });
+
+      idChunks.push(preDiscovered);
+      bar.tick();
+    }),
+  );
+
+  return idChunks.sort((a, b) => a[0] - b[0]).flat();
+}