abstract away some code

packages/scripts/src/core/blitzkit/gitObjectStorage.ts

@@ -0,0 +1,58 @@
+import { $ } from "bun";
+import { mkdir, rm } from "node:fs/promises";
+
+export class GitObjectStorage {
+  constructor(
+    private repo: string,
+    private base: string,
+    private token?: string,
+  ) {}
+
+  async init() {
+    await rm(this.base, { recursive: true, force: true });
+
+    console.log(`Cloning ${this.repo} to ${this.base}`);
+
+    const url = this.token
+      ? `https://x-access-token:${this.token}@github.com/${this.repo}.git`
+      : `https://github.com/${this.repo}.git`;
+
+    await $`git clone --depth 1 ${url} ${this.base}`;
+
+    return this;
+  }
+
+  async commit() {
+    const shell = $.cwd(this.base);
+
+    await shell`git add .`.quiet();
+    await shell`git commit --amend --reset-author -m "ids update ${new Date().toISOString()}"`.quiet();
+    await shell`git push --force-with-lease`;
+  }
+
+  async mkdir(path: string) {
+    await mkdir(`${this.base}/${path}`, { recursive: true });
+  }
+
+  async rm(path: string) {
+    await rm(`${this.base}/${path}`, { force: true });
+  }
+
+  async write(path: string, bytes: Uint8Array | string) {
+    await Bun.write(`${this.base}/${path}`, bytes);
+  }
+
+  file(path: string) {
+    return Bun.file(path);
+  }
+
+  async bytes(path: string) {
+    const file = this.file(path);
+    return await file.bytes();
+  }
+
+  async json<Type>(path: string) {
+    const file = this.file(path);
+    return (await file.json()) as Type;
+  }
+}

packages/scripts/src/discoverIds/constants.ts

@@ -0,0 +1,9 @@
+export const WORKING_DIR = `../../temp/ids`;
+export const REPO = "blitzkit/ids";
+
+export const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
+export const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
+
+export const MAX_IDS_PER_CALL = 100;
+export const API_RATE = 10; // 10Hz
+export const MAX_DRY_STREAK = 10_000;

packages/scripts/src/discoverIds/index.ts

@@ -1,73 +1,40 @@
-import { $ } from "bun";
 import { range } from "lodash-es";
 import { compress, decompress } from "lz4js";
-import { mkdir, rm } from "node:fs/promises";
 import { exit } from "node:process";
-import { IdArray } from "./idArray";
+import { GitObjectStorage } from "../core/blitzkit/gitObjectStorage";
+import { IdArray } from "../core/blitzkit/idArray";
+import {
+  ACCOMMODATED_ID_COUNT,
+  API_RATE,
+  MAX_BYTES_PER_CHUNK,
+  MAX_DRY_STREAK,
+  MAX_IDS_PER_CALL,
+  REPO,
+  WORKING_DIR,
+} from "./constants";
+import { ManifestV1, QuickInfo, RegionDescriptor } from "./types";
 
-interface ManifestV1 {
+const storage = await new GitObjectStorage(
-  version: 1;
+  REPO,
-  last_verified: number;
+  WORKING_DIR,
-  chunks: number;
+  import.meta.env.GH_TOKEN,
-  total_count: number;
+).init();
-}
-
-interface RegionDescriptor {
-  domain: string;
-  seed: number;
-  max: number;
-  dry_streak: number;
-}
-
-export interface QuickInfo {
-  status: string;
-  data: Record<string, {} | null>;
-}
-
-const TEMP = "../../temp";
-const WORKING_DIR = `${TEMP}/ids`;
-const CHUNKS_DIR = `${WORKING_DIR}/chunks`;
-const REPO = "blitzkit/ids";
-
-const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
-const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
-
-const MAX_IDS_PER_CALL = 100;
-const API_RATE = 10; // 10Hz
-const MAX_DRY_STREAK = 10_000;
 
 const t0 = performance.now();
 const maxT = 5.5 * 60 * 60 * 1000; // 5hr 30min
 const t1 = t0 + maxT;
 
-await mkdir(TEMP, { recursive: true });
+await storage.mkdir("chunks");
-await rm(WORKING_DIR, { recursive: true, force: true });
-
-console.log(`Cloning ${REPO} to ${WORKING_DIR}`);
-
-const token = process.env.GH_TOKEN;
-const repoUrl = token
-  ? `https://x-access-token:${token}@github.com/${REPO}.git`
-  : `https://github.com/${REPO}.git`;
-
-await $`git clone --depth 1 ${repoUrl} ${WORKING_DIR}`;
-await mkdir(CHUNKS_DIR, { recursive: true });
 
 console.log(`Discovering pre-existing chunks...`);
 
 const discoveredChunks: (IdArray | null)[] = [];
 
-const manifest: ManifestV1 = await Bun.file(
+const manifest = await storage.json<ManifestV1>("manifest.json");
-  `${WORKING_DIR}/manifest.json`,
-).json();
 
 for (let i = 0; i < manifest.chunks; i++) {
-  const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`;
+  const bytes = await storage.bytes(`chunks/chunk-${i}.dat.lz4`);
-  const file = Bun.file(path);
-
-  const bytes = await file.bytes();
   const uncompressed = decompress(bytes);
-
   const ids = IdArray.fromBytes(uncompressed);
 
   discoveredChunks.push(ids);
@@ -146,33 +113,25 @@ async function save() {
 
   console.log("Saving chunks...");
 
-  await rm(`${CHUNKS_DIR}/*`, { force: true });
+  await storage.rm("chunks/*");
 
   let totalFound = 0;
 
   for (let i = 0; i < chunks.length; i++) {
-    const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`;
     const chunk = chunks[i];
     const bytes = chunk.toBytes();
     const compressed = compress(bytes);
 
     totalFound += chunk.size();
 
-    await Bun.write(path, compressed);
+    await storage.write(`chunks/chunk-${i}.dat.lz4`, compressed);
   }
 
   manifest.total_count = totalFound;
 
-  await Bun.write(
+  await storage.write("manifest.json", JSON.stringify(manifest, null, 2));
-    `${WORKING_DIR}/manifest.json`,
-    JSON.stringify(manifest, null, 2),
-  );
 
-  const git = $.cwd(WORKING_DIR);
+  await storage.commit();
-
-  await git`git add .`.quiet();
-  await git`git commit --amend --reset-author -m "ids update ${new Date().toISOString()}"`.quiet();
-  await git`git push --force-with-lease`.quiet();
 
   console.log(
     `Committed ${totalFound.toLocaleString()} total ids across ${manifest.chunks} chunks`,

packages/scripts/src/discoverIds/types.ts

@@ -0,0 +1,18 @@
+export interface ManifestV1 {
+  version: 1;
+  last_verified: number;
+  chunks: number;
+  total_count: number;
+}
+
+export interface RegionDescriptor {
+  domain: string;
+  seed: number;
+  max: number;
+  dry_streak: number;
+}
+
+export interface QuickInfo {
+  status: string;
+  data: Record<string, {} | null>;
+}