diff --git a/packages/scripts/src/core/github/assetUploader.ts b/packages/scripts/src/core/github/assetUploader.ts new file mode 100644 index 000000000..0b9fd42fe --- /dev/null +++ b/packages/scripts/src/core/github/assetUploader.ts @@ -0,0 +1,216 @@ +import { assertSecret } from "@blitzkit/core"; +import { Octokit } from "@octokit/rest"; +import ProgressBar from "progress"; +import { compareUint8Arrays } from "./compareUint8Arrays"; + +export interface FileChange { + path: string; + content: Uint8Array; +} + +interface GithubChangeBlob { + sha: string; + path: string; + mode: "100644"; + type: "blob"; +} + +enum DiffStatus { + New, + Changed, + Unchanged, +} + +export class AssetUploader { + time_per_blob = 2 ** 4 * 1000; + time_between_blobs = (60 * 60 * 1000) / 5000 / 0.9; + + max_tree_size = 7_000_000; + max_file_count = 128; + + heuristicFormats = ["glb", "webp", "jpg", "png"]; + + owner: string; + repo: string; + branch: string; + + changes: FileChange[] = []; + changesSize = 0; + + private octokit = new Octokit({ + auth: assertSecret(import.meta.env.GH_TOKEN), + }); + + constructor(public message: string) { + [this.owner, this.repo] = assertSecret( + import.meta.env.PUBLIC_ASSET_REPO + ).split("/"); + this.branch = assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH); + } + + async add(change: FileChange) { + const blobPath = `${this.owner}/${this.repo}/${this.branch}/${change.path}`; + const response = await fetch( + `https://raw.githubusercontent.com/${blobPath}`, + { headers: { "Accept-Encoding": "identity" } } + ); + let diff: { change: number; status: DiffStatus }; + + if (response.status === 404) { + diff = { change: change.content.length, status: DiffStatus.New }; + } else if ( + response.headers.has("Content-Length") && + Number(response.headers.get("Content-Length")) !== change.content.length + ) { + diff = { + change: + change.content.length - + Number(response.headers.get("Content-Length")), + status: DiffStatus.Changed, + }; + } else if (response.status === 200) { + if ( + this.heuristicFormats.some((format) => + change.path.endsWith(`.${format}`) + ) + ) { + /** + * Heuristic: if it's a large blob like a .glb, a diff = 0 might end + * up being a false positive with isDifferent = true. So, it's okay + * to assume it's unchanged. + */ + diff = { change: 0, status: DiffStatus.Unchanged }; + } else { + const buffer = new Uint8Array(await response.arrayBuffer()); + + if (compareUint8Arrays(buffer, change.content)) { + diff = { change: 0, status: DiffStatus.Unchanged }; + } else { + diff = { + change: change.content.length - buffer.length, + status: DiffStatus.Changed, + }; + } + } + } else { + throw new Error( + `Unexpected status code ${response.status} for ${change.path}` + ); + } + + if (diff.status === DiffStatus.Unchanged) { + console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`); + } else { + console.log( + `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${ + diff.change > 0 ? "+" : "" + }${diff.change.toLocaleString()}B) ${change.path}` + ); + + // flush BEFORE going over limit + if ( + this.changesSize + change.content.length > this.max_tree_size || + this.changes.length + 1 > this.max_file_count + ) { + console.log(`Flushing ${this.changes.length} changes...`); + await this.flush(); + } + + this.changes.push(change); + this.changesSize += change.content.length; + } + } + + async flush() { + if (this.changes.length == 0) return; + + const latestCommitSha = ( + await this.octokit.git.getRef({ + owner: this.owner, + repo: this.repo, + ref: `heads/${this.branch}`, + }) + ).data.object.sha; + const treeSha = ( + await this.octokit.git.getCommit({ + owner: this.owner, + repo: this.repo, + commit_sha: latestCommitSha, + }) + ).data.tree.sha; + const blobs: GithubChangeBlob[] = []; + const bar = new ProgressBar("Blobs :bar", this.changes.length); + + for (const change of this.changes) { + while (true) { + try { + const start = Date.now(); + var { sha } = await this.octokit.git + .createBlob({ + owner: this.owner, + repo: this.repo, + encoding: "base64", + content: Buffer.from(change.content).toString("base64"), + }) + .then((response) => response.data); + const end = Date.now(); + + console.log(` blobbed ${change.path} ${end - start}ms`); + + await new Promise((resolve) => + setTimeout( + resolve, + Math.max(0, this.time_between_blobs + (start - end)) + ) + ); + + blobs.push({ sha, path: change.path, mode: "100644", type: "blob" }); + bar.tick(); + + break; + } catch { + console.warn( + ` failed ${change.path}; retrying in ${this.time_per_blob}ms...` + ); + + await new Promise((resolve) => + setTimeout(resolve, this.time_per_blob) + ); + } + } + } + + const { data: treeData } = await this.octokit.git.createTree({ + owner: this.owner, + repo: this.repo, + base_tree: treeSha, + tree: blobs, + }); + const { data: commitData } = await this.octokit.git.createCommit({ + owner: this.owner, + repo: this.repo, + message: this.message, + tree: treeData.sha, + parents: [latestCommitSha], + }); + + await this.octokit.git.updateRef({ + owner: this.owner, + repo: this.repo, + ref: `heads/${this.branch}`, + sha: commitData.sha, + force: true, + }); + + this.changes = []; + this.changesSize = 0; + } + + [Symbol.dispose]() { + if (this.changes.length > 0) { + throw new Error( + `${this.changes.length} uncommitted changes. Did you forget to flush?` + ); + } + } +} diff --git a/packages/scripts/src/core/github/commitAssets.ts b/packages/scripts/src/core/github/commitAssets.ts index 0d8a652f7..d1eab912c 100644 --- a/packages/scripts/src/core/github/commitAssets.ts +++ b/packages/scripts/src/core/github/commitAssets.ts @@ -1,6 +1,6 @@ -import { assertSecret } from '@blitzkit/core'; -import { chunk } from 'lodash-es'; -import { commitMultipleFiles, FileChange } from './commitMultipleFiles'; +import { assertSecret } from "@blitzkit/core"; +import { chunk } from "lodash-es"; +import { commitMultipleFiles, FileChange } from "./commitMultipleFiles"; export async function commitAssets(message: string, changes: FileChange[]) { console.log(`Committing ${message}...`); @@ -16,8 +16,10 @@ export async function commitAssets(message: string, changes: FileChange[]) { await commitMultipleFiles( assertSecret(import.meta.env.PUBLIC_ASSET_REPO), assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH), - `${message} - ${new Date().toDateString()} (${chunkIndex + 1}/${chunks.length})`, - chunk, + `${message} - ${new Date().toDateString()} (${chunkIndex + 1}/${ + chunks.length + })`, + chunk ); chunkIndex++; diff --git a/packages/scripts/src/core/github/commitMultipleFiles.ts b/packages/scripts/src/core/github/commitMultipleFiles.ts index b809e3937..bf6f510a3 100644 --- a/packages/scripts/src/core/github/commitMultipleFiles.ts +++ b/packages/scripts/src/core/github/commitMultipleFiles.ts @@ -77,9 +77,9 @@ export async function commitMultipleFiles( // console.log(`🔵 (${diff.change.toLocaleString()}B) ${change.path}`); } else { console.log( - `${ - diff.status === DiffStatus.New ? "🟢" : "🟡" - } (${diff.change > 0 ? "+" : ""}${diff.change.toLocaleString()}B) ${change.path}` + `${diff.status === DiffStatus.New ? "🟢" : "🟡"} (${ + diff.change > 0 ? "+" : "" + }${diff.change.toLocaleString()}B) ${change.path}` ); changes.push(change); } diff --git a/packages/scripts/src/core/github/octokit.ts b/packages/scripts/src/core/github/octokit.ts index 8ee0a04e4..b0b21eada 100644 --- a/packages/scripts/src/core/github/octokit.ts +++ b/packages/scripts/src/core/github/octokit.ts @@ -1,6 +1,6 @@ -import { assertSecret } from '@blitzkit/core'; -import { throttling } from '@octokit/plugin-throttling'; -import { Octokit } from '@octokit/rest'; +import { assertSecret } from "@blitzkit/core"; +import { throttling } from "@octokit/plugin-throttling"; +import { Octokit } from "@octokit/rest"; Octokit.plugin(throttling);