feat: optimized blobs

src/core/blitzkrieg/commitMultipleFiles.ts

@@ -1,3 +1,4 @@
+import { GithubChangeBlob, createBlob } from './createBlob';
 import { octokit } from './octokit';
 
 const TIME_PER_BLOB = 2 ** 4 * 1000;
@@ -30,41 +31,13 @@ export default async function commitMultipleFiles(
       commit_sha: latestCommitSha,
     })
   ).data.tree.sha;
-  const blobs: {
+  const blobs: GithubChangeBlob[] = [];
-    sha: string;
-    path: string;
-    mode: '100644';
-    type: 'blob';
-  }[] = [];
 
   for (const changeIndexString in changes) {
     const changeIndex = parseInt(changeIndexString);
     const change = changes[changeIndex];
-    let done = false;
+    const createdBlob = await createBlob(owner, repo, change);
-
+    blobs.push(createdBlob!);
-    while (!done) {
-      try {
-        const { sha } = (
-          await octokit.git.createBlob({
-            owner,
-            repo,
-            content: change.content,
-            encoding: change.encoding,
-          })
-        ).data;
-
-        blobs.push({ sha, path: change.path, mode: '100644', type: 'blob' });
-        done = true;
-      } catch (error) {
-        console.warn(
-          `Failed blob ${changeIndex + 1} / ${
-            changes.length
-          }; retrying in ${TIME_PER_BLOB}ms...`,
-        );
-
-        await new Promise((resolve) => setTimeout(resolve, TIME_PER_BLOB));
-      }
-    }
 
     if (verbose) {
       console.log(

src/core/blitzkrieg/createBlob.ts

@@ -0,0 +1,45 @@
+import { throttle } from 'lodash';
+import { FileChange } from './commitMultipleFiles';
+import { octokit } from './octokit';
+
+export interface GithubChangeBlob {
+  sha: string;
+  path: string;
+  mode: '100644';
+  type: 'blob';
+}
+
+const TIME_PER_BLOB = 2 ** 4 * 1000;
+
+// github rate limit: 5000 requests per hour
+export const createBlob = throttle(
+  async (owner: string, repo: string, change: FileChange) => {
+    while (true) {
+      try {
+        const { sha } = (
+          await octokit.git.createBlob({
+            owner,
+            repo,
+            content: change.content,
+            encoding: change.encoding,
+          })
+        ).data;
+
+        return {
+          sha,
+          path: change.path,
+          mode: '100644',
+          type: 'blob',
+        } satisfies GithubChangeBlob;
+      } catch (error) {
+        console.warn(
+          `Failed blob ${change.path}; retrying in ${TIME_PER_BLOB}ms...`,
+        );
+
+        await new Promise((resolve) => setTimeout(resolve, TIME_PER_BLOB));
+      }
+    }
+  },
+  5000 / (60 * 60 * 1000) * 0.95 // 5% less than max rate,
+  { trailing: true },
+);