feat: smart uploader

src/BlitzKit.CLI/Functions/Mangler.cs

@@ -20,14 +20,14 @@ namespace BlitzKit.CLI.Functions
   {
     readonly BlitzProvider provider = new(args.Contains("--depot"));
     Locales? locales;
-    readonly List<FileChange> changes = [];
     readonly Dictionary<string, Dictionary<string, string>> strings = [];
+    readonly AssetUploader assetUploader = new("mangled ue assets");
 
     public async Task Mangle()
     {
       await FetchStrings();
-      MangleNations();
-      await BlitzKitAssets.CommitAssets("ue assets", changes);
+      await MangleNations();
+      await assetUploader.Flush();
     }
 
     I18n GetString(string name)
@@ -64,7 +64,7 @@ namespace BlitzKit.CLI.Functions
     {
       using var client = new HttpClient();
       var text = File.ReadAllText("../../packages/i18n/locales.json");
-      this.locales =
+      locales =
         JsonSerializer.Deserialize<Locales>(text)
         ?? throw new Exception("Failed to deserialize locales");
 
@@ -81,7 +81,7 @@ namespace BlitzKit.CLI.Functions
       }
     }
 
-    void MangleNations()
+    async Task MangleNations()
     {
       var dirs = provider.RootDirectory.GetDirectory("Blitz/Content/Tanks").Directories;
       List<Tank> tanksRaw = [];
@@ -93,10 +93,11 @@ namespace BlitzKit.CLI.Functions
 
         foreach (var tankDir in dir.Value.Directories)
         {
-          if (tankDir.Value.Name != "R90_IS_4")
-            continue;
+          // if (tankDir.Value.Name != "R90_IS_4")
+          //   continue;
 
-          tanksRaw.Add(MangleTank(tankDir.Value));
+          var tank = await MangleTank(tankDir.Value);
+          tanksRaw.Add(tank);
         }
       }
 
@@ -110,7 +111,7 @@ namespace BlitzKit.CLI.Functions
         tanks.Tanks_.Add(tankMeta);
       }
 
-      changes.Add(new("definitions/tanks.pb", tanks.ToByteArray()));
+      await assetUploader.Add(new("definitions/tanks.pb", tanks.ToByteArray()));
     }
 
     void DeduplicateSlugs(List<Tank> tanks)
@@ -185,7 +186,7 @@ namespace BlitzKit.CLI.Functions
       }
     }
 
-    Tank MangleTank(VFS tankDir)
+    async Task<Tank> MangleTank(VFS tankDir)
     {
       var pdaName = $"PDA_{tankDir.Name}";
       var pda = provider.LoadObject($"{tankDir.Path}/{pdaName}.{pdaName}");
@@ -197,8 +198,8 @@ namespace BlitzKit.CLI.Functions
       slug = Regex.Replace(slug, "--+", "-");
       slug = Regex.Replace(slug, "-$", "");
 
-      // MangleHull(pda);
-      MangleIcon(pda);
+      await MangleHull(pda);
+      await MangleIcon(pda);
 
       Tank tank = new()
       {
@@ -207,22 +208,23 @@ namespace BlitzKit.CLI.Functions
         Slug = slug,
       };
 
-      changes.Add(new($"tanks/{id}/meta.pb", tank.ToByteArray()));
+      await assetUploader.Add(new($"tanks/{id}/meta.pb", tank.ToByteArray()));
 
       return tank;
     }
 
-    void MangleIcon(UObject pda)
+    async Task MangleIcon(UObject pda)
     {
       var id = PropertyUtil.Get<FName>(pda, "TankId").Text;
       var smallIcon = BlitzKitExporter.Texture2D(PropertyUtil.Get<UTexture2D>(pda, "SmallIcon"));
+      Console.WriteLine(id);
       var bigIcon = BlitzKitExporter.Texture2D(PropertyUtil.Get<UTexture2D>(pda, "BigIcon"));
 
-      changes.Add(new($"tanks/{id}/icons/small.png", smallIcon));
-      changes.Add(new($"tanks/{id}/icons/big.png", bigIcon));
+      await assetUploader.Add(new($"tanks/{id}/icons/small.png", smallIcon));
+      await assetUploader.Add(new($"tanks/{id}/icons/big.png", bigIcon));
     }
 
-    void MangleHull(UObject pda)
+    async Task MangleHull(UObject pda)
     {
       var id = PropertyUtil.Get<FName>(pda, "TankId").Text;
       var hulls = PropertyUtil.Get<UDataTable>(pda, "DT_Hulls");
@@ -232,7 +234,7 @@ namespace BlitzKit.CLI.Functions
       var hullCollision = PropertyUtil.Get<UStaticMesh>(hullMeshSettings, "CollisionMesh");
       MonoGltf hullCollisionGltf = new(hullCollision);
 
-      changes.Add(new($"tanks/{id}/collision/hull.glb", hullCollisionGltf.Write()));
+      await assetUploader.Add(new($"tanks/{id}/collision/hull.glb", hullCollisionGltf.Write()));
     }
   }
 }

src/BlitzKit.CLI/Models/AssetUploader.cs

@@ -0,0 +1,187 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Net;
+using System.Threading.Tasks;
+using BlitzKit.CLI.Utils;
+using DotNetEnv;
+using Octokit;
+
+namespace BlitzKit.CLI.Models
+{
+  public class AssetUploader(string message)
+  {
+    private static readonly int TIME_PER_BLOB = (int)Math.Pow(2, 4) * 10000;
+    private static readonly int TIME_BETWEEN_BLOBS = (int)(60 * 60 * 1000 / 5000 / 0.9);
+    private static readonly int MAX_TREE_SIZE = 7_000_000; // 7MB
+
+    private readonly GitHubClient octokit = new(new ProductHeaderValue("MyAmazingApp"))
+    {
+      Credentials = new(Env.GetString("GH_TOKEN")),
+    };
+    private readonly HttpClient httpClient = new();
+
+    private readonly string branch = Env.GetString("PUBLIC_ASSET_BRANCH");
+    private readonly string repo = Env.GetString("PUBLIC_ASSET_REPO");
+    public string message = message;
+    private static readonly List<FileChange> changes = [];
+    private int changesSize = 0;
+
+    public async Task Add(FileChange change)
+    {
+      var blobPath = $"{repo}/{branch}/{change.Path}";
+      HttpResponseMessage response = await httpClient.GetAsync(
+        $"https://raw.githubusercontent.com/{blobPath}"
+      );
+
+      if (response.StatusCode == HttpStatusCode.NotFound)
+      {
+        Console.WriteLine(
+          $"🟢 (+{change.Content.Count.ToString("N0", Program.Culture)}B) {blobPath}"
+        );
+        changes.Add(change);
+      }
+      else if (response.StatusCode == HttpStatusCode.OK)
+      {
+        int deltaSize = 0;
+        bool isDifferent = false;
+
+        // hopefully they just give you the file size to delta from
+        if (response.Content.Headers.ContentLength.HasValue)
+        {
+          deltaSize = change.Content.Count - (int)response.Content.Headers.ContentLength;
+
+          if (deltaSize == 0)
+          {
+            var bytes = await response.Content.ReadAsByteArrayAsync();
+            isDifferent = !bytes.SequenceEqual(change.Content);
+          }
+          else
+          {
+            isDifferent = true;
+          }
+        }
+        else
+        {
+          // the meanies didn't give us the file size so diff manually
+          var bytes = await response.Content.ReadAsByteArrayAsync();
+          isDifferent = !bytes.SequenceEqual(change.Content);
+          deltaSize = change.Content.Count - bytes.Length;
+        }
+
+        if (isDifferent)
+        {
+          Console.WriteLine(
+            $"🟡 ({(deltaSize > 0 ? "+" : "")}{deltaSize.ToString("N0", Program.Culture)}B) {blobPath}"
+          );
+          changes.Add(change);
+        }
+        else
+        {
+          return;
+        }
+      }
+      else
+      {
+        throw new Exception($"Unexpected status code {response.StatusCode} for {blobPath}");
+      }
+
+      // flush BEFORE going over limit
+      if (changesSize + change.Content.Count > MAX_TREE_SIZE)
+      {
+        Console.WriteLine("Flushing backlog...");
+        await Flush();
+      }
+
+      changes.Add(change);
+      changesSize += change.Content.Count;
+    }
+
+    public async Task Flush()
+    {
+      if (changes.Count == 0)
+        return;
+
+      var repoRawSplit = repo.Split('/');
+      var owner = repoRawSplit[0];
+      var repoName = repoRawSplit[1];
+      var latestCommitSha = (await octokit.Git.Reference.Get(owner, repoName, $"heads/{branch}"))
+        .Object
+        .Sha;
+      var treeSha = (await octokit.Git.Commit.Get(owner, repoName, latestCommitSha)).Tree.Sha;
+      NewTree newTree = new() { BaseTree = treeSha };
+
+      foreach (var change in changes)
+      {
+        while (true)
+        {
+          try
+          {
+            var stopwatch = Stopwatch.StartNew();
+            var blobSha = octokit
+              .Git.Blob.Create(
+                owner,
+                repoName,
+                new()
+                {
+                  Encoding = EncodingType.Base64,
+                  Content = Convert.ToBase64String(change.Content),
+                }
+              )
+              .Result.Sha;
+
+            await Task.Delay((int)Math.Max(0, TIME_BETWEEN_BLOBS - stopwatch.ElapsedMilliseconds));
+
+            Console.WriteLine($"blobbed {change.Path} (+{stopwatch.ElapsedMilliseconds}ms)");
+
+            newTree.Tree.Add(
+              new()
+              {
+                Sha = blobSha,
+                Path = change.Path,
+                Mode = "100644",
+                Type = TreeType.Blob,
+              }
+            );
+
+            break;
+          }
+          catch
+          {
+            PrettyLog.Warn($"Failed blob {change.Path}; retrying in {TIME_PER_BLOB}ms...");
+            await Task.Delay(TIME_PER_BLOB);
+          }
+        }
+      }
+
+      var treeData = await octokit.Git.Tree.Create(owner, repoName, newTree);
+      var newCommit = await octokit.Git.Commit.Create(
+        owner,
+        repoName,
+        new(message, treeData.Sha, latestCommitSha)
+      );
+      await octokit.Git.Reference.Update(
+        owner,
+        repoName,
+        $"heads/{branch}",
+        new(newCommit.Sha, true)
+      );
+
+      changes.Clear();
+      changesSize = 0;
+    }
+  }
+
+  public class GithubChangeBlob
+  {
+    public required string Sha;
+    public required string Path;
+    public required string Mode;
+    public required string Type;
+  }
+
+  public class FileChange(string path, ArraySegment<byte> content)
+  {
+    public string Path = path;
+    public ArraySegment<byte> Content = content;
+  }
+}

src/BlitzKit.CLI/Models/BlitzKitAssets.cs

@@ -1,169 +0,0 @@
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using System.Net;
-using BlitzKit.CLI.Utils;
-using DotNetEnv;
-using Octokit;
-
-namespace BlitzKit.CLI.Models
-{
-  public static class BlitzKitAssets
-  {
-    private static int TIME_PER_BLOB = (int)Math.Pow(2, 4) * 10000;
-    private static int TIME_BETWEEN_BLOBS = (int)(60 * 60 * 1000 / 5000 / 0.9);
-
-    public static async Task CommitAssets(string message, List<FileChange> changes)
-    {
-      Console.WriteLine(
-        $"Committing \"{message}\"... with {changes.Count} proposed changes to the \"{Env.GetString("PUBLIC_ASSET_BRANCH")}\" branch"
-      );
-
-      if (changes.Count == 0)
-        return;
-
-      await CommitMultipleFiles(
-        Env.GetString("PUBLIC_ASSET_REPO"),
-        Env.GetString("PUBLIC_ASSET_BRANCH"),
-        message,
-        changes
-      );
-    }
-
-    public static async Task CommitMultipleFiles(
-      string repoRaw,
-      string branch,
-      string message,
-      List<FileChange> changesRaw
-    )
-    {
-      using HttpClient httpClient = new();
-
-      GitHubClient octokit = new(new ProductHeaderValue("MyAmazingApp"))
-      {
-        Credentials = new(Env.GetString("GH_TOKEN")),
-      };
-      var seenPaths = new ConcurrentDictionary<string, bool>();
-      var changes = new ConcurrentBag<FileChange>(); // Thread-safe alternative
-
-      await Task.WhenAll(
-        changesRaw
-          .Select(async change =>
-          {
-            var blobPath = $"{repoRaw}/{branch}/{change.Path}";
-
-            HttpResponseMessage response = await httpClient.GetAsync(
-              $"https://raw.githubusercontent.com/{blobPath}"
-            );
-
-            if (response.StatusCode == HttpStatusCode.NotFound)
-            {
-              if (seenPaths.TryAdd(change.Path, true)) // Ensures uniqueness
-              {
-                Console.WriteLine(
-                  $"🟢 (+{change.Content.Count.ToString("N0", Program.Culture)}B) {blobPath}"
-                );
-                changes.Add(change);
-              }
-            }
-            else if (response.StatusCode == HttpStatusCode.OK)
-            {
-              var bytes = await response.Content.ReadAsByteArrayAsync();
-
-              if (!bytes.SequenceEqual([.. change.Content]) && seenPaths.TryAdd(change.Path, true))
-              {
-                var diff = change.Content.Count - bytes.Length;
-
-                Console.WriteLine(
-                  $"🟡 ({(diff > 0 ? '+' : "")}{diff.ToString("N0", Program.Culture)}B) {blobPath}"
-                );
-
-                changes.Add(change);
-              }
-            }
-            else
-            {
-              throw new Exception($"Unexpected status code {response.StatusCode} for {blobPath}");
-            }
-          })
-          .ToList()
-      );
-
-      if (changes.IsEmpty)
-        return;
-
-      var repoRawSplit = repoRaw.Split('/');
-      var owner = repoRawSplit[0];
-      var repo = repoRawSplit[1];
-      var latestCommitSha = (await octokit.Git.Reference.Get(owner, repo, $"heads/{branch}"))
-        .Object
-        .Sha;
-      var treeSha = (await octokit.Git.Commit.Get(owner, repo, latestCommitSha)).Tree.Sha;
-      NewTree newTree = new() { BaseTree = treeSha };
-
-      foreach (var change in changes)
-      {
-        while (true)
-        {
-          try
-          {
-            var stopwatch = Stopwatch.StartNew();
-            var blobSha = octokit
-              .Git.Blob.Create(
-                owner,
-                repo,
-                new()
-                {
-                  Encoding = EncodingType.Base64,
-                  Content = Convert.ToBase64String(change.Content),
-                }
-              )
-              .Result.Sha;
-
-            await Task.Delay((int)Math.Max(0, TIME_BETWEEN_BLOBS - stopwatch.ElapsedMilliseconds));
-
-            Console.WriteLine($"blobbed {change.Path} (+{stopwatch.ElapsedMilliseconds}ms)");
-
-            newTree.Tree.Add(
-              new()
-              {
-                Sha = blobSha,
-                Path = change.Path,
-                Mode = "100644",
-                Type = TreeType.Blob,
-              }
-            );
-
-            break;
-          }
-          catch
-          {
-            PrettyLog.Warn($"Failed blob {change.Path}; retrying in {TIME_PER_BLOB}ms...");
-            await Task.Delay(TIME_PER_BLOB);
-          }
-        }
-      }
-
-      var treeData = await octokit.Git.Tree.Create(owner, repo, newTree);
-      var newCommit = await octokit.Git.Commit.Create(
-        owner,
-        repo,
-        new(message, treeData.Sha, latestCommitSha)
-      );
-      await octokit.Git.Reference.Update(owner, repo, $"heads/{branch}", new(newCommit.Sha, true));
-    }
-  }
-
-  public class GithubChangeBlob
-  {
-    public required string Sha;
-    public required string Path;
-    public required string Mode;
-    public required string Type;
-  }
-
-  public class FileChange(string path, ArraySegment<byte> content)
-  {
-    public string Path = path;
-    public ArraySegment<byte> Content = content;
-  }
-}