From 9dfa5a7693a7b3bc22c326ee5c41c8dd893a2baa Mon Sep 17 00:00:00 2001 From: tresabhi <43380238+tresabhi@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:46:05 -0600 Subject: [PATCH] feat: smart uploader --- src/BlitzKit.CLI/Functions/Mangler.cs | 38 ++--- src/BlitzKit.CLI/Models/AssetUploader.cs | 187 ++++++++++++++++++++++ src/BlitzKit.CLI/Models/BlitzKitAssets.cs | 169 ------------------- 3 files changed, 207 insertions(+), 187 deletions(-) create mode 100644 src/BlitzKit.CLI/Models/AssetUploader.cs delete mode 100644 src/BlitzKit.CLI/Models/BlitzKitAssets.cs diff --git a/src/BlitzKit.CLI/Functions/Mangler.cs b/src/BlitzKit.CLI/Functions/Mangler.cs index c055ab57a..c2e75908b 100644 --- a/src/BlitzKit.CLI/Functions/Mangler.cs +++ b/src/BlitzKit.CLI/Functions/Mangler.cs @@ -20,14 +20,14 @@ namespace BlitzKit.CLI.Functions { readonly BlitzProvider provider = new(args.Contains("--depot")); Locales? locales; - readonly List changes = []; readonly Dictionary> strings = []; + readonly AssetUploader assetUploader = new("mangled ue assets"); public async Task Mangle() { await FetchStrings(); - MangleNations(); - await BlitzKitAssets.CommitAssets("ue assets", changes); + await MangleNations(); + await assetUploader.Flush(); } I18n GetString(string name) @@ -64,7 +64,7 @@ namespace BlitzKit.CLI.Functions { using var client = new HttpClient(); var text = File.ReadAllText("../../packages/i18n/locales.json"); - this.locales = + locales = JsonSerializer.Deserialize(text) ?? throw new Exception("Failed to deserialize locales"); @@ -81,7 +81,7 @@ namespace BlitzKit.CLI.Functions } } - void MangleNations() + async Task MangleNations() { var dirs = provider.RootDirectory.GetDirectory("Blitz/Content/Tanks").Directories; List tanksRaw = []; @@ -93,10 +93,11 @@ namespace BlitzKit.CLI.Functions foreach (var tankDir in dir.Value.Directories) { - if (tankDir.Value.Name != "R90_IS_4") - continue; + // if (tankDir.Value.Name != "R90_IS_4") + // continue; - tanksRaw.Add(MangleTank(tankDir.Value)); + var tank = await MangleTank(tankDir.Value); + tanksRaw.Add(tank); } } @@ -110,7 +111,7 @@ namespace BlitzKit.CLI.Functions tanks.Tanks_.Add(tankMeta); } - changes.Add(new("definitions/tanks.pb", tanks.ToByteArray())); + await assetUploader.Add(new("definitions/tanks.pb", tanks.ToByteArray())); } void DeduplicateSlugs(List tanks) @@ -185,7 +186,7 @@ namespace BlitzKit.CLI.Functions } } - Tank MangleTank(VFS tankDir) + async Task MangleTank(VFS tankDir) { var pdaName = $"PDA_{tankDir.Name}"; var pda = provider.LoadObject($"{tankDir.Path}/{pdaName}.{pdaName}"); @@ -197,8 +198,8 @@ namespace BlitzKit.CLI.Functions slug = Regex.Replace(slug, "--+", "-"); slug = Regex.Replace(slug, "-$", ""); - // MangleHull(pda); - MangleIcon(pda); + await MangleHull(pda); + await MangleIcon(pda); Tank tank = new() { @@ -207,22 +208,23 @@ namespace BlitzKit.CLI.Functions Slug = slug, }; - changes.Add(new($"tanks/{id}/meta.pb", tank.ToByteArray())); + await assetUploader.Add(new($"tanks/{id}/meta.pb", tank.ToByteArray())); return tank; } - void MangleIcon(UObject pda) + async Task MangleIcon(UObject pda) { var id = PropertyUtil.Get(pda, "TankId").Text; var smallIcon = BlitzKitExporter.Texture2D(PropertyUtil.Get(pda, "SmallIcon")); + Console.WriteLine(id); var bigIcon = BlitzKitExporter.Texture2D(PropertyUtil.Get(pda, "BigIcon")); - changes.Add(new($"tanks/{id}/icons/small.png", smallIcon)); - changes.Add(new($"tanks/{id}/icons/big.png", bigIcon)); + await assetUploader.Add(new($"tanks/{id}/icons/small.png", smallIcon)); + await assetUploader.Add(new($"tanks/{id}/icons/big.png", bigIcon)); } - void MangleHull(UObject pda) + async Task MangleHull(UObject pda) { var id = PropertyUtil.Get(pda, "TankId").Text; var hulls = PropertyUtil.Get(pda, "DT_Hulls"); @@ -232,7 +234,7 @@ namespace BlitzKit.CLI.Functions var hullCollision = PropertyUtil.Get(hullMeshSettings, "CollisionMesh"); MonoGltf hullCollisionGltf = new(hullCollision); - changes.Add(new($"tanks/{id}/collision/hull.glb", hullCollisionGltf.Write())); + await assetUploader.Add(new($"tanks/{id}/collision/hull.glb", hullCollisionGltf.Write())); } } } diff --git a/src/BlitzKit.CLI/Models/AssetUploader.cs b/src/BlitzKit.CLI/Models/AssetUploader.cs new file mode 100644 index 000000000..70df2d612 --- /dev/null +++ b/src/BlitzKit.CLI/Models/AssetUploader.cs @@ -0,0 +1,187 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Net; +using System.Threading.Tasks; +using BlitzKit.CLI.Utils; +using DotNetEnv; +using Octokit; + +namespace BlitzKit.CLI.Models +{ + public class AssetUploader(string message) + { + private static readonly int TIME_PER_BLOB = (int)Math.Pow(2, 4) * 10000; + private static readonly int TIME_BETWEEN_BLOBS = (int)(60 * 60 * 1000 / 5000 / 0.9); + private static readonly int MAX_TREE_SIZE = 7_000_000; // 7MB + + private readonly GitHubClient octokit = new(new ProductHeaderValue("MyAmazingApp")) + { + Credentials = new(Env.GetString("GH_TOKEN")), + }; + private readonly HttpClient httpClient = new(); + + private readonly string branch = Env.GetString("PUBLIC_ASSET_BRANCH"); + private readonly string repo = Env.GetString("PUBLIC_ASSET_REPO"); + public string message = message; + private static readonly List changes = []; + private int changesSize = 0; + + public async Task Add(FileChange change) + { + var blobPath = $"{repo}/{branch}/{change.Path}"; + HttpResponseMessage response = await httpClient.GetAsync( + $"https://raw.githubusercontent.com/{blobPath}" + ); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + Console.WriteLine( + $"🟢 (+{change.Content.Count.ToString("N0", Program.Culture)}B) {blobPath}" + ); + changes.Add(change); + } + else if (response.StatusCode == HttpStatusCode.OK) + { + int deltaSize = 0; + bool isDifferent = false; + + // hopefully they just give you the file size to delta from + if (response.Content.Headers.ContentLength.HasValue) + { + deltaSize = change.Content.Count - (int)response.Content.Headers.ContentLength; + + if (deltaSize == 0) + { + var bytes = await response.Content.ReadAsByteArrayAsync(); + isDifferent = !bytes.SequenceEqual(change.Content); + } + else + { + isDifferent = true; + } + } + else + { + // the meanies didn't give us the file size so diff manually + var bytes = await response.Content.ReadAsByteArrayAsync(); + isDifferent = !bytes.SequenceEqual(change.Content); + deltaSize = change.Content.Count - bytes.Length; + } + + if (isDifferent) + { + Console.WriteLine( + $"🟡 ({(deltaSize > 0 ? "+" : "")}{deltaSize.ToString("N0", Program.Culture)}B) {blobPath}" + ); + changes.Add(change); + } + else + { + return; + } + } + else + { + throw new Exception($"Unexpected status code {response.StatusCode} for {blobPath}"); + } + + // flush BEFORE going over limit + if (changesSize + change.Content.Count > MAX_TREE_SIZE) + { + Console.WriteLine("Flushing backlog..."); + await Flush(); + } + + changes.Add(change); + changesSize += change.Content.Count; + } + + public async Task Flush() + { + if (changes.Count == 0) + return; + + var repoRawSplit = repo.Split('/'); + var owner = repoRawSplit[0]; + var repoName = repoRawSplit[1]; + var latestCommitSha = (await octokit.Git.Reference.Get(owner, repoName, $"heads/{branch}")) + .Object + .Sha; + var treeSha = (await octokit.Git.Commit.Get(owner, repoName, latestCommitSha)).Tree.Sha; + NewTree newTree = new() { BaseTree = treeSha }; + + foreach (var change in changes) + { + while (true) + { + try + { + var stopwatch = Stopwatch.StartNew(); + var blobSha = octokit + .Git.Blob.Create( + owner, + repoName, + new() + { + Encoding = EncodingType.Base64, + Content = Convert.ToBase64String(change.Content), + } + ) + .Result.Sha; + + await Task.Delay((int)Math.Max(0, TIME_BETWEEN_BLOBS - stopwatch.ElapsedMilliseconds)); + + Console.WriteLine($"blobbed {change.Path} (+{stopwatch.ElapsedMilliseconds}ms)"); + + newTree.Tree.Add( + new() + { + Sha = blobSha, + Path = change.Path, + Mode = "100644", + Type = TreeType.Blob, + } + ); + + break; + } + catch + { + PrettyLog.Warn($"Failed blob {change.Path}; retrying in {TIME_PER_BLOB}ms..."); + await Task.Delay(TIME_PER_BLOB); + } + } + } + + var treeData = await octokit.Git.Tree.Create(owner, repoName, newTree); + var newCommit = await octokit.Git.Commit.Create( + owner, + repoName, + new(message, treeData.Sha, latestCommitSha) + ); + await octokit.Git.Reference.Update( + owner, + repoName, + $"heads/{branch}", + new(newCommit.Sha, true) + ); + + changes.Clear(); + changesSize = 0; + } + } + + public class GithubChangeBlob + { + public required string Sha; + public required string Path; + public required string Mode; + public required string Type; + } + + public class FileChange(string path, ArraySegment content) + { + public string Path = path; + public ArraySegment Content = content; + } +} diff --git a/src/BlitzKit.CLI/Models/BlitzKitAssets.cs b/src/BlitzKit.CLI/Models/BlitzKitAssets.cs deleted file mode 100644 index 4e4353ac8..000000000 --- a/src/BlitzKit.CLI/Models/BlitzKitAssets.cs +++ /dev/null @@ -1,169 +0,0 @@ -using System.Collections.Concurrent; -using System.Diagnostics; -using System.Net; -using BlitzKit.CLI.Utils; -using DotNetEnv; -using Octokit; - -namespace BlitzKit.CLI.Models -{ - public static class BlitzKitAssets - { - private static int TIME_PER_BLOB = (int)Math.Pow(2, 4) * 10000; - private static int TIME_BETWEEN_BLOBS = (int)(60 * 60 * 1000 / 5000 / 0.9); - - public static async Task CommitAssets(string message, List changes) - { - Console.WriteLine( - $"Committing \"{message}\"... with {changes.Count} proposed changes to the \"{Env.GetString("PUBLIC_ASSET_BRANCH")}\" branch" - ); - - if (changes.Count == 0) - return; - - await CommitMultipleFiles( - Env.GetString("PUBLIC_ASSET_REPO"), - Env.GetString("PUBLIC_ASSET_BRANCH"), - message, - changes - ); - } - - public static async Task CommitMultipleFiles( - string repoRaw, - string branch, - string message, - List changesRaw - ) - { - using HttpClient httpClient = new(); - - GitHubClient octokit = new(new ProductHeaderValue("MyAmazingApp")) - { - Credentials = new(Env.GetString("GH_TOKEN")), - }; - var seenPaths = new ConcurrentDictionary(); - var changes = new ConcurrentBag(); // Thread-safe alternative - - await Task.WhenAll( - changesRaw - .Select(async change => - { - var blobPath = $"{repoRaw}/{branch}/{change.Path}"; - - HttpResponseMessage response = await httpClient.GetAsync( - $"https://raw.githubusercontent.com/{blobPath}" - ); - - if (response.StatusCode == HttpStatusCode.NotFound) - { - if (seenPaths.TryAdd(change.Path, true)) // Ensures uniqueness - { - Console.WriteLine( - $"🟢 (+{change.Content.Count.ToString("N0", Program.Culture)}B) {blobPath}" - ); - changes.Add(change); - } - } - else if (response.StatusCode == HttpStatusCode.OK) - { - var bytes = await response.Content.ReadAsByteArrayAsync(); - - if (!bytes.SequenceEqual([.. change.Content]) && seenPaths.TryAdd(change.Path, true)) - { - var diff = change.Content.Count - bytes.Length; - - Console.WriteLine( - $"🟡 ({(diff > 0 ? '+' : "")}{diff.ToString("N0", Program.Culture)}B) {blobPath}" - ); - - changes.Add(change); - } - } - else - { - throw new Exception($"Unexpected status code {response.StatusCode} for {blobPath}"); - } - }) - .ToList() - ); - - if (changes.IsEmpty) - return; - - var repoRawSplit = repoRaw.Split('/'); - var owner = repoRawSplit[0]; - var repo = repoRawSplit[1]; - var latestCommitSha = (await octokit.Git.Reference.Get(owner, repo, $"heads/{branch}")) - .Object - .Sha; - var treeSha = (await octokit.Git.Commit.Get(owner, repo, latestCommitSha)).Tree.Sha; - NewTree newTree = new() { BaseTree = treeSha }; - - foreach (var change in changes) - { - while (true) - { - try - { - var stopwatch = Stopwatch.StartNew(); - var blobSha = octokit - .Git.Blob.Create( - owner, - repo, - new() - { - Encoding = EncodingType.Base64, - Content = Convert.ToBase64String(change.Content), - } - ) - .Result.Sha; - - await Task.Delay((int)Math.Max(0, TIME_BETWEEN_BLOBS - stopwatch.ElapsedMilliseconds)); - - Console.WriteLine($"blobbed {change.Path} (+{stopwatch.ElapsedMilliseconds}ms)"); - - newTree.Tree.Add( - new() - { - Sha = blobSha, - Path = change.Path, - Mode = "100644", - Type = TreeType.Blob, - } - ); - - break; - } - catch - { - PrettyLog.Warn($"Failed blob {change.Path}; retrying in {TIME_PER_BLOB}ms..."); - await Task.Delay(TIME_PER_BLOB); - } - } - } - - var treeData = await octokit.Git.Tree.Create(owner, repo, newTree); - var newCommit = await octokit.Git.Commit.Create( - owner, - repo, - new(message, treeData.Sha, latestCommitSha) - ); - await octokit.Git.Reference.Update(owner, repo, $"heads/{branch}", new(newCommit.Sha, true)); - } - } - - public class GithubChangeBlob - { - public required string Sha; - public required string Path; - public required string Mode; - public required string Type; - } - - public class FileChange(string path, ArraySegment content) - { - public string Path = path; - public ArraySegment Content = content; - } -}