diff --git a/.gitignore b/.gitignore index 5db5c98ec..a1e6cbbce 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ target *.sln.docstates *.userprefs mono_crash.* -[Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ diff --git a/.vscode/settings.json b/.vscode/settings.json index f5a086df5..11a6322cd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -69,6 +69,7 @@ "DIDS", "donotremove", "drei", + "dsapp", "dvpd", "dvpl", "DVPLCRC", diff --git a/bun.lockb b/bun.lockb index ce79beddc..85b4fad8f 100644 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/packages/core/src/blitz/catalogItemAccessor.ts b/packages/core/src/blitz/catalogItemAccessor.ts index 0dca3cf8f..fbe032a4e 100644 --- a/packages/core/src/blitz/catalogItemAccessor.ts +++ b/packages/core/src/blitz/catalogItemAccessor.ts @@ -1,14 +1,17 @@ import { CatalogItem } from '@protos/blitz_items'; import { Any } from '@protos/google/protobuf/any'; import { lowerFirst } from 'lodash-es'; +import { compareUint8Arrays } from '../../../scripts/src/core/github/compareUint8Arrays'; import { MessageFns } from '../protos'; export class CatalogItemAccessor { public id: string; components: Record = {}; + count: number; - constructor(item: CatalogItem) { + constructor(private item: CatalogItem) { this.id = item.catalog_id; + this.count = item.components.length; for (const component of item.components) { if ( @@ -77,4 +80,20 @@ export class CatalogItemAccessor { encode() { return CatalogItem.encode(this.pack()).finish(); } + + equals(other: CatalogItemAccessor) { + return ( + this.count === other.count && + this.item.components.every( + (component) => + component.value === undefined || + (component.key in other.components && + (component.value.component === undefined || + compareUint8Arrays( + other.components[component.key].value, + component.value.component.value, + ))), + ) + ); + } } diff --git a/packages/core/src/blitz/metadata.ts b/packages/core/src/blitz/metadata.ts index 3826893d5..ead38e30c 100644 --- a/packages/core/src/blitz/metadata.ts +++ b/packages/core/src/blitz/metadata.ts @@ -118,4 +118,14 @@ export class MetadataAccessor { return filtered; } + + toArray() { + return Object.values(this.items); + } + + toSortedArray() { + return this.toArray().sort((a, b) => + a.catalog_id.localeCompare(b.catalog_id), + ); + } } diff --git a/packages/website-ue/package.json b/packages/website-ue/package.json index 1994f4136..d1dd4b8f1 100644 --- a/packages/website-ue/package.json +++ b/packages/website-ue/package.json @@ -22,6 +22,7 @@ "@react-three/fiber": "^8.17.10", "@types/ffi-napi": "^4.0.10", "@types/lodash-es": "^4.17.12", + "@types/pako": "^2.0.3", "@types/react": "^19.1.5", "@types/react-dom": "^19.1.5", "@types/ref-napi": "^3.0.12", @@ -33,6 +34,8 @@ "immer": "^10.1.1", "lodash-es": "^4.17.21", "minimatch": "^10.0.1", + "minizlib": "^3.0.2", + "pako": "^2.1.0", "react": "^19.1.0", "react-dom": "^19.1.0", "ref-napi": "^3.0.3", diff --git a/packages/website-ue/src/components/PageWrapper.tsx b/packages/website-ue/src/components/PageWrapper.tsx index 7ec4e7101..665ccb174 100644 --- a/packages/website-ue/src/components/PageWrapper.tsx +++ b/packages/website-ue/src/components/PageWrapper.tsx @@ -1,13 +1,11 @@ import { Flex, type FlexProps, Theme } from '@radix-ui/themes'; import type { ComponentProps } from 'react'; -import type { RadixSize } from '../stores/embedState'; import { BlitzKitTheme } from './BlitzKitTheme'; type PageWrapperProps = FlexProps & { color?: ComponentProps['accentColor']; containerProps?: ComponentProps; noFlex1?: boolean; - padding?: RadixSize; noMinHeight?: boolean; }; @@ -18,7 +16,6 @@ export function PageWrapper({ noFlex1 = false, containerProps, noMinHeight = false, - padding = '4', ...props }: PageWrapperProps) { return ( @@ -35,8 +32,9 @@ export function PageWrapper({ > - BlitzKit FFI - - + + + + + Path + Memo + + + + + + + Debug + + + Some debug information to check if everything is up + + + + + + Bindings + + List of all active bindings + + + + + Discovery + + List of all active discovery lists + + + + + Differ + + Differentiate two branches + + + ); } diff --git a/packages/website-ue/src/pages/internal/bindings/_index.tsx b/packages/website-ue/src/pages/internal/bindings/_index.tsx new file mode 100644 index 000000000..57ba9a7e5 --- /dev/null +++ b/packages/website-ue/src/pages/internal/bindings/_index.tsx @@ -0,0 +1,89 @@ +import { Badge, Flex, Heading, Link, Separator, Text } from '@radix-ui/themes'; +import { PageWrapper } from 'packages/website-ue/src/components/PageWrapper'; +import { ffi } from 'packages/website-ue/src/core/blitzkit/ffi'; + +export interface Bindings { + bindings: Binding[]; + forbidden: string[]; +} + +interface Binding { + name: string; + versions: string[]; + discovery: string; + env: string; + show_server_selection: boolean; +} + +export async function Page() { + return ( + + + + + + ); +} + +async function Bindings({ url, dev }: { url: string; dev?: boolean }) { + const data = await fetch(url).then( + (response) => response.json() as Promise, + ); + + return ( + <> + + Bindings URL{' '} + + {url} + + + + {data.bindings.map((binding) => ( + + + + {binding.env} + + {!dev && ( + + Production + + )} + {dev && ( + + Dev + + )} + {binding.show_server_selection && ( + Public + )} + {!binding.show_server_selection && ( + Private + )} + + + + {binding.name} + + + Discovery: + + {binding.discovery} + + + + + Versions: + + {binding.versions.map((version) => ( + + {version} + + ))} + + + ))} + + ); +} diff --git a/packages/website-ue/src/pages/internal/bindings/index.astro b/packages/website-ue/src/pages/internal/bindings/index.astro new file mode 100644 index 000000000..dc56143d6 --- /dev/null +++ b/packages/website-ue/src/pages/internal/bindings/index.astro @@ -0,0 +1,8 @@ +--- +import InternalBaseLayout from 'packages/website-ue/src/layouts/InternalBaseLayout.astro'; +import { Page } from './_index'; +--- + + + + diff --git a/packages/website-ue/src/pages/internal/debug/_index.tsx b/packages/website-ue/src/pages/internal/debug/_index.tsx new file mode 100644 index 000000000..7fd8e5130 --- /dev/null +++ b/packages/website-ue/src/pages/internal/debug/_index.tsx @@ -0,0 +1,14 @@ +import { Heading } from '@radix-ui/themes'; +import { FFIEcho } from '../../../components/Internal/FFIEcho'; +import { FFIFileCount } from '../../../components/Internal/FFIFileCount'; +import { PageWrapper } from '../../../components/PageWrapper'; + +export function Page() { + return ( + + BlitzKit FFI + + + + ); +} diff --git a/packages/website-ue/src/pages/internal/debug/index.astro b/packages/website-ue/src/pages/internal/debug/index.astro new file mode 100644 index 000000000..dc56143d6 --- /dev/null +++ b/packages/website-ue/src/pages/internal/debug/index.astro @@ -0,0 +1,8 @@ +--- +import InternalBaseLayout from 'packages/website-ue/src/layouts/InternalBaseLayout.astro'; +import { Page } from './_index'; +--- + + + + diff --git a/packages/website-ue/src/pages/internal/differ/_index.tsx b/packages/website-ue/src/pages/internal/differ/_index.tsx new file mode 100644 index 000000000..d28f4ef61 --- /dev/null +++ b/packages/website-ue/src/pages/internal/differ/_index.tsx @@ -0,0 +1,289 @@ +import type { MessageFns } from '@protos/blitz_static_profile_avatar_component'; +import type { Any } from '@protos/google/protobuf/any'; +import * as protos from '@protos/index'; +import { CommitIcon, ReaderIcon } from '@radix-ui/react-icons'; +import { + Badge, + Box, + Code, + Flex, + IconButton, + Select, + Spinner, + Text, +} from '@radix-ui/themes'; +import { MetadataAccessor } from 'packages/core/src'; +import type { CatalogItemAccessor } from 'packages/core/src/blitz/catalogItemAccessor'; +import { PageWrapper } from 'packages/website-ue/src/components/PageWrapper'; +import { Var } from 'packages/website-ue/src/core/radix/var'; +import { useState } from 'react'; +import { fetchMetadata } from 'submodules/blitzkit-closed/src'; +import type { DiscoveryServer } from '../discovery/_index'; + +export function Page({ servers }: { servers: DiscoveryServer[] }) { + const [serverA, setSeverA] = useState(undefined); + const [serverB, setSeverB] = useState(undefined); + const [loading, setLoading] = useState(false); + const [diff, setDiff] = useState(null); + + return ( + + + { + setSeverA(servers.find((server) => server.name === value)); + }} + > + + + Select base server + + {servers.map((server) => ( + + {server.name} + + ))} + + + + { + setLoading(true); + + const diff = await diffServers(serverA!.addr, serverB!.addr); + + setLoading(false); + setDiff(diff); + }} + > + {loading ? : } + + + { + setSeverB(servers.find((server) => server.name === value)); + }} + > + + + Select base server + + {servers.map((server) => ( + + {server.name} + + ))} + + + + + {diff !== null && ( + entry.type !== CatalogItemDiffType.Unmodified, + )} + /> + )} + + ); +} + +function DiffDisplay({ diff }: { diff: CatalogDiffEntry[] }) { + return ( + + {diff.length === 0 && ( + + Branches are identical + + )} + + {diff.map((entry) => ( + + + + {diffEmojis[entry.type]} {entry.name} + + + + + {(entry.type === CatalogItemDiffType.Removed || + entry.type === CatalogItemDiffType.Unmodified) && ( + + )} + {entry.type === CatalogItemDiffType.Modified && ( + + )} + {(entry.type === CatalogItemDiffType.Added || + entry.type === CatalogItemDiffType.Unmodified) && ( + + )} + {entry.type === CatalogItemDiffType.Modified && ( + + )} + + + ))} + + ); +} + +function ItemDisplay({ item }: { item: CatalogItemAccessor }) { + const entries = Object.entries(item.components); + + protos; + + return ( + + {entries.length === 0 && No components} + + {entries.map(([key, value]) => ( + + ))} + + ); +} + +function ComponentDisplay({ name, value }: { name: string; value: Any }) { + const [code, setCode] = useState(null); + + return ( + + + { + const messageName = value.type_url + .slice(2) + .split('.') + .slice(1) + .join('.'); + const json = ( + protos[messageName as keyof typeof protos] as MessageFns + ).decode(value.value); + + setCode(JSON.stringify(json, null, 2)); + }} + > + + + + + {name} + + {value.value.length.toLocaleString()}B + + + {code !== null && ( + + {code} + + )} + + ); +} + +enum CatalogItemDiffType { + Unmodified, + Modified, + Added, + Removed, +} + +type CatalogDiffEntry = { name: string } & ( + | { + type: CatalogItemDiffType.Modified; + a: CatalogItemAccessor; + b: CatalogItemAccessor; + } + | { + type: Exclude; + item: CatalogItemAccessor; + } +); + +const diffEmojis: Record = { + [CatalogItemDiffType.Unmodified]: '🩶', + [CatalogItemDiffType.Modified]: '🟡', + [CatalogItemDiffType.Added]: '🟢', + [CatalogItemDiffType.Removed]: '🔴', +}; + +async function diffServers(urlA: string, urlB: string) { + const [metadataA, metadataB] = await Promise.all([ + fetchMetadata(urlA), + fetchMetadata(urlB), + ]).then((members) => + members.map((metadata) => new MetadataAccessor(metadata)), + ); + const itemNameSet = new Set(); + + for (const item in metadataA.items) itemNameSet.add(item); + for (const item in metadataB.items) itemNameSet.add(item); + + const itemNames = [...itemNameSet].sort(); + const diff: CatalogDiffEntry[] = []; + + for (const name of itemNames) { + if (name in metadataA.items) { + if (name in metadataB.items) { + const a = metadataA.get(name); + const b = metadataB.get(name); + + if (a.equals(b)) { + diff.push({ name, type: CatalogItemDiffType.Unmodified, item: a }); + } else { + diff.push({ + name, + type: CatalogItemDiffType.Modified, + a: a, + b: b, + }); + } + } else { + diff.push({ + name, + type: CatalogItemDiffType.Removed, + item: metadataA.get(name), + }); + } + } else if (name in metadataB.items) { + diff.push({ + name, + type: CatalogItemDiffType.Added, + item: metadataB.get(name), + }); + } + } + + return diff; +} diff --git a/packages/website-ue/src/pages/internal/differ/index.astro b/packages/website-ue/src/pages/internal/differ/index.astro new file mode 100644 index 000000000..57059f4b3 --- /dev/null +++ b/packages/website-ue/src/pages/internal/differ/index.astro @@ -0,0 +1,40 @@ +--- +import { ffi } from '../../../core/blitzkit/ffi'; +import InternalBaseLayout from '../../../layouts/InternalBaseLayout.astro'; +import { Bindings } from '../bindings/_index'; +import { Discovery, DiscoveryServer } from '../discovery/_index'; +import { Page } from './_index'; + +const serversMap = new Map(); + +await Promise.all( + [ffi.get_bindings_url_dev()!, ffi.get_bindings_url_production()!] + .map((raw) => JSON.parse(raw) as string) + .map(async (url) => { + const { bindings } = await fetch(url).then( + (response) => response.json() as Promise, + ); + + await Promise.all( + bindings.map(async (binding) => { + const discovery = await fetch(binding.discovery).then( + (response) => response.json() as Promise, + ); + + Object.values(discovery.servers).map((server) => { + if (serversMap.has(server.name)) return; + serversMap.set(server.name, server); + }); + }), + ); + }), +); + +const servers = [...serversMap.values()].sort((a, b) => + a.name.localeCompare(b.name), +); +--- + + + + diff --git a/packages/website-ue/src/pages/internal/discovery/_index.tsx b/packages/website-ue/src/pages/internal/discovery/_index.tsx new file mode 100644 index 000000000..2398de59c --- /dev/null +++ b/packages/website-ue/src/pages/internal/discovery/_index.tsx @@ -0,0 +1,99 @@ +import { Flex, Heading, Link, Text } from '@radix-ui/themes'; +import { PageWrapper } from 'packages/website-ue/src/components/PageWrapper'; +import { ffi } from 'packages/website-ue/src/core/blitzkit/ffi'; +import type { Bindings } from '../bindings/_index'; + +export interface Discovery { + version: string; + clusters: string[]; + default_server: string; + servers: Record; + icmp_servers: ICMPServers; + clusters_versions: Record; + countries_by_clusters: CountriesByClusters; + extensions: Extensions; +} + +interface Empty { + version: string; + server: string; +} + +type CountriesByClusters = Record; + +interface Extensions { + endpoints: Record; +} + +interface Endpoint { + playability_config_url: string; + fullscreen_notifications_url: string; + mgg_frontier_url: string; + mgg_pay_url: string; +} + +interface ICMPServers { + default: Record; +} + +export interface DiscoveryServer { + name: string; + addr: string; + addr_dsapp?: string; +} + +const servers: DiscoveryServer[] = []; + +async function populate(url: string) { + const { bindings } = await fetch(url).then( + (response) => response.json() as Promise, + ); + + await Promise.all( + bindings.map(async (binding) => { + const discovery = await fetch(binding.discovery).then( + (response) => response.json() as Promise, + ); + + Object.values(discovery.servers).map((server) => { + const doesInclude = servers.some((s) => s.addr === server.addr); + if (!doesInclude) servers.push(server); + }); + }), + ); +} + +await Promise.all([ + populate(JSON.parse(ffi.get_bindings_url_dev()!)), + populate(JSON.parse(ffi.get_bindings_url_production()!)), +]); + +servers.sort((a, b) => a.name.localeCompare(b.name)); + +export function Page() { + return ( + + {servers.map((server) => ( + + {server.name} + + + Address:{' '} + + {server.addr} + + + + {server.addr_dsapp && ( + + DSApp:{' '} + + {server.addr_dsapp} + + + )} + + ))} + + ); +} diff --git a/packages/website-ue/src/pages/internal/discovery/index.astro b/packages/website-ue/src/pages/internal/discovery/index.astro new file mode 100644 index 000000000..dc56143d6 --- /dev/null +++ b/packages/website-ue/src/pages/internal/discovery/index.astro @@ -0,0 +1,8 @@ +--- +import InternalBaseLayout from 'packages/website-ue/src/layouts/InternalBaseLayout.astro'; +import { Page } from './_index'; +--- + + + + diff --git a/src/BlitzKit.FFI/BlitzKit.FFI.csproj b/src/BlitzKit.FFI/BlitzKit.FFI.csproj index d393bab71..e4384e248 100644 --- a/src/BlitzKit.FFI/BlitzKit.FFI.csproj +++ b/src/BlitzKit.FFI/BlitzKit.FFI.csproj @@ -7,8 +7,10 @@ win-x64 true - + + + diff --git a/src/BlitzKit.FFI/FFI.cs b/src/BlitzKit.FFI/FFI.cs index 5a07f912b..d5f9ff321 100644 --- a/src/BlitzKit.FFI/FFI.cs +++ b/src/BlitzKit.FFI/FFI.cs @@ -2,7 +2,6 @@ using System.Runtime.InteropServices; using BlitzKit.FFI.Models; using BlitzKit.FFI.Utils; -using CUE4Parse.Compression; namespace BlitzKit.FFI; @@ -32,4 +31,18 @@ public static class FFI { return provider.Files.Count; } + + [UnmanagedCallersOnly(EntryPoint = "get_bindings_url_production")] + public static IntPtr GetBindingsURLProduction() + { + var url = provider.GetBindingsURLProduction(); + return Marshal.StringToHGlobalAnsi(url); + } + + [UnmanagedCallersOnly(EntryPoint = "get_bindings_url_dev")] + public static IntPtr GetBindingsURLDev() + { + var url = provider.GetBindingsURLDev(); + return Marshal.StringToHGlobalAnsi(url); + } } diff --git a/src/BlitzKit.FFI/Models/BlitzProvider.cs b/src/BlitzKit.FFI/Models/BlitzProvider.cs index 268720e24..2cb9d929a 100644 --- a/src/BlitzKit.FFI/Models/BlitzProvider.cs +++ b/src/BlitzKit.FFI/Models/BlitzProvider.cs @@ -1,6 +1,6 @@ using System; using System.IO; -using System.Text; +using BlitzKit.FFI.Utils; using CUE4Parse.FileProvider; using CUE4Parse.UE4.Versions; @@ -18,24 +18,14 @@ namespace BlitzKit.FFI.Models { Initialize(); Mount(); - - Console.WriteLine(DiscoveryURL()); } - public string DiscoveryURL() - { - if (TryGetGameFile("Blitz/Config/DefaultEngine.ini", out var defaultEngine)) - { - INIAccessor accessor = new(Encoding.UTF8.GetString(defaultEngine.Read())); + public INIAccessor GetDefaultEngine() => new(this["Blitz/Config/DefaultEngine.ini"]); - var url = (string)accessor.Get("DiscoveryURLDedicProduction"); + public string GetBindingsURLProduction() => + GetDefaultEngine()["/Script/MintSDK.MintSDKSettingsObject"]["DiscoveryBindingsURLProduction"]; - return url; - } - else - { - throw new FileNotFoundException("Could not find DefaultEngine.ini"); - } - } + public string GetBindingsURLDev() => + GetDefaultEngine()["/Script/MintSDK.MintSDKSettingsObject"]["DiscoveryBindingsURLDev"]; } } diff --git a/src/BlitzKit.FFI/Models/INIAccessor.cs b/src/BlitzKit.FFI/Models/INIAccessor.cs deleted file mode 100644 index 56b536320..000000000 --- a/src/BlitzKit.FFI/Models/INIAccessor.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace BlitzKit.FFI.Models -{ - public class INIAccessor - { - private Dictionary data = []; - - public INIAccessor(string content) - { - var lines = content.Split(['\n', '\r'], System.StringSplitOptions.RemoveEmptyEntries); - - foreach (var line in lines) - { - var trimmed = line.Trim(); - - if (string.IsNullOrWhiteSpace(trimmed) || trimmed.StartsWith(';')) - continue; - - var prefix = trimmed[0]; - string key, - value; - - if (prefix == '+' || prefix == '-') - { - var x = trimmed.IndexOf('='); - key = trimmed[1..x]; - value = trimmed[(x + 1)..]; - - if (!data.TryGetValue(key, out var existing)) - { - data[key] = existing = new List(); - } - - var list = (List)existing; - - if (prefix == '+') - list.Add(value); - else - list.Remove(value); - } - else - { - var x = trimmed.IndexOf('='); - key = trimmed[..x]; - value = trimmed[(x + 1)..]; - - data[key] = value; - } - } - } - - public bool TryGet(string key, out object value) => data.TryGetValue(key, out value); - - public object Get(string key) => data[key]; - } -} diff --git a/src/BlitzKit.FFI/Utils/INIAccessor.cs b/src/BlitzKit.FFI/Utils/INIAccessor.cs new file mode 100644 index 000000000..288af988d --- /dev/null +++ b/src/BlitzKit.FFI/Utils/INIAccessor.cs @@ -0,0 +1,30 @@ +using System.Text; +using CUE4Parse.FileProvider.Objects; +using IniParser.Model; +using IniParser.Model.Configuration; +using IniParser.Parser; + +namespace BlitzKit.FFI.Utils +{ + public class INIAccessor + { + private static readonly IniParserConfiguration configuration = new() + { + AllowDuplicateSections = true, + AllowDuplicateKeys = true, + ConcatenateDuplicateKeys = true, + }; + + private readonly IniData data; + + public INIAccessor(GameFile file) + { + var bytes = file.Read(); + var str = Encoding.UTF8.GetString(bytes); + IniDataParser parser = new(configuration); + data = parser.Parse(str); + } + + public KeyDataCollection this[string sectionName] => data[sectionName]; + } +} diff --git a/submodules/blitzkit-closed b/submodules/blitzkit-closed index 89dc337d2..69df3b07a 160000 --- a/submodules/blitzkit-closed +++ b/submodules/blitzkit-closed @@ -1 +1 @@ -Subproject commit 89dc337d2713dd4aaa8319d01c544f9e08756127 +Subproject commit 69df3b07a2e8cf52ca0235c180393e4315d6f940