From c5e6f5e2eb45e2c5c24b2b5aaa68fd98a0a2e94e Mon Sep 17 00:00:00 2001 From: tresabhi <43380238+tresabhi@users.noreply.github.com> Date: Sat, 19 Apr 2025 00:47:28 -0500 Subject: [PATCH] feat: glossary cron --- .../core => core/src}/blitz/fetchGlossary.ts | 3 +- packages/core/src/blitz/index.ts | 1 + packages/core/src/blitzkit/gallery.ts | 7 + packages/core/src/blitzkit/index.ts | 1 + packages/core/src/protobuf/fetchPB.ts | 1 - packages/core/src/protos/gallery.proto | 14 ++ packages/core/src/protos/gallery.ts | 192 ++++++++++++++++++ packages/core/src/protos/index.ts | 1 + packages/i18n/locales.json | 14 ++ packages/scripts/src/buildAssets/glossary.ts | 86 ++++++++ packages/scripts/src/buildAssets/index.ts | 2 + .../scripts/src/core/github/commitAssets.ts | 22 +- .../website/src/components/Gallery/List.tsx | 25 ++- .../website/src/core/awaitables/gallery.ts | 3 + .../[...locale]/tools/gallery/_index.tsx | 32 +-- .../[...locale]/tools/gallery/index.astro | 20 +- 16 files changed, 367 insertions(+), 57 deletions(-) rename packages/{website/src/core => core/src}/blitz/fetchGlossary.ts (76%) create mode 100644 packages/core/src/blitzkit/gallery.ts create mode 100644 packages/core/src/protos/gallery.proto create mode 100644 packages/core/src/protos/gallery.ts create mode 100644 packages/i18n/locales.json create mode 100644 packages/scripts/src/buildAssets/glossary.ts create mode 100644 packages/website/src/core/awaitables/gallery.ts diff --git a/packages/website/src/core/blitz/fetchGlossary.ts b/packages/core/src/blitz/fetchGlossary.ts similarity index 76% rename from packages/website/src/core/blitz/fetchGlossary.ts rename to packages/core/src/blitz/fetchGlossary.ts index de33cd15f..3428e3fb3 100644 --- a/packages/website/src/core/blitz/fetchGlossary.ts +++ b/packages/core/src/blitz/fetchGlossary.ts @@ -1,4 +1,5 @@ -import { assertSecret, type BlitzGlossary } from '@blitzkit/core'; +import { assertSecret } from '../blitzkit'; +import { BlitzGlossary } from '../types'; export async function fetchGlossary(locale: string) { const response = await fetch( diff --git a/packages/core/src/blitz/index.ts b/packages/core/src/blitz/index.ts index 5304654f2..d1bd1b37d 100644 --- a/packages/core/src/blitz/index.ts +++ b/packages/core/src/blitz/index.ts @@ -4,6 +4,7 @@ export * from './classes'; export * from './constants'; export * from './equipmentPrice'; export * from './fetchBlitz'; +export * from './fetchGlossary'; export * from './generateStats'; export * from './getAccountAchievements'; export * from './getAccountInfo'; diff --git a/packages/core/src/blitzkit/gallery.ts b/packages/core/src/blitzkit/gallery.ts new file mode 100644 index 000000000..18ddc672b --- /dev/null +++ b/packages/core/src/blitzkit/gallery.ts @@ -0,0 +1,7 @@ +import { fetchPB } from '../protobuf'; +import { Gallery } from '../protos'; +import { asset } from './asset'; + +export function fetchGallery() { + return fetchPB(asset('definitions/gallery.pb'), Gallery); +} diff --git a/packages/core/src/blitzkit/index.ts b/packages/core/src/blitzkit/index.ts index 4438882d3..2221072c9 100644 --- a/packages/core/src/blitzkit/index.ts +++ b/packages/core/src/blitzkit/index.ts @@ -44,6 +44,7 @@ export * from './unwrapBlitzkitResponse'; export * from './votes'; // export * from './withCORSProxy'; export * from './createDefaultProvisions'; +export * from './gallery'; export * from './metaSortTank'; export * from './popularTanks'; export * from './romanize'; diff --git a/packages/core/src/protobuf/fetchPB.ts b/packages/core/src/protobuf/fetchPB.ts index 330511bf0..4d725ee00 100644 --- a/packages/core/src/protobuf/fetchPB.ts +++ b/packages/core/src/protobuf/fetchPB.ts @@ -2,7 +2,6 @@ import { MessageFns } from '../protos'; export async function fetchPB(url: string, message: MessageFns) { const response = await fetch(url); - const buffer = await response.arrayBuffer(); const array = new Uint8Array(buffer); return message.decode(array); diff --git a/packages/core/src/protos/gallery.proto b/packages/core/src/protos/gallery.proto new file mode 100644 index 000000000..7adf88b53 --- /dev/null +++ b/packages/core/src/protos/gallery.proto @@ -0,0 +1,14 @@ +syntax = "proto2"; + +package blitzkit; + +import "packages/core/src/protos/i18n.proto"; + +message Gallery { + repeated Avatar avatars = 1; +} + +message Avatar { + required string id = 1; + required I18nString name = 2; +} \ No newline at end of file diff --git a/packages/core/src/protos/gallery.ts b/packages/core/src/protos/gallery.ts new file mode 100644 index 000000000..aef37981e --- /dev/null +++ b/packages/core/src/protos/gallery.ts @@ -0,0 +1,192 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v2.2.2 +// protoc v5.28.2 +// source: packages/core/src/protos/gallery.proto + +/* eslint-disable */ +import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire"; +import { I18nString } from "./i18n"; + +export const protobufPackage = "blitzkit"; + +export interface Gallery { + avatars: Avatar[]; +} + +export interface Avatar { + id: string; + name: I18nString; +} + +function createBaseGallery(): Gallery { + return { avatars: [] }; +} + +export const Gallery: MessageFns = { + encode(message: Gallery, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + for (const v of message.avatars) { + Avatar.encode(v!, writer.uint32(10).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): Gallery { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseGallery(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.avatars.push(Avatar.decode(reader, reader.uint32())); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): Gallery { + return { + avatars: globalThis.Array.isArray(object?.avatars) ? object.avatars.map((e: any) => Avatar.fromJSON(e)) : [], + }; + }, + + toJSON(message: Gallery): unknown { + const obj: any = {}; + if (message.avatars?.length) { + obj.avatars = message.avatars.map((e) => Avatar.toJSON(e)); + } + return obj; + }, + + create, I>>(base?: I): Gallery { + return Gallery.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): Gallery { + const message = createBaseGallery(); + message.avatars = object.avatars?.map((e) => Avatar.fromPartial(e)) || []; + return message; + }, +}; + +function createBaseAvatar(): Avatar { + return { id: "", name: createBaseI18nString() }; +} + +export const Avatar: MessageFns = { + encode(message: Avatar, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.id !== "") { + writer.uint32(10).string(message.id); + } + if (message.name !== undefined) { + I18nString.encode(message.name, writer.uint32(18).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): Avatar { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAvatar(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.id = reader.string(); + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.name = I18nString.decode(reader, reader.uint32()); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): Avatar { + return { + id: globalThis.String(assertSet("Avatar.id", object.id)), + name: I18nString.fromJSON(assertSet("Avatar.name", object.name)), + }; + }, + + toJSON(message: Avatar): unknown { + const obj: any = {}; + if (message.id !== "") { + obj.id = message.id; + } + if (message.name !== undefined) { + obj.name = I18nString.toJSON(message.name); + } + return obj; + }, + + create, I>>(base?: I): Avatar { + return Avatar.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): Avatar { + const message = createBaseAvatar(); + message.id = object.id ?? ""; + message.name = (object.name !== undefined && object.name !== null) + ? I18nString.fromPartial(object.name) + : createBaseI18nString(); + return message; + }, +}; + +type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; + +export type DeepPartial = T extends Builtin ? T + : T extends globalThis.Array ? globalThis.Array> + : T extends ReadonlyArray ? ReadonlyArray> + : T extends { $case: string; value: unknown } ? { $case: T["$case"]; value?: DeepPartial } + : T extends {} ? { [K in keyof T]?: DeepPartial } + : Partial; + +type KeysOfUnion = T extends T ? keyof T : never; +export type Exact = P extends Builtin ? P + : P & { [K in keyof P]: Exact } & { [K in Exclude>]: never }; + +function isSet(value: any): boolean { + return value !== null && value !== undefined; +} + +function assertSet(field: string, value: T | undefined): T { + if (!isSet(value)) { + throw new TypeError(`Required field ${field} is not set`); + } + + return value as T; +} + +export interface MessageFns { + encode(message: T, writer?: BinaryWriter): BinaryWriter; + decode(input: BinaryReader | Uint8Array, length?: number): T; + fromJSON(object: any): T; + toJSON(message: T): unknown; + create, I>>(base?: I): T; + fromPartial, I>>(object: I): T; +} diff --git a/packages/core/src/protos/index.ts b/packages/core/src/protos/index.ts index 1f2a0a70c..a6a400ac7 100644 --- a/packages/core/src/protos/index.ts +++ b/packages/core/src/protos/index.ts @@ -3,6 +3,7 @@ export * from './average_definitions.ts'; export * from './camouflage_definitions.ts'; export * from './consumable_definitions.ts'; export * from './equipment_definitions.ts'; +export * from './gallery.ts'; export * from './game_definitions.ts'; export * from './i18n.ts'; export * from './map_definitions.ts'; diff --git a/packages/i18n/locales.json b/packages/i18n/locales.json new file mode 100644 index 000000000..488c43563 --- /dev/null +++ b/packages/i18n/locales.json @@ -0,0 +1,14 @@ +{ + "default": "en", + "supported": [ + { "locale": "en" }, + { "locale": "es" }, + { "locale": "fr" }, + { "locale": "ja" }, + { "locale": "pl" }, + { "locale": "pt" }, + { "locale": "ru" }, + { "locale": "uk" }, + { "locale": "zh", "blitz": "zh_Hans_SG" } + ] +} diff --git a/packages/scripts/src/buildAssets/glossary.ts b/packages/scripts/src/buildAssets/glossary.ts new file mode 100644 index 000000000..73ba45c77 --- /dev/null +++ b/packages/scripts/src/buildAssets/glossary.ts @@ -0,0 +1,86 @@ +import { Avatar, encodePBBuffer, fetchGlossary, Gallery } from '@blitzkit/core'; +import locales from '@blitzkit/i18n/locales.json'; +import { extname } from 'path'; +import ProgressBar from 'progress'; +import { commitAssets } from '../core/github/commitAssets'; +import { FileChange } from '../core/github/commitMultipleFiles'; + +export async function glossary() { + console.log('Building glossary...'); + + const avatars: Record = {}; + + for (const supported of locales.supported) { + console.log(`Fetching glossary for ${supported.locale}`); + + const glossary = await fetchGlossary(supported.blitz ?? supported.locale); + + console.log(`Found ${Object.keys(glossary).length} things`); + + for (const key in glossary) { + const glossaryEntry = glossary[key]; + + if ( + !key.startsWith('avatar') || + key.endsWith('_part') || + glossaryEntry.image_url === null + ) { + continue; + } + + if (key in avatars) { + avatars[key].avatar.name.locales[supported.locale] = + glossaryEntry.title; + } else { + avatars[key] = { + url: glossaryEntry.image_url, + avatar: { + id: key, + name: { locales: { [supported.locale]: glossaryEntry.title } }, + }, + }; + } + } + } + + const changes: FileChange[] = []; + const gallery: Gallery = { avatars: [] }; + const totalCount = Object.keys(avatars).length; + + const bar = new ProgressBar( + `Fetching avatars of ${totalCount} things :bar`, + totalCount, + ); + + await Promise.all( + Object.entries(avatars).map(async ([key, { url, avatar }]) => { + if (!(locales.default in avatars[key].avatar.name.locales)) { + console.warn(`Avatar ${key} has no ${locales.default} name`); + bar.tick(); + + return; + } + + gallery.avatars.push(avatar); + + const fileExtension = extname(url); + const content = await fetch(url).then((response) => + response.arrayBuffer(), + ); + + changes.push({ + path: `gallery/avatars/${key}${fileExtension}`, + content: Buffer.from(content), + }); + + bar.tick(); + }), + ); + + changes.push({ + path: 'definitions/gallery.pb', + content: encodePBBuffer(Gallery, gallery), + }); + + await commitAssets('glossary', changes); +} diff --git a/packages/scripts/src/buildAssets/index.ts b/packages/scripts/src/buildAssets/index.ts index 1fda9a06f..51483a711 100644 --- a/packages/scripts/src/buildAssets/index.ts +++ b/packages/scripts/src/buildAssets/index.ts @@ -18,6 +18,7 @@ import { definitions } from './definitions'; import { equipmentIcons } from './equipmentIcons'; import { flags } from './flags'; import { gameModeBanners } from './gameModeBanners'; +import { glossary } from './glossary'; import { migration } from './migration'; import { moduleIcons } from './moduleIcons'; import { popularTanks } from './popularTanks'; @@ -47,6 +48,7 @@ const methods = [ tankIcons, blitzkitTankIcons, popularTanks, + glossary, ]; for (const method of methods) { diff --git a/packages/scripts/src/core/github/commitAssets.ts b/packages/scripts/src/core/github/commitAssets.ts index b8594581b..de1bc699e 100644 --- a/packages/scripts/src/core/github/commitAssets.ts +++ b/packages/scripts/src/core/github/commitAssets.ts @@ -1,4 +1,5 @@ import { assertSecret } from '@blitzkit/core'; +import { chunk } from 'lodash-es'; import { commitMultipleFiles, FileChange } from './commitMultipleFiles'; export async function commitAssets(message: string, changes: FileChange[]) { @@ -6,10 +7,19 @@ export async function commitAssets(message: string, changes: FileChange[]) { if (changes.length === 0) return; - await commitMultipleFiles( - assertSecret(import.meta.env.PUBLIC_ASSET_REPO), - assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH), - `${message} - ${new Date().toDateString()}`, - changes, - ); + const chunks = chunk(changes, 16); + let chunkIndex = 0; + + for (const chunk of chunks) { + console.log(`Committing chunk ${chunkIndex + 1}/${chunks.length}`); + + await commitMultipleFiles( + assertSecret(import.meta.env.PUBLIC_ASSET_REPO), + assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH), + `${message} - ${new Date().toDateString()} (${chunkIndex + 1}/${chunks.length})`, + chunk, + ); + + chunkIndex++; + } } diff --git a/packages/website/src/components/Gallery/List.tsx b/packages/website/src/components/Gallery/List.tsx index d4a7d21c2..64d66b299 100644 --- a/packages/website/src/components/Gallery/List.tsx +++ b/packages/website/src/components/Gallery/List.tsx @@ -1,8 +1,10 @@ +import locales from '@blitzkit/i18n/locales.json'; import { literals } from '@blitzkit/i18n/src/literals'; import { Flex, Text } from '@radix-ui/themes'; import fuzzysort from 'fuzzysort'; import { times } from 'lodash-es'; import { useMemo, useState } from 'react'; +import { awaitableGallery } from '../../core/awaitables/gallery'; import { useLocale } from '../../hooks/useLocale'; import { GalleryEphemeral } from '../../stores/galleryEphemeral'; import { GalleryCard } from './Card'; @@ -16,21 +18,30 @@ export interface Avatar { id: string; } +const gallery = await awaitableGallery; + const DEFAULT_LOADED = 42; const PREVIEW_COUNT = 28; -export function GalleryList({ avatars }: GalleryListProps) { +export function GalleryList() { const search = GalleryEphemeral.use((state) => state.search); const [loadedCards, setLoadedCards] = useState(DEFAULT_LOADED); - const { locale, strings } = useLocale(); + const { locale, strings, unwrap } = useLocale(); const filtered = useMemo(() => { setLoadedCards(DEFAULT_LOADED); if (search === undefined) { - return avatars; + return gallery.avatars; } else { return fuzzysort - .go(search, avatars, { keys: ['name', 'id'] }) + .go(search, gallery.avatars, { + keys: [ + ...locales.supported.map( + (supported) => `name.locales.${supported.locale}`, + ), + 'id', + ], + }) .map((result) => result.obj); } }, [search]); @@ -45,7 +56,11 @@ export function GalleryList({ avatars }: GalleryListProps) { {filtered.slice(0, loadedCards).map((avatar) => ( - + ))} {times( diff --git a/packages/website/src/core/awaitables/gallery.ts b/packages/website/src/core/awaitables/gallery.ts new file mode 100644 index 000000000..3571a940d --- /dev/null +++ b/packages/website/src/core/awaitables/gallery.ts @@ -0,0 +1,3 @@ +import { fetchGallery } from '@blitzkit/core'; + +export const awaitableGallery = fetchGallery(); diff --git a/packages/website/src/pages/[...locale]/tools/gallery/_index.tsx b/packages/website/src/pages/[...locale]/tools/gallery/_index.tsx index 06a07c78a..8e5f04f09 100644 --- a/packages/website/src/pages/[...locale]/tools/gallery/_index.tsx +++ b/packages/website/src/pages/[...locale]/tools/gallery/_index.tsx @@ -1,9 +1,7 @@ -import { literals } from '@blitzkit/i18n/src/literals'; -import { CalendarIcon } from '@radix-ui/react-icons'; -import { Callout, Flex } from '@radix-ui/themes'; -import { GalleryList, type Avatar } from '../../../../components/Gallery/List'; +import { GalleryList } from '../../../../components/Gallery/List'; import { GallerySearch } from '../../../../components/Gallery/Search'; import { PageWrapper } from '../../../../components/PageWrapper'; +import { awaitableGallery } from '../../../../core/awaitables/gallery'; import { LocaleProvider, useLocale, @@ -11,14 +9,8 @@ import { } from '../../../../hooks/useLocale'; import { GalleryEphemeral } from '../../../../stores/galleryEphemeral'; -interface ContentProps { - avatars: Avatar[]; - lastUpdated: string; -} -type PageProps = ContentProps & LocaleAcceptorProps; - -export function Page({ locale, ...props }: PageProps) { +export function Page({ locale, ...props }: LocaleAcceptorProps) { return ( @@ -26,27 +18,13 @@ export function Page({ locale, ...props }: PageProps) { ); } -function Content({ avatars, lastUpdated }: ContentProps) { - const { strings } = useLocale(); +function Content() { return ( - - - - - - - {literals(strings.website.tools.gallery.last_updated, [ - lastUpdated, - ])} - - - - - + ); diff --git a/packages/website/src/pages/[...locale]/tools/gallery/index.astro b/packages/website/src/pages/[...locale]/tools/gallery/index.astro index b32f542ef..2fb84640c 100644 --- a/packages/website/src/pages/[...locale]/tools/gallery/index.astro +++ b/packages/website/src/pages/[...locale]/tools/gallery/index.astro @@ -1,29 +1,15 @@ --- import type { Avatar } from '../../../../components/Gallery/List'; import { galleryTool } from '../../../../constants/tools'; -import { fetchGlossary } from '../../../../core/blitz/fetchGlossary'; -import ToolLayout from '../../../../layouts/ToolLayout.astro'; +import { awaitableGallery } from '../../../../core/awaitables/gallery'; + import ToolLayout from '../../../../layouts/ToolLayout.astro'; import { Page } from './_index'; export { getStaticPaths } from '../../index.astro'; const { locale } = Astro.params; -const glossary = await fetchGlossary(locale); -const avatars = Object.entries(glossary) - .filter( - ([key, value]) => - key.startsWith('avatar') && - !key.endsWith('_part') && - value.image_url !== null, - ) - .map( - ([key, value]) => - ({ id: key.replace('avatar_', ''), name: value.title }) satisfies Avatar, - ) - .sort((a, b) => a.name.localeCompare(b.name)); -const lastUpdated = new Date().toLocaleDateString(locale); --- - +