feat: glossary cron

packages/core/src/blitz/fetchGlossary.ts

@@ -1,4 +1,5 @@
-import { assertSecret, type BlitzGlossary } from '@blitzkit/core';
+import { assertSecret } from '../blitzkit';
+import { BlitzGlossary } from '../types';
 
 export async function fetchGlossary(locale: string) {
   const response = await fetch(

packages/core/src/blitz/index.ts

@@ -4,6 +4,7 @@ export * from './classes';
 export * from './constants';
 export * from './equipmentPrice';
 export * from './fetchBlitz';
+export * from './fetchGlossary';
 export * from './generateStats';
 export * from './getAccountAchievements';
 export * from './getAccountInfo';

packages/core/src/blitzkit/gallery.ts

@@ -0,0 +1,7 @@
+import { fetchPB } from '../protobuf';
+import { Gallery } from '../protos';
+import { asset } from './asset';
+
+export function fetchGallery() {
+  return fetchPB(asset('definitions/gallery.pb'), Gallery);
+}

packages/core/src/blitzkit/index.ts

@@ -44,6 +44,7 @@ export * from './unwrapBlitzkitResponse';
 export * from './votes';
 // export * from './withCORSProxy';
 export * from './createDefaultProvisions';
+export * from './gallery';
 export * from './metaSortTank';
 export * from './popularTanks';
 export * from './romanize';

packages/core/src/protobuf/fetchPB.ts

@@ -2,7 +2,6 @@ import { MessageFns } from '../protos';
 
 export async function fetchPB<Type>(url: string, message: MessageFns<Type>) {
   const response = await fetch(url);
-
   const buffer = await response.arrayBuffer();
   const array = new Uint8Array(buffer);
   return message.decode(array);

packages/core/src/protos/gallery.proto

@@ -0,0 +1,14 @@
+syntax = "proto2";
+
+package blitzkit;
+
+import "packages/core/src/protos/i18n.proto";
+
+message Gallery {
+  repeated Avatar avatars = 1;
+}
+
+message Avatar {
+  required string id = 1;
+  required I18nString name = 2;
+}

packages/core/src/protos/gallery.ts

@@ -0,0 +1,192 @@
+// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
+// versions:
+//   protoc-gen-ts_proto  v2.2.2
+//   protoc               v5.28.2
+// source: packages/core/src/protos/gallery.proto
+
+/* eslint-disable */
+import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire";
+import { I18nString } from "./i18n";
+
+export const protobufPackage = "blitzkit";
+
+export interface Gallery {
+  avatars: Avatar[];
+}
+
+export interface Avatar {
+  id: string;
+  name: I18nString;
+}
+
+function createBaseGallery(): Gallery {
+  return { avatars: [] };
+}
+
+export const Gallery: MessageFns<Gallery> = {
+  encode(message: Gallery, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
+    for (const v of message.avatars) {
+      Avatar.encode(v!, writer.uint32(10).fork()).join();
+    }
+    return writer;
+  },
+
+  decode(input: BinaryReader | Uint8Array, length?: number): Gallery {
+    const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseGallery();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1: {
+          if (tag !== 10) {
+            break;
+          }
+
+          message.avatars.push(Avatar.decode(reader, reader.uint32()));
+          continue;
+        }
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skip(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): Gallery {
+    return {
+      avatars: globalThis.Array.isArray(object?.avatars) ? object.avatars.map((e: any) => Avatar.fromJSON(e)) : [],
+    };
+  },
+
+  toJSON(message: Gallery): unknown {
+    const obj: any = {};
+    if (message.avatars?.length) {
+      obj.avatars = message.avatars.map((e) => Avatar.toJSON(e));
+    }
+    return obj;
+  },
+
+  create<I extends Exact<DeepPartial<Gallery>, I>>(base?: I): Gallery {
+    return Gallery.fromPartial(base ?? ({} as any));
+  },
+  fromPartial<I extends Exact<DeepPartial<Gallery>, I>>(object: I): Gallery {
+    const message = createBaseGallery();
+    message.avatars = object.avatars?.map((e) => Avatar.fromPartial(e)) || [];
+    return message;
+  },
+};
+
+function createBaseAvatar(): Avatar {
+  return { id: "", name: createBaseI18nString() };
+}
+
+export const Avatar: MessageFns<Avatar> = {
+  encode(message: Avatar, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
+    if (message.id !== "") {
+      writer.uint32(10).string(message.id);
+    }
+    if (message.name !== undefined) {
+      I18nString.encode(message.name, writer.uint32(18).fork()).join();
+    }
+    return writer;
+  },
+
+  decode(input: BinaryReader | Uint8Array, length?: number): Avatar {
+    const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseAvatar();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1: {
+          if (tag !== 10) {
+            break;
+          }
+
+          message.id = reader.string();
+          continue;
+        }
+        case 2: {
+          if (tag !== 18) {
+            break;
+          }
+
+          message.name = I18nString.decode(reader, reader.uint32());
+          continue;
+        }
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skip(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): Avatar {
+    return {
+      id: globalThis.String(assertSet("Avatar.id", object.id)),
+      name: I18nString.fromJSON(assertSet("Avatar.name", object.name)),
+    };
+  },
+
+  toJSON(message: Avatar): unknown {
+    const obj: any = {};
+    if (message.id !== "") {
+      obj.id = message.id;
+    }
+    if (message.name !== undefined) {
+      obj.name = I18nString.toJSON(message.name);
+    }
+    return obj;
+  },
+
+  create<I extends Exact<DeepPartial<Avatar>, I>>(base?: I): Avatar {
+    return Avatar.fromPartial(base ?? ({} as any));
+  },
+  fromPartial<I extends Exact<DeepPartial<Avatar>, I>>(object: I): Avatar {
+    const message = createBaseAvatar();
+    message.id = object.id ?? "";
+    message.name = (object.name !== undefined && object.name !== null)
+      ? I18nString.fromPartial(object.name)
+      : createBaseI18nString();
+    return message;
+  },
+};
+
+type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
+
+export type DeepPartial<T> = T extends Builtin ? T
+  : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>>
+  : T extends { $case: string; value: unknown } ? { $case: T["$case"]; value?: DeepPartial<T["value"]> }
+  : T extends {} ? { [K in keyof T]?: DeepPartial<T[K]> }
+  : Partial<T>;
+
+type KeysOfUnion<T> = T extends T ? keyof T : never;
+export type Exact<P, I extends P> = P extends Builtin ? P
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P>>]: never };
+
+function isSet(value: any): boolean {
+  return value !== null && value !== undefined;
+}
+
+function assertSet<T>(field: string, value: T | undefined): T {
+  if (!isSet(value)) {
+    throw new TypeError(`Required field ${field} is not set`);
+  }
+
+  return value as T;
+}
+
+export interface MessageFns<T> {
+  encode(message: T, writer?: BinaryWriter): BinaryWriter;
+  decode(input: BinaryReader | Uint8Array, length?: number): T;
+  fromJSON(object: any): T;
+  toJSON(message: T): unknown;
+  create<I extends Exact<DeepPartial<T>, I>>(base?: I): T;
+  fromPartial<I extends Exact<DeepPartial<T>, I>>(object: I): T;
+}

packages/core/src/protos/index.ts

@@ -3,6 +3,7 @@ export * from './average_definitions.ts';
 export * from './camouflage_definitions.ts';
 export * from './consumable_definitions.ts';
 export * from './equipment_definitions.ts';
+export * from './gallery.ts';
 export * from './game_definitions.ts';
 export * from './i18n.ts';
 export * from './map_definitions.ts';

packages/i18n/locales.json

@@ -0,0 +1,14 @@
+{
+  "default": "en",
+  "supported": [
+    { "locale": "en" },
+    { "locale": "es" },
+    { "locale": "fr" },
+    { "locale": "ja" },
+    { "locale": "pl" },
+    { "locale": "pt" },
+    { "locale": "ru" },
+    { "locale": "uk" },
+    { "locale": "zh", "blitz": "zh_Hans_SG" }
+  ]
+}

packages/scripts/src/buildAssets/glossary.ts

@@ -0,0 +1,86 @@
+import { Avatar, encodePBBuffer, fetchGlossary, Gallery } from '@blitzkit/core';
+import locales from '@blitzkit/i18n/locales.json';
+import { extname } from 'path';
+import ProgressBar from 'progress';
+import { commitAssets } from '../core/github/commitAssets';
+import { FileChange } from '../core/github/commitMultipleFiles';
+
+export async function glossary() {
+  console.log('Building glossary...');
+
+  const avatars: Record<string, { avatar: Avatar; url: string }> = {};
+
+  for (const supported of locales.supported) {
+    console.log(`Fetching glossary for ${supported.locale}`);
+
+    const glossary = await fetchGlossary(supported.blitz ?? supported.locale);
+
+    console.log(`Found ${Object.keys(glossary).length} things`);
+
+    for (const key in glossary) {
+      const glossaryEntry = glossary[key];
+
+      if (
+        !key.startsWith('avatar') ||
+        key.endsWith('_part') ||
+        glossaryEntry.image_url === null
+      ) {
+        continue;
+      }
+
+      if (key in avatars) {
+        avatars[key].avatar.name.locales[supported.locale] =
+          glossaryEntry.title;
+      } else {
+        avatars[key] = {
+          url: glossaryEntry.image_url,
+          avatar: {
+            id: key,
+            name: { locales: { [supported.locale]: glossaryEntry.title } },
+          },
+        };
+      }
+    }
+  }
+
+  const changes: FileChange[] = [];
+  const gallery: Gallery = { avatars: [] };
+  const totalCount = Object.keys(avatars).length;
+
+  const bar = new ProgressBar(
+    `Fetching avatars of ${totalCount} things :bar`,
+    totalCount,
+  );
+
+  await Promise.all(
+    Object.entries(avatars).map(async ([key, { url, avatar }]) => {
+      if (!(locales.default in avatars[key].avatar.name.locales)) {
+        console.warn(`Avatar ${key} has no ${locales.default} name`);
+        bar.tick();
+
+        return;
+      }
+
+      gallery.avatars.push(avatar);
+
+      const fileExtension = extname(url);
+      const content = await fetch(url).then((response) =>
+        response.arrayBuffer(),
+      );
+
+      changes.push({
+        path: `gallery/avatars/${key}${fileExtension}`,
+        content: Buffer.from(content),
+      });
+
+      bar.tick();
+    }),
+  );
+
+  changes.push({
+    path: 'definitions/gallery.pb',
+    content: encodePBBuffer(Gallery, gallery),
+  });
+
+  await commitAssets('glossary', changes);
+}

packages/scripts/src/buildAssets/index.ts

@@ -18,6 +18,7 @@ import { definitions } from './definitions';
 import { equipmentIcons } from './equipmentIcons';
 import { flags } from './flags';
 import { gameModeBanners } from './gameModeBanners';
+import { glossary } from './glossary';
 import { migration } from './migration';
 import { moduleIcons } from './moduleIcons';
 import { popularTanks } from './popularTanks';
@@ -47,6 +48,7 @@ const methods = [
   tankIcons,
   blitzkitTankIcons,
   popularTanks,
+  glossary,
 ];
 
 for (const method of methods) {

packages/scripts/src/core/github/commitAssets.ts

@@ -1,4 +1,5 @@
 import { assertSecret } from '@blitzkit/core';
+import { chunk } from 'lodash-es';
 import { commitMultipleFiles, FileChange } from './commitMultipleFiles';
 
 export async function commitAssets(message: string, changes: FileChange[]) {
@@ -6,10 +7,19 @@ export async function commitAssets(message: string, changes: FileChange[]) {
 
   if (changes.length === 0) return;
 
-  await commitMultipleFiles(
-    assertSecret(import.meta.env.PUBLIC_ASSET_REPO),
-    assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH),
-    `${message} - ${new Date().toDateString()}`,
-    changes,
-  );
+  const chunks = chunk(changes, 16);
+  let chunkIndex = 0;
+
+  for (const chunk of chunks) {
+    console.log(`Committing chunk ${chunkIndex + 1}/${chunks.length}`);
+
+    await commitMultipleFiles(
+      assertSecret(import.meta.env.PUBLIC_ASSET_REPO),
+      assertSecret(import.meta.env.PUBLIC_ASSET_BRANCH),
+      `${message} - ${new Date().toDateString()} (${chunkIndex + 1}/${chunks.length})`,
+      chunk,
+    );
+
+    chunkIndex++;
+  }
 }

packages/website/src/components/Gallery/List.tsx

@@ -1,8 +1,10 @@
+import locales from '@blitzkit/i18n/locales.json';
 import { literals } from '@blitzkit/i18n/src/literals';
 import { Flex, Text } from '@radix-ui/themes';
 import fuzzysort from 'fuzzysort';
 import { times } from 'lodash-es';
 import { useMemo, useState } from 'react';
+import { awaitableGallery } from '../../core/awaitables/gallery';
 import { useLocale } from '../../hooks/useLocale';
 import { GalleryEphemeral } from '../../stores/galleryEphemeral';
 import { GalleryCard } from './Card';
@@ -16,21 +18,30 @@ export interface Avatar {
   id: string;
 }
 
+const gallery = await awaitableGallery;
+
 const DEFAULT_LOADED = 42;
 const PREVIEW_COUNT = 28;
 
-export function GalleryList({ avatars }: GalleryListProps) {
+export function GalleryList() {
   const search = GalleryEphemeral.use((state) => state.search);
   const [loadedCards, setLoadedCards] = useState(DEFAULT_LOADED);
-  const { locale, strings } = useLocale();
+  const { locale, strings, unwrap } = useLocale();
   const filtered = useMemo(() => {
     setLoadedCards(DEFAULT_LOADED);
 
     if (search === undefined) {
-      return avatars;
+      return gallery.avatars;
     } else {
       return fuzzysort
-        .go(search, avatars, { keys: ['name', 'id'] })
+        .go(search, gallery.avatars, {
+          keys: [
+            ...locales.supported.map(
+              (supported) => `name.locales.${supported.locale}`,
+            ),
+            'id',
+          ],
+        })
         .map((result) => result.obj);
     }
   }, [search]);
@@ -45,7 +56,11 @@ export function GalleryList({ avatars }: GalleryListProps) {
 
       <Flex wrap="wrap" gap="4" justify="center">
         {filtered.slice(0, loadedCards).map((avatar) => (
-          <GalleryCard key={avatar.id} id={avatar.id} name={avatar.name} />
+          <GalleryCard
+            key={avatar.id}
+            id={avatar.id}
+            name={unwrap(avatar.name)}
+          />
         ))}
 
         {times(

packages/website/src/core/awaitables/gallery.ts

@@ -0,0 +1,3 @@
+import { fetchGallery } from '@blitzkit/core';
+
+export const awaitableGallery = fetchGallery();

packages/website/src/pages/[...locale]/tools/gallery/_index.tsx

@@ -1,9 +1,7 @@
-import { literals } from '@blitzkit/i18n/src/literals';
-import { CalendarIcon } from '@radix-ui/react-icons';
-import { Callout, Flex } from '@radix-ui/themes';
-import { GalleryList, type Avatar } from '../../../../components/Gallery/List';
+import { GalleryList } from '../../../../components/Gallery/List';
 import { GallerySearch } from '../../../../components/Gallery/Search';
 import { PageWrapper } from '../../../../components/PageWrapper';
+import { awaitableGallery } from '../../../../core/awaitables/gallery';
 import {
   LocaleProvider,
   useLocale,
@@ -11,14 +9,8 @@ import {
 } from '../../../../hooks/useLocale';
 import { GalleryEphemeral } from '../../../../stores/galleryEphemeral';
 
-interface ContentProps {
-  avatars: Avatar[];
-  lastUpdated: string;
-}
 
-type PageProps = ContentProps & LocaleAcceptorProps;
-
-export function Page({ locale, ...props }: PageProps) {
+export function Page({ locale, ...props }: LocaleAcceptorProps) {
   return (
     <LocaleProvider locale={locale}>
       <Content {...props} />
@@ -26,27 +18,13 @@ export function Page({ locale, ...props }: PageProps) {
   );
 }
 
-function Content({ avatars, lastUpdated }: ContentProps) {
-  const { strings } = useLocale();
+function Content() {
 
   return (
     <PageWrapper maxWidth="80rem" color="gold">
-      <Flex justify="center">
-        <Callout.Root size="1">
-          <Callout.Icon>
-            <CalendarIcon />
-          </Callout.Icon>
-          <Callout.Text>
-            {literals(strings.website.tools.gallery.last_updated, [
-              lastUpdated,
-            ])}
-          </Callout.Text>
-        </Callout.Root>
-      </Flex>
-
       <GalleryEphemeral.Provider>
         <GallerySearch />
-        <GalleryList avatars={avatars} />
+        <GalleryList   />
       </GalleryEphemeral.Provider>
     </PageWrapper>
   );

packages/website/src/pages/[...locale]/tools/gallery/index.astro

@@ -1,29 +1,15 @@
 ---
 import type { Avatar } from '../../../../components/Gallery/List';
 import { galleryTool } from '../../../../constants/tools';
-import { fetchGlossary } from '../../../../core/blitz/fetchGlossary';
-import ToolLayout from '../../../../layouts/ToolLayout.astro';
+import { awaitableGallery } from '../../../../core/awaitables/gallery';
+ import ToolLayout from '../../../../layouts/ToolLayout.astro';
 import { Page } from './_index';
 
 export { getStaticPaths } from '../../index.astro';
 
 const { locale } = Astro.params;
-const glossary = await fetchGlossary(locale);
-const avatars = Object.entries(glossary)
-  .filter(
-    ([key, value]) =>
-      key.startsWith('avatar') &&
-      !key.endsWith('_part') &&
-      value.image_url !== null,
-  )
-  .map(
-    ([key, value]) =>
-      ({ id: key.replace('avatar_', ''), name: value.title }) satisfies Avatar,
-  )
-  .sort((a, b) => a.name.localeCompare(b.name));
-const lastUpdated = new Date().toLocaleDateString(locale);
 ---
 
 <ToolLayout tool={galleryTool}>
-  <Page {locale} {avatars} {lastUpdated} client:load />
+  <Page {locale}  client:load />
 </ToolLayout>