feat: new reviews

.github/workflows/reviews.yml

@@ -0,0 +1,24 @@
+name: Reviews
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: 0 0 * * 2
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v1
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Find reviews
+        run: DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} GH_TOKEN=${{ secrets.GH_TOKEN }} DATABASE_URL=${{ secrets.DATABASE_URL }} bun scripts/reviews.ts

public/protos/reviews.proto

@@ -7,7 +7,8 @@ message Reviews {
 }
 
 message Review {
-  repeated Video video = 1;
+  float last_updated = 1;
+  repeated Video videos = 2;
 }
 
 message Video {

scripts/buildAssets/index.ts

@@ -24,7 +24,6 @@ import { skillIcons } from './skillIcons';
 import { tankArmors } from './tankArmors';
 import { tankIcons } from './tankIcons';
 import { tankModels } from './tankModels';
-import { videos } from './videos';
 
 if (!targets && !allTargets) throw new Error('No target(s) specified');
 
@@ -32,7 +31,6 @@ const methods = [
   gameModeBanners,
   definitions,
   tankIcons,
-  videos,
   currencies,
   skillIcons,
   flags,

scripts/buildAssets/videos.ts

@@ -1,139 +0,0 @@
-import { readdir } from 'fs/promises';
-import { google } from 'googleapis';
-import { cloneDeep, uniqBy } from 'lodash';
-import { parse as parseYaml } from 'yaml';
-import { youtubers } from '../../src/constants/youtubers';
-import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
-import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
-import { toUniqueId } from '../../src/core/blitz/toUniqueId';
-import { commitAssets } from '../../src/core/blitzkit/commitAssets';
-import { VideoDefinitions } from '../../src/core/blitzkit/videos';
-import { DATA } from './constants';
-import { BlitzStrings, botPattern, VehicleDefinitionList } from './definitions';
-
-export async function videos(production: boolean) {
-  console.log('Building videos...');
-
-  const currentVideos = await fetch(
-    'https://raw.githubusercontent.com/tresabhi/blitzkit-assets/main/definitions/videos.csv',
-  )
-    .then((response) => response.text())
-    .then((lines) => {
-      return lines
-        .split('\n')
-        .map((line) => {
-          const [idString, lastUpdatedString, ...videos] = line.split(',');
-          const id = Number(idString);
-          const lastUpdated = Number(lastUpdatedString);
-          return { id, lastUpdated, videos };
-        })
-        .reduce<VideoDefinitions>(
-          (accumulator, { id, lastUpdated, videos }) => {
-            accumulator[id] = { lastUpdated, videos };
-            return accumulator;
-          },
-          {},
-        );
-    });
-  const auth = await google.auth.getClient({
-    scopes: ['https://www.googleapis.com/auth/youtube.readonly'],
-  });
-  const youtube = google.youtube({ version: 'v3', auth });
-
-  const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
-    (nations) => nations.filter((nation) => nation !== 'common'),
-  );
-  const stringsPreInstalled = await readYAMLDVPL<BlitzStrings>(
-    `${DATA}/Strings/en.yaml.dvpl`,
-  );
-  const stringsCache = await fetch(
-    'https://stufficons.wgcdn.co/localizations/en.yaml',
-  )
-    .then((response) => response.text())
-    .then((string) => parseYaml(string) as BlitzStrings);
-  const strings = {
-    ...stringsPreInstalled,
-    ...stringsCache,
-  };
-  const tanks: { id: number; name: string; tier: number }[] = [];
-  await Promise.all(
-    nations.map(async (nation) => {
-      const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>(
-        `${DATA}/XML/item_defs/vehicles/${nation}/list.xml.dvpl`,
-      );
-
-      await Promise.all(
-        Object.entries(tankList.root).map(async ([tankKey, tank]) => {
-          if (botPattern.test(tankKey)) return null;
-
-          const id = toUniqueId(nation, tank.id);
-          const name =
-            (tank.shortUserString
-              ? strings[tank.shortUserString]
-              : undefined) ?? strings[tank.userString];
-          const tier = tank.level;
-
-          tanks.push({ id, name, tier });
-        }),
-      );
-    }),
-  );
-
-  const tanksSanitized = tanks
-    .sort((a, b) => b.tier - a.tier)
-    .sort((a, b) => {
-      const lastUpdatedA = currentVideos[a.id]?.lastUpdated ?? 0;
-      const lastUpdatedB = currentVideos[b.id]?.lastUpdated ?? 0;
-      return lastUpdatedA - lastUpdatedB;
-    });
-  const videos: VideoDefinitions = cloneDeep(currentVideos);
-
-  let done = 0;
-  for (const tank of tanksSanitized) {
-    try {
-      const results = await youtube.search.list({
-        part: ['snippet'],
-        q: `World of Tanks Blitz ${tank.name}`,
-      });
-
-      if (!results.data.items || results.data.items.length === 0) continue;
-
-      const items = uniqBy(
-        results.data.items.filter(
-          (item) =>
-            item.snippet?.channelId &&
-            item.id?.videoId &&
-            youtubers.some(({ id }) => id === item.snippet?.channelId),
-        ),
-        (item) => item.snippet!.channelId,
-      );
-
-      videos[tank.id] = {
-        lastUpdated: Math.round(Date.now() / 1000),
-        videos: items.map((item) => item.id!.videoId!),
-      };
-
-      console.log(
-        `${++done} / ${tanks.length} (${Math.round(100 * (done / tanks.length))}%) done; found ${items.length} for ${tank.name}`,
-      );
-
-      await new Promise((resolve) => setTimeout(resolve, 1000 / 5));
-    } catch (error) {
-      console.warn('An error occurred, safely exiting prematurely...', error);
-      break;
-    }
-  }
-
-  const content = Object.entries(videos)
-    .map(
-      ([id, { videos, lastUpdated }]) =>
-        `${id},${lastUpdated},${videos.join(',')}`,
-    )
-    .join('\n');
-
-  await commitAssets(
-    'videos',
-    [{ content, encoding: 'utf-8', path: 'definitions/videos.csv' }],
-    production,
-  );
-}

scripts/reviews.ts

@@ -0,0 +1,135 @@
+import { readdir } from 'fs/promises';
+import { google } from 'googleapis';
+import { cloneDeep, uniqBy } from 'lodash';
+import { parse as parseYaml } from 'yaml';
+import { youtubers } from '../src/constants/youtubers';
+import { readXMLDVPL } from '../src/core/blitz/readXMLDVPL';
+import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL';
+import { toUniqueId } from '../src/core/blitz/toUniqueId';
+import { asset } from '../src/core/blitzkit/asset';
+import { commitAssets } from '../src/core/blitzkit/commitAssets';
+import { Reviews, Video } from '../src/core/blitzkit/reviews';
+import { decode } from '../src/core/protobuf/decode';
+import { encode } from '../src/core/protobuf/encode';
+import { DATA } from './buildAssets/constants';
+import {
+  BlitzStrings,
+  botPattern,
+  VehicleDefinitionList,
+} from './buildAssets/definitions';
+
+const MAX_QUERIES = 128;
+
+console.log('Finding reviews...');
+
+const currentReviews = await fetch(asset('definitions/reviews.pb', false))
+  .then((response) => response.arrayBuffer())
+  .then((buffer) =>
+    decode<Reviews>('reviews/blitzkit.Reviews', new Uint8Array(buffer)),
+  );
+const auth = await google.auth.getClient({
+  scopes: ['https://www.googleapis.com/auth/youtube.readonly'],
+});
+const youtube = google.youtube({ version: 'v3', auth });
+
+const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
+  (nations) => nations.filter((nation) => nation !== 'common'),
+);
+const stringsPreInstalled = await readYAMLDVPL<BlitzStrings>(
+  `${DATA}/Strings/en.yaml.dvpl`,
+);
+const stringsCache = await fetch(
+  'https://stufficons.wgcdn.co/localizations/en.yaml',
+)
+  .then((response) => response.text())
+  .then((string) => parseYaml(string) as BlitzStrings);
+const strings = {
+  ...stringsPreInstalled,
+  ...stringsCache,
+};
+const tanks: { id: number; name: string; tier: number }[] = [];
+await Promise.all(
+  nations.map(async (nation) => {
+    const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>(
+      `${DATA}/XML/item_defs/vehicles/${nation}/list.xml.dvpl`,
+    );
+
+    await Promise.all(
+      Object.entries(tankList.root).map(async ([tankKey, tank]) => {
+        if (botPattern.test(tankKey)) return null;
+
+        const id = toUniqueId(nation, tank.id);
+        const name =
+          (tank.shortUserString ? strings[tank.shortUserString] : undefined) ??
+          strings[tank.userString];
+        const tier = tank.level;
+
+        tanks.push({ id, name, tier });
+      }),
+    );
+  }),
+);
+
+const tanksSanitized = tanks
+  .sort((a, b) => b.tier - a.tier)
+  .sort((a, b) => {
+    const lastUpdatedA = currentReviews.reviews[a.id]?.last_updated ?? 0;
+    const lastUpdatedB = currentReviews.reviews[b.id]?.last_updated ?? 0;
+    return lastUpdatedA - lastUpdatedB;
+  });
+const reviews: Reviews = cloneDeep(currentReviews);
+
+let done = 0;
+for (const tank of tanksSanitized) {
+  if (done >= MAX_QUERIES) {
+    console.log(`Limit reached with ${done} queries; stopping...`);
+    break;
+  }
+
+  try {
+    const results = await youtube.search.list({
+      part: ['snippet'],
+      q: `World of Tanks Blitz ${tank.name}`,
+    });
+
+    if (!results.data.items || results.data.items.length === 0) continue;
+
+    const items = uniqBy(
+      results.data.items.filter(
+        (item) =>
+          item.snippet?.channelId &&
+          item.id?.videoId &&
+          youtubers.some(({ id }) => id === item.snippet?.channelId),
+      ),
+      (item) => item.snippet!.channelId,
+    );
+
+    reviews.reviews[tank.id] = {
+      last_updated: Date.now(),
+      videos: items.map(
+        (item) =>
+          ({
+            id: item.id!.videoId!,
+            author: item.snippet!.channelId!,
+          }) satisfies Video,
+      ),
+    };
+
+    console.log(`${++done} done; found ${items.length} for ${tank.name}`);
+
+    await new Promise((resolve) => setTimeout(resolve, 1000 / 5));
+  } catch (error) {
+    console.warn(`Limit reached with ${done} queries`);
+    break;
+  }
+}
+
+const content = await encode('reviews/blitzkit.Reviews', reviews).then(
+  (array) => Buffer.from(array).toString('base64'),
+);
+
+await commitAssets(
+  'reviews',
+  [{ content, encoding: 'base64', path: 'definitions/reviews.pb' }],
+  true,
+);

src/app/tools/tankopedia/[id]/components/VideoSection.tsx

@@ -2,13 +2,15 @@ import { InfoCircledIcon } from '@radix-ui/react-icons';
 import { Callout, Flex, Heading, Link } from '@radix-ui/themes';
 import { use } from 'react';
 import PageWrapper from '../../../../../components/PageWrapper';
-import { videoDefinitions } from '../../../../../core/blitzkit/videos';
+import { reviews } from '../../../../../core/blitzkit/reviews';
 import * as Duel from '../../../../../stores/duel';
 
 export function VideoSection() {
   const tank = Duel.use((state) => state.protagonist.tank);
-  const awaitedVideoDefinitions = use(videoDefinitions);
+  const awaitedReviews = use(reviews);
-  const videos = awaitedVideoDefinitions[tank.id]?.videos ?? [];
+  const videos = awaitedReviews.reviews[tank.id]?.videos;
+
+  if (!videos) return null;
 
   return (
     <PageWrapper noFlex1>
@@ -29,13 +31,13 @@ export function VideoSection() {
         <Flex gap="3" wrap="wrap">
           {videos.map((video) => (
             <Link
-              key={video}
+              key={video.id}
-              href={`https://www.youtube.com/watch?v=${video}`}
+              href={`https://www.youtube.com/watch?v=${video.id}`}
               target="_blank"
             >
               <img
                 alt="Video thumbnail"
-                src={`https://i.ytimg.com/vi/${video}/hqdefault.jpg`}
+                src={`https://i.ytimg.com/vi/${video.id}/hqdefault.jpg`}
                 style={{
                   aspectRatio: '16 / 9',
                   objectFit: 'cover',

src/core/blitzkit/reviews.ts

@@ -0,0 +1,22 @@
+import { decode } from '../protobuf/decode';
+import { asset } from './asset';
+
+export interface Reviews {
+  reviews: Record<number, Review>;
+}
+
+interface Review {
+  last_updated: number;
+  videos?: Video[];
+}
+
+export interface Video {
+  id: string;
+  author: string;
+}
+
+export const reviews = fetch(asset('definitions/reviews.pb'))
+  .then((response) => response.arrayBuffer())
+  .then((buffer) =>
+    decode<Reviews>('reviews/blitzkit.Reviews', new Uint8Array(buffer)),
+  );

src/core/blitzkit/videos.ts

@@ -1,32 +0,0 @@
-import { asset } from './asset';
-
-export interface VideoDefinitions {
-  [id: number]: {
-    videos: string[];
-    lastUpdated: number;
-  };
-}
-
-export const videoDefinitions = fetch(asset('definitions/videos.csv'))
-  .then((response) => response.text())
-  .then((text) =>
-    text
-      .split('\n')
-      .map((line) => {
-        const [idString, lastUpdatedString, ...videos] = line
-          .split(',')
-          .filter(Boolean);
-        const id = Number(idString);
-        const lastUpdated = Number(lastUpdatedString);
-        return { id, videos, lastUpdated };
-      })
-      .filter(({ videos }) => videos.length > 0)
-      .reduce<VideoDefinitions>((accumulator, { id, videos, lastUpdated }) => {
-        accumulator[id] = {
-          videos,
-          lastUpdated,
-        };
-
-        return accumulator;
-      }, {}),
-  );

src/core/protobuf/encode.ts

@@ -2,5 +2,9 @@ import { lookup } from './lookup';
 
 export async function encode(proto: string, data: any) {
   const Message = await lookup(proto);
+  const verification = Message.verify(data);
+
+  if (verification !== null) throw new Error(verification);
+
   return Message.encode(Message.create(data)).finish();
 }

src/core/protobuf/lookup.ts

@@ -2,7 +2,7 @@ import { Root } from 'protobufjs';
 import { context } from '../blitzkit/context';
 
 export async function lookup(proto: string) {
-  const [file, message] = proto.split('/')[0];
+  const [file, message] = proto.split('/');
   const root = await new Root().load(
     context === 'server'
       ? `public/protos/${file}.proto`

src/test.ts

@@ -1,7 +0,0 @@
-import { youtubers } from './constants/youtubers';
-import { decodeYouTubeId } from './core/youtube/decodeYouTubeId';
-import { encodeYouTubeId } from './core/youtube/encodeYouTubeId';
-
-youtubers.forEach((yt) => {
-  console.log(yt.id, decodeYouTubeId(encodeYouTubeId(yt.id)));
-});