From d9521ce1e5d0c4ab4d2bf0af232c6c2f57607194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Sun, 25 Aug 2024 18:10:56 -0500 Subject: [PATCH] feat: new reviews --- .github/workflows/reviews.yml | 24 +++ public/protos/reviews.proto | 3 +- scripts/buildAssets/index.ts | 2 - scripts/buildAssets/videos.ts | 139 ------------------ scripts/reviews.ts | 135 +++++++++++++++++ .../[id]/components/VideoSection.tsx | 14 +- src/core/blitzkit/reviews.ts | 22 +++ src/core/blitzkit/videos.ts | 32 ---- src/core/protobuf/encode.ts | 4 + src/core/protobuf/lookup.ts | 2 +- src/test.ts | 7 - 11 files changed, 196 insertions(+), 188 deletions(-) create mode 100644 .github/workflows/reviews.yml delete mode 100644 scripts/buildAssets/videos.ts create mode 100644 scripts/reviews.ts create mode 100644 src/core/blitzkit/reviews.ts delete mode 100644 src/core/blitzkit/videos.ts delete mode 100644 src/test.ts diff --git a/.github/workflows/reviews.yml b/.github/workflows/reviews.yml new file mode 100644 index 000000000..c9d0a1f6e --- /dev/null +++ b/.github/workflows/reviews.yml @@ -0,0 +1,24 @@ +name: Reviews + +on: + workflow_dispatch: + schedule: + - cron: 0 0 * * 2 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v1 + + - name: Install dependencies + run: bun install + + - name: Find reviews + run: DISCORD_TOKEN=${{ secrets.DISCORD_TOKEN }} GH_TOKEN=${{ secrets.GH_TOKEN }} DATABASE_URL=${{ secrets.DATABASE_URL }} bun scripts/reviews.ts diff --git a/public/protos/reviews.proto b/public/protos/reviews.proto index ba4f3391f..482daeb75 100644 --- a/public/protos/reviews.proto +++ b/public/protos/reviews.proto @@ -7,7 +7,8 @@ message Reviews { } message Review { - repeated Video video = 1; + float last_updated = 1; + repeated Video videos = 2; } message Video { diff --git a/scripts/buildAssets/index.ts b/scripts/buildAssets/index.ts index 4d6b5abae..32a886455 100644 --- a/scripts/buildAssets/index.ts +++ b/scripts/buildAssets/index.ts @@ -24,7 +24,6 @@ import { skillIcons } from './skillIcons'; import { tankArmors } from './tankArmors'; import { tankIcons } from './tankIcons'; import { tankModels } from './tankModels'; -import { videos } from './videos'; if (!targets && !allTargets) throw new Error('No target(s) specified'); @@ -32,7 +31,6 @@ const methods = [ gameModeBanners, definitions, tankIcons, - videos, currencies, skillIcons, flags, diff --git a/scripts/buildAssets/videos.ts b/scripts/buildAssets/videos.ts deleted file mode 100644 index 6f4ab1fd2..000000000 --- a/scripts/buildAssets/videos.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { readdir } from 'fs/promises'; -import { google } from 'googleapis'; -import { cloneDeep, uniqBy } from 'lodash'; -import { parse as parseYaml } from 'yaml'; -import { youtubers } from '../../src/constants/youtubers'; -import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL'; -import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL'; -import { toUniqueId } from '../../src/core/blitz/toUniqueId'; -import { commitAssets } from '../../src/core/blitzkit/commitAssets'; -import { VideoDefinitions } from '../../src/core/blitzkit/videos'; -import { DATA } from './constants'; -import { BlitzStrings, botPattern, VehicleDefinitionList } from './definitions'; - -export async function videos(production: boolean) { - console.log('Building videos...'); - - const currentVideos = await fetch( - 'https://raw.githubusercontent.com/tresabhi/blitzkit-assets/main/definitions/videos.csv', - ) - .then((response) => response.text()) - .then((lines) => { - return lines - .split('\n') - .map((line) => { - const [idString, lastUpdatedString, ...videos] = line.split(','); - const id = Number(idString); - const lastUpdated = Number(lastUpdatedString); - return { id, lastUpdated, videos }; - }) - .reduce( - (accumulator, { id, lastUpdated, videos }) => { - accumulator[id] = { lastUpdated, videos }; - return accumulator; - }, - {}, - ); - }); - const auth = await google.auth.getClient({ - scopes: ['https://www.googleapis.com/auth/youtube.readonly'], - }); - const youtube = google.youtube({ version: 'v3', auth }); - - const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then( - (nations) => nations.filter((nation) => nation !== 'common'), - ); - const stringsPreInstalled = await readYAMLDVPL( - `${DATA}/Strings/en.yaml.dvpl`, - ); - const stringsCache = await fetch( - 'https://stufficons.wgcdn.co/localizations/en.yaml', - ) - .then((response) => response.text()) - .then((string) => parseYaml(string) as BlitzStrings); - const strings = { - ...stringsPreInstalled, - ...stringsCache, - }; - const tanks: { id: number; name: string; tier: number }[] = []; - await Promise.all( - nations.map(async (nation) => { - const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>( - `${DATA}/XML/item_defs/vehicles/${nation}/list.xml.dvpl`, - ); - - await Promise.all( - Object.entries(tankList.root).map(async ([tankKey, tank]) => { - if (botPattern.test(tankKey)) return null; - - const id = toUniqueId(nation, tank.id); - const name = - (tank.shortUserString - ? strings[tank.shortUserString] - : undefined) ?? strings[tank.userString]; - const tier = tank.level; - - tanks.push({ id, name, tier }); - }), - ); - }), - ); - - const tanksSanitized = tanks - .sort((a, b) => b.tier - a.tier) - .sort((a, b) => { - const lastUpdatedA = currentVideos[a.id]?.lastUpdated ?? 0; - const lastUpdatedB = currentVideos[b.id]?.lastUpdated ?? 0; - return lastUpdatedA - lastUpdatedB; - }); - const videos: VideoDefinitions = cloneDeep(currentVideos); - - let done = 0; - for (const tank of tanksSanitized) { - try { - const results = await youtube.search.list({ - part: ['snippet'], - q: `World of Tanks Blitz ${tank.name}`, - }); - - if (!results.data.items || results.data.items.length === 0) continue; - - const items = uniqBy( - results.data.items.filter( - (item) => - item.snippet?.channelId && - item.id?.videoId && - youtubers.some(({ id }) => id === item.snippet?.channelId), - ), - (item) => item.snippet!.channelId, - ); - - videos[tank.id] = { - lastUpdated: Math.round(Date.now() / 1000), - videos: items.map((item) => item.id!.videoId!), - }; - - console.log( - `${++done} / ${tanks.length} (${Math.round(100 * (done / tanks.length))}%) done; found ${items.length} for ${tank.name}`, - ); - - await new Promise((resolve) => setTimeout(resolve, 1000 / 5)); - } catch (error) { - console.warn('An error occurred, safely exiting prematurely...', error); - break; - } - } - - const content = Object.entries(videos) - .map( - ([id, { videos, lastUpdated }]) => - `${id},${lastUpdated},${videos.join(',')}`, - ) - .join('\n'); - - await commitAssets( - 'videos', - [{ content, encoding: 'utf-8', path: 'definitions/videos.csv' }], - production, - ); -} diff --git a/scripts/reviews.ts b/scripts/reviews.ts new file mode 100644 index 000000000..d3a232644 --- /dev/null +++ b/scripts/reviews.ts @@ -0,0 +1,135 @@ +import { readdir } from 'fs/promises'; +import { google } from 'googleapis'; +import { cloneDeep, uniqBy } from 'lodash'; +import { parse as parseYaml } from 'yaml'; +import { youtubers } from '../src/constants/youtubers'; +import { readXMLDVPL } from '../src/core/blitz/readXMLDVPL'; +import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL'; +import { toUniqueId } from '../src/core/blitz/toUniqueId'; +import { asset } from '../src/core/blitzkit/asset'; +import { commitAssets } from '../src/core/blitzkit/commitAssets'; +import { Reviews, Video } from '../src/core/blitzkit/reviews'; +import { decode } from '../src/core/protobuf/decode'; +import { encode } from '../src/core/protobuf/encode'; +import { DATA } from './buildAssets/constants'; +import { + BlitzStrings, + botPattern, + VehicleDefinitionList, +} from './buildAssets/definitions'; + +const MAX_QUERIES = 128; + +console.log('Finding reviews...'); + +const currentReviews = await fetch(asset('definitions/reviews.pb', false)) + .then((response) => response.arrayBuffer()) + .then((buffer) => + decode('reviews/blitzkit.Reviews', new Uint8Array(buffer)), + ); +const auth = await google.auth.getClient({ + scopes: ['https://www.googleapis.com/auth/youtube.readonly'], +}); +const youtube = google.youtube({ version: 'v3', auth }); + +const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then( + (nations) => nations.filter((nation) => nation !== 'common'), +); +const stringsPreInstalled = await readYAMLDVPL( + `${DATA}/Strings/en.yaml.dvpl`, +); +const stringsCache = await fetch( + 'https://stufficons.wgcdn.co/localizations/en.yaml', +) + .then((response) => response.text()) + .then((string) => parseYaml(string) as BlitzStrings); +const strings = { + ...stringsPreInstalled, + ...stringsCache, +}; +const tanks: { id: number; name: string; tier: number }[] = []; +await Promise.all( + nations.map(async (nation) => { + const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>( + `${DATA}/XML/item_defs/vehicles/${nation}/list.xml.dvpl`, + ); + + await Promise.all( + Object.entries(tankList.root).map(async ([tankKey, tank]) => { + if (botPattern.test(tankKey)) return null; + + const id = toUniqueId(nation, tank.id); + const name = + (tank.shortUserString ? strings[tank.shortUserString] : undefined) ?? + strings[tank.userString]; + const tier = tank.level; + + tanks.push({ id, name, tier }); + }), + ); + }), +); + +const tanksSanitized = tanks + .sort((a, b) => b.tier - a.tier) + .sort((a, b) => { + const lastUpdatedA = currentReviews.reviews[a.id]?.last_updated ?? 0; + const lastUpdatedB = currentReviews.reviews[b.id]?.last_updated ?? 0; + return lastUpdatedA - lastUpdatedB; + }); +const reviews: Reviews = cloneDeep(currentReviews); + +let done = 0; +for (const tank of tanksSanitized) { + if (done >= MAX_QUERIES) { + console.log(`Limit reached with ${done} queries; stopping...`); + break; + } + + try { + const results = await youtube.search.list({ + part: ['snippet'], + q: `World of Tanks Blitz ${tank.name}`, + }); + + if (!results.data.items || results.data.items.length === 0) continue; + + const items = uniqBy( + results.data.items.filter( + (item) => + item.snippet?.channelId && + item.id?.videoId && + youtubers.some(({ id }) => id === item.snippet?.channelId), + ), + (item) => item.snippet!.channelId, + ); + + reviews.reviews[tank.id] = { + last_updated: Date.now(), + videos: items.map( + (item) => + ({ + id: item.id!.videoId!, + author: item.snippet!.channelId!, + }) satisfies Video, + ), + }; + + console.log(`${++done} done; found ${items.length} for ${tank.name}`); + + await new Promise((resolve) => setTimeout(resolve, 1000 / 5)); + } catch (error) { + console.warn(`Limit reached with ${done} queries`); + break; + } +} + +const content = await encode('reviews/blitzkit.Reviews', reviews).then( + (array) => Buffer.from(array).toString('base64'), +); + +await commitAssets( + 'reviews', + [{ content, encoding: 'base64', path: 'definitions/reviews.pb' }], + true, +); diff --git a/src/app/tools/tankopedia/[id]/components/VideoSection.tsx b/src/app/tools/tankopedia/[id]/components/VideoSection.tsx index ac826ae28..19d3f0527 100644 --- a/src/app/tools/tankopedia/[id]/components/VideoSection.tsx +++ b/src/app/tools/tankopedia/[id]/components/VideoSection.tsx @@ -2,13 +2,15 @@ import { InfoCircledIcon } from '@radix-ui/react-icons'; import { Callout, Flex, Heading, Link } from '@radix-ui/themes'; import { use } from 'react'; import PageWrapper from '../../../../../components/PageWrapper'; -import { videoDefinitions } from '../../../../../core/blitzkit/videos'; +import { reviews } from '../../../../../core/blitzkit/reviews'; import * as Duel from '../../../../../stores/duel'; export function VideoSection() { const tank = Duel.use((state) => state.protagonist.tank); - const awaitedVideoDefinitions = use(videoDefinitions); - const videos = awaitedVideoDefinitions[tank.id]?.videos ?? []; + const awaitedReviews = use(reviews); + const videos = awaitedReviews.reviews[tank.id]?.videos; + + if (!videos) return null; return ( @@ -29,13 +31,13 @@ export function VideoSection() { {videos.map((video) => ( Video thumbnail; +} + +interface Review { + last_updated: number; + videos?: Video[]; +} + +export interface Video { + id: string; + author: string; +} + +export const reviews = fetch(asset('definitions/reviews.pb')) + .then((response) => response.arrayBuffer()) + .then((buffer) => + decode('reviews/blitzkit.Reviews', new Uint8Array(buffer)), + ); diff --git a/src/core/blitzkit/videos.ts b/src/core/blitzkit/videos.ts deleted file mode 100644 index 69c3e2508..000000000 --- a/src/core/blitzkit/videos.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { asset } from './asset'; - -export interface VideoDefinitions { - [id: number]: { - videos: string[]; - lastUpdated: number; - }; -} - -export const videoDefinitions = fetch(asset('definitions/videos.csv')) - .then((response) => response.text()) - .then((text) => - text - .split('\n') - .map((line) => { - const [idString, lastUpdatedString, ...videos] = line - .split(',') - .filter(Boolean); - const id = Number(idString); - const lastUpdated = Number(lastUpdatedString); - return { id, videos, lastUpdated }; - }) - .filter(({ videos }) => videos.length > 0) - .reduce((accumulator, { id, videos, lastUpdated }) => { - accumulator[id] = { - videos, - lastUpdated, - }; - - return accumulator; - }, {}), - ); diff --git a/src/core/protobuf/encode.ts b/src/core/protobuf/encode.ts index a7c3034c6..7ff1b18d1 100644 --- a/src/core/protobuf/encode.ts +++ b/src/core/protobuf/encode.ts @@ -2,5 +2,9 @@ import { lookup } from './lookup'; export async function encode(proto: string, data: any) { const Message = await lookup(proto); + const verification = Message.verify(data); + + if (verification !== null) throw new Error(verification); + return Message.encode(Message.create(data)).finish(); } diff --git a/src/core/protobuf/lookup.ts b/src/core/protobuf/lookup.ts index fd0ce752a..388ed29da 100644 --- a/src/core/protobuf/lookup.ts +++ b/src/core/protobuf/lookup.ts @@ -2,7 +2,7 @@ import { Root } from 'protobufjs'; import { context } from '../blitzkit/context'; export async function lookup(proto: string) { - const [file, message] = proto.split('/')[0]; + const [file, message] = proto.split('/'); const root = await new Root().load( context === 'server' ? `public/protos/${file}.proto` diff --git a/src/test.ts b/src/test.ts deleted file mode 100644 index 266ce6616..000000000 --- a/src/test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { youtubers } from './constants/youtubers'; -import { decodeYouTubeId } from './core/youtube/decodeYouTubeId'; -import { encodeYouTubeId } from './core/youtube/encodeYouTubeId'; - -youtubers.forEach((yt) => { - console.log(yt.id, decodeYouTubeId(encodeYouTubeId(yt.id))); -});