From c281e9b66fa4d6107b294b49349f4fd07edbd817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:38:00 -0500 Subject: [PATCH] feat: cumulative video indexing --- scripts/buildAssets/videos.ts | 80 +++++++++++++------ .../tankopedia/[id]/components/Videos.tsx | 2 +- src/core/blitzkrieg/videos.ts | 20 ++++- 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/scripts/buildAssets/videos.ts b/scripts/buildAssets/videos.ts index 5fdcd6fd7..a96b9b69b 100644 --- a/scripts/buildAssets/videos.ts +++ b/scripts/buildAssets/videos.ts @@ -1,10 +1,15 @@ import { readdir } from 'fs/promises'; import { google } from 'googleapis'; +import { cloneDeep, uniqBy } from 'lodash'; import { parse as parseYaml } from 'yaml'; import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL'; import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL'; import { toUniqueId } from '../../src/core/blitz/toUniqueId'; import { commitAssets } from '../../src/core/blitzkrieg/commitAssets'; +import { + VideoDefinitions, + videoDefinitions, +} from '../../src/core/blitzkrieg/videos'; import { DATA, POI } from './constants'; import { BlitzStrings, botPattern, VehicleDefinitionList } from './definitions'; @@ -16,22 +21,12 @@ const youtubers = [ export async function videos(production: boolean) { console.log('Building videos...'); + const currentVideos = await videoDefinitions; const auth = await google.auth.getClient({ scopes: ['https://www.googleapis.com/auth/youtube.readonly'], }); const youtube = google.youtube({ version: 'v3', auth }); - async function search(channel: string, search: string) { - ( - await youtube.search.list({ - channelId: channel, - maxResults: 1, - part: ['snippet'], - q: search, - }) - ).data.items?.at(0)?.id ?? null; - } - const nations = await readdir(`${DATA}/${POI.vehicleDefinitions}`).then( (nations) => nations.filter((nation) => nation !== 'common'), ); @@ -45,7 +40,7 @@ export async function videos(production: boolean) { ...stringsPreInstalled, ...stringsCache, }; - const tanks: { id: number; name: string }[] = []; + const tanks: { id: number; name: string; tier: number }[] = []; await Promise.all( nations.map(async (nation) => { const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>( @@ -61,32 +56,65 @@ export async function videos(production: boolean) { (tank.shortUserString ? strings[tank.shortUserString] : undefined) ?? strings[tank.userString]; + const tier = tank.level; - tanks.push({ id, name }); + tanks.push({ id, name, tier }); }), ); }), ); - let content = ''; + const tanksSanitized = tanks + .sort((a, b) => b.tier - a.tier) + .sort((a, b) => { + const lastUpdatedA = currentVideos[a.id]?.lastUpdated ?? 0; + const lastUpdatedB = currentVideos[b.id]?.lastUpdated ?? 0; + return lastUpdatedA - lastUpdatedB; + }); + const videos: VideoDefinitions = cloneDeep(currentVideos); + let done = 0; - for (const tank of tanks) { - const results = await Promise.all( - youtubers.map((channel) => search(channel, tank.name)), - ).then((results) => results.filter((result) => result !== null)); + for (const tank of tanksSanitized) { + try { + const results = await youtube.search.list({ + part: ['snippet'], + q: `World of Tanks Blitz ${tank.name}`, + }); - if (results.length === 0) continue; + if (!results.data.items || results.data.items.length === 0) continue; - const line = `${tank.id},${results.join(',')}`; - content += `${line}\n`; - console.log(`${++done} / ${tanks.length} done`, tank.name, line); + const items = uniqBy( + results.data.items.filter( + (item) => + item.snippet?.channelId && + item.id?.videoId && + youtubers.includes(item.snippet.channelId), + ), + (item) => item.snippet!.channelId, + ); - await new Promise((resolve) => - setTimeout(resolve, (1000 / 5) * youtubers.length), - ); + videos[tank.id] = { + lastUpdated: Math.round(Date.now() / 1000), + videos: items.map((item) => item.id!.videoId!), + }; + + console.log( + `${++done} / ${tanks.length} (${Math.round(100 * (done / tanks.length))}%) done; found ${items.length} for ${tank.name}`, + ); + + await new Promise((resolve) => setTimeout(resolve, 1000 / 5)); + } catch (error) { + console.warn('An error occurred, safely exiting prematurely...', error); + break; + } } - console.log(content); + const content = Object.entries(videos) + .map( + ([id, { videos, lastUpdated }]) => + `${id},${lastUpdated},${videos.join(',')}`, + ) + .join('\n'); await commitAssets( 'videos', diff --git a/src/app/tools/tankopedia/[id]/components/Videos.tsx b/src/app/tools/tankopedia/[id]/components/Videos.tsx index ded8fae2f..c7e5475ea 100644 --- a/src/app/tools/tankopedia/[id]/components/Videos.tsx +++ b/src/app/tools/tankopedia/[id]/components/Videos.tsx @@ -9,7 +9,7 @@ export function Videos() { const isFullScreen = useFullScreen(); const tank = useDuel((state) => state.protagonist!.tank); const awaitedVideoDefinitions = use(videoDefinitions); - const videos = awaitedVideoDefinitions[tank.id] ?? []; + const videos = awaitedVideoDefinitions[tank.id].videos ?? []; if (isFullScreen) return null; diff --git a/src/core/blitzkrieg/videos.ts b/src/core/blitzkrieg/videos.ts index dca8788b8..def7bbe35 100644 --- a/src/core/blitzkrieg/videos.ts +++ b/src/core/blitzkrieg/videos.ts @@ -1,18 +1,30 @@ import { asset } from './asset'; +export interface VideoDefinitions { + [id: number]: { + videos: string[]; + lastUpdated: number; + }; +} + export const videoDefinitions = fetch(asset('definitions/videos.csv')) .then((response) => response.text()) .then((text) => text .split('\n') .map((line) => { - const [idString, ...videos] = line.split(','); + const [idString, lastUpdatedString, ...videos] = line.split(','); const id = Number(idString); - return { id, videos }; + const lastUpdated = Number(lastUpdatedString); + return { id, videos, lastUpdated }; }) .filter(({ videos }) => videos.length > 0) - .reduce>((accumulator, { id, videos }) => { - accumulator[id] = videos; + .reduce((accumulator, { id, videos, lastUpdated }) => { + accumulator[id] = { + videos, + lastUpdated, + }; + return accumulator; }, {}), );