mirror of
https://github.com/blitzkit/blitzkit.git
synced 2026-10-22 03:07:04 +02:00
feat: new reviews
This commit is contained in:
TrèsAbhi committed
1 parent
f91dd1e2c0
commit
d9521ce1e5
11 files changed
+196
-188
No files matched your search
@@ -24,7 +24,6 @@ import { skillIcons } from './skillIcons';
|
||||
import { tankArmors } from './tankArmors';
|
||||
import { tankIcons } from './tankIcons';
|
||||
import { tankModels } from './tankModels';
|
||||
import { videos } from './videos';
|
||||
|
||||
if (!targets && !allTargets) throw new Error('No target(s) specified');
|
||||
|
||||
@@ -32,7 +31,6 @@ const methods = [
|
||||
gameModeBanners,
|
||||
definitions,
|
||||
tankIcons,
|
||||
videos,
|
||||
currencies,
|
||||
skillIcons,
|
||||
flags,
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import { readdir } from 'fs/promises';
|
||||
import { google } from 'googleapis';
|
||||
import { cloneDeep, uniqBy } from 'lodash';
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import { youtubers } from '../../src/constants/youtubers';
|
||||
import { readXMLDVPL } from '../../src/core/blitz/readXMLDVPL';
|
||||
import { readYAMLDVPL } from '../../src/core/blitz/readYAMLDVPL';
|
||||
import { toUniqueId } from '../../src/core/blitz/toUniqueId';
|
||||
import { commitAssets } from '../../src/core/blitzkit/commitAssets';
|
||||
import { VideoDefinitions } from '../../src/core/blitzkit/videos';
|
||||
import { DATA } from './constants';
|
||||
import { BlitzStrings, botPattern, VehicleDefinitionList } from './definitions';
|
||||
|
||||
export async function videos(production: boolean) {
|
||||
console.log('Building videos...');
|
||||
|
||||
const currentVideos = await fetch(
|
||||
'https://raw.githubusercontent.com/tresabhi/blitzkit-assets/main/definitions/videos.csv',
|
||||
)
|
||||
.then((response) => response.text())
|
||||
.then((lines) => {
|
||||
return lines
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
const [idString, lastUpdatedString, ...videos] = line.split(',');
|
||||
const id = Number(idString);
|
||||
const lastUpdated = Number(lastUpdatedString);
|
||||
return { id, lastUpdated, videos };
|
||||
})
|
||||
.reduce<VideoDefinitions>(
|
||||
(accumulator, { id, lastUpdated, videos }) => {
|
||||
accumulator[id] = { lastUpdated, videos };
|
||||
return accumulator;
|
||||
},
|
||||
{},
|
||||
);
|
||||
});
|
||||
const auth = await google.auth.getClient({
|
||||
scopes: ['https://www.googleapis.com/auth/youtube.readonly'],
|
||||
});
|
||||
const youtube = google.youtube({ version: 'v3', auth });
|
||||
|
||||
const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
|
||||
(nations) => nations.filter((nation) => nation !== 'common'),
|
||||
);
|
||||
const stringsPreInstalled = await readYAMLDVPL<BlitzStrings>(
|
||||
`${DATA}/Strings/en.yaml.dvpl`,
|
||||
);
|
||||
const stringsCache = await fetch(
|
||||
'https://stufficons.wgcdn.co/localizations/en.yaml',
|
||||
)
|
||||
.then((response) => response.text())
|
||||
.then((string) => parseYaml(string) as BlitzStrings);
|
||||
const strings = {
|
||||
...stringsPreInstalled,
|
||||
...stringsCache,
|
||||
};
|
||||
const tanks: { id: number; name: string; tier: number }[] = [];
|
||||
await Promise.all(
|
||||
nations.map(async (nation) => {
|
||||
const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>(
|
||||
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml.dvpl`,
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
Object.entries(tankList.root).map(async ([tankKey, tank]) => {
|
||||
if (botPattern.test(tankKey)) return null;
|
||||
|
||||
const id = toUniqueId(nation, tank.id);
|
||||
const name =
|
||||
(tank.shortUserString
|
||||
? strings[tank.shortUserString]
|
||||
: undefined) ?? strings[tank.userString];
|
||||
const tier = tank.level;
|
||||
|
||||
tanks.push({ id, name, tier });
|
||||
}),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const tanksSanitized = tanks
|
||||
.sort((a, b) => b.tier - a.tier)
|
||||
.sort((a, b) => {
|
||||
const lastUpdatedA = currentVideos[a.id]?.lastUpdated ?? 0;
|
||||
const lastUpdatedB = currentVideos[b.id]?.lastUpdated ?? 0;
|
||||
return lastUpdatedA - lastUpdatedB;
|
||||
});
|
||||
const videos: VideoDefinitions = cloneDeep(currentVideos);
|
||||
|
||||
let done = 0;
|
||||
for (const tank of tanksSanitized) {
|
||||
try {
|
||||
const results = await youtube.search.list({
|
||||
part: ['snippet'],
|
||||
q: `World of Tanks Blitz ${tank.name}`,
|
||||
});
|
||||
|
||||
if (!results.data.items || results.data.items.length === 0) continue;
|
||||
|
||||
const items = uniqBy(
|
||||
results.data.items.filter(
|
||||
(item) =>
|
||||
item.snippet?.channelId &&
|
||||
item.id?.videoId &&
|
||||
youtubers.some(({ id }) => id === item.snippet?.channelId),
|
||||
),
|
||||
(item) => item.snippet!.channelId,
|
||||
);
|
||||
|
||||
videos[tank.id] = {
|
||||
lastUpdated: Math.round(Date.now() / 1000),
|
||||
videos: items.map((item) => item.id!.videoId!),
|
||||
};
|
||||
|
||||
console.log(
|
||||
`${++done} / ${tanks.length} (${Math.round(100 * (done / tanks.length))}%) done; found ${items.length} for ${tank.name}`,
|
||||
);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000 / 5));
|
||||
} catch (error) {
|
||||
console.warn('An error occurred, safely exiting prematurely...', error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const content = Object.entries(videos)
|
||||
.map(
|
||||
([id, { videos, lastUpdated }]) =>
|
||||
`${id},${lastUpdated},${videos.join(',')}`,
|
||||
)
|
||||
.join('\n');
|
||||
|
||||
await commitAssets(
|
||||
'videos',
|
||||
[{ content, encoding: 'utf-8', path: 'definitions/videos.csv' }],
|
||||
production,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import { readdir } from 'fs/promises';
|
||||
import { google } from 'googleapis';
|
||||
import { cloneDeep, uniqBy } from 'lodash';
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import { youtubers } from '../src/constants/youtubers';
|
||||
import { readXMLDVPL } from '../src/core/blitz/readXMLDVPL';
|
||||
import { readYAMLDVPL } from '../src/core/blitz/readYAMLDVPL';
|
||||
import { toUniqueId } from '../src/core/blitz/toUniqueId';
|
||||
import { asset } from '../src/core/blitzkit/asset';
|
||||
import { commitAssets } from '../src/core/blitzkit/commitAssets';
|
||||
import { Reviews, Video } from '../src/core/blitzkit/reviews';
|
||||
import { decode } from '../src/core/protobuf/decode';
|
||||
import { encode } from '../src/core/protobuf/encode';
|
||||
import { DATA } from './buildAssets/constants';
|
||||
import {
|
||||
BlitzStrings,
|
||||
botPattern,
|
||||
VehicleDefinitionList,
|
||||
} from './buildAssets/definitions';
|
||||
|
||||
const MAX_QUERIES = 128;
|
||||
|
||||
console.log('Finding reviews...');
|
||||
|
||||
const currentReviews = await fetch(asset('definitions/reviews.pb', false))
|
||||
.then((response) => response.arrayBuffer())
|
||||
.then((buffer) =>
|
||||
decode<Reviews>('reviews/blitzkit.Reviews', new Uint8Array(buffer)),
|
||||
);
|
||||
const auth = await google.auth.getClient({
|
||||
scopes: ['https://www.googleapis.com/auth/youtube.readonly'],
|
||||
});
|
||||
const youtube = google.youtube({ version: 'v3', auth });
|
||||
|
||||
const nations = await readdir(`${DATA}/XML/item_defs/vehicles`).then(
|
||||
(nations) => nations.filter((nation) => nation !== 'common'),
|
||||
);
|
||||
const stringsPreInstalled = await readYAMLDVPL<BlitzStrings>(
|
||||
`${DATA}/Strings/en.yaml.dvpl`,
|
||||
);
|
||||
const stringsCache = await fetch(
|
||||
'https://stufficons.wgcdn.co/localizations/en.yaml',
|
||||
)
|
||||
.then((response) => response.text())
|
||||
.then((string) => parseYaml(string) as BlitzStrings);
|
||||
const strings = {
|
||||
...stringsPreInstalled,
|
||||
...stringsCache,
|
||||
};
|
||||
const tanks: { id: number; name: string; tier: number }[] = [];
|
||||
await Promise.all(
|
||||
nations.map(async (nation) => {
|
||||
const tankList = await readXMLDVPL<{ root: VehicleDefinitionList }>(
|
||||
`${DATA}/XML/item_defs/vehicles/${nation}/list.xml.dvpl`,
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
Object.entries(tankList.root).map(async ([tankKey, tank]) => {
|
||||
if (botPattern.test(tankKey)) return null;
|
||||
|
||||
const id = toUniqueId(nation, tank.id);
|
||||
const name =
|
||||
(tank.shortUserString ? strings[tank.shortUserString] : undefined) ??
|
||||
strings[tank.userString];
|
||||
const tier = tank.level;
|
||||
|
||||
tanks.push({ id, name, tier });
|
||||
}),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const tanksSanitized = tanks
|
||||
.sort((a, b) => b.tier - a.tier)
|
||||
.sort((a, b) => {
|
||||
const lastUpdatedA = currentReviews.reviews[a.id]?.last_updated ?? 0;
|
||||
const lastUpdatedB = currentReviews.reviews[b.id]?.last_updated ?? 0;
|
||||
return lastUpdatedA - lastUpdatedB;
|
||||
});
|
||||
const reviews: Reviews = cloneDeep(currentReviews);
|
||||
|
||||
let done = 0;
|
||||
for (const tank of tanksSanitized) {
|
||||
if (done >= MAX_QUERIES) {
|
||||
console.log(`Limit reached with ${done} queries; stopping...`);
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
const results = await youtube.search.list({
|
||||
part: ['snippet'],
|
||||
q: `World of Tanks Blitz ${tank.name}`,
|
||||
});
|
||||
|
||||
if (!results.data.items || results.data.items.length === 0) continue;
|
||||
|
||||
const items = uniqBy(
|
||||
results.data.items.filter(
|
||||
(item) =>
|
||||
item.snippet?.channelId &&
|
||||
item.id?.videoId &&
|
||||
youtubers.some(({ id }) => id === item.snippet?.channelId),
|
||||
),
|
||||
(item) => item.snippet!.channelId,
|
||||
);
|
||||
|
||||
reviews.reviews[tank.id] = {
|
||||
last_updated: Date.now(),
|
||||
videos: items.map(
|
||||
(item) =>
|
||||
({
|
||||
id: item.id!.videoId!,
|
||||
author: item.snippet!.channelId!,
|
||||
}) satisfies Video,
|
||||
),
|
||||
};
|
||||
|
||||
console.log(`${++done} done; found ${items.length} for ${tank.name}`);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000 / 5));
|
||||
} catch (error) {
|
||||
console.warn(`Limit reached with ${done} queries`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const content = await encode('reviews/blitzkit.Reviews', reviews).then(
|
||||
(array) => Buffer.from(array).toString('base64'),
|
||||
);
|
||||
|
||||
await commitAssets(
|
||||
'reviews',
|
||||
[{ content, encoding: 'base64', path: 'definitions/reviews.pb' }],
|
||||
true,
|
||||
);
|
||||
Reference in new issue
Block a user