From 365c68aa4d4d6ded2dde0ac77e7d656870c57b30 Mon Sep 17 00:00:00 2001 From: tresabhi <43380238+tresabhi@users.noreply.github.com> Date: Fri, 18 Apr 2025 23:57:04 -0500 Subject: [PATCH] feat: move popular tanks to cron job --- package.json | 1 + packages/core/src/blitzkit/index.ts | 1 + packages/core/src/blitzkit/popularTanks.ts | 7 + packages/core/src/protobuf/fetchPB.ts | 1 + packages/core/src/protos/index.ts | 1 + packages/core/src/protos/popular_tanks.proto | 12 ++ packages/core/src/protos/popular_tanks.ts | 190 ++++++++++++++++++ packages/scripts/src/buildAssets/index.ts | 2 + .../scripts/src/buildAssets/popularTanks.ts | 76 +++++++ packages/website/package.json | 1 - .../src/components/HomePageHotTanks.tsx | 68 +++---- .../src/core/awaitables/popularTanks.ts | 3 + 12 files changed, 318 insertions(+), 45 deletions(-) create mode 100644 packages/core/src/blitzkit/popularTanks.ts create mode 100644 packages/core/src/protos/popular_tanks.proto create mode 100644 packages/core/src/protos/popular_tanks.ts create mode 100644 packages/scripts/src/buildAssets/popularTanks.ts create mode 100644 packages/website/src/core/awaitables/popularTanks.ts diff --git a/package.json b/package.json index 8206b459f..e7ba97d32 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "license": "MIT", "dependencies": { "sharp": "^0.33.5", + "googleapis": "^144.0.0", "lodash-es": "^4.17.21" }, "devDependencies": { diff --git a/packages/core/src/blitzkit/index.ts b/packages/core/src/blitzkit/index.ts index 983def4d7..4438882d3 100644 --- a/packages/core/src/blitzkit/index.ts +++ b/packages/core/src/blitzkit/index.ts @@ -45,5 +45,6 @@ export * from './votes'; // export * from './withCORSProxy'; export * from './createDefaultProvisions'; export * from './metaSortTank'; +export * from './popularTanks'; export * from './romanize'; export * from './youtubers'; diff --git a/packages/core/src/blitzkit/popularTanks.ts b/packages/core/src/blitzkit/popularTanks.ts new file mode 100644 index 000000000..92d6e372b --- /dev/null +++ b/packages/core/src/blitzkit/popularTanks.ts @@ -0,0 +1,7 @@ +import { fetchPB } from '../protobuf'; +import { PopularTanks } from '../protos'; +import { asset } from './asset'; + +export function fetchPopularTanks() { + return fetchPB(asset('definitions/popular-tanks.pb'), PopularTanks); +} diff --git a/packages/core/src/protobuf/fetchPB.ts b/packages/core/src/protobuf/fetchPB.ts index 4d725ee00..330511bf0 100644 --- a/packages/core/src/protobuf/fetchPB.ts +++ b/packages/core/src/protobuf/fetchPB.ts @@ -2,6 +2,7 @@ import { MessageFns } from '../protos'; export async function fetchPB(url: string, message: MessageFns) { const response = await fetch(url); + const buffer = await response.arrayBuffer(); const array = new Uint8Array(buffer); return message.decode(array); diff --git a/packages/core/src/protos/index.ts b/packages/core/src/protos/index.ts index a0b2eb4fc..1f2a0a70c 100644 --- a/packages/core/src/protos/index.ts +++ b/packages/core/src/protos/index.ts @@ -7,6 +7,7 @@ export * from './game_definitions.ts'; export * from './i18n.ts'; export * from './map_definitions.ts'; export * from './model_definitions.ts'; +export * from './popular_tanks.ts'; export * from './provision_definitions.ts'; export * from './rating_leaderboard.ts'; export * from './reviews.ts'; diff --git a/packages/core/src/protos/popular_tanks.proto b/packages/core/src/protos/popular_tanks.proto new file mode 100644 index 000000000..d4e127a67 --- /dev/null +++ b/packages/core/src/protos/popular_tanks.proto @@ -0,0 +1,12 @@ +syntax = "proto2"; + +package blitzkit; + +message PopularTanks { + repeated PopularTank tanks = 1; +} + +message PopularTank { + required uint32 id = 1; + required uint32 views = 2; // views in the last 7 days +} \ No newline at end of file diff --git a/packages/core/src/protos/popular_tanks.ts b/packages/core/src/protos/popular_tanks.ts new file mode 100644 index 000000000..1112efb65 --- /dev/null +++ b/packages/core/src/protos/popular_tanks.ts @@ -0,0 +1,190 @@ +// Code generated by protoc-gen-ts_proto. DO NOT EDIT. +// versions: +// protoc-gen-ts_proto v2.2.2 +// protoc v5.28.2 +// source: packages/core/src/protos/popular_tanks.proto + +/* eslint-disable */ +import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire"; + +export const protobufPackage = "blitzkit"; + +export interface PopularTanks { + tanks: PopularTank[]; +} + +export interface PopularTank { + id: number; + /** views in the last 7 days */ + views: number; +} + +function createBasePopularTanks(): PopularTanks { + return { tanks: [] }; +} + +export const PopularTanks: MessageFns = { + encode(message: PopularTanks, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + for (const v of message.tanks) { + PopularTank.encode(v!, writer.uint32(10).fork()).join(); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): PopularTanks { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBasePopularTanks(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.tanks.push(PopularTank.decode(reader, reader.uint32())); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): PopularTanks { + return { + tanks: globalThis.Array.isArray(object?.tanks) ? object.tanks.map((e: any) => PopularTank.fromJSON(e)) : [], + }; + }, + + toJSON(message: PopularTanks): unknown { + const obj: any = {}; + if (message.tanks?.length) { + obj.tanks = message.tanks.map((e) => PopularTank.toJSON(e)); + } + return obj; + }, + + create, I>>(base?: I): PopularTanks { + return PopularTanks.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): PopularTanks { + const message = createBasePopularTanks(); + message.tanks = object.tanks?.map((e) => PopularTank.fromPartial(e)) || []; + return message; + }, +}; + +function createBasePopularTank(): PopularTank { + return { id: 0, views: 0 }; +} + +export const PopularTank: MessageFns = { + encode(message: PopularTank, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.id !== 0) { + writer.uint32(8).uint32(message.id); + } + if (message.views !== 0) { + writer.uint32(16).uint32(message.views); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): PopularTank { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBasePopularTank(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 8) { + break; + } + + message.id = reader.uint32(); + continue; + } + case 2: { + if (tag !== 16) { + break; + } + + message.views = reader.uint32(); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): PopularTank { + return { + id: globalThis.Number(assertSet("PopularTank.id", object.id)), + views: globalThis.Number(assertSet("PopularTank.views", object.views)), + }; + }, + + toJSON(message: PopularTank): unknown { + const obj: any = {}; + if (message.id !== 0) { + obj.id = Math.round(message.id); + } + if (message.views !== 0) { + obj.views = Math.round(message.views); + } + return obj; + }, + + create, I>>(base?: I): PopularTank { + return PopularTank.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>(object: I): PopularTank { + const message = createBasePopularTank(); + message.id = object.id ?? 0; + message.views = object.views ?? 0; + return message; + }, +}; + +type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; + +export type DeepPartial = T extends Builtin ? T + : T extends globalThis.Array ? globalThis.Array> + : T extends ReadonlyArray ? ReadonlyArray> + : T extends { $case: string; value: unknown } ? { $case: T["$case"]; value?: DeepPartial } + : T extends {} ? { [K in keyof T]?: DeepPartial } + : Partial; + +type KeysOfUnion = T extends T ? keyof T : never; +export type Exact = P extends Builtin ? P + : P & { [K in keyof P]: Exact } & { [K in Exclude>]: never }; + +function isSet(value: any): boolean { + return value !== null && value !== undefined; +} + +function assertSet(field: string, value: T | undefined): T { + if (!isSet(value)) { + throw new TypeError(`Required field ${field} is not set`); + } + + return value as T; +} + +export interface MessageFns { + encode(message: T, writer?: BinaryWriter): BinaryWriter; + decode(input: BinaryReader | Uint8Array, length?: number): T; + fromJSON(object: any): T; + toJSON(message: T): unknown; + create, I>>(base?: I): T; + fromPartial, I>>(object: I): T; +} diff --git a/packages/scripts/src/buildAssets/index.ts b/packages/scripts/src/buildAssets/index.ts index 1c9c76a97..1fda9a06f 100644 --- a/packages/scripts/src/buildAssets/index.ts +++ b/packages/scripts/src/buildAssets/index.ts @@ -20,6 +20,7 @@ import { flags } from './flags'; import { gameModeBanners } from './gameModeBanners'; import { migration } from './migration'; import { moduleIcons } from './moduleIcons'; +import { popularTanks } from './popularTanks'; import { shellIcons } from './shellIcons'; import { skillIcons } from './skillIcons'; import { tankArmors } from './tankArmors'; @@ -45,6 +46,7 @@ const methods = [ boosterIcons, tankIcons, blitzkitTankIcons, + popularTanks, ]; for (const method of methods) { diff --git a/packages/scripts/src/buildAssets/popularTanks.ts b/packages/scripts/src/buildAssets/popularTanks.ts new file mode 100644 index 000000000..f434022d2 --- /dev/null +++ b/packages/scripts/src/buildAssets/popularTanks.ts @@ -0,0 +1,76 @@ +import { + assertSecret, + encodePBBuffer, + fetchTankDefinitions, + PopularTanks, +} from '@blitzkit/core'; +import { google } from 'googleapis'; +import { commitAssets } from '../core/github/commitAssets'; + +export async function popularTanks() { + const tankDefinitions = await fetchTankDefinitions(); + + const auth = await google.auth.getClient({ + keyFile: import.meta.env.GOOGLE_APPLICATION_CREDENTIALS, + scopes: ['https://www.googleapis.com/auth/analytics.readonly'], + }); + const analytics = google.analyticsdata({ version: 'v1beta', auth }); + const report = await analytics.properties.runReport({ + property: `properties/${assertSecret(import.meta.env.PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID)}`, + requestBody: { + dateRanges: [{ startDate: '7daysAgo', endDate: 'today' }], + dimensions: [{ name: 'pagePath' }], + metrics: [{ name: 'screenPageViews' }], + orderBys: [{ metric: { metricName: 'screenPageViews' }, desc: true }], + dimensionFilter: { + filter: { + fieldName: 'pagePath', + stringFilter: { + matchType: 'BEGINS_WITH', + value: '/tools/tankopedia/', + }, + }, + }, + }, + }); + + if (!report.data.rows) { + throw new Error('No rows in report'); + } + + const tanks = report.data.rows + .filter( + (row) => + row.dimensionValues && + row.dimensionValues[0].value && + row.dimensionValues[0].value !== '/tools/tankopedia/' && + row.metricValues && + row.metricValues[0].value, + ) + .map((row) => ({ + id: Number( + row.dimensionValues![0].value!.match( + /\/tools\/tankopedia\/(\d+)\/?/, + )?.[1], + ), + views: Number(row.metricValues![0].value!), + })) + .filter((row) => { + if (!(row.id in tankDefinitions.tanks)) { + console.warn('Skipping unknown popular tank', row.id); + return false; + } + + return row.id in tankDefinitions.tanks; + }) + .slice(0, 8) + .map(({ id, views }) => ({ id, views })); + const popularTanks: PopularTanks = { tanks }; + + await commitAssets('popular tanks', [ + { + path: 'definitions/popular-tanks.pb', + content: encodePBBuffer(PopularTanks, popularTanks), + }, + ]); +} diff --git a/packages/website/package.json b/packages/website/package.json index d57c08095..d3602e71d 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -25,7 +25,6 @@ "@react-three/fiber": "^8.17.10", "compare-versions": "^6.1.1", "echarts": "^5.5.1", - "googleapis": "^144.0.0", "immer": "^10.1.1", "js-cookie": "^3.0.5", "markdown-to-jsx": "^7.5.0", diff --git a/packages/website/src/components/HomePageHotTanks.tsx b/packages/website/src/components/HomePageHotTanks.tsx index 9cac91ea5..246d5eb4d 100644 --- a/packages/website/src/components/HomePageHotTanks.tsx +++ b/packages/website/src/components/HomePageHotTanks.tsx @@ -3,6 +3,7 @@ import { literals } from '@blitzkit/i18n/src/literals'; import { EyeOpenIcon } from '@radix-ui/react-icons'; import { Box, Flex, Heading, Text } from '@radix-ui/themes'; import { google } from 'googleapis'; +import { awaitablePopularTanks } from '../core/awaitables/popularTanks'; import { awaitableTankDefinitions } from '../core/awaitables/tankDefinitions'; import { LocaleProvider, @@ -12,7 +13,10 @@ import { import { TankopediaPersistent } from '../stores/tankopediaPersistent'; import { TankCard } from './TankCard'; -const [tankDefinitions] = await Promise.all([awaitableTankDefinitions, ,]); +const [tankDefinitions, popularTanks] = await Promise.all([ + awaitableTankDefinitions, + awaitablePopularTanks, +]); const auth = await google.auth.getClient({ keyFile: import.meta.env.GOOGLE_APPLICATION_CREDENTIALS, @@ -39,36 +43,6 @@ if (!report.data.rows) { throw new Error('No rows in report'); } -const hotTanks = report.data.rows - .filter( - (row) => - row.dimensionValues && - row.dimensionValues[0].value && - row.dimensionValues[0].value !== '/tools/tankopedia/' && - row.metricValues && - row.metricValues[0].value, - ) - .map((row) => ({ - id: Number( - row.dimensionValues![0].value!.match( - /\/tools\/tankopedia\/(\d+)\/?/, - )?.[1], - ), - views: Number(row.metricValues![0].value!), - })) - .filter((row) => { - if (!(row.id in tankDefinitions.tanks)) { - // console.log('Unknown tank', row.id); - } - - return row.id in tankDefinitions.tanks; - }) - .slice(0, 8) - .map(({ id, views }) => ({ - tank: tankDefinitions.tanks[id], - views, - })); - export function HomePageHotTanks({ locale }: LocaleAcceptorProps) { return ( @@ -92,19 +66,25 @@ function Content() { - {hotTanks.map((row) => ( - - - - {Math.round(row.views * (30 / 7)).toLocaleString(locale)} - - } - /> - - ))} + {popularTanks.tanks.map(({ id, views }) => { + const tank = tankDefinitions.tanks[id]; + + if (tank === undefined) return null; + + return ( + + + + {Math.round(views * (30 / 7)).toLocaleString(locale)} + + } + /> + + ); + })} ); diff --git a/packages/website/src/core/awaitables/popularTanks.ts b/packages/website/src/core/awaitables/popularTanks.ts new file mode 100644 index 000000000..0c5d31348 --- /dev/null +++ b/packages/website/src/core/awaitables/popularTanks.ts @@ -0,0 +1,3 @@ +import { fetchPopularTanks } from '@blitzkit/core'; + +export const awaitablePopularTanks = fetchPopularTanks();