From c943baf6f312fc5736e0b990ff7ad9cead18540a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=A8sAbhi?= <43380238+tresabhi@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:37:50 -0500 Subject: [PATCH] feat: discover ids --- .github/workflows/cd-discover-ids.yml | 26 +++++++++++ package.json | 2 +- scripts/discoverIds.ts | 65 ++++++++++++++++----------- src/core/blitz/fetchBlitz.ts | 4 ++ 4 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/cd-discover-ids.yml diff --git a/.github/workflows/cd-discover-ids.yml b/.github/workflows/cd-discover-ids.yml new file mode 100644 index 000000000..2b9b711f3 --- /dev/null +++ b/.github/workflows/cd-discover-ids.yml @@ -0,0 +1,26 @@ +name: CD Discover IDs + +on: + push: + branches: main + workflow_dispatch: + schedule: + - cron: '0 8 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v1 + + - name: Install dependencies + run: bun install + + - name: Discover IDs + run: bun scripts/discoverIds.ts diff --git a/package.json b/package.json index 3f42628a4..1e6c503f5 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "update": "yarn upgrade --latest", "setup:depotdownloader": "vite-node --script scripts/setupDepotDownloader", "tunnel": "cloudflared tunnel --url http://localhost:3000", - "discoverids": "bun run scripts/discoverIds" + "discoverids": "scripts/discoverIds.ts" }, "license": "MIT", "dependencies": { diff --git a/scripts/discoverIds.ts b/scripts/discoverIds.ts index 418603f5f..1a7cbdf0e 100644 --- a/scripts/discoverIds.ts +++ b/scripts/discoverIds.ts @@ -1,19 +1,40 @@ import { times } from 'lodash'; import { argv } from 'process'; import { Region, REGIONS } from '../src/constants/regions'; +import { requestLimitExceededEvent } from '../src/core/blitz/fetchBlitz'; import { getAccountInfo } from '../src/core/blitz/getAccountInfo'; import { idToRegion, MIN_IDS } from '../src/core/blitz/idToRegion'; import { asset } from '../src/core/blitzkit/asset'; +import { commitAssets } from '../src/core/blitzkit/commitAssets'; import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids'; -// const RUN_TIME = 1000 * 60 * 60 * 5; -const RUN_TIME = 1000 * 5; +const RUN_TIME = 1000 * 60 * 60 * 5.5; const MAX_REQUESTS = 10; const ACCOUNTS_PER_CALL = 100; const REMOVE_REGION_THRESHOLD = 10; const production = argv.includes('--production'); const startTime = Date.now(); +const indexableRegions = [...REGIONS]; +const preDiscoveredRaw = await fetch( + asset('averages/discovered.dids.lz4', !production), +).then(async (response) => { + if (response.status === 200) { + const buffer = await response.arrayBuffer(); + return new DidsReadStream(buffer).dids(); + } + + return undefined; +}); +const ids: number[] = [...(preDiscoveredRaw ?? [])]; // TODO: re-validate these ids +const regionalIdIndex: Record = { + asia: ids.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, + com: ids.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com, + eu: ids.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu, +}; +const zeroStreak: Record = { asia: 0, com: 0, eu: 0 }; +let regionIndex = 0; +let outOfTimeFlagged = false; async function verify(region: Region, ids: number[]) { const infos = await getAccountInfo(region, ids, undefined, { @@ -28,31 +49,14 @@ async function verify(region: Region, ids: number[]) { return filtered; } -const indexableRegions = [...REGIONS]; -const ids: number[] = []; - -const preDiscoveredRaw = await fetch( - asset('averages/discovered.dids.lz4', !production), -).then(async (response) => { - if (response.status === 200) { - const buffer = await response.arrayBuffer(); - return new DidsReadStream(buffer).dids(); - } - - return undefined; +let discardAttempts = 0; +requestLimitExceededEvent.on(() => { + discardAttempts++; }); -const regionalIdIndex: Record = { - asia: - preDiscoveredRaw?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia, - com: preDiscoveredRaw?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com, - eu: preDiscoveredRaw?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu, -}; -const zeroStreak: Record = { asia: 0, com: 0, eu: 0 }; -let regionIndex = 0; -let outOfTimeFlagged = false; - const interval = setInterval(async () => { + if (discardAttempts > 0) return discardAttempts--; + const region = indexableRegions[regionIndex]; const idsToVerify = times( ACCOUNTS_PER_CALL, @@ -100,6 +104,17 @@ function post() { console.log(`Uploading ${ids.length} ids...`); const didsWriteStream = new DidsWriteStream().dids(ids); + const content = Buffer.from(didsWriteStream.uint8Array).toString(); - console.log(didsWriteStream.uint8Array.length); + commitAssets( + 'discovered ids', + [ + { + content, + encoding: 'base64', + path: 'definitions/ids.dids', + }, + ], + production, + ); } diff --git a/src/core/blitz/fetchBlitz.ts b/src/core/blitz/fetchBlitz.ts index 64a05e453..b6e6e3ae7 100644 --- a/src/core/blitz/fetchBlitz.ts +++ b/src/core/blitz/fetchBlitz.ts @@ -1,7 +1,10 @@ import { Region } from '../../constants/regions'; import { WARGAMING_APPLICATION_ID } from '../../constants/wargamingApplicationID'; +import { EventManager } from '../blitzkit/eventManager'; import { patientFetch } from '../blitzkit/patientFetch'; +export const requestLimitExceededEvent = new EventManager(); + type BlitzResponse = | { status: 'error'; @@ -41,6 +44,7 @@ async function manageQueue() { `Rate limit exceeded, putting "${request.url}" back in queue...`, ); + requestLimitExceededEvent.emit(undefined); clearTimeout(timeout); setTimeout(() => { queue.push(request);