feat: discover ids

.github/workflows/cd-discover-ids.yml

@@ -0,0 +1,26 @@
+name: CD Discover IDs
+
+on:
+  push:
+    branches: main
+  workflow_dispatch:
+  schedule:
+    - cron: '0 8 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v1
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Discover IDs
+        run: bun scripts/discoverIds.ts

package.json

@@ -31,7 +31,7 @@
     "update": "yarn upgrade --latest",
     "setup:depotdownloader": "vite-node --script scripts/setupDepotDownloader",
     "tunnel": "cloudflared tunnel --url http://localhost:3000",
-    "discoverids": "bun run scripts/discoverIds"
+    "discoverids": "scripts/discoverIds.ts"
   },
   "license": "MIT",
   "dependencies": {

scripts/discoverIds.ts

@@ -1,19 +1,40 @@
 import { times } from 'lodash';
 import { argv } from 'process';
 import { Region, REGIONS } from '../src/constants/regions';
+import { requestLimitExceededEvent } from '../src/core/blitz/fetchBlitz';
 import { getAccountInfo } from '../src/core/blitz/getAccountInfo';
 import { idToRegion, MIN_IDS } from '../src/core/blitz/idToRegion';
 import { asset } from '../src/core/blitzkit/asset';
+import { commitAssets } from '../src/core/blitzkit/commitAssets';
 import { DidsReadStream, DidsWriteStream } from '../src/core/streams/dids';
 
-// const RUN_TIME = 1000 * 60 * 60 * 5;
+const RUN_TIME = 1000 * 60 * 60 * 5.5;
-const RUN_TIME = 1000 * 5;
 const MAX_REQUESTS = 10;
 const ACCOUNTS_PER_CALL = 100;
 const REMOVE_REGION_THRESHOLD = 10;
 
 const production = argv.includes('--production');
 const startTime = Date.now();
+const indexableRegions = [...REGIONS];
+const preDiscoveredRaw = await fetch(
+  asset('averages/discovered.dids.lz4', !production),
+).then(async (response) => {
+  if (response.status === 200) {
+    const buffer = await response.arrayBuffer();
+    return new DidsReadStream(buffer).dids();
+  }
+
+  return undefined;
+});
+const ids: number[] = [...(preDiscoveredRaw ?? [])]; // TODO: re-validate these ids
+const regionalIdIndex: Record<Region, number> = {
+  asia: ids.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
+  com: ids.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com,
+  eu: ids.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu,
+};
+const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
+let regionIndex = 0;
+let outOfTimeFlagged = false;
 
 async function verify(region: Region, ids: number[]) {
   const infos = await getAccountInfo(region, ids, undefined, {
@@ -28,31 +49,14 @@ async function verify(region: Region, ids: number[]) {
   return filtered;
 }
 
-const indexableRegions = [...REGIONS];
+let discardAttempts = 0;
-const ids: number[] = [];
+requestLimitExceededEvent.on(() => {
-
+  discardAttempts++;
-const preDiscoveredRaw = await fetch(
-  asset('averages/discovered.dids.lz4', !production),
-).then(async (response) => {
-  if (response.status === 200) {
-    const buffer = await response.arrayBuffer();
-    return new DidsReadStream(buffer).dids();
-  }
-
-  return undefined;
 });
 
-const regionalIdIndex: Record<Region, number> = {
-  asia:
-    preDiscoveredRaw?.find((id) => idToRegion(id) === 'asia') ?? MIN_IDS.asia,
-  com: preDiscoveredRaw?.find((id) => idToRegion(id) === 'com') ?? MIN_IDS.com,
-  eu: preDiscoveredRaw?.find((id) => idToRegion(id) === 'eu') ?? MIN_IDS.eu,
-};
-const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
-let regionIndex = 0;
-let outOfTimeFlagged = false;
-
 const interval = setInterval(async () => {
+  if (discardAttempts > 0) return discardAttempts--;
+
   const region = indexableRegions[regionIndex];
   const idsToVerify = times(
     ACCOUNTS_PER_CALL,
@@ -100,6 +104,17 @@ function post() {
   console.log(`Uploading ${ids.length} ids...`);
 
   const didsWriteStream = new DidsWriteStream().dids(ids);
+  const content = Buffer.from(didsWriteStream.uint8Array).toString();
 
-  console.log(didsWriteStream.uint8Array.length);
+  commitAssets(
+    'discovered ids',
+    [
+      {
+        content,
+        encoding: 'base64',
+        path: 'definitions/ids.dids',
+      },
+    ],
+    production,
+  );
 }

src/core/blitz/fetchBlitz.ts

@@ -1,7 +1,10 @@
 import { Region } from '../../constants/regions';
 import { WARGAMING_APPLICATION_ID } from '../../constants/wargamingApplicationID';
+import { EventManager } from '../blitzkit/eventManager';
 import { patientFetch } from '../blitzkit/patientFetch';
 
+export const requestLimitExceededEvent = new EventManager();
+
 type BlitzResponse<Data extends object> =
   | {
       status: 'error';
@@ -41,6 +44,7 @@ async function manageQueue() {
           `Rate limit exceeded, putting "${request.url}" back in queue...`,
         );
 
+        requestLimitExceededEvent.emit(undefined);
         clearTimeout(timeout);
         setTimeout(() => {
           queue.push(request);