mirror of
https://github.com/blitzkit/blitzkit.git
synced 2026-10-21 10:47:04 +02:00
more modern way of handling id discovery
This commit is contained in:
tresabhi committed
1 parent
ef84d5b3b5
commit
5c3077033e
2 files changed
+76
-153
No files matched your search
@@ -1,165 +1,87 @@
|
||||
import {
|
||||
DidsReadStream,
|
||||
DidsWriteStream,
|
||||
DiscoveredIdsDefinitions,
|
||||
MIN_REGION_IDS,
|
||||
REGIONS,
|
||||
Region,
|
||||
asset,
|
||||
getAccountInfo,
|
||||
idToRegion,
|
||||
retryAbleBlitzFetchEvent,
|
||||
} from "@blitzkit/core";
|
||||
import { chunk, times, uniq } from "lodash-es";
|
||||
import { compress, decompress } from "lz4js";
|
||||
import { AssetUploader } from "./core/github/assetUploader";
|
||||
const MAX_IDS = 100;
|
||||
const API_RATE = 10;
|
||||
|
||||
const CHUNK_SIZE = 2 ** 21;
|
||||
const RUN_TIME = 1000 * 60 * 60 * 5.5;
|
||||
const PROGRESS_UPDATE_FREQUENCY = 1000 * 60;
|
||||
const MAX_REQUESTS = 10;
|
||||
const ACCOUNTS_PER_CALL = 100;
|
||||
const TERMINATION_THRESHOLD = 1000;
|
||||
const startTime = Date.now();
|
||||
const indexableRegions = [...REGIONS];
|
||||
let ids: number[] = [];
|
||||
const preDiscoveredManifest = (await fetch(asset("ids/manifest.json")).then(
|
||||
(response) => response.json()
|
||||
)) as DiscoveredIdsDefinitions;
|
||||
const ACCOMMODATED_ID_COUNT = 10_000_000_000;
|
||||
const BYTES_PER_ID = 64 / 8;
|
||||
const MAX_BYTES_PER_BATCH = 100 * 1_000_000;
|
||||
|
||||
console.log(
|
||||
`Fetching ${preDiscoveredManifest.chunks} pre-discovered chunks...`
|
||||
const regions = ["eu", "com", "asia"];
|
||||
const seeds = [5e8, 10e8, 20e8].map(BigInt);
|
||||
|
||||
export interface QuickInfo {
|
||||
status: string;
|
||||
data: Record<string, {} | null>;
|
||||
}
|
||||
|
||||
const batches = BigInt(
|
||||
Math.ceil((ACCOMMODATED_ID_COUNT * BYTES_PER_ID) / MAX_BYTES_PER_BATCH),
|
||||
);
|
||||
const ids: bigint[] = [];
|
||||
|
||||
let chunkIndex = 0;
|
||||
while (chunkIndex < preDiscoveredManifest.chunks) {
|
||||
const preDiscovered = await fetch(asset(`ids/${chunkIndex}.dids.lz4`)).then(
|
||||
async (response) => {
|
||||
const buffer = await response.arrayBuffer();
|
||||
const decompressed = decompress(new Uint8Array(buffer)).buffer;
|
||||
return new DidsReadStream(decompressed as ArrayBuffer).dids();
|
||||
const t0 = performance.now();
|
||||
const maxT = 5 * 60 * 60 * 1000;
|
||||
const t1 = t0 + maxT;
|
||||
|
||||
const unfilled = Array.from({ length: seeds.length });
|
||||
const offsets = unfilled.map(() => 0n);
|
||||
const queues = unfilled.map(() => [] as bigint[]);
|
||||
|
||||
function mountIds() {
|
||||
for (let i = 0; i < regions.length; i++) {
|
||||
const queue = queues[i];
|
||||
|
||||
if (queue.length >= MAX_IDS) continue;
|
||||
|
||||
const availableSpace = MAX_IDS - queue.length;
|
||||
|
||||
const seed = seeds[i];
|
||||
const offset = offsets[i]++;
|
||||
|
||||
for (let j = 0n; j < availableSpace; j++) {
|
||||
const id = seed + offset + j;
|
||||
queue.push(id);
|
||||
}
|
||||
);
|
||||
|
||||
// no spread syntax: https://github.com/oven-sh/bun/issues/11734
|
||||
ids = ids.concat(preDiscovered);
|
||||
|
||||
console.log(
|
||||
`Pre-discovered ${preDiscovered.length} ids (chunk ${chunkIndex})`
|
||||
);
|
||||
|
||||
chunkIndex++;
|
||||
offsets[i] += BigInt(availableSpace);
|
||||
}
|
||||
}
|
||||
|
||||
const regionalIdIndex: Record<Region, number> = {
|
||||
asia: ids.findLast((id) => idToRegion(id) === "asia") ?? MIN_REGION_IDS.asia,
|
||||
com: ids.findLast((id) => idToRegion(id) === "com") ?? MIN_REGION_IDS.com,
|
||||
eu: ids.findLast((id) => idToRegion(id) === "eu") ?? MIN_REGION_IDS.eu,
|
||||
};
|
||||
const zeroStreak: Record<Region, number> = { asia: 0, com: 0, eu: 0 };
|
||||
let regionIndex = 0;
|
||||
let outOfTimeFlagged = false;
|
||||
let i = 0;
|
||||
while (performance.now() < t1) {
|
||||
mountIds();
|
||||
await sleep();
|
||||
|
||||
async function verify(region: Region, ids: number[]) {
|
||||
const infos = await getAccountInfo(region, ids, undefined, {
|
||||
// cheeky way of requesting nothing
|
||||
fields: "account_id,-account_id",
|
||||
});
|
||||
const filtered = ids.filter((_, index) => {
|
||||
const info = infos[index];
|
||||
return info !== null;
|
||||
});
|
||||
const region = regions[i];
|
||||
const queue = queues[i];
|
||||
|
||||
return filtered;
|
||||
const accountIds = queue.join(",");
|
||||
const fields = "account_id%2C-account_id";
|
||||
const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
|
||||
|
||||
const url = `https://api.wotblitz.${
|
||||
region
|
||||
}/wotb/account/info/?application_id=${applicationId}&fields=${
|
||||
fields
|
||||
}&account_id=${accountIds}`;
|
||||
const response = await fetch(url);
|
||||
const data = (await response.json()) as QuickInfo;
|
||||
|
||||
if (data.status !== "ok") continue;
|
||||
|
||||
for (const key in data.data) {
|
||||
const value = data.data[key];
|
||||
|
||||
if (value === null) continue;
|
||||
|
||||
const id = BigInt(key);
|
||||
|
||||
ids.push(id);
|
||||
}
|
||||
|
||||
queues[i] = [];
|
||||
i = (i + 1) % regions.length;
|
||||
}
|
||||
|
||||
let discardAttempts = 0;
|
||||
let lastProgressUpdate = 0;
|
||||
|
||||
retryAbleBlitzFetchEvent.on(() => discardAttempts++);
|
||||
|
||||
const interval = setInterval(async () => {
|
||||
if (discardAttempts > 0) return discardAttempts--;
|
||||
|
||||
const region = indexableRegions[regionIndex];
|
||||
const idsToVerify = times(
|
||||
ACCOUNTS_PER_CALL,
|
||||
(index) => regionalIdIndex[region] + index
|
||||
);
|
||||
const verified = await verify(region, idsToVerify);
|
||||
|
||||
if (Date.now() - lastProgressUpdate > PROGRESS_UPDATE_FREQUENCY) {
|
||||
lastProgressUpdate = Date.now();
|
||||
console.log(`Discovered ${ids.length} ids in ${Date.now() - startTime}ms`);
|
||||
}
|
||||
|
||||
if (verified.length === 0) {
|
||||
zeroStreak[region]++;
|
||||
|
||||
if (zeroStreak[region] >= TERMINATION_THRESHOLD) {
|
||||
const sliceableIndex = indexableRegions.indexOf(region);
|
||||
|
||||
if (sliceableIndex !== -1) {
|
||||
console.log(`Stopped discovery for ${region}`);
|
||||
indexableRegions.splice(sliceableIndex, 1);
|
||||
|
||||
if (indexableRegions.length === 0) {
|
||||
console.log("All players discovered exhaustively!");
|
||||
await post();
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
zeroStreak[region] = 0;
|
||||
ids.push(...verified);
|
||||
}
|
||||
|
||||
if (Date.now() - startTime > RUN_TIME && !outOfTimeFlagged) {
|
||||
outOfTimeFlagged = true;
|
||||
console.log("Out of time");
|
||||
await post();
|
||||
}
|
||||
|
||||
regionIndex = (regionIndex + 1) % indexableRegions.length;
|
||||
regionalIdIndex[region] += ACCOUNTS_PER_CALL;
|
||||
}, 1000 / MAX_REQUESTS);
|
||||
|
||||
async function post() {
|
||||
clearInterval(interval);
|
||||
ids = uniq(ids).sort((a, b) => a - b);
|
||||
console.log(`Uploading ${ids.length} ids...`);
|
||||
|
||||
using uploader = new AssetUploader("discovered ids");
|
||||
const idsChunked = chunk(ids, CHUNK_SIZE);
|
||||
|
||||
let index = 0;
|
||||
for (const chunk of idsChunked) {
|
||||
const didsWriteStream = new DidsWriteStream().dids(chunk);
|
||||
const buffer = didsWriteStream.uint8Array;
|
||||
const content = compress(buffer);
|
||||
|
||||
console.log(
|
||||
`Chunk ${index} with ${chunk.length} ids (compressed: ${content.length}; uncompressed: ${buffer.length})`
|
||||
);
|
||||
|
||||
await uploader.add({
|
||||
content,
|
||||
path: `ids/${index}.dids.lz4`,
|
||||
});
|
||||
|
||||
index++;
|
||||
}
|
||||
|
||||
await uploader.add({
|
||||
content: new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
chunks: idsChunked.length,
|
||||
count: ids.length,
|
||||
time: Date.now(),
|
||||
} satisfies DiscoveredIdsDefinitions)
|
||||
),
|
||||
path: "ids/manifest.json",
|
||||
});
|
||||
|
||||
await uploader.flush();
|
||||
function sleep() {
|
||||
return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
|
||||
}
|
||||
Reference in new issue
Block a user