This commit is contained in:
tresabhi committed 2026-08-08 12:28:45 -05:00
1 parent 5c3077033e
commit 09edd39a06
5 files changed
+194 -108

No files matched your search

+43 -20
View File
@@ -63,7 +63,7 @@
"@gltf-transform/functions": "^4.2.1",
"@octokit/plugin-throttling": "^11.0.2",
"@octokit/rest": "^22.0.0",
"@types/bun": "^1.3.0",
"@types/bun": "^1.3.14",
"@types/decompress": "^4.2.7",
"@types/lodash-es": "^4.17.12",
"@types/lz4js": "^0.2.1",
@@ -78,6 +78,7 @@
"fast-xml-parser": "^5.2.5",
"googleapis": "^162.0.0",
"lodash-es": "^4.17.21",
"lz4-napi": "^2.9.1",
"progress": "^2.0.3",
"puppeteer": "^24.22.3",
"sharp": "^0.34.4",
@@ -144,6 +145,40 @@
},
},
"packages": {
"@antoniomuso/lz4-napi-android-arm-eabi": ["@antoniomuso/[email protected]", "", { "os": "android", "cpu": "arm" }, "sha512-KcRvkSOymDvxFDT9AGFBc7g+J/Xv4qExMMHhtxjZ45qE2sEIVCXoSFmyLDiVVKztkVzoV24VXPPRLOa4WQOi2Q=="],
"@antoniomuso/lz4-napi-android-arm64": ["@antoniomuso/[email protected]", "", { "os": "android", "cpu": "arm64" }, "sha512-BNiddtI+5Z83w1jIufTFhTPrtxMVKb0UPezTq47XIE/7vkB6y5mHSWSlcHqEIFFIg2/PVdqyqec1CXv0bkpCPA=="],
"@antoniomuso/lz4-napi-darwin-arm64": ["@antoniomuso/[email protected]", "", { "os": "darwin", "cpu": "arm64" }, "sha512-nhLu+8g8Q6Yqd0rxRn4JkCMJBUs6mByiyT3Mhha2EgorsA2txx18JKm9qLhnXpDgFGWc94tLEVHqzDk0tLD9yA=="],
"@antoniomuso/lz4-napi-darwin-x64": ["@antoniomuso/[email protected]", "", { "os": "darwin", "cpu": "x64" }, "sha512-DqkN1D/IrqOaUBppCP7BrF9j5Usb6RqFyz16kf3SHPDxLG7pmemgwXmGWpcNrKQfp4Shx1VkbFWA61gMyE1+uw=="],
"@antoniomuso/lz4-napi-freebsd-x64": ["@antoniomuso/[email protected]", "", { "os": "freebsd", "cpu": "x64" }, "sha512-ORtZ27l+j0yDgPMgt9q/j10LCchtKS44gFohEWl8fokgr5rzDAYAtMyQZzHV8sqE2LUWHAU3Gk6Wh9s1GRl9ow=="],
"@antoniomuso/lz4-napi-linux-arm-gnueabihf": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "arm" }, "sha512-GBZgQ/rY0AzjZA4jOKFXmLQKVPGXglVZ3U7SLcCu2nmuHTO73vsqiPeAHnYu7/org1km7yw3ii6XFlBRzyNKIA=="],
"@antoniomuso/lz4-napi-linux-arm64-gnu": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "arm64" }, "sha512-9lvGts14KHNT4pc9lKYUPI1PRdBkfbyT4HCjyDk8fJJzEG9ryvLvyhTeFr/IVpJOFmkSodZw1nxJMQDT5+zvlQ=="],
"@antoniomuso/lz4-napi-linux-arm64-musl": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "arm64" }, "sha512-SsidKyLRHGB/rrgic+N7ZxFjjl+Cbav2sbuSj4AJsc37oVhyjzbiI5EEL9Rl+zL4ajL0d3utG3UbIUooThhykQ=="],
"@antoniomuso/lz4-napi-linux-ppc64-gnu": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1Joz9RmBLiLIfI5Gk/wmJI/WwQy74prKDhLcCPFoa/FcTOJavJAopAzs1MQdnM7nGklEPOpKZWmKHZz1yQ4LMQ=="],
"@antoniomuso/lz4-napi-linux-riscv64-gnu": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-3z6jSvhEpnSMlubcHwSn6W8Xziq5LYSuIcSpSo7wTEOopnr6GNnKcH2uTPFsRiqC/aC2TLpm17ktXbYTrYL7jg=="],
"@antoniomuso/lz4-napi-linux-s390x-gnu": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "s390x" }, "sha512-wtQg/ikF1JutKdWfUdGXvR0LG+eG49mS01luwJXG6sg40DX737T3BAsbjM9Fccvif5jQwC+KmzP8+tvxy2eMtQ=="],
"@antoniomuso/lz4-napi-linux-x64-gnu": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "x64" }, "sha512-Vw7vb3i1Q6QU2sI5GdgZdi4cVwN2xZ44Gw/WawIxCno1gX39VIF/j8T1pYwS7Dc67NeoA37LsYIPb3h49CiRRA=="],
"@antoniomuso/lz4-napi-linux-x64-musl": ["@antoniomuso/[email protected]", "", { "os": "linux", "cpu": "x64" }, "sha512-oZVrvOPjnI7c07v1lx6HMLFi8xlW9WS0pHmkaisldybNUWmNI6vR44KoqBctGCFplvF5YVMK2xPs3W04UGzF8w=="],
"@antoniomuso/lz4-napi-openharmony-arm64": ["@antoniomuso/[email protected]", "", { "os": "none", "cpu": "arm64" }, "sha512-mkDQv6C0JbMbCDn070Dv67SWfkDy8uD6DkqUT373LHjdaEXj/0eK2gAVykKyPdnC1RBhONgUCaV4HQOp63oH8Q=="],
"@antoniomuso/lz4-napi-win32-arm64-msvc": ["@antoniomuso/[email protected]", "", { "os": "win32", "cpu": "arm64" }, "sha512-EBsN9YzV3xpSPr6h/t6ntBetpdtO0nZw9o4sZ1vNOfLJ8FMdHxHiY7FonzhEyTnc4N3x3yMqPudD6zLVy/oNXg=="],
"@antoniomuso/lz4-napi-win32-ia32-msvc": ["@antoniomuso/[email protected]", "", { "os": "win32", "cpu": "ia32" }, "sha512-Fqe2JmcfjvVrC+BHjHPzoiIzNgjHpQvDO7duMbJh5fW9VvF7pKkjRAPDwhtFhkU/fBLVQywq5WbsFqtJ3pYG/w=="],
"@antoniomuso/lz4-napi-win32-x64-msvc": ["@antoniomuso/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-N6bAJ4pcYfbiuysHNyEl6uVnYxl0CBboPeA6Iz1GQWJb6mp8J/z4+E8bTvasMuRhIhEW5F6X+oJIwmOwSzHoIQ=="],
"@apideck/better-ajv-errors": ["@apideck/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0", "jsonpointer": "^5.0.0", "leven": "^3.1.0" }, "peerDependencies": { "ajv": ">=8" } }, "sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA=="],
"@astrojs/check": ["@astrojs/[email protected]", "", { "dependencies": { "@astrojs/language-server": "^2.15.0", "chokidar": "^4.0.1", "kleur": "^4.1.5", "yargs": "^17.7.2" }, "peerDependencies": { "typescript": "^5.0.0" }, "bin": { "astro-check": "dist/bin.js" } }, "sha512-IOheHwCtpUfvogHHsvu0AbeRZEnjJg3MopdLddkJE70mULItS/Vh37BHcI00mcOJcH1vhD3odbpvWokpxam7xA=="],
@@ -456,11 +491,11 @@
"@fontsource/inter": ["@fontsource/[email protected]", "", {}, "sha512-P6r5WnJoKiNVV+zvW2xM13gNdFhAEpQ9dQJHt3naLvfg+LkF2ldgSLiF4T41lf1SQCM9QmkqPTn4TH568IRagg=="],
"@gltf-transform/core": ["@gltf-transform/core@4.2.1", "", { "dependencies": { "property-graph": "^3.0.0" } }, "sha512-qKhrQ29KJ9K6sz7xGoGRllqPBlROjPJMQPvCLBOfgMPH3S/Ph5E0e6fD5WmwTY5QPyBsDMRqKd1Vl0ncThTXGw=="],
"@gltf-transform/core": ["@gltf-transform/core@4.4.2", "", { "dependencies": { "property-graph": "^4.1.0" } }, "sha512-qsWKwNSwK+2s834Mt4xbYcyHqCrgNFP7hIv5s487JxebngRfDgelpghNF+kSswGb2/NuapasfK3UViFoSJJoMg=="],
"@gltf-transform/extensions": ["@gltf-transform/[email protected]", "", { "dependencies": { "@gltf-transform/core": "^4.4.2", "ktx-parse": "^1.1.0" } }, "sha512-HJH1FM+edC5eNvl6xO0SOXJ/j/3oDoIpSu150OTdJaLBoM3TgCCGIfh4wyhgWAqZrkvgHKVGiZKxcKV5LkgPCQ=="],
"@gltf-transform/functions": ["@gltf-transform/functions@4.2.1", "", { "dependencies": { "@gltf-transform/core": "^4.2.1", "@gltf-transform/extensions": "^4.2.1", "ktx-parse": "^1.0.1", "ndarray": "^1.0.19", "ndarray-lanczos": "^0.3.0", "ndarray-pixels": "^5.0.1" } }, "sha512-hFCI80N6zt7yHlqXbzlYEYQbFwK+2OOZNvacrRtrWiCWKI0k1Ad5+nZFsmuIZZV03aXb5XG6UF7JINT3wETS1Q=="],
"@gltf-transform/functions": ["@gltf-transform/functions@4.4.2", "", { "dependencies": { "@gltf-transform/core": "^4.4.2", "@gltf-transform/extensions": "^4.4.2", "ktx-parse": "^1.1.0", "ndarray": "^1.0.19", "ndarray-lanczos": "^0.3.0", "ndarray-pixels": "^5.0.1" } }, "sha512-dclXgv9TshMaWBqPDUYd4xTwBQ2PpuR8p0Y9pokrRzGQDUPXRP6lTDzbqT0UmEmxFSvRyPJjvOWUmzeiRafpvw=="],
"@img/colour": ["@img/[email protected]", "", {}, "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw=="],
@@ -860,7 +895,7 @@
"@types/babel__traverse": ["@types/[email protected]", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="],
"@types/bun": ["@types/[email protected].9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
"@types/bun": ["@types/[email protected].14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="],
"@types/bytebuffer": ["@types/[email protected]", "", { "dependencies": { "@types/long": "^3.0.0", "@types/node": "*" } }, "sha512-lV4YLiolMdD4upDmr4vnfiwV/FN9Jg33eNTSFMkHqyMKqTIAn5TmFTYsARwXwUXeFU7yzRpECmWV1SOhIvdkRQ=="],
@@ -1076,7 +1111,7 @@
"buffer-from": ["[email protected]", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="],
"bun-types": ["[email protected].9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
"bun-types": ["[email protected].14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],
"bytebuffer": ["[email protected]", "", { "dependencies": { "long": "~3" } }, "sha512-IuzSdmADppkZ6DlpycMkm8l9zeEq16fWtLvunEwFiYciR/BHo4E8/xs5piFquG+Za8OWmMqHF8zuRviz2LHvRQ=="],
@@ -1664,6 +1699,8 @@
"lru-cache": ["[email protected]", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
"lz4-napi": ["[email protected]", "", { "optionalDependencies": { "@antoniomuso/lz4-napi-android-arm-eabi": "2.9.1", "@antoniomuso/lz4-napi-android-arm64": "2.9.1", "@antoniomuso/lz4-napi-darwin-arm64": "2.9.1", "@antoniomuso/lz4-napi-darwin-x64": "2.9.1", "@antoniomuso/lz4-napi-freebsd-x64": "2.9.1", "@antoniomuso/lz4-napi-linux-arm-gnueabihf": "2.9.1", "@antoniomuso/lz4-napi-linux-arm64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-arm64-musl": "2.9.1", "@antoniomuso/lz4-napi-linux-ppc64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-riscv64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-s390x-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-x64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-x64-musl": "2.9.1", "@antoniomuso/lz4-napi-openharmony-arm64": "2.9.1", "@antoniomuso/lz4-napi-win32-arm64-msvc": "2.9.1", "@antoniomuso/lz4-napi-win32-ia32-msvc": "2.9.1", "@antoniomuso/lz4-napi-win32-x64-msvc": "2.9.1" } }, "sha512-VjY1TAUQa3bo8j4vCcpQ0RBXh4mbC1OFHWH9rpOvx8qLYwaZTEzwxW/648ufD2cmz5/pFfPLDkX6N8+TqI9BYQ=="],
"lz4js": ["[email protected]", "", {}, "sha512-gY2Ia9Lm7Ep8qMiuGRhvUq0Q7qUereeldZPP1PMEJxPtEWHJLqw9pgX68oHajBH0nzJK4MaZEA/YNV3jT8u8Bg=="],
"lzma": ["[email protected]", "", { "bin": { "lzma.js": "bin/lzma.js" } }, "sha512-DcfiawQ1avYbW+hsILhF38IKAlnguc/fjHrychs9hdxe4qLykvhT5VTGNs5YRWgaNePh7NTxGD4uv4gKsRomCQ=="],
@@ -1930,7 +1967,7 @@
"prompts": ["[email protected]", "", { "dependencies": { "kleur": "^3.0.3", "sisteransi": "^1.0.5" } }, "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q=="],
"property-graph": ["property-graph@3.0.0", "", {}, "sha512-TnzxUsttmGtw+OiU0LDw+0FlMbJ8vV8pOjyDI7+Kdni4Tj0hW5BFh7TatQu7Y68hcvvFmiFOHilKShsA4R82fA=="],
"property-graph": ["property-graph@4.1.0", "", {}, "sha512-AvPcP7XECNWy4LGmFQ77k7un4lSKM4eS29PTvW4ck95uYeLxXPWJM7hLuBqK91FaHqCcgJvIUCuNJjjxKE7VKQ=="],
"property-information": ["[email protected]", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="],
@@ -2524,16 +2561,6 @@
"@babel/preset-env/semver": ["[email protected]", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
"@blitzkit/scripts/@gltf-transform/extensions": ["@gltf-transform/[email protected]", "", { "dependencies": { "@gltf-transform/core": "^4.2.1", "ktx-parse": "^1.0.1" } }, "sha512-ieHSJU9qvb3ucWDxgHFcff+C7fLtFPa6G0Wtc/ki7GsN6Rh7o9eoyqi7Ggj2LO4i3ynCwLLPb/onbJKw2fBtsA=="],
"@blitzkit/website/@gltf-transform/core": ["@gltf-transform/[email protected]", "", { "dependencies": { "property-graph": "^4.1.0" } }, "sha512-qsWKwNSwK+2s834Mt4xbYcyHqCrgNFP7hIv5s487JxebngRfDgelpghNF+kSswGb2/NuapasfK3UViFoSJJoMg=="],
"@blitzkit/website/@gltf-transform/functions": ["@gltf-transform/[email protected]", "", { "dependencies": { "@gltf-transform/core": "^4.4.2", "@gltf-transform/extensions": "^4.4.2", "ktx-parse": "^1.1.0", "ndarray": "^1.0.19", "ndarray-lanczos": "^0.3.0", "ndarray-pixels": "^5.0.1" } }, "sha512-dclXgv9TshMaWBqPDUYd4xTwBQ2PpuR8p0Y9pokrRzGQDUPXRP6lTDzbqT0UmEmxFSvRyPJjvOWUmzeiRafpvw=="],
"@gltf-transform/extensions/@gltf-transform/core": ["@gltf-transform/[email protected]", "", { "dependencies": { "property-graph": "^4.1.0" } }, "sha512-qsWKwNSwK+2s834Mt4xbYcyHqCrgNFP7hIv5s487JxebngRfDgelpghNF+kSswGb2/NuapasfK3UViFoSJJoMg=="],
"@gltf-transform/functions/@gltf-transform/extensions": ["@gltf-transform/[email protected]", "", { "dependencies": { "@gltf-transform/core": "^4.2.1", "ktx-parse": "^1.0.1" } }, "sha512-ieHSJU9qvb3ucWDxgHFcff+C7fLtFPa6G0Wtc/ki7GsN6Rh7o9eoyqi7Ggj2LO4i3ynCwLLPb/onbJKw2fBtsA=="],
"@jimp/core/file-type": ["[email protected]", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="],
"@octokit/endpoint/@octokit/types": ["@octokit/[email protected]", "", { "dependencies": { "@octokit/openapi-types": "^25.1.0" } }, "sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g=="],
@@ -2714,10 +2741,6 @@
"yaml-language-server/yaml": ["[email protected]", "", {}, "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA=="],
"@blitzkit/website/@gltf-transform/core/property-graph": ["[email protected]", "", {}, "sha512-AvPcP7XECNWy4LGmFQ77k7un4lSKM4eS29PTvW4ck95uYeLxXPWJM7hLuBqK91FaHqCcgJvIUCuNJjjxKE7VKQ=="],
"@gltf-transform/extensions/@gltf-transform/core/property-graph": ["[email protected]", "", {}, "sha512-AvPcP7XECNWy4LGmFQ77k7un4lSKM4eS29PTvW4ck95uYeLxXPWJM7hLuBqK91FaHqCcgJvIUCuNJjjxKE7VKQ=="],
"@octokit/endpoint/@octokit/types/@octokit/openapi-types": ["@octokit/[email protected]", "", {}, "sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA=="],
"@octokit/graphql/@octokit/types/@octokit/openapi-types": ["@octokit/[email protected]", "", {}, "sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA=="],
+2 -1
View File
@@ -9,7 +9,7 @@
"@gltf-transform/functions": "^4.2.1",
"@octokit/plugin-throttling": "^11.0.2",
"@octokit/rest": "^22.0.0",
"@types/bun": "^1.3.0",
"@types/bun": "^1.3.14",
"@types/decompress": "^4.2.7",
"@types/lodash-es": "^4.17.12",
"@types/lz4js": "^0.2.1",
@@ -24,6 +24,7 @@
"fast-xml-parser": "^5.2.5",
"googleapis": "^162.0.0",
"lodash-es": "^4.17.21",
"lz4-napi": "^2.9.1",
"progress": "^2.0.3",
"puppeteer": "^24.22.3",
"sharp": "^0.34.4",
-87
View File
@@ -1,87 +0,0 @@
const MAX_IDS = 100;
const API_RATE = 10;
const ACCOMMODATED_ID_COUNT = 10_000_000_000;
const BYTES_PER_ID = 64 / 8;
const MAX_BYTES_PER_BATCH = 100 * 1_000_000;
const regions = ["eu", "com", "asia"];
const seeds = [5e8, 10e8, 20e8].map(BigInt);
export interface QuickInfo {
status: string;
data: Record<string, {} | null>;
}
const batches = BigInt(
Math.ceil((ACCOMMODATED_ID_COUNT * BYTES_PER_ID) / MAX_BYTES_PER_BATCH),
);
const ids: bigint[] = [];
const t0 = performance.now();
const maxT = 5 * 60 * 60 * 1000;
const t1 = t0 + maxT;
const unfilled = Array.from({ length: seeds.length });
const offsets = unfilled.map(() => 0n);
const queues = unfilled.map(() => [] as bigint[]);
function mountIds() {
for (let i = 0; i < regions.length; i++) {
const queue = queues[i];
if (queue.length >= MAX_IDS) continue;
const availableSpace = MAX_IDS - queue.length;
const seed = seeds[i];
const offset = offsets[i]++;
for (let j = 0n; j < availableSpace; j++) {
const id = seed + offset + j;
queue.push(id);
}
offsets[i] += BigInt(availableSpace);
}
}
let i = 0;
while (performance.now() < t1) {
mountIds();
await sleep();
const region = regions[i];
const queue = queues[i];
const accountIds = queue.join(",");
const fields = "account_id%2C-account_id";
const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
const url = `https://api.wotblitz.${
region
}/wotb/account/info/?application_id=${applicationId}&fields=${
fields
}&account_id=${accountIds}`;
const response = await fetch(url);
const data = (await response.json()) as QuickInfo;
if (data.status !== "ok") continue;
for (const key in data.data) {
const value = data.data[key];
if (value === null) continue;
const id = BigInt(key);
ids.push(id);
}
queues[i] = [];
i = (i + 1) % regions.length;
}
function sleep() {
return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
}
@@ -0,0 +1,5 @@
export class IdArray {
constructor(
private array: Uint32Array<ArrayBufferLike> = new Uint32Array(),
) {}
}
+144
View File
@@ -0,0 +1,144 @@
import { $ } from "bun";
import { uncompress } from "lz4-napi";
import { mkdir, rm } from "node:fs/promises";
import { IdArray } from "./idArray";
interface ManifestV1 {
version: 1;
last_verified: number;
}
const TEMP = "../../temp";
const WORKING_DIR = `${TEMP}/ids`;
const CHUNKS_DIR = `${WORKING_DIR}/chunks`;
const REPO = "blitzkit/ids";
const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids
const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB
const chunkCount = Math.ceil(
(ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK,
);
console.log(
`${chunkCount.toLocaleString()} will be needed for ${ACCOMMODATED_ID_COUNT.toLocaleString()} ids`,
);
await mkdir(TEMP, { recursive: true });
await rm(WORKING_DIR, { recursive: true, force: true });
await $`git clone https://github.com/${REPO} ${WORKING_DIR}`;
await mkdir(CHUNKS_DIR, { recursive: true });
const discoveredChunks: IdArray[] = [];
for (let i = 0; true; i++) {
const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`;
const file = Bun.file(path);
if (!(await file.exists())) break;
const bytes = await file.bytes();
const uncompressed = await uncompress(bytes);
const array = new Uint32Array(
uncompressed.buffer,
uncompressed.byteOffset,
uncompressed.byteLength / Uint32Array.BYTES_PER_ELEMENT,
);
const ids = new IdArray(array);
discoveredChunks.push(ids);
}
let chunks: IdArray[] = [];
if (discoveredChunks.length === chunkCount) {
chunks = discoveredChunks;
} else {
chunks = Array.from({ length: chunkCount }).map(() => new IdArray());
}
// const MAX_IDS = 100;
// const API_RATE = 10;
// const BYTES_PER_ID = 64 / 8;
// const regions = ["eu", "com", "asia"];
// const seeds = [5e8, 10e8, 20e8].map(BigInt);
// export interface QuickInfo {
// status: string;
// data: Record<string, {} | null>;
// }
// const ids: bigint[] = [];
// const t0 = performance.now();
// const maxT = 5 * 60 * 60 * 1000;
// const t1 = t0 + maxT;
// const unfilled = Array.from({ length: seeds.length });
// const offsets = unfilled.map(() => 0n);
// const queues = unfilled.map(() => [] as bigint[]);
// function mountIds() {
// for (let i = 0; i < regions.length; i++) {
// const queue = queues[i];
// if (queue.length >= MAX_IDS) continue;
// const availableSpace = MAX_IDS - queue.length;
// const seed = seeds[i];
// const offset = offsets[i]++;
// for (let j = 0n; j < availableSpace; j++) {
// const id = seed + offset + j;
// queue.push(id);
// }
// offsets[i] += BigInt(availableSpace);
// }
// }
// let i = 0;
// while (performance.now() < t1) {
// mountIds();
// await sleep();
// const region = regions[i];
// const queue = queues[i];
// const accountIds = queue.join(",");
// const fields = "account_id%2C-account_id";
// const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID;
// const url = `https://api.wotblitz.${
// region
// }/wotb/account/info/?application_id=${applicationId}&fields=${
// fields
// }&account_id=${accountIds}`;
// const response = await fetch(url);
// const data = (await response.json()) as QuickInfo;
// if (data.status !== "ok") continue;
// for (const key in data.data) {
// const value = data.data[key];
// if (value === null) continue;
// const id = BigInt(key);
// ids.push(id);
// }
// queues[i] = [];
// i = (i + 1) % regions.length;
// }
// function sleep() {
// return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE));
// }