diff --git a/bun.lock b/bun.lock index ff4e2722a..73b200941 100644 --- a/bun.lock +++ b/bun.lock @@ -63,7 +63,7 @@ "@gltf-transform/functions": "^4.2.1", "@octokit/plugin-throttling": "^11.0.2", "@octokit/rest": "^22.0.0", - "@types/bun": "^1.3.0", + "@types/bun": "^1.3.14", "@types/decompress": "^4.2.7", "@types/lodash-es": "^4.17.12", "@types/lz4js": "^0.2.1", @@ -78,6 +78,7 @@ "fast-xml-parser": "^5.2.5", "googleapis": "^162.0.0", "lodash-es": "^4.17.21", + "lz4-napi": "^2.9.1", "progress": "^2.0.3", "puppeteer": "^24.22.3", "sharp": "^0.34.4", @@ -144,6 +145,40 @@ }, }, "packages": { + "@antoniomuso/lz4-napi-android-arm-eabi": ["@antoniomuso/lz4-napi-android-arm-eabi@2.9.1", "", { "os": "android", "cpu": "arm" }, "sha512-KcRvkSOymDvxFDT9AGFBc7g+J/Xv4qExMMHhtxjZ45qE2sEIVCXoSFmyLDiVVKztkVzoV24VXPPRLOa4WQOi2Q=="], + + "@antoniomuso/lz4-napi-android-arm64": ["@antoniomuso/lz4-napi-android-arm64@2.9.1", "", { "os": "android", "cpu": "arm64" }, "sha512-BNiddtI+5Z83w1jIufTFhTPrtxMVKb0UPezTq47XIE/7vkB6y5mHSWSlcHqEIFFIg2/PVdqyqec1CXv0bkpCPA=="], + + "@antoniomuso/lz4-napi-darwin-arm64": ["@antoniomuso/lz4-napi-darwin-arm64@2.9.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-nhLu+8g8Q6Yqd0rxRn4JkCMJBUs6mByiyT3Mhha2EgorsA2txx18JKm9qLhnXpDgFGWc94tLEVHqzDk0tLD9yA=="], + + "@antoniomuso/lz4-napi-darwin-x64": ["@antoniomuso/lz4-napi-darwin-x64@2.9.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-DqkN1D/IrqOaUBppCP7BrF9j5Usb6RqFyz16kf3SHPDxLG7pmemgwXmGWpcNrKQfp4Shx1VkbFWA61gMyE1+uw=="], + + "@antoniomuso/lz4-napi-freebsd-x64": ["@antoniomuso/lz4-napi-freebsd-x64@2.9.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-ORtZ27l+j0yDgPMgt9q/j10LCchtKS44gFohEWl8fokgr5rzDAYAtMyQZzHV8sqE2LUWHAU3Gk6Wh9s1GRl9ow=="], + + "@antoniomuso/lz4-napi-linux-arm-gnueabihf": ["@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.9.1", "", { "os": "linux", "cpu": "arm" }, "sha512-GBZgQ/rY0AzjZA4jOKFXmLQKVPGXglVZ3U7SLcCu2nmuHTO73vsqiPeAHnYu7/org1km7yw3ii6XFlBRzyNKIA=="], + + "@antoniomuso/lz4-napi-linux-arm64-gnu": ["@antoniomuso/lz4-napi-linux-arm64-gnu@2.9.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-9lvGts14KHNT4pc9lKYUPI1PRdBkfbyT4HCjyDk8fJJzEG9ryvLvyhTeFr/IVpJOFmkSodZw1nxJMQDT5+zvlQ=="], + + "@antoniomuso/lz4-napi-linux-arm64-musl": ["@antoniomuso/lz4-napi-linux-arm64-musl@2.9.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-SsidKyLRHGB/rrgic+N7ZxFjjl+Cbav2sbuSj4AJsc37oVhyjzbiI5EEL9Rl+zL4ajL0d3utG3UbIUooThhykQ=="], + + "@antoniomuso/lz4-napi-linux-ppc64-gnu": ["@antoniomuso/lz4-napi-linux-ppc64-gnu@2.9.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1Joz9RmBLiLIfI5Gk/wmJI/WwQy74prKDhLcCPFoa/FcTOJavJAopAzs1MQdnM7nGklEPOpKZWmKHZz1yQ4LMQ=="], + + "@antoniomuso/lz4-napi-linux-riscv64-gnu": ["@antoniomuso/lz4-napi-linux-riscv64-gnu@2.9.1", "", { "os": "linux", "cpu": "none" }, "sha512-3z6jSvhEpnSMlubcHwSn6W8Xziq5LYSuIcSpSo7wTEOopnr6GNnKcH2uTPFsRiqC/aC2TLpm17ktXbYTrYL7jg=="], + + "@antoniomuso/lz4-napi-linux-s390x-gnu": ["@antoniomuso/lz4-napi-linux-s390x-gnu@2.9.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-wtQg/ikF1JutKdWfUdGXvR0LG+eG49mS01luwJXG6sg40DX737T3BAsbjM9Fccvif5jQwC+KmzP8+tvxy2eMtQ=="], + + "@antoniomuso/lz4-napi-linux-x64-gnu": ["@antoniomuso/lz4-napi-linux-x64-gnu@2.9.1", "", { "os": "linux", "cpu": "x64" }, "sha512-Vw7vb3i1Q6QU2sI5GdgZdi4cVwN2xZ44Gw/WawIxCno1gX39VIF/j8T1pYwS7Dc67NeoA37LsYIPb3h49CiRRA=="], + + "@antoniomuso/lz4-napi-linux-x64-musl": ["@antoniomuso/lz4-napi-linux-x64-musl@2.9.1", "", { "os": "linux", "cpu": "x64" }, "sha512-oZVrvOPjnI7c07v1lx6HMLFi8xlW9WS0pHmkaisldybNUWmNI6vR44KoqBctGCFplvF5YVMK2xPs3W04UGzF8w=="], + + "@antoniomuso/lz4-napi-openharmony-arm64": ["@antoniomuso/lz4-napi-openharmony-arm64@2.9.1", "", { "os": "none", "cpu": "arm64" }, "sha512-mkDQv6C0JbMbCDn070Dv67SWfkDy8uD6DkqUT373LHjdaEXj/0eK2gAVykKyPdnC1RBhONgUCaV4HQOp63oH8Q=="], + + "@antoniomuso/lz4-napi-win32-arm64-msvc": ["@antoniomuso/lz4-napi-win32-arm64-msvc@2.9.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-EBsN9YzV3xpSPr6h/t6ntBetpdtO0nZw9o4sZ1vNOfLJ8FMdHxHiY7FonzhEyTnc4N3x3yMqPudD6zLVy/oNXg=="], + + "@antoniomuso/lz4-napi-win32-ia32-msvc": ["@antoniomuso/lz4-napi-win32-ia32-msvc@2.9.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-Fqe2JmcfjvVrC+BHjHPzoiIzNgjHpQvDO7duMbJh5fW9VvF7pKkjRAPDwhtFhkU/fBLVQywq5WbsFqtJ3pYG/w=="], + + "@antoniomuso/lz4-napi-win32-x64-msvc": ["@antoniomuso/lz4-napi-win32-x64-msvc@2.9.1", "", { "os": "win32", "cpu": "x64" }, "sha512-N6bAJ4pcYfbiuysHNyEl6uVnYxl0CBboPeA6Iz1GQWJb6mp8J/z4+E8bTvasMuRhIhEW5F6X+oJIwmOwSzHoIQ=="], + "@apideck/better-ajv-errors": ["@apideck/better-ajv-errors@0.3.6", "", { "dependencies": { "json-schema": "^0.4.0", "jsonpointer": "^5.0.0", "leven": "^3.1.0" }, "peerDependencies": { "ajv": ">=8" } }, "sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA=="], "@astrojs/check": ["@astrojs/check@0.9.4", "", { "dependencies": { "@astrojs/language-server": "^2.15.0", "chokidar": "^4.0.1", "kleur": "^4.1.5", "yargs": "^17.7.2" }, "peerDependencies": { "typescript": "^5.0.0" }, "bin": { "astro-check": "dist/bin.js" } }, "sha512-IOheHwCtpUfvogHHsvu0AbeRZEnjJg3MopdLddkJE70mULItS/Vh37BHcI00mcOJcH1vhD3odbpvWokpxam7xA=="], @@ -456,11 +491,11 @@ "@fontsource/inter": ["@fontsource/inter@5.2.8", "", {}, "sha512-P6r5WnJoKiNVV+zvW2xM13gNdFhAEpQ9dQJHt3naLvfg+LkF2ldgSLiF4T41lf1SQCM9QmkqPTn4TH568IRagg=="], - "@gltf-transform/core": ["@gltf-transform/core@4.2.1", "", { "dependencies": { "property-graph": "^3.0.0" } }, "sha512-qKhrQ29KJ9K6sz7xGoGRllqPBlROjPJMQPvCLBOfgMPH3S/Ph5E0e6fD5WmwTY5QPyBsDMRqKd1Vl0ncThTXGw=="], + "@gltf-transform/core": ["@gltf-transform/core@4.4.2", "", { "dependencies": { "property-graph": "^4.1.0" } }, "sha512-qsWKwNSwK+2s834Mt4xbYcyHqCrgNFP7hIv5s487JxebngRfDgelpghNF+kSswGb2/NuapasfK3UViFoSJJoMg=="], "@gltf-transform/extensions": ["@gltf-transform/extensions@4.4.2", "", { "dependencies": { "@gltf-transform/core": "^4.4.2", "ktx-parse": "^1.1.0" } }, "sha512-HJH1FM+edC5eNvl6xO0SOXJ/j/3oDoIpSu150OTdJaLBoM3TgCCGIfh4wyhgWAqZrkvgHKVGiZKxcKV5LkgPCQ=="], - "@gltf-transform/functions": ["@gltf-transform/functions@4.2.1", "", { "dependencies": { "@gltf-transform/core": "^4.2.1", "@gltf-transform/extensions": "^4.2.1", "ktx-parse": "^1.0.1", "ndarray": "^1.0.19", "ndarray-lanczos": "^0.3.0", "ndarray-pixels": "^5.0.1" } }, "sha512-hFCI80N6zt7yHlqXbzlYEYQbFwK+2OOZNvacrRtrWiCWKI0k1Ad5+nZFsmuIZZV03aXb5XG6UF7JINT3wETS1Q=="], + "@gltf-transform/functions": ["@gltf-transform/functions@4.4.2", "", { "dependencies": { "@gltf-transform/core": "^4.4.2", "@gltf-transform/extensions": "^4.4.2", "ktx-parse": "^1.1.0", "ndarray": "^1.0.19", "ndarray-lanczos": "^0.3.0", "ndarray-pixels": "^5.0.1" } }, "sha512-dclXgv9TshMaWBqPDUYd4xTwBQ2PpuR8p0Y9pokrRzGQDUPXRP6lTDzbqT0UmEmxFSvRyPJjvOWUmzeiRafpvw=="], "@img/colour": ["@img/colour@1.0.0", "", {}, "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw=="], @@ -860,7 +895,7 @@ "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], - "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/bytebuffer": ["@types/bytebuffer@5.0.49", "", { "dependencies": { "@types/long": "^3.0.0", "@types/node": "*" } }, "sha512-lV4YLiolMdD4upDmr4vnfiwV/FN9Jg33eNTSFMkHqyMKqTIAn5TmFTYsARwXwUXeFU7yzRpECmWV1SOhIvdkRQ=="], @@ -1076,7 +1111,7 @@ "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "bytebuffer": ["bytebuffer@5.0.1", "", { "dependencies": { "long": "~3" } }, "sha512-IuzSdmADppkZ6DlpycMkm8l9zeEq16fWtLvunEwFiYciR/BHo4E8/xs5piFquG+Za8OWmMqHF8zuRviz2LHvRQ=="], @@ -1664,6 +1699,8 @@ "lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "lz4-napi": ["lz4-napi@2.9.1", "", { "optionalDependencies": { "@antoniomuso/lz4-napi-android-arm-eabi": "2.9.1", "@antoniomuso/lz4-napi-android-arm64": "2.9.1", "@antoniomuso/lz4-napi-darwin-arm64": "2.9.1", "@antoniomuso/lz4-napi-darwin-x64": "2.9.1", "@antoniomuso/lz4-napi-freebsd-x64": "2.9.1", "@antoniomuso/lz4-napi-linux-arm-gnueabihf": "2.9.1", "@antoniomuso/lz4-napi-linux-arm64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-arm64-musl": "2.9.1", "@antoniomuso/lz4-napi-linux-ppc64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-riscv64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-s390x-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-x64-gnu": "2.9.1", "@antoniomuso/lz4-napi-linux-x64-musl": "2.9.1", "@antoniomuso/lz4-napi-openharmony-arm64": "2.9.1", "@antoniomuso/lz4-napi-win32-arm64-msvc": "2.9.1", "@antoniomuso/lz4-napi-win32-ia32-msvc": "2.9.1", "@antoniomuso/lz4-napi-win32-x64-msvc": "2.9.1" } }, "sha512-VjY1TAUQa3bo8j4vCcpQ0RBXh4mbC1OFHWH9rpOvx8qLYwaZTEzwxW/648ufD2cmz5/pFfPLDkX6N8+TqI9BYQ=="], + "lz4js": ["lz4js@0.2.0", "", {}, "sha512-gY2Ia9Lm7Ep8qMiuGRhvUq0Q7qUereeldZPP1PMEJxPtEWHJLqw9pgX68oHajBH0nzJK4MaZEA/YNV3jT8u8Bg=="], "lzma": ["lzma@2.3.2", "", { "bin": { "lzma.js": "bin/lzma.js" } }, "sha512-DcfiawQ1avYbW+hsILhF38IKAlnguc/fjHrychs9hdxe4qLykvhT5VTGNs5YRWgaNePh7NTxGD4uv4gKsRomCQ=="], @@ -1930,7 +1967,7 @@ "prompts": ["prompts@2.4.2", "", { "dependencies": { "kleur": "^3.0.3", "sisteransi": "^1.0.5" } }, "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q=="], - "property-graph": ["property-graph@3.0.0", "", {}, "sha512-TnzxUsttmGtw+OiU0LDw+0FlMbJ8vV8pOjyDI7+Kdni4Tj0hW5BFh7TatQu7Y68hcvvFmiFOHilKShsA4R82fA=="], + "property-graph": ["property-graph@4.1.0", "", {}, "sha512-AvPcP7XECNWy4LGmFQ77k7un4lSKM4eS29PTvW4ck95uYeLxXPWJM7hLuBqK91FaHqCcgJvIUCuNJjjxKE7VKQ=="], "property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], @@ -2524,16 +2561,6 @@ "@babel/preset-env/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - "@blitzkit/scripts/@gltf-transform/extensions": ["@gltf-transform/extensions@4.2.1", "", { "dependencies": { "@gltf-transform/core": "^4.2.1", "ktx-parse": "^1.0.1" } }, "sha512-ieHSJU9qvb3ucWDxgHFcff+C7fLtFPa6G0Wtc/ki7GsN6Rh7o9eoyqi7Ggj2LO4i3ynCwLLPb/onbJKw2fBtsA=="], - - "@blitzkit/website/@gltf-transform/core": ["@gltf-transform/core@4.4.2", "", { "dependencies": { "property-graph": "^4.1.0" } }, "sha512-qsWKwNSwK+2s834Mt4xbYcyHqCrgNFP7hIv5s487JxebngRfDgelpghNF+kSswGb2/NuapasfK3UViFoSJJoMg=="], - - "@blitzkit/website/@gltf-transform/functions": ["@gltf-transform/functions@4.4.2", "", { "dependencies": { "@gltf-transform/core": "^4.4.2", "@gltf-transform/extensions": "^4.4.2", "ktx-parse": "^1.1.0", "ndarray": "^1.0.19", "ndarray-lanczos": "^0.3.0", "ndarray-pixels": "^5.0.1" } }, "sha512-dclXgv9TshMaWBqPDUYd4xTwBQ2PpuR8p0Y9pokrRzGQDUPXRP6lTDzbqT0UmEmxFSvRyPJjvOWUmzeiRafpvw=="], - - "@gltf-transform/extensions/@gltf-transform/core": ["@gltf-transform/core@4.4.2", "", { "dependencies": { "property-graph": "^4.1.0" } }, "sha512-qsWKwNSwK+2s834Mt4xbYcyHqCrgNFP7hIv5s487JxebngRfDgelpghNF+kSswGb2/NuapasfK3UViFoSJJoMg=="], - - "@gltf-transform/functions/@gltf-transform/extensions": ["@gltf-transform/extensions@4.2.1", "", { "dependencies": { "@gltf-transform/core": "^4.2.1", "ktx-parse": "^1.0.1" } }, "sha512-ieHSJU9qvb3ucWDxgHFcff+C7fLtFPa6G0Wtc/ki7GsN6Rh7o9eoyqi7Ggj2LO4i3ynCwLLPb/onbJKw2fBtsA=="], - "@jimp/core/file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="], "@octokit/endpoint/@octokit/types": ["@octokit/types@14.1.0", "", { "dependencies": { "@octokit/openapi-types": "^25.1.0" } }, "sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g=="], @@ -2714,10 +2741,6 @@ "yaml-language-server/yaml": ["yaml@2.2.2", "", {}, "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA=="], - "@blitzkit/website/@gltf-transform/core/property-graph": ["property-graph@4.1.0", "", {}, "sha512-AvPcP7XECNWy4LGmFQ77k7un4lSKM4eS29PTvW4ck95uYeLxXPWJM7hLuBqK91FaHqCcgJvIUCuNJjjxKE7VKQ=="], - - "@gltf-transform/extensions/@gltf-transform/core/property-graph": ["property-graph@4.1.0", "", {}, "sha512-AvPcP7XECNWy4LGmFQ77k7un4lSKM4eS29PTvW4ck95uYeLxXPWJM7hLuBqK91FaHqCcgJvIUCuNJjjxKE7VKQ=="], - "@octokit/endpoint/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@25.1.0", "", {}, "sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA=="], "@octokit/graphql/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@25.1.0", "", {}, "sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA=="], diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 41678e3ad..ca48d6942 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -9,7 +9,7 @@ "@gltf-transform/functions": "^4.2.1", "@octokit/plugin-throttling": "^11.0.2", "@octokit/rest": "^22.0.0", - "@types/bun": "^1.3.0", + "@types/bun": "^1.3.14", "@types/decompress": "^4.2.7", "@types/lodash-es": "^4.17.12", "@types/lz4js": "^0.2.1", @@ -24,6 +24,7 @@ "fast-xml-parser": "^5.2.5", "googleapis": "^162.0.0", "lodash-es": "^4.17.21", + "lz4-napi": "^2.9.1", "progress": "^2.0.3", "puppeteer": "^24.22.3", "sharp": "^0.34.4", diff --git a/packages/scripts/src/discoverIds.ts b/packages/scripts/src/discoverIds.ts deleted file mode 100644 index 8bb59e588..000000000 --- a/packages/scripts/src/discoverIds.ts +++ /dev/null @@ -1,87 +0,0 @@ -const MAX_IDS = 100; -const API_RATE = 10; - -const ACCOMMODATED_ID_COUNT = 10_000_000_000; -const BYTES_PER_ID = 64 / 8; -const MAX_BYTES_PER_BATCH = 100 * 1_000_000; - -const regions = ["eu", "com", "asia"]; -const seeds = [5e8, 10e8, 20e8].map(BigInt); - -export interface QuickInfo { - status: string; - data: Record; -} - -const batches = BigInt( - Math.ceil((ACCOMMODATED_ID_COUNT * BYTES_PER_ID) / MAX_BYTES_PER_BATCH), -); -const ids: bigint[] = []; - -const t0 = performance.now(); -const maxT = 5 * 60 * 60 * 1000; -const t1 = t0 + maxT; - -const unfilled = Array.from({ length: seeds.length }); -const offsets = unfilled.map(() => 0n); -const queues = unfilled.map(() => [] as bigint[]); - -function mountIds() { - for (let i = 0; i < regions.length; i++) { - const queue = queues[i]; - - if (queue.length >= MAX_IDS) continue; - - const availableSpace = MAX_IDS - queue.length; - - const seed = seeds[i]; - const offset = offsets[i]++; - - for (let j = 0n; j < availableSpace; j++) { - const id = seed + offset + j; - queue.push(id); - } - - offsets[i] += BigInt(availableSpace); - } -} - -let i = 0; -while (performance.now() < t1) { - mountIds(); - await sleep(); - - const region = regions[i]; - const queue = queues[i]; - - const accountIds = queue.join(","); - const fields = "account_id%2C-account_id"; - const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID; - - const url = `https://api.wotblitz.${ - region - }/wotb/account/info/?application_id=${applicationId}&fields=${ - fields - }&account_id=${accountIds}`; - const response = await fetch(url); - const data = (await response.json()) as QuickInfo; - - if (data.status !== "ok") continue; - - for (const key in data.data) { - const value = data.data[key]; - - if (value === null) continue; - - const id = BigInt(key); - - ids.push(id); - } - - queues[i] = []; - i = (i + 1) % regions.length; -} - -function sleep() { - return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE)); -} diff --git a/packages/scripts/src/discoverIds/idArray.ts b/packages/scripts/src/discoverIds/idArray.ts new file mode 100644 index 000000000..a9800c71a --- /dev/null +++ b/packages/scripts/src/discoverIds/idArray.ts @@ -0,0 +1,5 @@ +export class IdArray { + constructor( + private array: Uint32Array = new Uint32Array(), + ) {} +} diff --git a/packages/scripts/src/discoverIds/index.ts b/packages/scripts/src/discoverIds/index.ts new file mode 100644 index 000000000..9eb6e6603 --- /dev/null +++ b/packages/scripts/src/discoverIds/index.ts @@ -0,0 +1,144 @@ +import { $ } from "bun"; +import { uncompress } from "lz4-napi"; +import { mkdir, rm } from "node:fs/promises"; +import { IdArray } from "./idArray"; + +interface ManifestV1 { + version: 1; + last_verified: number; +} + +const TEMP = "../../temp"; +const WORKING_DIR = `${TEMP}/ids`; +const CHUNKS_DIR = `${WORKING_DIR}/chunks`; +const REPO = "blitzkit/ids"; + +const ACCOMMODATED_ID_COUNT = 10_000_000_000; // 10B ids +const MAX_BYTES_PER_CHUNK = 100 * 1_000_000; // 100MB + +const chunkCount = Math.ceil( + (ACCOMMODATED_ID_COUNT * Uint32Array.BYTES_PER_ELEMENT) / MAX_BYTES_PER_CHUNK, +); + +console.log( + `${chunkCount.toLocaleString()} will be needed for ${ACCOMMODATED_ID_COUNT.toLocaleString()} ids`, +); + +await mkdir(TEMP, { recursive: true }); +await rm(WORKING_DIR, { recursive: true, force: true }); + +await $`git clone https://github.com/${REPO} ${WORKING_DIR}`; + +await mkdir(CHUNKS_DIR, { recursive: true }); + +const discoveredChunks: IdArray[] = []; + +for (let i = 0; true; i++) { + const path = `${CHUNKS_DIR}/chunk-${i}.dat.lz4`; + const file = Bun.file(path); + + if (!(await file.exists())) break; + + const bytes = await file.bytes(); + const uncompressed = await uncompress(bytes); + + const array = new Uint32Array( + uncompressed.buffer, + uncompressed.byteOffset, + uncompressed.byteLength / Uint32Array.BYTES_PER_ELEMENT, + ); + const ids = new IdArray(array); + + discoveredChunks.push(ids); +} + +let chunks: IdArray[] = []; + +if (discoveredChunks.length === chunkCount) { + chunks = discoveredChunks; +} else { + chunks = Array.from({ length: chunkCount }).map(() => new IdArray()); +} + +// const MAX_IDS = 100; +// const API_RATE = 10; + +// const BYTES_PER_ID = 64 / 8; + +// const regions = ["eu", "com", "asia"]; +// const seeds = [5e8, 10e8, 20e8].map(BigInt); + +// export interface QuickInfo { +// status: string; +// data: Record; +// } + +// const ids: bigint[] = []; + +// const t0 = performance.now(); +// const maxT = 5 * 60 * 60 * 1000; +// const t1 = t0 + maxT; + +// const unfilled = Array.from({ length: seeds.length }); +// const offsets = unfilled.map(() => 0n); +// const queues = unfilled.map(() => [] as bigint[]); + +// function mountIds() { +// for (let i = 0; i < regions.length; i++) { +// const queue = queues[i]; + +// if (queue.length >= MAX_IDS) continue; + +// const availableSpace = MAX_IDS - queue.length; + +// const seed = seeds[i]; +// const offset = offsets[i]++; + +// for (let j = 0n; j < availableSpace; j++) { +// const id = seed + offset + j; +// queue.push(id); +// } + +// offsets[i] += BigInt(availableSpace); +// } +// } + +// let i = 0; +// while (performance.now() < t1) { +// mountIds(); +// await sleep(); + +// const region = regions[i]; +// const queue = queues[i]; + +// const accountIds = queue.join(","); +// const fields = "account_id%2C-account_id"; +// const applicationId = import.meta.env.PUBLIC_WARGAMING_APPLICATION_ID; + +// const url = `https://api.wotblitz.${ +// region +// }/wotb/account/info/?application_id=${applicationId}&fields=${ +// fields +// }&account_id=${accountIds}`; +// const response = await fetch(url); +// const data = (await response.json()) as QuickInfo; + +// if (data.status !== "ok") continue; + +// for (const key in data.data) { +// const value = data.data[key]; + +// if (value === null) continue; + +// const id = BigInt(key); + +// ids.push(id); +// } + +// queues[i] = []; +// i = (i + 1) % regions.length; +// } + +// function sleep() { +// return new Promise((resolve) => setTimeout(resolve, 1000 / API_RATE)); +// }