From 5628cd4f7e2cf45e3e7e62ece597e2c90449684a Mon Sep 17 00:00:00 2001 From: jylpah Date: Sun, 10 Sep 2023 23:33:12 +0300 Subject: [PATCH 1/3] WG(): Use blitzutils.Maps() --- blitzutils1.py | 477 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 404 insertions(+), 73 deletions(-) diff --git a/blitzutils1.py b/blitzutils1.py index 5d43555..31d7264 100755 --- a/blitzutils1.py +++ b/blitzutils1.py @@ -10,7 +10,8 @@ from progress.counter import Counter from decimal import Decimal from datetime import datetime -from blitzutils.wg_api import WGApiTankopedia, WGTank +from pydantic import ValidationError +from blitzutils import WGApiTankopedia, WGTank, Maps, Map MAX_RETRIES = 3 SLEEP = 1.5 @@ -886,38 +887,357 @@ class WG: # Default data. Please use the latest maps.json - maps = { - "Random": "Random map", - "amigosville": "Falls Creek", - "asia": "Lost Temple", - "canal": "Canal", - "canyon": "Canyon", - "desert_train": "Desert Sands", - "erlenberg": "Middleburg", - "faust": "Faust", - "fort": "Macragge", - "grossberg": "Dynasty's Pearl", - "himmelsdorf": "Himmelsdorf", - "italy": "Vineyards", - "karelia": "Rockfield", - "karieri": "Copperfield", - "lake": "Mirage", - "lumber": "Alpenstadt", - "malinovka": "Winter Malinovka", - "medvedkovo": "Dead Rail", - "milbase": "Yamato Harbor", - "mountain": "Black Goldville", - "north": "North", - "ordeal": "Trial by Fire", - "pliego": "Castilla", - "port": "Port Bay", - "rock": "Mayan Ruins", - "rudniki": "Mines", - "savanna": "Oasis Palms", - "skit": "Naval Frontier", - "test": "World of Ducks", - "tutorial": "Proving Grounds", - } + MAPS_JSON: str = """ + { + "Random": { + "key": "Random", + "name": "Random map", + "id": -1, + "mode": 1 + }, + "amigosville": { + "key": "amigosville", + "name": "Falls Creek", + "id": -1, + "mode": 1 + }, + "amigosville_02": { + "key": "amigosville_02", + "name": "Falls Creek - Bridge", + "id": -1, + "mode": 3 + }, + "amigosville_03": { + "key": "amigosville_03", + "name": "Falls Creek - Factory", + "id": -1, + "mode": 3 + }, + "asia": { + "key": "asia", + "name": "Lost Temple", + "id": -1, + "mode": 1 + }, + "canal": { + "key": "canal", + "name": "Canal", + "id": -1, + "mode": 1 + }, + "canal_01": { + "key": "canal_01", + "name": "Canal - Factory", + "id": -1, + "mode": 3 + }, + "canyon": { + "key": "canyon", + "name": "Canyon", + "id": -1, + "mode": 1 + }, + "desert_train": { + "key": "desert_train", + "name": "Desert Sands", + "id": -1, + "mode": 1 + }, + "desert_train_02": { + "key": "desert_train_02", + "name": "Desert Sands - Town", + "id": -1, + "mode": 3 + }, + "desert_train_03": { + "key": "desert_train_03", + "name": "Desert Sands - Dunes", + "id": -1, + "mode": 3 + }, + "erlenberg": { + "key": "erlenberg", + "name": "Middleburg", + "id": -1, + "mode": 1 + }, + "erlenberg_01": { + "key": "erlenberg_01", + "name": "Middleburg - Town", + "id": -1, + "mode": 3 + }, + "erlenberg_02": { + "key": "erlenberg_02", + "name": "Middleburg - Hill", + "id": -1, + "mode": 3 + }, + "faust": { + "key": "faust", + "name": "Faust", + "id": -1, + "mode": 1 + }, + "forgecity": { + "key": "forgecity", + "name": "New Bay", + "id": -1, + "mode": 1 + }, + "fort": { + "key": "fort", + "name": "Fort Despair", + "id": -1, + "mode": 1 + }, + "grossberg": { + "key": "grossberg", + "name": "Dynasty's Pearl", + "id": -1, + "mode": 1 + }, + "himmelsdorf": { + "key": "himmelsdorf", + "name": "Himmelsdorf", + "id": -1, + "mode": 1 + }, + "holland": { + "key": "holland", + "name": "Molendijk", + "id": -1, + "mode": 1 + }, + "holmeisk": { + "key": "holmeisk", + "name": "Wasteland", + "id": -1, + "mode": 1 + }, + "iceworld": { + "key": "iceworld", + "name": "Everfrost", + "id": -1, + "mode": 2 + }, + "idle": { + "key": "idle", + "name": "Yukon", + "id": -1, + "mode": 1 + }, + "italy": { + "key": "italy", + "name": "Vineyards", + "id": -1, + "mode": 1 + }, + "karelia": { + "key": "karelia", + "name": "Rockfield", + "id": -1, + "mode": 1 + }, + "karieri": { + "key": "karieri", + "name": "Copperfield", + "id": -1, + "mode": 1 + }, + "lake": { + "key": "lake", + "name": "Mirage", + "id": -1, + "mode": 1 + }, + "lumber": { + "key": "lumber", + "name": "Horrorstadt", + "id": -1, + "mode": 1 + }, + "lumber_01": { + "key": "lumber_01", + "name": "Alpenstadt - Town", + "id": -1, + "mode": 3 + }, + "malinovka": { + "key": "malinovka", + "name": "Winter Malinovka", + "id": -1, + "mode": 1 + }, + "medvedkovo": { + "key": "medvedkovo", + "name": "Dead Rail", + "id": -1, + "mode": 1 + }, + "medvedkovo_02": { + "key": "medvedkovo_02", + "name": "Dead Rail - Valley", + "id": -1, + "mode": 3 + }, + "medvedkovo_03": { + "key": "medvedkovo_03", + "name": "Dead Rail - Railroad", + "id": -1, + "mode": 3 + }, + "milbase": { + "key": "milbase", + "name": "Yamato Harbor", + "id": -1, + "mode": 1 + }, + "milbase_02": { + "key": "milbase_02", + "name": "Yamato Harbor - Battleship", + "id": -1, + "mode": 3 + }, + "milbase_03": { + "key": "milbase_03", + "name": "Yamato Harbor - Hills", + "id": -1, + "mode": 3 + }, + "milbase_04": { + "key": "milbase_04", + "name": "Yamato Harbor - Center", + "id": -1, + "mode": 3 + }, + "milbase_05": { + "key": "milbase_05", + "name": "Yamato Harbor - Dock", + "id": -1, + "mode": 3 + }, + "moon": { + "key": "moon", + "name": "Sea of Tranquility", + "id": -1, + "mode": 2 + }, + "mountain": { + "key": "mountain", + "name": "Black Goldville", + "id": -1, + "mode": 1 + }, + "neptune": { + "key": "neptune", + "name": "Normandy", + "id": -1, + "mode": 1 + }, + "neptune_01": { + "key": "neptune_01", + "name": "Normandy - Hills", + "id": -1, + "mode": 3 + }, + "neptune_02": { + "key": "neptune_02", + "name": "Normandy - Beach", + "id": -1, + "mode": 3 + }, + "north": { + "key": "north", + "name": "North", + "id": -1, + "mode": 1 + }, + "ordeal": { + "key": "ordeal", + "name": "Trial by Fire", + "id": -1, + "mode": 1 + }, + "plant": { + "key": "plant", + "name": "Ghost Factory", + "id": -1, + "mode": 1 + }, + "pliego": { + "key": "pliego", + "name": "Castilla", + "id": -1, + "mode": 1 + }, + "port": { + "key": "port", + "name": "Port Bay", + "id": -1, + "mode": 1 + }, + "rift": { + "key": "rift", + "name": "Hellas", + "id": -1, + "mode": 1 + }, + "rock": { + "key": "rock", + "name": "Mayan Ruins", + "id": -1, + "mode": 1 + }, + "rudniki": { + "key": "rudniki", + "name": "Mines", + "id": -1, + "mode": 1 + }, + "rudniki_01": { + "key": "rudniki_01", + "name": "Mines - Hill", + "id": -1, + "mode": 3 + }, + "rudniki_02": { + "key": "rudniki_02", + "name": "Mines - Village", + "id": -1, + "mode": 3 + }, + "rudniki_03": { + "key": "rudniki_03", + "name": "Mines - Center", + "id": -1, + "mode": 3 + }, + "savanna": { + "key": "savanna", + "name": "Oasis Palms", + "id": -1, + "mode": 1 + }, + "skit": { + "key": "skit", + "name": "Naval Frontier", + "id": -1, + "mode": 1 + }, + "test": { + "key": "test", + "name": "World of Ducks", + "id": -1, + "mode": 2 + }, + "tutorial": { + "key": "tutorial", + "name": "Proving Grounds", + "id": -1, + "mode": 0 + } + }""" tanks: WGApiTankopedia tanks_by_tier = None @@ -979,33 +1299,44 @@ class WG: def __init__( self, - tankopedia_fn: str, + tankopedia_fn: str = "tanks.json", WG_app_id: str = _DEFAULT_WG_APP_ID, - maps_fn: str | None = None, + maps_fn: str = "maps.json", stats_cache: bool = False, rate_limit: int = 10, global_rate_limit=True, ): - with open(tankopedia_fn, mode="r") as file: + self.WG_app_id: str = WG_app_id + self.tanks: WGApiTankopedia + self.maps: Maps + self.global_rate_limit = global_rate_limit + + # read tankopedia + with open(tankopedia_fn, mode="rt") as file: try: self.tanks: WGApiTankopedia = WGApiTankopedia.parse_raw(file.read()) except Exception as err: error(f"could not parse Tankopedia: {tankopedia_fn}") - self.WG_app_id = WG_app_id - # self.load_tanks(tankopedia_fn) - WG.tanks = self.tanks - self.global_rate_limit = global_rate_limit + WG.tanks = self.tanks # WHY? + + # read maps + if (maps := Maps.parse_str(self.MAPS_JSON)) is not None: + self.maps = maps + else: + error("could not parse default map data") + sys.exit(1) + + if os.path.isfile(maps_fn): + try: + with open(maps_fn, "rt", encoding="utf8") as file: + self.maps = Maps.parse_raw(file.read()) + except ValidationError as err: + error(f"invalid maps file format: {maps_fn}") + except IOError as err: + error(f"Could not read maps file: {maps_fn}") + error(err) - if maps_fn is not None: - if os.path.exists(maps_fn) and os.path.isfile(maps_fn): - try: - with open(maps_fn, "rt", encoding="utf8") as f: - WG.maps = json.loads(f.read()) - except Exception as err: - error("Could not read maps file: " + maps_fn + "\n" + str(err)) - else: - verbose("Could not find maps file: " + maps_fn) if self.WG_app_id is not None: headers = {"Accept-Encoding": "gzip, deflate"} if self.global_rate_limit: @@ -1082,20 +1413,20 @@ class WG: return "eu" return None - @classmethod - def update_maps(cls, map_data: dict): - """Update maps data""" - cls.maps = map_data - return None + # @classmethod + # def update_maps(cls, map_data: dict): + # """Update maps data""" + # cls.maps = map_data + # return None @classmethod def get_map(self, map_str: str) -> str: """Return map name from short map string in replays""" try: - return self.maps[map_str] + return self.maps[map_str].name except: error("Map " + map_str + " not found") - return None + return "UNKNOWN MAP" # @classmethod # def get_tank_user_strs(self) -> str: @@ -1256,25 +1587,25 @@ class WG: # error("Could not read tankopedia: " + tankopedia_fn, err) # return False - def get_replay_filename(self, replay: dict): - try: - summary = replay["data"]["summary"] - timestamp = int(summary["battle_start_timestamp"]) - timestamp = get_date_str(timestamp, date_format="%Y%m%d_%H%M") - player = summary["player_name"] - arena_id = int(summary["arena_unique_id"]) - vehicle = self.name2tank_str(summary["vehicle"]) - vehicle.replace(" ", "_") + # def get_replay_filename(self, replay: dict): + # try: + # summary = replay["data"]["summary"] + # timestamp = int(summary["battle_start_timestamp"]) + # timestamp = get_date_str(timestamp, date_format="%Y%m%d_%H%M") + # player = summary["player_name"] + # arena_id = int(summary["arena_unique_id"]) + # vehicle = self.name2tank_str(summary["vehicle"]) + # vehicle.replace(" ", "_") - return "{:s}_{:s}_{:s}_{:d}.wotbreplay.json".format( - timestamp, player, vehicle, arena_id - ) - except Exception as err: - error("Unexpected Exception", err) - return None + # return "{:s}_{:s}_{:s}_{:d}.wotbreplay.json".format( + # timestamp, player, vehicle, arena_id + # ) + # except Exception as err: + # error("Unexpected Exception", err) + # return None - def get_map_user_strs(self) -> str: - return self.maps.keys() + # def get_map_user_strs(self) -> str: + # return self.maps.keys() # def tank_str2name(self, tank_str: str) -> str: # """Return tank name from short tank string in replays""" From f1a876ea4bd476be615b2d66538b1ee68372bb17 Mon Sep 17 00:00:00 2001 From: jylpah Date: Sun, 10 Sep 2023 23:33:24 +0300 Subject: [PATCH 2/3] update 10.2 --- maps.json | 61 +------------------------------------------------------ 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/maps.json b/maps.json index 046e9ac..6d0e664 100644 --- a/maps.json +++ b/maps.json @@ -1,60 +1 @@ -{ - "Random": "Random map", - "amigosville": "Falls Creek", - "amigosville_02": "Falls Creek - Bridge", - "amigosville_03": "Falls Creek - Factory", - "asia": "Lost Temple", - "canal": "Canal", - "canal_01": "Canal - Factory", - "canyon": "Canyon", - "desert_train": "Desert Sands", - "desert_train_02": "Desert Sands - Town", - "desert_train_03": "Desert Sands - Dunes", - "erlenberg": "Middleburg", - "erlenberg_01": "Middleburg - Town", - "erlenberg_02": "Middleburg - Hill", - "faust": "Faust", - "forgecity": "New Bay", - "fort": "Fort Despair", - "grossberg": "Dynasty's Pearl", - "himmelsdorf": "Himmelsdorf", - "holland": "Molendijk", - "holmeisk": "Wasteland", - "iceworld": "Everfrost", - "idle": "Yukon", - "italy": "Vineyards", - "karelia": "Rockfield", - "karieri": "Copperfield", - "lake": "Mirage", - "lumber": "Alpenstadt", - "lumber_01": "Alpenstadt - Town", - "malinovka": "Winter Malinovka", - "medvedkovo": "Dead Rail", - "medvedkovo_02": "Dead Rail - Valley", - "medvedkovo_03": "Dead Rail - Railroad", - "milbase": "Yamato Harbor", - "milbase_02": "Yamato Harbor - Battleship", - "milbase_03": "Yamato Harbor - Hills", - "milbase_04": "Yamato Harbor - Center", - "milbase_05": "Yamato Harbor - Dock", - "moon": "Sea of Tranquility", - "mountain": "Black Goldville", - "neptune": "Normandy", - "neptune_01": "Normandy - Hills", - "neptune_02": "Normandy - Beach", - "north": "North", - "ordeal": "Trial by Fire", - "plant": "Ghost Factory", - "pliego": "Castilla", - "port": "Port Bay", - "rift": "Hellas", - "rock": "Mayan Ruins", - "rudniki": "Mines", - "rudniki_01": "Mines - Hill", - "rudniki_02": "Mines - Village", - "rudniki_03": "Mines - Center", - "savanna": "Oasis Palms", - "skit": "Naval Frontier", - "test": "World of Ducks", - "tutorial": "Proving Grounds" -} +{"Random": {"key": "Random", "name": "Random map", "id": -1, "mode": 1}, "amigosville": {"key": "amigosville", "name": "Falls Creek", "id": -1, "mode": 1}, "amigosville_02": {"key": "amigosville_02", "name": "Falls Creek - Bridge", "id": -1, "mode": 3}, "amigosville_03": {"key": "amigosville_03", "name": "Falls Creek - Factory", "id": -1, "mode": 3}, "asia": {"key": "asia", "name": "Lost Temple", "id": -1, "mode": 1}, "canal": {"key": "canal", "name": "Canal", "id": -1, "mode": 1}, "canal_01": {"key": "canal_01", "name": "Canal - Factory", "id": -1, "mode": 3}, "canyon": {"key": "canyon", "name": "Canyon", "id": -1, "mode": 1}, "desert_train": {"key": "desert_train", "name": "Desert Sands", "id": -1, "mode": 1}, "desert_train_02": {"key": "desert_train_02", "name": "Desert Sands - Town", "id": -1, "mode": 3}, "desert_train_03": {"key": "desert_train_03", "name": "Desert Sands - Dunes", "id": -1, "mode": 3}, "erlenberg": {"key": "erlenberg", "name": "Middleburg", "id": -1, "mode": 1}, "erlenberg_01": {"key": "erlenberg_01", "name": "Middleburg - Town", "id": -1, "mode": 3}, "erlenberg_02": {"key": "erlenberg_02", "name": "Middleburg - Hill", "id": -1, "mode": 3}, "faust": {"key": "faust", "name": "Faust", "id": -1, "mode": 1}, "forgecity": {"key": "forgecity", "name": "New Bay", "id": -1, "mode": 1}, "fort": {"key": "fort", "name": "Fort Despair", "id": -1, "mode": 1}, "grossberg": {"key": "grossberg", "name": "Dynasty's Pearl", "id": -1, "mode": 1}, "himmelsdorf": {"key": "himmelsdorf", "name": "Himmelsdorf", "id": -1, "mode": 1}, "holland": {"key": "holland", "name": "Molendijk", "id": -1, "mode": 1}, "holmeisk": {"key": "holmeisk", "name": "Wasteland", "id": -1, "mode": 1}, "iceworld": {"key": "iceworld", "name": "Everfrost", "id": -1, "mode": 2}, "idle": {"key": "idle", "name": "Yukon", "id": -1, "mode": 1}, "italy": {"key": "italy", "name": "Vineyards", "id": -1, "mode": 1}, "karelia": {"key": "karelia", "name": "Rockfield", "id": -1, "mode": 1}, "karieri": {"key": "karieri", "name": "Copperfield", "id": -1, "mode": 1}, "lake": {"key": "lake", "name": "Mirage", "id": -1, "mode": 1}, "lumber": {"key": "lumber", "name": "Horrorstadt", "id": -1, "mode": 1}, "lumber_01": {"key": "lumber_01", "name": "Alpenstadt - Town", "id": -1, "mode": 3}, "malinovka": {"key": "malinovka", "name": "Winter Malinovka", "id": -1, "mode": 1}, "medvedkovo": {"key": "medvedkovo", "name": "Dead Rail", "id": -1, "mode": 1}, "medvedkovo_02": {"key": "medvedkovo_02", "name": "Dead Rail - Valley", "id": -1, "mode": 3}, "medvedkovo_03": {"key": "medvedkovo_03", "name": "Dead Rail - Railroad", "id": -1, "mode": 3}, "milbase": {"key": "milbase", "name": "Yamato Harbor", "id": -1, "mode": 1}, "milbase_02": {"key": "milbase_02", "name": "Yamato Harbor - Battleship", "id": -1, "mode": 3}, "milbase_03": {"key": "milbase_03", "name": "Yamato Harbor - Hills", "id": -1, "mode": 3}, "milbase_04": {"key": "milbase_04", "name": "Yamato Harbor - Center", "id": -1, "mode": 3}, "milbase_05": {"key": "milbase_05", "name": "Yamato Harbor - Dock", "id": -1, "mode": 3}, "moon": {"key": "moon", "name": "Sea of Tranquility", "id": -1, "mode": 2}, "mountain": {"key": "mountain", "name": "Black Goldville", "id": -1, "mode": 1}, "neptune": {"key": "neptune", "name": "Normandy", "id": -1, "mode": 1}, "neptune_01": {"key": "neptune_01", "name": "Normandy - Hills", "id": -1, "mode": 3}, "neptune_02": {"key": "neptune_02", "name": "Normandy - Beach", "id": -1, "mode": 3}, "north": {"key": "north", "name": "North", "id": -1, "mode": 1}, "ordeal": {"key": "ordeal", "name": "Trial by Fire", "id": -1, "mode": 1}, "plant": {"key": "plant", "name": "Ghost Factory", "id": -1, "mode": 1}, "pliego": {"key": "pliego", "name": "Castilla", "id": -1, "mode": 1}, "port": {"key": "port", "name": "Port Bay", "id": -1, "mode": 1}, "rift": {"key": "rift", "name": "Hellas", "id": -1, "mode": 1}, "rock": {"key": "rock", "name": "Mayan Ruins", "id": -1, "mode": 1}, "rudniki": {"key": "rudniki", "name": "Mines", "id": -1, "mode": 1}, "rudniki_01": {"key": "rudniki_01", "name": "Mines - Hill", "id": -1, "mode": 3}, "rudniki_02": {"key": "rudniki_02", "name": "Mines - Village", "id": -1, "mode": 3}, "rudniki_03": {"key": "rudniki_03", "name": "Mines - Center", "id": -1, "mode": 3}, "savanna": {"key": "savanna", "name": "Oasis Palms", "id": -1, "mode": 1}, "skit": {"key": "skit", "name": "Naval Frontier", "id": -1, "mode": 1}, "test": {"key": "test", "name": "World of Ducks", "id": -1, "mode": 2}, "tutorial": {"key": "tutorial", "name": "Proving Grounds", "id": -1, "mode": 0}} \ No newline at end of file From de85e95f96fa3b8385f4d62f4f53b0aa00162620 Mon Sep 17 00:00:00 2001 From: jylpah Date: Sun, 10 Sep 2023 23:33:55 +0300 Subject: [PATCH 3/3] Fix replay search with deep dir trees --- analyze_wotb_replays.py | 122 ++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 68 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 5033771..a669f27 100755 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -5,11 +5,16 @@ import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole import configparser, ssl, re, logging, time, xmltodict, collections, csv +from asyncio import Queue, Task, create_task from datetime import datetime, timedelta, date -from motor.motor_asyncio import AsyncIOMotorDatabase, AsyncIOMotorClient +from motor.motor_asyncio import ( + AsyncIOMotorDatabase, + AsyncIOMotorClient, +) import blitzutils1 as bu from blitzutils1 import WG from blitzutils1 import WoTinspector +from pyutils import FileQueue logging.getLogger("asyncio").setLevel(logging.DEBUG) @@ -2006,7 +2011,7 @@ async def main(argv): # try: BattleCategory.set_fields(args.mode) - replayQ = asyncio.Queue(maxsize=1000) + replayQ = Queue(maxsize=1000) reader_tasks = [] # Make replay Queue @@ -2072,8 +2077,8 @@ async def main(argv): if args.csv: await export_csv(args, blt_cat_list, histograms) - if args.replays: - await export_replays(replays) + # if args.replays: + # await export_replays(replays) bu.debug("Finished. Cleaning up..................") # except Exception as err: @@ -2146,8 +2151,8 @@ async def export_json( async def help_extended( - db: AsyncIOMotorDatabase = None, - parser: argparse.ArgumentParser = None, + db: AsyncIOMotorDatabase | None = None, + parser: argparse.ArgumentParser | None = None, ): """Help for --filters""" if parser is not None: @@ -2208,26 +2213,26 @@ async def help_extended( bu.error(exception=err) -async def export_replays(replays: dict): - """Export replays""" - global wg - try: - replay_dir = OPT_EXPORT_REPLAY_DIR + "_" + bu.get_date_str() - bu.verbose_std("Exporting replays to " + replay_dir) - if not os.path.isdir(replay_dir): - os.mkdir(replay_dir) +# async def export_replays(replays: dict): +# """Export replays""" +# global wg +# try: +# replay_dir = OPT_EXPORT_REPLAY_DIR + "_" + bu.get_date_str() +# bu.verbose_std("Exporting replays to " + replay_dir) +# if not os.path.isdir(replay_dir): +# os.mkdir(replay_dir) - i = 0 - for replay in replays.values(): - filename = wg.get_replay_filename(replay) - async with aiofiles.open( - os.path.join(replay_dir, filename), "w", encoding="utf8" - ) as f: - await f.write(json.dumps(replay, indent=4)) - i += 1 - bu.verbose_std(str(i) + " replays exported") - except Exception as err: - bu.error(exception=err) +# i = 0 +# for replay in replays.values(): +# filename = wg.get_replay_filename(replay) +# async with aiofiles.open( +# os.path.join(replay_dir, filename), "w", encoding="utf8" +# ) as f: +# await f.write(json.dumps(replay, indent=4)) +# i += 1 +# bu.verbose_std(str(i) + " replays exported") +# except Exception as err: +# bu.error(exception=err) ## move the class PlayerHistogram? @@ -2426,7 +2431,7 @@ async def process_player_stats( """Start stat_workers to retrieve and store players' stats""" ## Create queue of player-tank pairs to find stats for try: - statsQ = asyncio.Queue() + statsQ = Queue() bu.debug("Create player stats queue: " + str(len(players)) + " players") stat_id_map = dict() stat_ids = set() @@ -2569,7 +2574,7 @@ def calc_team_stats( async def stat_worker( - queue: asyncio.Queue, + queue: Queue, workerID: int, args: argparse.Namespace, db: AsyncIOMotorDatabase, @@ -2968,14 +2973,15 @@ async def player_stats_helper(player_stats: dict): async def mk_replayQ( - queue: asyncio.Queue, + replayQ: Queue, args: argparse.Namespace, - db: AsyncIOMotorDatabase = None, + db: AsyncIOMotorDatabase | None = None, ): """Create queue of replays to post""" p_replayfile = re.compile(".*\\.wotbreplay\\.json$") - files = args.files + files: list[str] = args.files Nreplays = 0 + if files[0] == "db:": if db is None: bu.error("No database connection opened") @@ -2995,7 +3001,7 @@ async def mk_replayQ( async for replay_json in cursor: _id = replay_json["_id"] # del(replay_json['_id']) - await queue.put(await mk_readerQ_item(replay_json, _id=_id)) + await replayQ.put(await mk_readerQ_item(replay_json, _id=_id)) Nreplays += 1 bu.debug("All the matching replays have been read from the DB") except Exception as err: @@ -3004,53 +3010,33 @@ async def mk_replayQ( elif files[0] == "-": bu.debug("reading replay file list from STDIN") stdin, _ = await aioconsole.get_standard_streams() - while True: + while (line := (await stdin.readline()).decode("utf-8").rstrip()) is not None: try: - line = (await stdin.readline()).decode("utf-8").rstrip() - if not line: - break - if p_replayfile.match(line) is not None: replay_json = await bu.open_JSON(line, wi.chk_JSON_replay) - await queue.put(await mk_readerQ_item(replay_json, filename=line)) + await replayQ.put(await mk_readerQ_item(replay_json, filename=line)) except Exception as err: bu.error(exception=err) else: - for fn in files: + jsonQ = FileQueue(maxsize=500, filter="*.wotbreplay.json") + scanner: Task = create_task(jsonQ.mk_queue(files=files)) + async for fn in jsonQ: try: - # bu.debug('Filename: ' + fn) - if fn.endswith('"'): - fn = fn[:-1] - if os.path.isfile(fn) and (p_replayfile.match(fn) is not None): - replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay) - await queue.put(await mk_readerQ_item(replay_json, filename=fn)) - bu.debug("File added to queue: " + fn) - Nreplays += 1 - elif os.path.isdir(fn): - with os.scandir(fn) as dirEntry: - for entry in dirEntry: - if entry.is_file() and ( - p_replayfile.match(entry.name) is not None - ): - bu.debug(entry.name) - replay_json = await bu.open_JSON( - entry.path, wi.chk_JSON_replay - ) - await queue.put( - await mk_readerQ_item( - replay_json, filename=entry.name - ) - ) - bu.debug("File added to queue: " + entry.path) - Nreplays += 1 + replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay) + await replayQ.put(await mk_readerQ_item(replay_json, filename=fn)) + bu.debug("File added to queue: " + fn) + Nreplays += 1 except Exception as err: bu.error(exception=err) + await asyncio.wait([scanner]) return Nreplays -async def mk_readerQ_item(replay_json, filename: str = None, _id: str = None) -> list: +async def mk_readerQ_item( + replay_json, filename: str | None = None, _id: str | None = None +) -> list | None: """Make an item to replay queue""" - global REPLAY_N + global REPLAY_N, wi REPLAY_N += 1 try: if wi.chk_JSON_replay(replay_json): @@ -3071,7 +3057,7 @@ async def mk_readerQ_item(replay_json, filename: str = None, _id: str = None) -> return None -async def replay_reader(queue: asyncio.Queue, readerID: int, args: argparse.Namespace): +async def replay_reader(queue: Queue, readerID: int, args: argparse.Namespace): """Async Worker to process the replay queue""" # global SKIPPED_N global wi @@ -3187,7 +3173,7 @@ async def read_replay_JSON(replay_json: dict, args: argparse.Namespace) -> dict: btl_tier = 0 protagonist_tank = None for player in replay_summary["details"]: - tank_id : int = player["vehicle_descr"] + tank_id: int = player["vehicle_descr"] btl_duration = max(btl_duration, player["time_alive"]) player_tank_tier = wg.get_tank_tier(tank_id) btl_tier = max(btl_tier, player_tank_tier) @@ -3195,7 +3181,7 @@ async def read_replay_JSON(replay_json: dict, args: argparse.Namespace) -> dict: if player["dbid"] == account_id: # player itself is not put in results['allies'] tmp = dict() - tmp["account_id"] = account_id + tmp["account_id"] = account_id tmp["tank_id"] = tank_id tmp["tank_tier"] = player_tank_tier tmp["tank_name"] = wg.get_tank_name(tank_id)