From 8cce091fa8b4afbaaf0325325d752f8de6353ce4 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sun, 9 Feb 2020 23:19:55 +0200 Subject: [PATCH 01/18] First version of DB support (not tested) --- analyze_wotb_replays.py | 86 ++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 9296bc5..328a975 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -15,6 +15,7 @@ WG_APP_ID = '81381d3f45fa4aa75b78a7198eb216ad' FILE_CONFIG = 'blitzstats.ini' DB_C_TANK_STATS = 'WG_TankStats' DB_C_PLAYER_STATS = 'WG_PlayerStats' +DB_C_REPLAYS = 'Replays' wg = None wi = None @@ -384,10 +385,11 @@ async def main(argv): parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"') parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT') parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') + parser.add_argument('--filters', type=str, default=None, help='Filters for DB based analyses. MongoDB find() filter JSON format.') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') - parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"') + parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files/dirs to read. Use \'-\' for STDIN, "db:" for database') args = parser.parse_args() bu.set_log_level(args.silent, args.verbose, args.debug) @@ -418,15 +420,16 @@ async def main(argv): bu.error("Could no initiate DB connection: Disabling DB", err) args.db = False pass - else: + + if not(args.db): bu.debug('No DB in use') try: replayQ = asyncio.Queue(maxsize=1000) reader_tasks = [] # Make replay Queue - - scanner_task = asyncio.create_task(mk_replayQ(replayQ, args.files)) + + scanner_task = asyncio.create_task(mk_replayQ(replayQ, args.files, db)) bu.debug('Replay scanner started') # Start tasks to process the Queue for i in range(TASK_N): @@ -852,43 +855,65 @@ async def player_stats_helper(player_stats: dict): return None -async def mk_replayQ(queue : asyncio.Queue, files : list): +async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase = None): """Create queue of replays to post""" p_replayfile = re.compile('.*\\.wotbreplay\\.json$') - if files[0] == '-': + files = args.files + if files[0] == 'db:': + if db == None: + bu.error('No database connection opened') + sys.exit(1) + try: + dbc = db[DB_C_REPLAYS] + filters = args.filters + cursor = dbc.find(filters) + async for replay_json in cursor: + await mk_readerQ_item(replay_json) + except Exception as err: + bu.error(exception=err) + + elif files[0] == '-': bu.debug('reading replay file list from STDIN') stdin, _ = await aioconsole.get_standard_streams() while True: - line = (await stdin.readline()).decode('utf-8').rstrip() - if not line: - break - else: + try: + line = (await stdin.readline()).decode('utf-8').rstrip() + if not line: break + if (p_replayfile.match(line) != None): - await queue.put(await mk_readerQ_item(line)) + replay_json = await bu.open_JSON(line, wi.chk_JSON_replay) + await queue.put(await mk_readerQ_item(replay_json)) + except Exception as err: + bu.error(exception=err) else: for fn in files: - # bu.debug('Filename: ' + fn) - if fn.endswith('"'): - fn = fn[:-1] - if os.path.isfile(fn) and (p_replayfile.match(fn) != None): - await queue.put(await mk_readerQ_item(fn)) - bu.debug('File added to queue: ' + fn) - elif os.path.isdir(fn): - with os.scandir(fn) as dirEntry: - for entry in dirEntry: - if entry.is_file() and (p_replayfile.match(entry.name) != None): - bu.debug(entry.name) - await queue.put(await mk_readerQ_item(entry.path)) - bu.debug('File added to queue: ' + entry.path) + try: + # bu.debug('Filename: ' + fn) + if fn.endswith('"'): + fn = fn[:-1] + if os.path.isfile(fn) and (p_replayfile.match(fn) != None): + replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay) + await queue.put(await mk_readerQ_item(fn)) + bu.debug('File added to queue: ' + fn) + elif os.path.isdir(fn): + with os.scandir(fn) as dirEntry: + for entry in dirEntry: + if entry.is_file() and (p_replayfile.match(entry.name) != None): + bu.debug(entry.name) + replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay) + await queue.put(await mk_readerQ_item(replay_json)) + bu.debug('File added to queue: ' + entry.path) + except Exception as err: + bu.error(exception=err) bu.debug('Finished: ' + str(queue.qsize()) + ' replays to process') return None -async def mk_readerQ_item(filename : str) -> list: +async def mk_readerQ_item(replay_json : dict) -> list: """Make an item to replay queue""" global REPLAY_N REPLAY_N +=1 - return [filename, REPLAY_N] + return [replay_json, REPLAY_N] async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace): @@ -900,23 +925,22 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam try: while True: item = await queue.get() - filename = item[0] + replay_json = item[0] replayID = item[1] try: - replay_json = await bu.open_JSON(filename, wi.chk_JSON_replay) if replay_json == None: - bu.verbose('Replay[' + str(replayID) + ']: ' + filename + ' is invalid. Skipping.' ) + bu.verbose('Replay[' + str(replayID) + ']: is empty. Skipping.' ) #SKIPPED_N += 1 queue.task_done() continue ## Read the replay JSON - bu.debug('reading replay: ' + filename, readerID) + bu.debug('reading replay', readerID) result = await read_replay_JSON(replay_json, args) bu.print_progress() if result == None: - bu.error('Replay[' + str(replayID) + ']: Error reading: ' + filename, id=readerID) + bu.error('Replay[' + str(replayID) + ']: Invalid replay', id=readerID) queue.task_done() continue From 4259f967effb06521f07ee9fa8bf0e881a6a0cb0 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 11 Feb 2020 21:22:18 +0200 Subject: [PATCH 02/18] Removing database support. It breaks the replay fetching logic. --- upload_wotb_replays.py | 110 +++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py index ec41ee2..088cfa0 100644 --- a/upload_wotb_replays.py +++ b/upload_wotb_replays.py @@ -9,7 +9,7 @@ from blitzutils import WoTinspector logging.getLogger("asyncio").setLevel(logging.DEBUG) FILE_CONFIG = 'blitzstats.ini' -DB_C_REPLAYS = 'Replays' +# DB_C_REPLAYS = 'Replays' DEBUG = False VERBOSE = False @@ -31,29 +31,29 @@ async def main(argv): ## Read config config = configparser.ConfigParser() config.read(FILE_CONFIG) - configDB = config['DATABASE'] - DB_SERVER = configDB.get('db_server', 'localhost') - DB_PORT = configDB.getint('db_port', 27017) - DB_SSL = configDB.getboolean('db_ssl', False) - DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) - DB_AUTH = configDB.get('db_auth', 'admin') - DB_NAME = configDB.get('db_name', 'BlitzStats') - DB_USER = configDB.get('db_user', 'mongouser') - DB_PASSWD = configDB.get('db_password', "PASSWORD") - DB_CERT = configDB.get('db_ssl_cert_file', None) - DB_CA = configDB.get('db_ssl_ca_file', None) + # configDB = config['DATABASE'] + # DB_SERVER = configDB.get('db_server', 'localhost') + # DB_PORT = configDB.getint('db_port', 27017) + # DB_SSL = configDB.getboolean('db_ssl', False) + # DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) + # DB_AUTH = configDB.get('db_auth', 'admin') + # DB_NAME = configDB.get('db_name', 'BlitzStats') + # DB_USER = configDB.get('db_user', 'mongouser') + # DB_PASSWD = configDB.get('db_password', "PASSWORD") + # DB_CERT = configDB.get('db_ssl_cert_file', None) + # DB_CA = configDB.get('db_ssl_ca_file', None) - bu.debug('DB_SERVER: ' + DB_SERVER) - bu.debug('DB_PORT: ' + str(DB_PORT)) - bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") - bu.debug('DB_AUTH: ' + DB_AUTH) - bu.debug('DB_NAME: ' + DB_NAME) + # bu.debug('DB_SERVER: ' + DB_SERVER) + # bu.debug('DB_PORT: ' + str(DB_PORT)) + # bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") + # bu.debug('DB_AUTH: ' + DB_AUTH) + # bu.debug('DB_NAME: ' + DB_NAME) configGeneral = config['GENERAL'] # WG account id of the uploader: # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ WG_ID = configGeneral.getint('wg_id', 0) - USE_DB = configGeneral.getint('use_DB', False) +# USE_DB = configGeneral.getint('use_DB', False) parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data') #parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database') @@ -64,7 +64,7 @@ async def main(argv): parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') - parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') +# parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') @@ -78,19 +78,19 @@ async def main(argv): wi = WoTinspector() #### Connect to MongoDB (TBD) - client = None - db = None - if args.db: - try: - client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) - db = client[DB_NAME] - bu.debug('Database connection initiated') - except Exception as err: - bu.error("Could no initiate DB connection: Disabling DB", err) - args.db = False - pass - else: - bu.debug('No DB in use') + # client = None + # db = None + # if args.db: + # try: + # client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) + # db = client[DB_NAME] + # bu.debug('Database connection initiated') + # except Exception as err: + # bu.error("Could no initiate DB connection: Disabling DB", err) + # args.db = False + # pass + # else: + # bu.debug('No DB in use') if args.account != None: args.accountID = await wg.get_account_id(args.account) @@ -107,7 +107,8 @@ async def main(argv): tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) # Start tasks to process the Queue for i in range(args.N_tasks): - tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private))) + #tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private))) + tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private))) bu.debug('Task ' + str(i) + ' started') bu.debug('Waiting for the replay scanner to finish') @@ -175,7 +176,8 @@ async def mkQueueItem(filename : str, title : str) -> list: REPLAY_N +=1 return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)] -async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False): +#async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False): +async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False): """Async Worker to process the replay queue""" global SKIPPED_N global ERROR_N @@ -199,7 +201,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto if wi.chk_JSON_replay(replay_json): bu.verbose(msg_str + title + ' has already been posted. Skipping.' ) SKIPPED_N += 1 - await saveReplay2DB(db, replay_json) + #await saveReplay2DB(db, replay_json) queue.task_done() continue else: @@ -221,7 +223,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto else: bu.error(msg_str + 'Error saving replay: ' + filename) ERROR_N += 1 - await saveReplay2DB(db, json_resp) + # await saveReplay2DB(db, json_resp) except Exception as err: bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.debug(msg_str + 'Marking task done') @@ -230,24 +232,24 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto return None -async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool: - """Save retriever replay JSON into DB (mongo)""" - try: - if db == None: - return False - dbc = db[DB_C_REPLAYS] - replay_link = replay['data']['download_url'] - replay_id = wi.get_replay_id(replay_link) - replay['_id'] = replay_id - await dbc.insert_one(replay) - bu.debug('Replay added to database') - except pymongo.errors.DuplicateKeyError as err: - bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) - return False - except Exception as err: - bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) - return False - return True +# async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool: +# """Save retriever replay JSON into DB (mongo)""" +# try: +# if db == None: +# return False +# dbc = db[DB_C_REPLAYS] +# replay_link = replay['data']['download_url'] +# replay_id = wi.get_replay_id(replay_link) +# replay['_id'] = replay_id +# await dbc.insert_one(replay) +# bu.debug('Replay added to database') +# except pymongo.errors.DuplicateKeyError as err: +# bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) +# return False +# except Exception as err: +# bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) +# return False +# return True def getTitle(replayfile: str, title: str, i : int) -> str: From 7e7f29b48f8669cd8f81a5123546e7cfdd746456 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 00:20:54 +0200 Subject: [PATCH 03/18] Added handling of Chinese account_ids Chinese server account_ids >= 3e9 --- blitzutils.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index 620cadb..932e03c 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -261,6 +261,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = ## To avoid excessive use of servers for retry in range(1,max_tries+1): try: + await asyncio.sleep(SLEEP) async with session.get(url) as resp: if resp.status == 200: debug('HTTP request OK') @@ -272,7 +273,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = if retry == max_tries: break debug('Retrying URL [' + str(retry) + '/' + str(max_tries) + ']: ' + url ) - await asyncio.sleep(SLEEP) + except aiohttp.ClientError as err: debug("Could not retrieve URL: " + url) @@ -541,8 +542,11 @@ class WG: @classmethod def get_server(cls, account_id: int) -> str: """Get Realm/server of an account based on account ID""" - if account_id > 1e9: - if account_id > 2e9: + if account_id >= 1e9: + if account_id >= 3e9: + debug('Chinese account/server: not stats available') + return None + if account_id >= 2e9: return 'asia' return 'na' else: @@ -736,7 +740,9 @@ class WG: def get_url_clan_info(self, server: str, clan_id: int) -> str: try: - return self.URL_WG_SERVER[server] + self.URL_WG_CLAN_INFO + self.WG_app_id + '&clan_id=' + str(clan_id) + if server == None: + return None + return self.URL_WG_SERVER[server] + self.URL_WG_CLAN_INFO + self.WG_app_id + '&clan_id=' + str(clan_id) except Exception as err: if (server == None) or (server.lower() not in WG.ACCOUNT_ID_SERVER.keys()): error('No server name or invalid server name given: ' + server if (server != None) else '') @@ -751,7 +757,8 @@ class WG: def get_url_player_tanks_stats(self, account_id: int, tank_ids = [], fields = []) -> str: server = self.get_server(account_id) - + if server == None: + return None if (tank_ids != None) and (len(tank_ids) > 0): tank_id_str= '&tank_id=' + '%2C'.join([ str(x) for x in tank_ids]) else: @@ -770,6 +777,8 @@ class WG: def get_url_player_stats(self, account_id, fields) -> str: try: server = self.get_server(account_id) + if server == None: + return None if (fields != None) and (len(fields) > 0): field_str = '&fields=' + '%2C'.join(fields) else: From 8f8e41f2dd86bc77090d5400819376fa1cb79701 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 00:21:36 +0200 Subject: [PATCH 04/18] Support for analysis of replays in DB --- analyze_wotb_replays.py | 82 ++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 328a975..50c9f2d 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3.7 +#!/usr/bin/env python3.8 # Script Analyze WoT Blitz replays @@ -11,12 +11,15 @@ from blitzutils import WoTinspector logging.getLogger("asyncio").setLevel(logging.DEBUG) -WG_APP_ID = '81381d3f45fa4aa75b78a7198eb216ad' +WG_APP_ID = 'df25c506c7944e48fbec67e8b74fd34e' FILE_CONFIG = 'blitzstats.ini' DB_C_TANK_STATS = 'WG_TankStats' DB_C_PLAYER_STATS = 'WG_PlayerStats' DB_C_REPLAYS = 'Replays' +N_PLAYERS = 'Nplayers' +MISSING_STATS = 'missing_stats' + wg = None wi = None REPLAY_N = 0 @@ -110,6 +113,7 @@ result_fields = { 'survived' : [ 'Surv%', 'Survival rate', 6, '{:6.1%}' ], 'time_alive%' : [ 'T alive%', 'Percentage of time being alive in a battle', 8, '{:8.0%}' ], 'top_tier' : [ 'Top tier', 'Share of games as top tier', 8, '{:8.0%}' ], + MISSING_STATS : [ 'No stats', 'Players without stats avail', 8, '{:8.1%}'], 'allies_wins' : [ 'Allies WR', 'Average WR of allies at the tier of their tank', 9, '{:9.2%}' ], 'enemies_wins' : [ 'Enemies WR', 'Average WR of enemies at the tier of their tank', 10, '{:10.2%}' ], 'allies_battles' : [ 'Allies Btls', 'Average number battles of the allies', 11, '{:11.0f}' ], @@ -187,12 +191,15 @@ class PlayerHistogram(): class BattleRecord(): def __init__(self): try: - self.battles = 0 + self.battles = 0 + self.missing_stats = 0 + self.n_players = 0 self.ratios_ready = False self.results_ready = False self.results = collections.defaultdict(def_value_zero) self.avg_fields = set(result_fields.keys()) self.avg_fields.remove('battles') + self.avg_fields.remove(MISSING_STATS) self.avg_fields.difference_update(result_ratios.keys()) self.ratio_fields = set() for ratio in result_ratios.keys(): @@ -205,12 +212,15 @@ class BattleRecord(): def record_result(self, result : dict) -> bool: try: for field in self.avg_fields | self.ratio_fields: - if result[field] != None: + if (field in result) and (result[field] != None): self.results[field] += result[field] self.battles += 1 + self.n_players += result[N_PLAYERS] + self.missing_stats += result[MISSING_STATS] return True except KeyError as err: bu.error('Key not found', err) + bu.error(str(result)) return False def calc_ratios(self) -> bool: @@ -239,6 +249,7 @@ class BattleRecord(): for field in self.avg_fields: self.results[field] = self.results[field] / max(self.battles,1) self.results['battles'] = self.battles + self.results[MISSING_STATS] = self.missing_stats / self.n_players self.results_ready = True return True except KeyError as err: @@ -283,6 +294,7 @@ class BattleRecordCategory(): def record_result(self, result: dict): try: + cat = None if self.type == 'total': cat = 'Total' elif self.type == 'number': @@ -294,7 +306,9 @@ class BattleRecordCategory(): self.category[cat].record_result(result) return True except KeyError as err: - bu.error('Key not found', err) + bu.error('Key not found', err) + bu.error('Category: ', str(cat)) + bu.error(str(result)) except Exception as err: bu.error(str(err)) return False @@ -415,6 +429,7 @@ async def main(argv): try: client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) db = client[DB_NAME] + args.account_id = None bu.debug('Database connection initiated') except Exception as err: bu.error("Could no initiate DB connection: Disabling DB", err) @@ -429,7 +444,7 @@ async def main(argv): reader_tasks = [] # Make replay Queue - scanner_task = asyncio.create_task(mk_replayQ(replayQ, args.files, db)) + scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db)) bu.debug('Replay scanner started') # Start tasks to process the Queue for i in range(TASK_N): @@ -578,9 +593,9 @@ async def process_player_stats(players, N_workers: int, args : argparse.Namespac def remap_stat_id(stat_id_map: dict, stat_id_remap: dict): bu.debug('Remapping stat_ids: stat_id_map: ' + str(len(stat_id_map)) + ' remap needed: ' + str(len(stat_id_remap))) for stat_id in stat_id_map: - # bu.debug(stat_id) - if stat_id_map[stat_id] in stat_id_remap: - stat_id_map[stat_id] = stat_id_remap[stat_id_map[stat_id]] + # bu.debug(stat_id) + if stat_id_map[stat_id] in stat_id_remap.keys(): + stat_id_map[stat_id] = stat_id_remap[stat_id_map[stat_id]] return stat_id_map @@ -589,18 +604,29 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg return_list = [] - stat_types = player_stats[list(player_stats.keys())[0]].keys() + ## Bug here?? + #stat_types = player_stats[list(player_stats.keys())[0]].keys() + stat_types = list() + p_team_fields = re.compile('^allies_(.+)$') + for field in result_fields: + m = p_team_fields.match(field) + if m != None: + stat_types.append(m.group(1)) + bu.debug('stat_types: ' + ','.join(stat_types)) for result in results: try: + missing_stats = 0 + n_players = len(result['allies']) + len(result['enemies']) n_allies = collections.defaultdict(def_value_zero) allies_stats = collections.defaultdict(def_value_zero) #bu.debug('Processing Allies') for ally in result['allies']: # Player itself is not in 'allies': see read_replay_JSON() ally_mapped = stat_id_map[ally] - if ally_mapped not in player_stats: + if ally_mapped not in player_stats.keys(): + missing_stats += 1 continue for stat in stat_types: if player_stats[ally_mapped][stat] != None: @@ -611,13 +637,17 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg for stat in stat_types: if n_allies[stat] > 0: result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat] + else: + bu.debug('No allies stats for: ' + str(result)) #bu.debug('Processing Enemies') n_enemies = collections.defaultdict(def_value_zero) enemies_stats = collections.defaultdict(def_value_zero) for enemy in result['enemies']: enemy_mapped = stat_id_map[enemy] - if enemy_mapped not in player_stats: continue + if enemy_mapped not in player_stats: + missing_stats += 1 + continue for stat in stat_types: if player_stats[enemy_mapped][stat] != None: enemies_stats[stat] += player_stats[enemy_mapped][stat] @@ -627,7 +657,10 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg for stat in stat_types: if n_enemies[stat] > 0: result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat] - + else: + bu.debug('No allies stats for: ' + str(result)) + result[N_PLAYERS] = n_players + result[MISSING_STATS] = missing_stats return_list.append(result) except KeyError as err: bu.error('Key not found', err) @@ -658,6 +691,7 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name stats[stat_id] = await stat_db_func(db, stat_id) bu.debug('get_db_' + args.stat_func + '_stats returned: ' + str(stats[stat_id]), workerID) + # no DB stats found, trying WG if (stats[stat_id] == None): pruned_stat_id = prune_stat_id(stat_id) if (pruned_stat_id not in stats): @@ -683,7 +717,7 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name if stats[key] == None: keys_2_del.append(key) for key in keys_2_del: - del stats[key] + del stats[key] except KeyError as err: bu.error('Error in pruning empty stats', err) # bu.debug('Returning stats & exiting') @@ -859,16 +893,27 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto """Create queue of replays to post""" p_replayfile = re.compile('.*\\.wotbreplay\\.json$') files = args.files + if files[0] == 'db:': if db == None: bu.error('No database connection opened') sys.exit(1) try: dbc = db[DB_C_REPLAYS] - filters = args.filters - cursor = dbc.find(filters) + cursor = None + if args.filters != None: + bu.debug(str(args.filters)) + filters = json.loads(args.filters) + bu.debug(json.dumps(filters, indent=2)) + cursor = dbc.find(filters, {'_id': 0 }) + else: + # select all + cursor = dbc.find({}, {'_id': 0 }) + bu.debug('Reading replays...') async for replay_json in cursor: - await mk_readerQ_item(replay_json) + #bu.debug(json.dumps(replay_json, indent=2)) + await queue.put(await mk_readerQ_item(replay_json)) + bu.debug('All the matching replays have been read from the DB') except Exception as err: bu.error(exception=err) @@ -909,7 +954,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto return None -async def mk_readerQ_item(replay_json : dict) -> list: +async def mk_readerQ_item(replay_json) -> list: """Make an item to replay queue""" global REPLAY_N REPLAY_N +=1 @@ -1061,6 +1106,7 @@ def str2ints(stat_id_str: str) -> list: """Convert stat_id string e.g. 'account_id:tank_id' to list of ints""" return [ int(x) for x in stat_id_str.split(':') ] + def prune_stat_id(stat_id_str: str) -> str: stat_id = stat_id_str.split(':') del stat_id[-1] From eeb1d7fc6d0f53dd46fc7426b7d93ae3f73d535e Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 00:52:16 +0200 Subject: [PATCH 05/18] Add result_display_fields list to make editing the reports easier --- analyze_wotb_replays.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 50c9f2d..fcded45 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -120,6 +120,25 @@ result_fields = { 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ] } +# fields to display in results +result_display_fields = [ + 'battles', + 'win', + 'damage_made', + 'DR', + 'KDR', + 'enemies_spotted', + 'hit_rate', + 'pen_rate', + 'survived', + 'time_alive%', + 'top_tier', + 'allies_wins', + 'enemies_wins', + 'allies_battles', + 'enemies_battles' +] + ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY) ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ] histogram_fields = { @@ -263,7 +282,7 @@ class BattleRecord(): self.calc_results() results = [] try: - for field in result_fields.keys(): + for field in result_display_fields: results.append( result_fields[field][3].format(self.results[field]) ) return results except KeyError as err: From 33bfce221540097131733a4f7c0659d479d8fb1b Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 07:47:07 +0200 Subject: [PATCH 06/18] Fix handling of None stats --- blitzutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blitzutils.py b/blitzutils.py index 932e03c..8a613ff 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -252,7 +252,7 @@ async def open_JSON(filename: str, chk_JSON_func = None) -> dict: return None -async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = None, max_tries = MAX_RETRIES) -> dict: +async def get_url_JSON(session: aiohttp.ClientSession, url, chk_JSON_func = None, max_tries = MAX_RETRIES) -> dict: """Retrieve (GET) an URL and return JSON object""" if session == None: error('Session must be initialized first') From a349d8ebea160a8dc5dc63689c58e97e9f8ebb0f Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 07:47:19 +0200 Subject: [PATCH 07/18] SImpler default reports --- analyze_wotb_replays.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index fcded45..4f3dac6 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -90,10 +90,10 @@ result_categories = { result_categories_default = [ 'total', 'battle_result', + 'battle_type', 'tank_tier', 'top_tier', - 'tank_name', - 'map_name' + ] RESULT_CAT_HEADER_FRMT = '{:_<17s}' From 8b3f6d0262e5f8380caccef89f8026f090dece3c Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 20:52:54 +0200 Subject: [PATCH 08/18] get_url_JSON(): fix handling of url = None --- analyze_wotb_replays.py | 2 +- blitzutils.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 4f3dac6..cf3acb9 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -335,7 +335,7 @@ class BattleRecordCategory(): def get_headers(self): try: headers = [ RESULT_CAT_HEADER_FRMT.format(result_categories[self.category_name][0]) ] - for field in result_fields.keys(): + for field in result_display_fields: print_format = '{:^' + str(result_fields[field][2]) + 's}' headers.append(print_format.format(result_fields[field][0])) return headers diff --git a/blitzutils.py b/blitzutils.py index 8a613ff..7484456 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -252,13 +252,15 @@ async def open_JSON(filename: str, chk_JSON_func = None) -> dict: return None -async def get_url_JSON(session: aiohttp.ClientSession, url, chk_JSON_func = None, max_tries = MAX_RETRIES) -> dict: +async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = None, max_tries = MAX_RETRIES) -> dict: """Retrieve (GET) an URL and return JSON object""" if session == None: error('Session must be initialized first') sys.exit(1) - - ## To avoid excessive use of servers + if url == None: + return None + + ## To avoid excessive use of servers for retry in range(1,max_tries+1): try: await asyncio.sleep(SLEEP) @@ -830,10 +832,7 @@ class WG: async def get_player_tank_stats(self, account_id: int, tank_ids = [], fields = [], cache=True) -> dict: """Get player's stats (WR, # of battles) in a tank or all tanks (empty tank_ids[])""" - # debug('Started') - try: - #debug('account_id: ' + str(account_id) + ' TankID: ' + ','.join([ str(id) for id in tank_ids])) stats = None # try cached stats first: From b8ef7f42da5a113aed6e1e258500111a881ac506 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 22:39:54 +0200 Subject: [PATCH 09/18] Optimization: stats are put into 2 week buckets --- analyze_wotb_replays.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index cf3acb9..04585a0 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -25,6 +25,7 @@ wi = None REPLAY_N = 0 REPLAY_I = 0 STAT_TANK_BATTLE_MIN = 100 +BATTLE_TIME_BUCKET = 3600*24*14 ## For different player stat functions (global stats, tank tier stats, etc) # 1st function = for forming stat_id, 2nd for DB stats query, 3rd for WG API stats query @@ -92,8 +93,7 @@ result_categories_default = [ 'battle_result', 'battle_type', 'tank_tier', - 'top_tier', - + 'top_tier' ] RESULT_CAT_HEADER_FRMT = '{:_<17s}' @@ -136,7 +136,8 @@ result_display_fields = [ 'allies_wins', 'enemies_wins', 'allies_battles', - 'enemies_battles' + 'enemies_battles', + MISSING_STATS ] ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY) @@ -1085,8 +1086,7 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict else: tmp_account_id = player['dbid'] tmp_tank_id = player['vehicle_descr'] - # tmp_tank_tier = wg.get_tank_tier(tmp_tank_id) - tmp_battletime = int(replay_json['data']['summary']['battle_start_timestamp']) + tmp_battletime = result['battle_start_timestamp'] if player['dbid'] in replay_json['data']['summary']['allies']: result['allies'].add(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) @@ -1102,9 +1102,10 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict tmp_account_id = player['dbid'] tmp_tank_id = player['vehicle_descr'] #tmp_tank_tier = str(wg.get_tank_tier(tmp_tank_id)) - tmp_battletime = int(replay_json['data']['summary']['battle_start_timestamp']) + tmp_battletime = result['battle_start_timestamp'] # platoon buddy removed from stats result['allies'].remove(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) + break result['time_alive%'] = result['time_alive'] / btl_duration result['battle_tier'] = btl_tier @@ -1140,13 +1141,15 @@ def get_stat_id_tank_tier(stat_id_str: str) -> str: """Return stat_id = account_id:tank_tier""" stat_id = stat_id_str.split(':') tank_tier = wg.get_tank_tier(int(stat_id[1])) - return ':'.join([ stat_id[0], str(tank_tier), stat_id[2] ]) + battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET + return ':'.join([ stat_id[0], str(tank_tier), str(battle_time) ]) def get_stat_id_player(stat_id_str: str) -> str: """Return stat_id = account_id:tank_tier""" stat_id = stat_id_str.split(':') - return ':'.join([ stat_id[0], stat_id[2] ]) + battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET + return ':'.join([ stat_id[0], str(battle_time) ]) ### main() From eb1583f76e75d7e1145fdfe297e7e0d95be07a8d Mon Sep 17 00:00:00 2001 From: Jylpah Date: Thu, 13 Feb 2020 23:33:38 +0200 Subject: [PATCH 10/18] Update analyze_wotb_replays.py --- analyze_wotb_replays.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 04585a0..b5fc206 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -241,6 +241,8 @@ class BattleRecord(): except KeyError as err: bu.error('Key not found', err) bu.error(str(result)) + except Exception as err: + bu.error('BattleRecord:' ,exception=err) return False def calc_ratios(self) -> bool: @@ -573,6 +575,8 @@ async def process_player_stats(players, N_workers: int, args : argparse.Namespac stat_id_map = {} stat_ids = set() + bu.set_progress_bar('Fetching player stats', len(players), 25, True) + stat_id_map_func = globals()[STAT_FUNC[args.stat_func][0]] for player in players: @@ -603,6 +607,8 @@ async def process_player_stats(players, N_workers: int, args : argparse.Namespac player_stats = {**player_stats, **stats} stat_id_remap = {**stat_id_remap, **id_remap} + bu.finish_progress_bar() + ## DO REMAPPING stat_id_map = remap_stat_id(stat_id_map, stat_id_remap) @@ -1033,6 +1039,12 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict #db = args.db result = {} try: + result['battle_start_timestamp'] = int(replay_json['data']['summary']['battle_start_timestamp']) + # TBD... + # protagonist = int(replay_json['data']['summary']['protagonist']) + # protagonist_tank = int() + # result['protagonist'] = get_stat_id(protagonist, protagonist_tank, result['battle_start_timestamp']) + if account_id == None: account_id = replay_json['data']['summary']['protagonist'] elif replay_json['data']['summary']['protagonist'] != account_id: @@ -1054,7 +1066,7 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict result['url'] = replay_json['data']['view_url'] for key in replay_summary_flds: result[key] = replay_json['data']['summary'][key] - result['battle_start_timestamp'] = int(replay_json['data']['summary']['battle_start_timestamp']) + result['allies'] = set() result['enemies'] = set() btl_duration = 0 From 6ecddb3d95a3049e13089c26cbbc19b9b86faa9d Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 00:00:19 +0200 Subject: [PATCH 11/18] Not finished BattleRecordCategory.print_results() --- analyze_wotb_replays.py | 533 ++++++++++++++++++++++------------------ 1 file changed, 291 insertions(+), 242 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index b5fc206..d6bc638 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -69,76 +69,6 @@ replay_details_flds = [ 'enemies_spotted' ] -result_ratios = { - 'KDR' : [ 'enemies_destroyed', 'destroyed' ], - 'DR' : [ 'damage_made', 'damage_received' ], - 'hit_rate' : [ 'shots_hit', 'shots_made' ], - 'pen_rate' : [ 'shots_pen', 'shots_hit' ], - 'dmg_block_rate' : [ 'damage_blocked', 'damage_received' ] -} - -result_categories = { - 'total' : [ 'TOTAL', 'total' ], - 'battle_result' : [ 'Result', [ 'Loss', 'Win', 'Draw']], - 'battle_type' : [ 'Battle Type',['Encounter', 'Supremacy']], - 'tank_tier' : [ 'Tank Tier', 'number' ], - 'top_tier' : [ 'Tier', ['Bottom tier', 'Top tier']], - 'tank_name' : [ 'Tank', 'string' ], - 'map_name' : [ 'Map', 'string' ], - 'battle_i' : [ 'Battle #', 'number'] - } - -result_categories_default = [ - 'total', - 'battle_result', - 'battle_type', - 'tank_tier', - 'top_tier' -] - -RESULT_CAT_HEADER_FRMT = '{:_<17s}' -RESULT_CAT_FRMT = '{:>17s}' - - -## Syntax: Check how the replay JSON files look. The algorithm is counting/recording fields -result_fields = { - 'battles' : [ 'Battles', 'Number of battles', 8, '{:^8.0f}' ], - 'win' : [ 'WR', 'Win rate', 6, '{:6.1%}' ], - 'damage_made' : [ 'DPB', 'Average Damage', 5, '{:5.0f}' ], - 'DR' : [ 'DR', 'Damage Ratio', 5, '{:5.1f}' ], - 'KDR' : [ 'KDR', 'Kills / Death', 4, '{:4.1f}' ], - 'enemies_spotted' : [ 'Spot', 'Enemies spotted per battle', 4, '{:4.1f}' ], - 'hit_rate' : [ 'Hit rate', 'Shots hit / all shots made', 8, '{:8.1%}' ], - 'pen_rate' : [ 'Pen rate', 'Shots pen / shots hit', 8, '{:8.1%}' ], - 'survived' : [ 'Surv%', 'Survival rate', 6, '{:6.1%}' ], - 'time_alive%' : [ 'T alive%', 'Percentage of time being alive in a battle', 8, '{:8.0%}' ], - 'top_tier' : [ 'Top tier', 'Share of games as top tier', 8, '{:8.0%}' ], - MISSING_STATS : [ 'No stats', 'Players without stats avail', 8, '{:8.1%}'], - 'allies_wins' : [ 'Allies WR', 'Average WR of allies at the tier of their tank', 9, '{:9.2%}' ], - 'enemies_wins' : [ 'Enemies WR', 'Average WR of enemies at the tier of their tank', 10, '{:10.2%}' ], - 'allies_battles' : [ 'Allies Btls', 'Average number battles of the allies', 11, '{:11.0f}' ], - 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ] -} - -# fields to display in results -result_display_fields = [ - 'battles', - 'win', - 'damage_made', - 'DR', - 'KDR', - 'enemies_spotted', - 'hit_rate', - 'pen_rate', - 'survived', - 'time_alive%', - 'top_tier', - 'allies_wins', - 'enemies_wins', - 'allies_battles', - 'enemies_battles', - MISSING_STATS -] ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY) ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ] @@ -155,6 +85,296 @@ def def_value_zero(): def def_value_BattleRecord(): return BattleRecord() +class BattleRecordCategory(): + _result_categories = { + 'total' : [ 'TOTAL', 'total' ], + 'battle_result' : [ 'Result', [ 'Loss', 'Win', 'Draw']], + 'battle_type' : [ 'Battle Type',['Encounter', 'Supremacy']], + 'tank_tier' : [ 'Tank Tier', 'number' ], + 'top_tier' : [ 'Tier', ['Bottom tier', 'Top tier']], + 'tank_name' : [ 'Tank', 'string' ], + 'map_name' : [ 'Map', 'string' ], + 'battle_i' : [ 'Battle #', 'number'] + } + + _result_categories_default = [ + 'total', + 'battle_result', + 'battle_type', + 'tank_tier', + 'top_tier' + ] + + _result_categories_extended = [ + 'tank_name', + 'map_name', + 'battle_i' + ] + + + RESULT_CAT_FRMT = '{:>17s}' + + @classmethod + def get_result_categories(cls, extended_stats: bool = False): + if extended_stats: + return cls._result_categories_default + cls._result_categories_extended + else: + return cls._result_categories_default + + def __init__(self, cat_name : str): + self.category_name = cat_name + self.category = collections.defaultdict(def_value_BattleRecord) + if self._result_categories[self.category_name][1] == 'string': + self.type = 'string' + elif self._result_categories[self.category_name][1] == 'number': + self.type = 'number' + elif self._result_categories[self.category_name][1] == 'total': + self.type = 'total' + else: + self.type = 'category' + + + def get_sub_categories(self): + return self.category.keys() + + def record_result(self, result: dict): + try: + cat = None + if self.type == 'total': + cat = 'Total' + elif self.type == 'number': + cat = str(result[self.category_name]) + elif self.type == 'string': + cat = result[self.category_name] + else: + cat = self._result_categories[self.category_name][1][result[self.category_name]] + self.category[cat].record_result(result) + return True + except KeyError as err: + bu.error('Key not found', err) + bu.error('Category: ', str(cat)) + bu.error(str(result)) + except Exception as err: + bu.error(str(err)) + return False + + + def print_results(self): + try: + print(' '.join(self.get_headers())) + for row in self.get_results(): + print(' : '.join(row)) + except KeyError as err: + bu.error('Key not found', err) + except Exception as err: + bu.error(str(err)) + return None + + def get_results(self): + try: + results = [] + # results.append(self.get_headers()) + if self.type == 'number': + for cat in sorted( [ int(s) for s in self.category.keys() ] ): + cat = str(cat) + row = [ self.RESULT_CAT_FRMT.format(cat) ] + row.extend(self.category[cat].get_results()) + results.append(row) + else: + for cat in sorted(self.category.keys() , key=str.casefold): + row = [ self.RESULT_CAT_FRMT.format(cat) ] + row.extend(self.category[cat].get_results()) + results.append(row) + return results + except KeyError as err: + bu.error('Key not found', err) + return None + + +class BattleRecord(): + + ## Syntax: Check how the replay JSON files look. The algorithm is counting/recording fields + _result_fields = { + 'battles' : [ 'Battles', 'Number of battles', 8, '{:^8.0f}' ], + 'win' : [ 'WR', 'Win rate', 6, '{:6.1%}' ], + 'damage_made' : [ 'DPB', 'Average Damage', 5, '{:5.0f}' ], + 'DR' : [ 'DR', 'Damage Ratio', 5, '{:5.1f}' ], + 'KDR' : [ 'KDR', 'Kills / Death', 4, '{:4.1f}' ], + 'enemies_spotted' : [ 'Spot', 'Enemies spotted per battle', 4, '{:4.1f}' ], + 'hit_rate' : [ 'Hit rate', 'Shots hit / all shots made', 8, '{:8.1%}' ], + 'pen_rate' : [ 'Pen rate', 'Shots pen / shots hit', 8, '{:8.1%}' ], + 'survived' : [ 'Surv%', 'Survival rate', 6, '{:6.1%}' ], + 'time_alive%' : [ 'T alive%', 'Percentage of time being alive in a battle', 8, '{:8.0%}' ], + 'top_tier' : [ 'Top tier', 'Share of games as top tier', 8, '{:8.0%}' ], + MISSING_STATS : [ 'No stats', 'Players without stats avail', 8, '{:8.1%}'], + 'allies_wins' : [ 'Allies WR', 'Average WR of allies at the tier of their tank', 9, '{:9.2%}' ], + 'enemies_wins' : [ 'Enemies WR', 'Average WR of enemies at the tier of their tank', 10, '{:10.2%}' ], + 'allies_battles' : [ 'Allies Btls', 'Average number battles of the allies', 11, '{:11.0f}' ], + 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ] + } + + # fields to display in results + _result_fields_default = [ + 'battles', + 'win', + 'damage_made', + 'enemies_spotted', + 'top_tier', + 'allies_wins', + 'enemies_wins', + 'allies_battles', + 'enemies_battles' + ] + + _result_fields_extended = [ + 'DR', + 'KDR', + 'hit_rate', + 'pen_rate', + 'survived', + 'time_alive%', + MISSING_STATS + ] + + _result_ratios = { + 'KDR' : [ 'enemies_destroyed', 'destroyed' ], + 'DR' : [ 'damage_made', 'damage_received' ], + 'hit_rate' : [ 'shots_hit', 'shots_made' ], + 'pen_rate' : [ 'shots_pen', 'shots_hit' ], + 'dmg_block_rate' : [ 'damage_blocked', 'damage_received' ] + } + + + RESULT_CAT_HEADER_FRMT = '{:_<17s}' + + + @classmethod + def get_result_fields(cls, extended_stats: bool = False): + if extended_stats: + return cls._result_fields_default + cls._result_fields_extended + else: + return cls._result_fields_default + + + @classmethod + def get_fields_avg(cls) -> set: + avg_fields = set(cls.get_result_fields(True)) + avg_fields.remove('battles') + avg_fields.remove(MISSING_STATS) + avg_fields.difference_update(cls._result_ratios.keys()) + return avg_fields + + + @classmethod + def get_fields_ratio(cls) -> set: + ratio_fields = set() + for ratio in cls._result_ratios.keys(): + ratio_fields.add(cls._result_ratios[ratio][0]) + ratio_fields.add(cls._result_ratios[ratio][1]) + return ratio_fields + + + def __init__(self, extended_stats: bool = False): + try: + self.battles = 0 + self.missing_stats = 0 + self.n_players = 0 + self.ratios_ready = False + self.results_ready = False + self.results = collections.defaultdict(def_value_zero) + self.avg_fields = self.get_fields_avg() + self.ratio_fields = self.get_fields_ratio() + self.result_fields = self.get_result_fields(extended_stats) + self.result_fields_ratio = set(self._result_ratios.keys()) & set(self.get_result_fields(extended_stats)) + + + except KeyError as err: + bu.error('Key not found', err) + + + def record_result(self, result : dict) -> bool: + try: + for field in self.avg_fields | self.ratio_fields: + if (field in result) and (result[field] != None): + self.results[field] += result[field] + self.battles += 1 + self.n_players += result[N_PLAYERS] + self.missing_stats += result[MISSING_STATS] + return True + except KeyError as err: + bu.error('Key not found', err) + bu.error(str(result)) + except Exception as err: + bu.error('BattleRecord:' ,exception=err) + return False + + + def calc_ratios(self) -> bool: + if self.ratios_ready: + return True + try: + for field in self.result_fields_ratio: + if self.results[self._result_ratios[field][1]] != 0: + self.results[field] = self.results[self._result_ratios[field][0]] / self.results[self._result_ratios[field][1]] + else: + self.results[field] = float('Inf') + self.ratios_ready = True + return True + except KeyError as err: + bu.error('Key not found', err) + return False + + + def calc_results(self): + if self.results_ready == True: + return True + if not self.ratios_ready: + self.calc_ratios() + try: + for field in self.avg_fields: + self.results[field] = self.results[field] / max(self.battles,1) + self.results['battles'] = self.battles + self.results[MISSING_STATS] = self.missing_stats / self.n_players + self.results_ready = True + return True + except KeyError as err: + bu.error('Key not found', err) + except Exception as err: + bu.error(exception=err) + return False + + + def get_headers(self, cat_name: str): + try: + headers = [ self.RESULT_CAT_HEADER_FRMT.format(cat_name) ] + for field in self.result_fields: + print_format = '{:^' + str(self._result_fields[field][2]) + 's}' + headers.append(print_format.format(self._result_fields[field][0])) + return headers + except KeyError as err: + bu.error('Key not found', err) + except Exception as err: + bu.error(str(err)) + return None + + def get_results(self): + if not self.results_ready: + self.calc_results() + results = [] + try: + for field in self.result_fields: + results.append(self._result_fields[field][3].format(self.results[field]) ) + return results + except KeyError as err: + bu.error('Key not found', err) + except Exception as err: + bu.error(exception=err) + return None + + + def print_results(self): + print(' : '.join(self.get_results())) + class PlayerHistogram(): def __init__(self, field: str, name: str, fields : list, factor: float, format: str ): @@ -162,7 +382,7 @@ class PlayerHistogram(): self.name = name self.fields = fields self.cat_factor = factor - self.cat_format = format + self.cat_format = format self.ncat = len(self.fields) - 1 self.allies = [0] * self.ncat self.enemies = [0] * self.ncat @@ -208,177 +428,6 @@ class PlayerHistogram(): return None -class BattleRecord(): - def __init__(self): - try: - self.battles = 0 - self.missing_stats = 0 - self.n_players = 0 - self.ratios_ready = False - self.results_ready = False - self.results = collections.defaultdict(def_value_zero) - self.avg_fields = set(result_fields.keys()) - self.avg_fields.remove('battles') - self.avg_fields.remove(MISSING_STATS) - self.avg_fields.difference_update(result_ratios.keys()) - self.ratio_fields = set() - for ratio in result_ratios.keys(): - self.ratio_fields.add(result_ratios[ratio][0]) - self.ratio_fields.add(result_ratios[ratio][1]) - - except KeyError as err: - bu.error('Key not found', err) - - def record_result(self, result : dict) -> bool: - try: - for field in self.avg_fields | self.ratio_fields: - if (field in result) and (result[field] != None): - self.results[field] += result[field] - self.battles += 1 - self.n_players += result[N_PLAYERS] - self.missing_stats += result[MISSING_STATS] - return True - except KeyError as err: - bu.error('Key not found', err) - bu.error(str(result)) - except Exception as err: - bu.error('BattleRecord:' ,exception=err) - return False - - def calc_ratios(self) -> bool: - if self.ratios_ready: - return True - try: - ratio_result_flds = set(result_ratios.keys()) & set(result_fields.keys()) - - for field in ratio_result_flds: - if self.results[result_ratios[field][1]] != 0: - self.results[field] = self.results[result_ratios[field][0]] / self.results[result_ratios[field][1]] - else: - self.results[field] = float('Inf') - self.ratios_ready = True - return True - except KeyError as err: - bu.error('Key not found', err) - return False - - def calc_results(self): - if self.results_ready == True: - return True - if not self.ratios_ready: - self.calc_ratios() - try: - for field in self.avg_fields: - self.results[field] = self.results[field] / max(self.battles,1) - self.results['battles'] = self.battles - self.results[MISSING_STATS] = self.missing_stats / self.n_players - self.results_ready = True - return True - except KeyError as err: - bu.error('Key not found', err) - except Exception as err: - bu.error(exception=err) - return False - - def get_results(self): - if not self.results_ready: - self.calc_results() - results = [] - try: - for field in result_display_fields: - results.append( result_fields[field][3].format(self.results[field]) ) - return results - except KeyError as err: - bu.error('Key not found', err) - except Exception as err: - bu.error(exception=err) - return None - - def print_results(self): - print(' : '.join(self.get_results())) - - -class BattleRecordCategory(): - def __init__(self, cat_name : str): - self.category_name = cat_name - self.category = collections.defaultdict(def_value_BattleRecord) - if result_categories[self.category_name][1] == 'string': - self.type = 'string' - elif result_categories[self.category_name][1] == 'number': - self.type = 'number' - elif result_categories[self.category_name][1] == 'total': - self.type = 'total' - else: - self.type = 'category' - - def get_sub_categories(self): - return self.category.keys() - - def record_result(self, result: dict): - try: - cat = None - if self.type == 'total': - cat = 'Total' - elif self.type == 'number': - cat = str(result[self.category_name]) - elif self.type == 'string': - cat = result[self.category_name] - else: - cat = result_categories[self.category_name][1][result[self.category_name]] - self.category[cat].record_result(result) - return True - except KeyError as err: - bu.error('Key not found', err) - bu.error('Category: ', str(cat)) - bu.error(str(result)) - except Exception as err: - bu.error(str(err)) - return False - - def get_headers(self): - try: - headers = [ RESULT_CAT_HEADER_FRMT.format(result_categories[self.category_name][0]) ] - for field in result_display_fields: - print_format = '{:^' + str(result_fields[field][2]) + 's}' - headers.append(print_format.format(result_fields[field][0])) - return headers - except KeyError as err: - bu.error('Key not found', err) - except Exception as err: - bu.error(str(err)) - return None - - def print_results(self): - try: - print(' '.join(self.get_headers())) - for row in self.get_results(): - print(' : '.join(row)) - except KeyError as err: - bu.error('Key not found', err) - except Exception as err: - bu.error(str(err)) - return None - - def get_results(self): - try: - results = [] - # results.append(self.get_headers()) - if self.type == 'number': - for cat in sorted( [ int(s) for s in self.category.keys() ] ): - cat = str(cat) - row = [ RESULT_CAT_FRMT.format(cat) ] - row.extend(self.category[cat].get_results()) - results.append(row) - else: - for cat in sorted(self.category.keys() , key=str.casefold): - row = [ RESULT_CAT_FRMT.format(cat) ] - row.extend(self.category[cat].get_results()) - results.append(row) - return results - except KeyError as err: - bu.error('Key not found', err) - return None - ## main() ------------------------------------------------------------- async def main(argv): From 15dddcf1b7c52ffcd81b0b979118d4070cef9077 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 15:26:06 +0200 Subject: [PATCH 12/18] Class SlowBar: ETA estimate with minute accuracy --- blitzutils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index 7484456..9245ed6 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -312,10 +312,15 @@ def bld_dict_hierarcy(d : dict, key : str, value) -> dict: ## ----------------------------------------------------------- class SlowBar(IncrementalBar): - suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).1f hours' + suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).0f h %(remaining_mins).0f mins' @property def remaining_hours(self): - return Decimal(self.eta / 3600).quantize(Decimal('1.0')) + return self.eta // 3600 + + @property + def remaining_mins(self): + return (self.eta - (self.eta // 3600)*3600) // 60 + ## ----------------------------------------------------------- From 691deb7c80cd846b924f8e9c314935d4221c09a2 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 15:26:32 +0200 Subject: [PATCH 13/18] Replay player's WR added to stats --- analyze_wotb_replays.py | 247 ++++++++++++++++++++++++---------------- 1 file changed, 148 insertions(+), 99 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index d6bc638..49f901e 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -40,7 +40,8 @@ replay_summary_flds = [ 'map_name', 'battle_duration', 'protagonist', - 'title' + 'title', + 'mastery_badge' ] replay_details_flds = [ @@ -115,12 +116,16 @@ class BattleRecordCategory(): RESULT_CAT_FRMT = '{:>17s}' @classmethod - def get_result_categories(cls, extended_stats: bool = False): - if extended_stats: - return cls._result_categories_default + cls._result_categories_extended + def get_result_categories(cls, extended_cats: list = None): + if extended_cats != None: + return cls._result_categories_default + extended_cats else: return cls._result_categories_default + @classmethod + def get_extra_categories(cls) -> list: + return cls._result_categories_extended + def __init__(self, cat_name : str): self.category_name = cat_name self.category = collections.defaultdict(def_value_BattleRecord) @@ -137,6 +142,7 @@ class BattleRecordCategory(): def get_sub_categories(self): return self.category.keys() + def record_result(self, result: dict): try: cat = None @@ -161,7 +167,8 @@ class BattleRecordCategory(): def print_results(self): try: - print(' '.join(self.get_headers())) + first_btl_record = list(self.category.values())[0] + print(' '.join(first_btl_record.get_headers(self.category_name))) for row in self.get_results(): print(' : '.join(row)) except KeyError as err: @@ -170,6 +177,7 @@ class BattleRecordCategory(): bu.error(str(err)) return None + def get_results(self): try: results = [] @@ -210,9 +218,13 @@ class BattleRecord(): 'allies_wins' : [ 'Allies WR', 'Average WR of allies at the tier of their tank', 9, '{:9.2%}' ], 'enemies_wins' : [ 'Enemies WR', 'Average WR of enemies at the tier of their tank', 10, '{:10.2%}' ], 'allies_battles' : [ 'Allies Btls', 'Average number battles of the allies', 11, '{:11.0f}' ], - 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ] + 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ], + 'player_wins' : [ 'Player WR', 'Average WR of the player', 9, '{:9.2%}' ], + 'player_battles' : [ 'Player Btls', 'Average number battles of the allies', 11, '{:11.0f}' ] } + _team_fields = [ 'wins', 'battles' ] + # fields to display in results _result_fields_default = [ 'battles', @@ -220,6 +232,7 @@ class BattleRecord(): 'damage_made', 'enemies_spotted', 'top_tier', + 'player_wins', 'allies_wins', 'enemies_wins', 'allies_battles', @@ -244,50 +257,65 @@ class BattleRecord(): 'dmg_block_rate' : [ 'damage_blocked', 'damage_received' ] } + _extended_stats = False + result_fields = list() + avg_fields = set() + ratio_fields = set() + + RESULT_CAT_HEADER_FRMT = '{:_<17s}' @classmethod - def get_result_fields(cls, extended_stats: bool = False): - if extended_stats: - return cls._result_fields_default + cls._result_fields_extended - else: - return cls._result_fields_default + def set_fields(cls, extended_stats: bool = False): + try: + cls._extended_stats = extended_stats + if cls._extended_stats: + cls.result_fields = cls._result_fields_default + cls._result_fields_extended + else: + cls.result_fields = cls._result_fields_default + cls.result_fields_ratio = set(cls._result_ratios.keys()) & set(cls.result_fields) + + cls.avg_fields = set(cls.result_fields) + for field in ['battles', MISSING_STATS]: + if field in cls.avg_fields: + cls.avg_fields.remove(field) + cls.avg_fields.difference_update(set(cls._result_ratios.keys())) + for ratio in cls.result_fields_ratio: + cls.ratio_fields.add(cls._result_ratios[ratio][0]) + cls.ratio_fields.add(cls._result_ratios[ratio][1]) + except Exception as err: + bu.error('BattleRecord:' ,exception=err) + + + @classmethod + def get_result_fields(cls) -> set: + return cls.result_fields - @classmethod def get_fields_avg(cls) -> set: - avg_fields = set(cls.get_result_fields(True)) - avg_fields.remove('battles') - avg_fields.remove(MISSING_STATS) - avg_fields.difference_update(cls._result_ratios.keys()) - return avg_fields - + return cls.avg_fields @classmethod def get_fields_ratio(cls) -> set: - ratio_fields = set() - for ratio in cls._result_ratios.keys(): - ratio_fields.add(cls._result_ratios[ratio][0]) - ratio_fields.add(cls._result_ratios[ratio][1]) - return ratio_fields + return cls.ratio_fields + + @classmethod + def get_team_fields(cls) -> list: + return cls._team_fields - def __init__(self, extended_stats: bool = False): + def __init__(self): try: + self.battles = 0 self.missing_stats = 0 self.n_players = 0 self.ratios_ready = False self.results_ready = False self.results = collections.defaultdict(def_value_zero) - self.avg_fields = self.get_fields_avg() - self.ratio_fields = self.get_fields_ratio() - self.result_fields = self.get_result_fields(extended_stats) - self.result_fields_ratio = set(self._result_ratios.keys()) & set(self.get_result_fields(extended_stats)) - except KeyError as err: bu.error('Key not found', err) @@ -297,7 +325,7 @@ class BattleRecord(): for field in self.avg_fields | self.ratio_fields: if (field in result) and (result[field] != None): self.results[field] += result[field] - self.battles += 1 + self.battles += 1 self.n_players += result[N_PLAYERS] self.missing_stats += result[MISSING_STATS] return True @@ -463,6 +491,7 @@ async def main(argv): parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze') parser.add_argument('-a', '--account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-x', '--extended', action='store_true', default=False, help='Print Extended stats') + parser.add_argument('-X', '--extra_categories', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print Extended categories') parser.add_argument('--hist', action='store_true', default=False, help='Print player histograms (WR/battles)') parser.add_argument('--stat_func', default='tank_tier', choices=STAT_FUNC.keys(), help='Select how to calculate for ally/enemy performance: tank-tier stats, global player stats') parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs') @@ -487,6 +516,8 @@ async def main(argv): args.account_id = await wg.get_account_id(args.account) bu.debug('WG account_id: ' + str(args.account_id)) + BattleRecord.set_fields(args.extended) + #### Connect to MongoDB (TBD) bu.debug('DB_SERVER: ' + DB_SERVER) bu.debug('DB_PORT: ' + str(DB_PORT)) @@ -544,7 +575,7 @@ async def main(argv): teamresults = calc_team_stats(results, player_stats, stat_id_map, args) process_battle_results(teamresults, args) if args.hist: - print('\nPlayer Historgrams______', end='', flush=True) + print('\nPlayer Histograms______', end='', flush=True) process_player_dist(results, player_stats, stat_id_map) bu.debug('Finished. Cleaning up..................') except Exception as err: @@ -588,9 +619,7 @@ def process_battle_results(results: dict, args : argparse.Namespace): urls = collections.OrderedDict() categories = {} - cats = result_categories_default - if args.extended: - cats = result_categories.keys() + cats = BattleRecordCategory.get_result_categories(args.extra_categories) for cat in cats: categories[cat] = BattleRecordCategory(cat) @@ -619,50 +648,54 @@ def process_battle_results(results: dict, args : argparse.Namespace): async def process_player_stats(players, N_workers: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict: """Start stat_workers to retrieve and store players' stats""" ## Create queue of player-tank pairs to find stats for - statsQ = asyncio.Queue() - bu.debug('Create player stats queue: ' + str(len(players)) + ' players') - stat_id_map = {} - stat_ids = set() + try: + statsQ = asyncio.Queue() + bu.debug('Create player stats queue: ' + str(len(players)) + ' players') + stat_id_map = {} + stat_ids = set() - bu.set_progress_bar('Fetching player stats', len(players), 25, True) + bu.set_progress_bar('Fetching player stats', len(players), 25, True) - stat_id_map_func = globals()[STAT_FUNC[args.stat_func][0]] + stat_id_map_func = globals()[STAT_FUNC[args.stat_func][0]] - for player in players: - stat_id_map[player] = stat_id_map_func(player) - stat_ids.add(stat_id_map[player]) + for player in players: + stat_id_map[player] = stat_id_map_func(player) + stat_ids.add(stat_id_map[player]) + + # create statsQ + for stat_id in stat_ids: + await statsQ.put(stat_id) + + # Process player WR / battle stats + stats_tasks = [] + bu.debug('Starting player stats workers') + for i in range(N_workers): + bu.debug("Starting worker " + str(i)) + stats_tasks.append(asyncio.create_task(stat_worker(statsQ, i, args, db))) - # create statsQ - for stat_id in stat_ids: - await statsQ.put(stat_id) + bu.debug('Waiting stats workers to finish') + await statsQ.join() + bu.debug('Cancelling stats workers') + for task in stats_tasks: + task.cancel() + player_stats = {} + stat_id_remap = {} - # Process player WR / battle stats - stats_tasks = [] - bu.debug('Starting player stats workers') - for i in range(N_workers): - bu.debug("Starting worker " + str(i)) - stats_tasks.append(asyncio.create_task(stat_worker(statsQ, i, args, db))) - - bu.debug('Waiting stats workers to finish') - await statsQ.join() - bu.debug('Cancelling stats workers') - for task in stats_tasks: - task.cancel() - player_stats = {} - stat_id_remap = {} + bu.debug('Gathering stats worker outputs') + for (stats, id_remap) in await asyncio.gather(*stats_tasks): + player_stats = {**player_stats, **stats} + stat_id_remap = {**stat_id_remap, **id_remap} - bu.debug('Gathering stats worker outputs') - for (stats, id_remap) in await asyncio.gather(*stats_tasks): - player_stats = {**player_stats, **stats} - stat_id_remap = {**stat_id_remap, **id_remap} + bu.finish_progress_bar() + + ## DO REMAPPING + stat_id_map = remap_stat_id(stat_id_map, stat_id_remap) - bu.finish_progress_bar() - - ## DO REMAPPING - stat_id_map = remap_stat_id(stat_id_map, stat_id_remap) - - bu.debug('Returning player_stats') - return (player_stats, stat_id_map) + bu.debug('Returning player_stats') + return (player_stats, stat_id_map) + except Exception as err: + bu.error(exception=err) + return None def remap_stat_id(stat_id_map: dict, stat_id_remap: dict): @@ -682,14 +715,8 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg ## Bug here?? #stat_types = player_stats[list(player_stats.keys())[0]].keys() stat_types = list() - p_team_fields = re.compile('^allies_(.+)$') - for field in result_fields: - m = p_team_fields.match(field) - if m != None: - stat_types.append(m.group(1)) - - bu.debug('stat_types: ' + ','.join(stat_types)) - + stat_types = BattleRecord.get_team_fields() + for result in results: try: missing_stats = 0 @@ -697,6 +724,8 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg n_allies = collections.defaultdict(def_value_zero) allies_stats = collections.defaultdict(def_value_zero) #bu.debug('Processing Allies') + + for ally in result['allies']: # Player itself is not in 'allies': see read_replay_JSON() ally_mapped = stat_id_map[ally] @@ -708,13 +737,6 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg allies_stats[stat] += player_stats[ally_mapped][stat] n_allies[stat] += 1 - #bu.debug('Processing Allies\' avg stats') - for stat in stat_types: - if n_allies[stat] > 0: - result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat] - else: - bu.debug('No allies stats for: ' + str(result)) - #bu.debug('Processing Enemies') n_enemies = collections.defaultdict(def_value_zero) enemies_stats = collections.defaultdict(def_value_zero) @@ -728,12 +750,23 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg enemies_stats[stat] += player_stats[enemy_mapped][stat] n_enemies[stat] += 1 - #bu.debug('Processing Enemies\' avg stats') + #bu.debug('Processing avg stats') + player_mapped = stat_id_map[result['player']] + for stat in stat_types: + if n_allies[stat] > 0: + result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat] + else: + bu.debug('No allies stats for: ' + str(result)) + if n_enemies[stat] > 0: result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat] else: - bu.debug('No allies stats for: ' + str(result)) + bu.debug('No enemies stats for: ' + str(result)) + + if player_stats[player_mapped][stat] != None: + result['player_' + str(stat)] = player_stats[player_mapped][stat] + result[N_PLAYERS] = n_players result[MISSING_STATS] = missing_stats return_list.append(result) @@ -1067,6 +1100,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam # if (account_id != None): playerstanks.update(set(result['allies'])) playerstanks.update(set(result['enemies'])) + playerstanks.update(set([ result['player'] ])) results.append(result) bu.debug('Marking task ' + str(replayID) + ' done') @@ -1090,9 +1124,7 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict try: result['battle_start_timestamp'] = int(replay_json['data']['summary']['battle_start_timestamp']) # TBD... - # protagonist = int(replay_json['data']['summary']['protagonist']) - # protagonist_tank = int() - # result['protagonist'] = get_stat_id(protagonist, protagonist_tank, result['battle_start_timestamp']) + protagonist = int(replay_json['data']['summary']['protagonist']) if account_id == None: account_id = replay_json['data']['summary']['protagonist'] @@ -1120,11 +1152,15 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict result['enemies'] = set() btl_duration = 0 btl_tier = 0 + protagonist_tank = None for player in replay_json['data']['summary']['details']: btl_duration = max(btl_duration, player['time_alive']) player_tank_tier = wg.get_tank_data(player['vehicle_descr'], 'tier') btl_tier = max(btl_tier, player_tank_tier) + if (protagonist != None) and (player['dbid'] == protagonist): + protagonist_tank = player['vehicle_descr'] + if player['dbid'] == account_id: # player itself is not put in results['allies'] tmp = {} @@ -1153,7 +1189,10 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict result['allies'].add(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) else: result['enemies'].add(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) - + + ## Rather use 'player' than incomprehensible 'protagonist'... + result['player'] = get_stat_id(protagonist, protagonist_tank, result['battle_start_timestamp']) + bu.debug('Player stats_id: ' + result['player']) # remove platoon buddy from stats if result['squad_index'] != None: for player in replay_json['data']['summary']['details']: @@ -1200,17 +1239,27 @@ def get_stat_id(account_id: int, tank_id: int, battletime: int) -> str: def get_stat_id_tank_tier(stat_id_str: str) -> str: """Return stat_id = account_id:tank_tier""" - stat_id = stat_id_str.split(':') - tank_tier = wg.get_tank_tier(int(stat_id[1])) - battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET - return ':'.join([ stat_id[0], str(tank_tier), str(battle_time) ]) + try: + stat_id = stat_id_str.split(':') + tank_tier = wg.get_tank_tier(int(stat_id[1])) + battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET + return ':'.join([ stat_id[0], str(tank_tier), str(battle_time) ]) + except Exception as err: + bu.error('Stats_id: ' + stat_id_str) + bu.error(exception=err) + return None def get_stat_id_player(stat_id_str: str) -> str: """Return stat_id = account_id:tank_tier""" - stat_id = stat_id_str.split(':') - battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET - return ':'.join([ stat_id[0], str(battle_time) ]) + try: + stat_id = stat_id_str.split(':') + battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET + return ':'.join([ stat_id[0], str(battle_time) ]) + except Exception as err: + bu.error('Stats_id: ' + stat_id_str) + bu.error(exception=err) + return None ### main() From 2603c1611c791700369e3d221de6220ec268dec8 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 15:58:04 +0200 Subject: [PATCH 14/18] Fix --- blitzutils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index 9245ed6..b98c29d 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -263,7 +263,6 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = ## To avoid excessive use of servers for retry in range(1,max_tries+1): try: - await asyncio.sleep(SLEEP) async with session.get(url) as resp: if resp.status == 200: debug('HTTP request OK') @@ -275,7 +274,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = if retry == max_tries: break debug('Retrying URL [' + str(retry) + '/' + str(max_tries) + ']: ' + url ) - + await asyncio.sleep(SLEEP) except aiohttp.ClientError as err: debug("Could not retrieve URL: " + url) From 43dee15de5fa2846435bdb084cb3911297454d02 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 15:58:52 +0200 Subject: [PATCH 15/18] Fix counting missing stats + stats per battle class (1/2/3rd, Ace) --- analyze_wotb_replays.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 49f901e..8302682 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -93,6 +93,7 @@ class BattleRecordCategory(): 'battle_type' : [ 'Battle Type',['Encounter', 'Supremacy']], 'tank_tier' : [ 'Tank Tier', 'number' ], 'top_tier' : [ 'Tier', ['Bottom tier', 'Top tier']], + 'mastery_badge' : [ 'Battle Class', ['-', '3rd Class', '2nd Class', '1st Class', 'Mastery' ]], 'tank_name' : [ 'Tank', 'string' ], 'map_name' : [ 'Map', 'string' ], 'battle_i' : [ 'Battle #', 'number'] @@ -103,7 +104,8 @@ class BattleRecordCategory(): 'battle_result', 'battle_type', 'tank_tier', - 'top_tier' + 'top_tier', + 'mastery_badge' ] _result_categories_extended = [ @@ -142,6 +144,10 @@ class BattleRecordCategory(): def get_sub_categories(self): return self.category.keys() + + def get_category_name(self) -> str: + return self._result_categories[self.category_name][0] + def record_result(self, result: dict): try: @@ -168,7 +174,7 @@ class BattleRecordCategory(): def print_results(self): try: first_btl_record = list(self.category.values())[0] - print(' '.join(first_btl_record.get_headers(self.category_name))) + print(' '.join(first_btl_record.get_headers(self.get_category_name()))) for row in self.get_results(): print(' : '.join(row)) except KeyError as err: @@ -233,6 +239,7 @@ class BattleRecord(): 'enemies_spotted', 'top_tier', 'player_wins', + 'player_battles', 'allies_wins', 'enemies_wins', 'allies_battles', @@ -249,6 +256,9 @@ class BattleRecord(): MISSING_STATS ] + _result_counts = [ + 'battles', MISSING_STATS, 'mastery_badge' + ] _result_ratios = { 'KDR' : [ 'enemies_destroyed', 'destroyed' ], 'DR' : [ 'damage_made', 'damage_received' ], @@ -278,7 +288,7 @@ class BattleRecord(): cls.result_fields_ratio = set(cls._result_ratios.keys()) & set(cls.result_fields) cls.avg_fields = set(cls.result_fields) - for field in ['battles', MISSING_STATS]: + for field in cls._result_counts: if field in cls.avg_fields: cls.avg_fields.remove(field) cls.avg_fields.difference_update(set(cls._result_ratios.keys())) @@ -362,7 +372,7 @@ class BattleRecord(): for field in self.avg_fields: self.results[field] = self.results[field] / max(self.battles,1) self.results['battles'] = self.battles - self.results[MISSING_STATS] = self.missing_stats / self.n_players + self.results[MISSING_STATS] = self.missing_stats / max(self.n_players, 1) self.results_ready = True return True except KeyError as err: @@ -385,6 +395,7 @@ class BattleRecord(): bu.error(str(err)) return None + def get_results(self): if not self.results_ready: self.calc_results() From aedb67f4e03c7e466149b51b074382d161486737 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 16:02:55 +0200 Subject: [PATCH 16/18] less frequent progress update --- analyze_wotb_replays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 8302682..fef4720 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -518,7 +518,7 @@ async def main(argv): args = parser.parse_args() bu.set_log_level(args.silent, args.verbose, args.debug) - bu.set_progress_step(30) # Set the frequency of the progress dots. + bu.set_progress_step(200) # Set the frequency of the progress dots. wg = await WG.create(WG_APP_ID, args.tankfile, args.mapfile, True) wi = WoTinspector() From 6b01b6cd476edb2c26624a86281e66f20d3fb4c2 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 20:57:03 +0200 Subject: [PATCH 17/18] Fixes. Should work now. --- analyze_wotb_replays.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index fef4720..ee5685b 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -93,7 +93,7 @@ class BattleRecordCategory(): 'battle_type' : [ 'Battle Type',['Encounter', 'Supremacy']], 'tank_tier' : [ 'Tank Tier', 'number' ], 'top_tier' : [ 'Tier', ['Bottom tier', 'Top tier']], - 'mastery_badge' : [ 'Battle Class', ['-', '3rd Class', '2nd Class', '1st Class', 'Mastery' ]], + 'mastery_badge' : [ 'Battle Medal', ['-', '3rd Class', '2nd Class', '1st Class', 'Mastery' ]], 'tank_name' : [ 'Tank', 'string' ], 'map_name' : [ 'Map', 'string' ], 'battle_i' : [ 'Battle #', 'number'] @@ -167,6 +167,7 @@ class BattleRecordCategory(): bu.error('Category: ', str(cat)) bu.error(str(result)) except Exception as err: + bu.error('Category: ', str(cat)) bu.error(str(err)) return False @@ -239,7 +240,7 @@ class BattleRecord(): 'enemies_spotted', 'top_tier', 'player_wins', - 'player_battles', +# 'player_battles', 'allies_wins', 'enemies_wins', 'allies_battles', @@ -518,7 +519,7 @@ async def main(argv): args = parser.parse_args() bu.set_log_level(args.silent, args.verbose, args.debug) - bu.set_progress_step(200) # Set the frequency of the progress dots. + bu.set_progress_step(250) # Set the frequency of the progress dots. wg = await WG.create(WG_APP_ID, args.tankfile, args.mapfile, True) wi = WoTinspector() @@ -763,6 +764,12 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg #bu.debug('Processing avg stats') player_mapped = stat_id_map[result['player']] + if player_mapped not in player_stats: + missing_stats += 1 + else: + for stat in stat_types: + if player_stats[player_mapped][stat] != None: + result['player_' + str(stat)] = player_stats[player_mapped][stat] for stat in stat_types: if n_allies[stat] > 0: @@ -775,9 +782,7 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg else: bu.debug('No enemies stats for: ' + str(result)) - if player_stats[player_mapped][stat] != None: - result['player_' + str(stat)] = player_stats[player_mapped][stat] - + result[N_PLAYERS] = n_players result[MISSING_STATS] = missing_stats return_list.append(result) @@ -1104,7 +1109,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam result = await read_replay_JSON(replay_json, args) bu.print_progress() if result == None: - bu.error('Replay[' + str(replayID) + ']: Invalid replay', id=readerID) + bu.debug('Replay[' + str(replayID) + ']: Invalid replay', id=readerID) queue.task_done() continue @@ -1133,6 +1138,10 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict #db = args.db result = {} try: + if (replay_json['data']['summary']['mastery_badge'] == None) or (replay_json['data']['summary']['battle_result'] < 0): + bu.debug('Invalid replay') + return None + result['battle_start_timestamp'] = int(replay_json['data']['summary']['battle_start_timestamp']) # TBD... protagonist = int(replay_json['data']['summary']['protagonist']) @@ -1156,8 +1165,10 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict return None if url: result['url'] = replay_json['data']['view_url'] + for key in replay_summary_flds: result[key] = replay_json['data']['summary'][key] + result['allies'] = set() result['enemies'] = set() From e51ab20f1e70f93e7bd05bdf07d3b17d2f6a407c Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sat, 15 Feb 2020 21:09:10 +0200 Subject: [PATCH 18/18] Widen the category name column to fit long tank names --- analyze_wotb_replays.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index ee5685b..d931a0c 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -115,7 +115,7 @@ class BattleRecordCategory(): ] - RESULT_CAT_FRMT = '{:>17s}' + RESULT_CAT_FRMT = '{:>20s}' @classmethod def get_result_categories(cls, extended_cats: list = None): @@ -275,7 +275,7 @@ class BattleRecord(): ratio_fields = set() - RESULT_CAT_HEADER_FRMT = '{:_<17s}' + RESULT_CAT_HEADER_FRMT = '{:_<20s}' @classmethod