diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index f9608f0..615a672 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -74,9 +74,9 @@ replay_details_flds = [ ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY) ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ] histogram_fields = { - 'wins' : [ 'Win rate', [0, .35, .40, .45, .5, .55, .60, .65, 1], 100, '{:.0f}%' ], - 'damage_dealt' : [ 'Avg. Dmg.', [0, .25e3, .5e3, 1e3, 1.5e3, 2e3,2.5e3,3e3,100e3], 1, '{:.0f}' ], - 'battles' : [ 'Battles', [0, 1e3, 3e3, 5e3, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k'] # battles is a mandatory stat to include + 'wins' : [ 'Win rate', [0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:.0f}%' ], + 'damage_dealt' : [ 'Avg. Dmg.', [0, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ], + 'battles' : [ 'Battles', [0, 1000, 2500, 5000, 7000, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k'] # battles is a mandatory stat to include } @@ -499,6 +499,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser): async def main(argv): global wg, wi, WG_APP_ID # set the directory for the script + current_dir = os.getcwd() os.chdir(os.path.dirname(sys.argv[0])) ## Default options: @@ -508,8 +509,11 @@ async def main(argv): OPT_STAT_FUNC = 'player' OPT_WORKERS_N = 10 - WG_ID = None - #WG_APP_ID = WG_APP_ID + WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. + # China is not supported since WG API stats are not available there + WG_ID = None # WG account_id in integer format. + # WG_ACCOUNT will be used to retrieve the account_id, but it can be set directly too + # WG_APP_ID = WG_APP_ID WG_RATE_LIMIT = 10 ## WG standard. Do not edit unless you have your ## own server app ID, it will REDUCE the performance @@ -532,45 +536,49 @@ async def main(argv): config.read(FILE_CONFIG) try: - configOptions = config['OPTIONS'] - # WG account id of the uploader: - # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ - OPT_DB = configOptions.getboolean('opt_DB', OPT_DB) - OPT_EXTENDED = configOptions.getboolean('opt_analyzer_extended', OPT_EXTENDED) - OPT_HIST = configOptions.getboolean('opt_analyzer_hist', OPT_HIST) - OPT_STAT_FUNC = configOptions.get('opt_analyzer_stat_func', fallback=OPT_STAT_FUNC) - OPT_WORKERS_N = configOptions.getint('opt_analyzer_workers', OPT_WORKERS_N) - except configparser.NoSectionError as err: + if 'OPTIONS' in config.sections(): + configOptions = config['OPTIONS'] + # WG account id of the uploader: + # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ + OPT_DB = configOptions.getboolean('opt_DB', OPT_DB) + OPT_EXTENDED = configOptions.getboolean('opt_analyzer_extended', OPT_EXTENDED) + OPT_HIST = configOptions.getboolean('opt_analyzer_hist', OPT_HIST) + OPT_STAT_FUNC = configOptions.get('opt_analyzer_stat_func', fallback=OPT_STAT_FUNC) + OPT_WORKERS_N = configOptions.getint('opt_analyzer_workers', OPT_WORKERS_N) + except (KeyError, configparser.NoSectionError) as err: bu.error(exception=err) try: - configWG = config['WG'] - WG_ID = configWG.getint('wg_id', WG_ID) - WG_APP_ID = configWG.get('wg_app_id', WG_APP_ID) - WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT) - except configparser.NoSectionError as err: + if 'WG' in config.sections(): + configWG = config['WG'] + WG_ID = configWG.getint('wg_id', WG_ID) + WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT) + WG_APP_ID = configWG.get('wg_app_id', WG_APP_ID) + WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT) + except (KeyError, configparser.NoSectionError) as err: bu.error(exception=err) try: - configDB = config['DATABASE'] - DB_SERVER = configDB.get('db_server', DB_SERVER) - DB_PORT = configDB.getint('db_port', DB_PORT) - DB_SSL = configDB.getboolean('db_ssl', DB_SSL) - DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ) - DB_AUTH = configDB.get('db_auth', DB_AUTH) - DB_NAME = configDB.get('db_name', DB_NAME) - DB_USER = configDB.get('db_user', DB_USER) - DB_PASSWD = configDB.get('db_password', DB_PASSWD) - DB_CERT = configDB.get('db_ssl_cert_file', DB_CERT) - DB_CA = configDB.get('db_ssl_ca_file', DB_CA) - except configparser.NoSectionError as err: + if 'DATABASE' in config.sections(): + configDB = config['DATABASE'] + DB_SERVER = configDB.get('db_server', DB_SERVER) + DB_PORT = configDB.getint('db_port', DB_PORT) + DB_SSL = configDB.getboolean('db_ssl', DB_SSL) + DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ) + DB_AUTH = configDB.get('db_auth', DB_AUTH) + DB_NAME = configDB.get('db_name', DB_NAME) + DB_USER = configDB.get('db_user', DB_USER) + DB_PASSWD = configDB.get('db_password', DB_PASSWD) + DB_CERT = configDB.get('db_ssl_cert_file', DB_CERT) + DB_CA = configDB.get('db_ssl_ca_file', DB_CA) + except (KeyError, configparser.NoSectionError) as err: bu.error(exception=err) parser = ErrorCatchingArgumentParser(description='Analyze Blitz replay JSON files from WoTinspector.com. Use \'upload_wotb_replays.py\' to upload the replay files first.') parser.add_argument('--output', default='plain', choices=['plain', 'db'], help='Select output mode: plain text or database') parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze') - parser.add_argument('-a', '--account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER') + parser.add_argument('-a', '--account', type=str, default=WG_ACCOUNT, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-x', '--extended', action='store_true', default=OPT_EXTENDED, help='Print Extended stats') parser.add_argument('-X', '--extra_categories', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print Extended categories') parser.add_argument('--hist', action='store_true', default=OPT_HIST, help='Print player histograms (WR/battles)') @@ -622,10 +630,12 @@ async def main(argv): bu.error("Could no initiate DB connection: Disabling DB", err) args.db = False pass - - if not(args.db): + else: bu.debug('No DB in use') + # rebase file arguments due to moving the working directory to the script location + args.files = bu.rebase_file_args(current_dir, args.files) + try: replayQ = asyncio.Queue(maxsize=1000) reader_tasks = [] @@ -1149,7 +1159,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto if (p_replayfile.match(line) != None): replay_json = await bu.open_JSON(line, wi.chk_JSON_replay) - await queue.put(await mk_readerQ_item(replay_json)) + await queue.put(await mk_readerQ_item(replay_json, line)) except Exception as err: bu.error(exception=err) else: @@ -1160,7 +1170,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto fn = fn[:-1] if os.path.isfile(fn) and (p_replayfile.match(fn) != None): replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay) - await queue.put(await mk_readerQ_item(replay_json)) + await queue.put(await mk_readerQ_item(replay_json, fn)) bu.debug('File added to queue: ' + fn) Nreplays += 1 elif os.path.isdir(fn): @@ -1169,7 +1179,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto if entry.is_file() and (p_replayfile.match(entry.name) != None): bu.debug(entry.name) replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay) - await queue.put(await mk_readerQ_item(replay_json)) + await queue.put(await mk_readerQ_item(replay_json, entry.name)) bu.debug('File added to queue: ' + entry.path) Nreplays += 1 except Exception as err: @@ -1178,28 +1188,29 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto return Nreplays -async def mk_readerQ_item(replay_json) -> list: +async def mk_readerQ_item(replay_json, filename : str = None) -> list: """Make an item to replay queue""" global REPLAY_N REPLAY_N +=1 - return [replay_json, REPLAY_N] + return [replay_json, REPLAY_N, os.path.basename(filename) ] async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace): """Async Worker to process the replay queue""" #global SKIPPED_N - # account_id = args.account_id results = [] playerstanks = set() try: while True: item = await queue.get() - replay_json = item[0] - replayID = item[1] + replay_json = item[0] + replayID = item[1] + replay_file = item[2] try: + msg_str = 'Replay[' + str(replayID) + ']: ' if replay_json == None: - bu.verbose('Replay[' + str(replayID) + ']: is empty. Skipping.' ) + bu.warning(msg_str + 'Invalid replay. Skipping: ' + (replay_file if replay_file != None else '') ) #SKIPPED_N += 1 queue.task_done() continue @@ -1209,7 +1220,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam result = await read_replay_JSON(replay_json, args) bu.print_progress() if result == None: - bu.debug('Replay[' + str(replayID) + ']: Invalid replay', id=readerID) + bu.warning(msg_str + 'Invalid replay' + (replay_file if replay_file != None else '') ) queue.task_done() continue diff --git a/blitzstats.ini.default b/blitzstats.ini.default index 10cc583..8793231 100644 --- a/blitzstats.ini.default +++ b/blitzstats.ini.default @@ -5,8 +5,14 @@ #opt_analyzer_stat_func = player [WG] + ## Find your WG account_id : https://developers.wargaming.net/reference/all/wotb/account/list/ -#wg_id = 0 +# wg_id = 0 + +## wg_account format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. +## China is not supported since WG API stats are not available there +# wg_account = your_nick@eu + ## WG API limits requests / sec to 10 for mobile apps. ## You can register your "own" server app and ## upgrade the rate limit to 20 requests / sec. diff --git a/blitzutils.py b/blitzutils.py index 939181d..298e4df 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -9,7 +9,7 @@ from progress.counter import Counter from decimal import Decimal MAX_RETRIES= 3 -SLEEP = 3 +SLEEP = 1.5 LOG_LEVELS = { 'silent': 0, 'normal': 1, 'verbose': 2, 'debug': 3 } SILENT = 0 @@ -210,6 +210,13 @@ def verbose_std(msg = "", id = None) -> bool: return True return False +def warning(msg = "", id = None) -> bool: + """Print a warning message""" + if _log_level >= NORMAL: + _print_log_msg('', 'Warning: ' + msg, None, id) + return True + return False + def debug(msg = "", id = None, exception = None, force: bool = False) -> bool: """print a conditional debug message""" if (_log_level >= DEBUG) or force: @@ -341,6 +348,10 @@ def NOW() -> int: return int(time.time()) +def rebase_file_args(current_dir, files): + """REbase file command line params after moving working dir to the script's dir""" + return [ os.path.join(current_dir, fn) for fn in files ] + async def read_int_list(filename: str) -> list(): """Read file to a list and return list of integers in the input file""" @@ -445,6 +456,40 @@ def bld_dict_hierarcy(d : dict, key : str, value) -> dict: error('Unexpected Exception', err) return None +def get_JSON_keypath(keypath: str, key: str): + if keypath == None: + return key + else: + return '.'.join([keypath, key]) + +def get_JSON_value(json, key : str = None, keys : list = None, keypath = None): + if keys == None: + keys = key.split('.') + + if len(keys) == 0: + return json + + key = keys.pop(0) + if type(json) == dict: + if key in json: + return get_JSON_value(json[key], keys=keys, keypath=get_JSON_keypath(keypath, key)) + else: + raise KeyError('Key: '+ get_JSON_keypath(keypath, key) + ' not found') + + if type(json) == list: + p = re.compile(r'^\[(\d+)\]$') + m = p.match(key) + if len(m.groups()) != 1: + raise KeyError('Invalid key given: ' + get_JSON_keypath(keypath, key)) + ndx = m.group(1) + try: + return get_JSON_value(json[ndx], keys=keys, keypath=get_JSON_keypath(keypath, key)) + except IndexError: + raise KeyError('JSON array index out of range: ' + get_JSON_keypath(keypath, key)) + + raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0])) + + ## ----------------------------------------------------------- #### Class SlowBar ## ----------------------------------------------------------- @@ -1462,41 +1507,39 @@ class WoTinspector: error(msg_str + 'Unexpected Exception', err) return None + json_resp = None for retry in range(MAX_RETRIES): debug(msg_str + 'Posting: ' + title + ' Try #: ' + str(retry + 1) + '/' + str(MAX_RETRIES) ) try: async with self.session.post(url, headers=headers, data=payload) as resp: debug(msg_str + 'HTTP response: '+ str(resp.status)) if resp.status == 200: - debug('HTTP POST 200 = Success. Reading response data') + debug(msg_str + 'HTTP POST 200 = Success. Reading response data') json_resp = await resp.json() - if json_resp.get('status', None) == None: - error(msg_str +' : ' + title + ' : Received invalid JSON') - elif (json_resp['status'] == 'ok'): - debug('Response data read. Status OK') + if self.chk_JSON_replay(json_resp): + debug(msg_str + 'Response data read. Status OK') return json_resp - elif (json_resp['status'] == 'error'): - error(msg_str + json_resp['error']['message'] + ' : ' + title) - else: - error(msg_str + ' Unspecified error: ' + title) + debug(msg_str +' : ' + title + ' : Receive invalid JSON') else: debug(msg_str + 'Got HTTP/' + str(resp.status)) except Exception as err: - error(exception=err) + debug(msg_str, exception=err) await asyncio.sleep(SLEEP) - error(msg_str + ' Could not post replay: ' + title) - return None + debug(msg_str + ' Could not post replay: ' + title) + return json_resp async def get_replay_listing(self, page: int = 0) -> aiohttp.ClientResponse: url = self.get_url_replay_listing(page) return await self.session.get(url) + @classmethod def get_url_replay_listing(cls, page : int): return cls.URL_REPLAY_LIST + str(page) + '?vt=#filters' + @classmethod def get_replay_links(cls, doc: str): """Get replay download links from WoTinspector.com replay listing page""" @@ -1513,21 +1556,24 @@ class WoTinspector: error(exception=err) return replay_links + @classmethod def get_replay_id(cls, url): return url.rsplit('/', 1)[-1] + @classmethod def chk_JSON_replay(cls, json_resp): """"Check String for being a valid JSON file""" try: - if ('status' in json_resp) and json_resp['status'] == 'ok' and ('data' in json_resp) and json_resp['data'] != None: + if ('status' in json_resp) and json_resp['status'] == 'ok' and \ + (get_JSON_value(json_resp, key='data.summary.exp_base') != None) : debug("JSON check OK") return True except KeyError as err: - error('Key not found', err) + debug('Replay JSON check failed', exception=err) except: - debug("JSON check failed: " + str(json_resp)) + debug("Replay JSON check failed: " + str(json_resp)) return False class BlitzStars: diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py index 4d38edd..15a1d2f 100644 --- a/upload_wotb_replays.py +++ b/upload_wotb_replays.py @@ -25,42 +25,48 @@ wi = None async def main(argv): global wg, wi + current_dir = os.getcwd() # set the directory for the script os.chdir(os.path.dirname(sys.argv[0])) - + # options defaults - OPT_WORKERS_N = 5 - WG_ID = None - WG_RATE_LIMIT = 10 + OPT_WORKERS_N = 5 + WG_ID = None + WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. + # China is not supported since WG API stats are not available there + WG_RATE_LIMIT = 10 try: ## Read config if os.path.isfile(FILE_CONFIG): + bu.debug('Reading config file: ' + FILE_CONFIG) config = configparser.ConfigParser() config.read(FILE_CONFIG) - try: - configOptions = config['OPTIONS'] - OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', OPT_WORKERS_N) + if 'OPTIONS' in config.sections(): + configOptions = config['OPTIONS'] + OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', OPT_WORKERS_N) except (KeyError, configparser.NoSectionError) as err: - bu.debug("No section 'OPTIONS' found in configuration file: " + FILE_CONFIG) + bu.error(exception=err) try: - configWG = config['WG'] - # WG account id of the uploader: - # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ - WG_ID = configWG.getint('wg_id', WG_ID) - ## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay - WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT) + if 'WG' in config.sections(): + configWG = config['WG'] + # WG account id of the uploader: + # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ + WG_ID = configWG.getint('wg_id', WG_ID) + WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT) + ## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay + WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT) except (KeyError, configparser.NoSectionError) as err: - bu.debug("No section 'WG' found in configuration file: " + FILE_CONFIG) - + bu.error(exception=err) + except Exception as err: bu.error(exception=err) parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON') parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id') - parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') + parser.add_argument('-a', '--account', dest='account', type=str, default=WG_ACCOUNT, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering') parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') @@ -86,7 +92,9 @@ async def main(argv): try: queue = asyncio.Queue() - + # rebase file arguments due to moving the working directory to the script location + args.files = bu.rebase_file_args(current_dir, args.files) + tasks = [] # Make replay Queue tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) @@ -104,7 +112,7 @@ async def main(argv): task.cancel() bu.debug('Waiting for workers to cancel') await asyncio.gather(*tasks, return_exceptions=True) - bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors') + bu.verbose_std(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors') except KeyboardInterrupt: print('Ctrl-c pressed ...') @@ -186,7 +194,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' ) else: os.remove(replay_json_fn) - bu.debug("Replay JSON not valid: Deleting " + replay_json_fn) + bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID) SKIPPED_N += 1 queue.task_done() continue @@ -203,11 +211,18 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N) if json_resp != None: if (await bu.save_JSON(replay_json_fn,json_resp)): - if not bu.debug(msg_str + 'Replay saved OK: ' + filename): - bu.verbose_std(msg_str + title + ' posted') + if wi.chk_JSON_replay(json_resp): + if not bu.debug(msg_str + 'Replay saved OK: ' + filename): + bu.verbose_std(msg_str + title + ' posted') + else: + bu.warning(msg_str +'Replay file is not valid: ' + filename) + ERROR_N += 1 else: bu.error(msg_str + 'Error saving replay: ' + filename) - ERROR_N += 1 + ERROR_N += 1 + else: + bu.error(msg_str + 'Replay file is not valid: ' + filename) + ERROR_N += 1 except Exception as err: bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.debug(msg_str + 'Marking task done')