From 45756a451fd5d5e4a0e2fceb00fd5b2a6ecd6968 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 25 Feb 2020 20:22:00 +0200 Subject: [PATCH 1/2] Renaming of config file params, clean-up --- analyze_wotb_replays.py | 15 +++++++-------- blitzstats.ini.default | 7 +++---- upload_wotb_replays.py | 11 ++++------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 379a354..efda73b 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -483,9 +483,10 @@ async def main(argv): # WG account id of the uploader: # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ OPT_DB = configOptions.getboolean('opt_DB', False) - OPT_EXTENDED = configOptions.getboolean('opt_extended', False) - OPT_HIST = configOptions.getboolean('opt_hist', False) - OPT_STAT_FUNC = configOptions.get('opt_stat_func', fallback='tank_tier') + OPT_EXTENDED = configOptions.getboolean('opt_analyzer_extended', False) + OPT_HIST = configOptions.getboolean('opt_analyzer_hist', False) + OPT_STAT_FUNC = configOptions.get('opt_analyzer_stat_func', fallback='player') + OPT_WORKERS_N = configOptions.getint('opt_analyzer_workers', 10) configWG = config['WG'] WG_ID = configWG.getint('wg_id', None) @@ -503,9 +504,7 @@ async def main(argv): DB_PASSWD = configDB.get('db_password', "PASSWORD") DB_CERT = configDB.get('db_ssl_cert_file', None) DB_CA = configDB.get('db_ssl_ca_file', None) - - TASK_N = 7 - + parser = argparse.ArgumentParser(description='Analyze Blitz replay JSONs from WoTinspector.com') parser.add_argument('--output', default='plain', choices=['json', 'plain', 'db'], help='Select output mode: JSON, plain text or database') parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze') @@ -569,7 +568,7 @@ async def main(argv): scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db)) bu.debug('Replay scanner started') # Start tasks to process the Queue - for i in range(TASK_N): + for i in range(OPT_WORKERS_N): reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args))) bu.debug('ReplayReader ' + str(i) + ' started') @@ -589,7 +588,7 @@ async def main(argv): results.extend(res[0]) players.update(res[1]) - (player_stats, stat_id_map) = await process_player_stats(players, TASK_N, args, db) + (player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db) bu.verbose('') bu.debug('Number of player stats: ' + str(len(player_stats))) teamresults = calc_team_stats(results, player_stats, stat_id_map, args) diff --git a/blitzstats.ini.default b/blitzstats.ini.default index d4fd4e1..809d341 100644 --- a/blitzstats.ini.default +++ b/blitzstats.ini.default @@ -1,14 +1,13 @@ [OPTIONS] ## You may set your preferred commandline param defaults here -#opt_extended = False -#opt_hist = False -#opt_stat_func = tank_tier +#opt_analyzer_extended = False +#opt_analyzer_hist = False +#opt_analyzer_stat_func = player [WG] ## Find your WG account_id : https://developers.wargaming.net/reference/all/wotb/account/list/ #wg_id = 0 ## WG API limits requests / sec to 10 for client apps ## Check https://developers.wargaming.net/documentation/guide/getting-started/ -#wg_rate_limit = 10 #wg_app_id = XXXXXXXXXXX diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py index 5fd5b81..08f8cbf 100644 --- a/upload_wotb_replays.py +++ b/upload_wotb_replays.py @@ -14,7 +14,6 @@ FILE_CONFIG = 'blitzstats.ini' DEBUG = False VERBOSE = False SLEEP = 2 -N_WORKERS = 5 MAX_RETRIES = 3 REPLAY_N = 0 SKIPPED_N = 0 @@ -33,6 +32,9 @@ async def main(argv): config = configparser.ConfigParser() config.read(FILE_CONFIG) + configOptions = config['OPTIONS'] + OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', 5) + configWG = config['WG'] # WG account id of the uploader: # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ @@ -41,15 +43,12 @@ async def main(argv): WG_RATE_LIMIT = configWG.getint('wg_rate_limit', 10) parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data') - #parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database') parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id') parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering') parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com') - parser.add_argument('--tasks', dest='N_tasks', type=int, default=N_WORKERS, help='Number of worker threads') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') -# parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') @@ -76,8 +75,7 @@ async def main(argv): # Make replay Queue tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) # Start tasks to process the Queue - for i in range(args.N_tasks): - #tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private))) + for i in range(OPT_WORKERS_N): tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private))) bu.debug('Task ' + str(i) + ' started') @@ -164,7 +162,6 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri #bu.debug(msg_str + replay_json_fn) try: - #if os.path.exists(replay_json_fn) and os.path.isfile(replay_json_fn): if os.path.isfile(replay_json_fn): async with aiofiles.open(replay_json_fn) as fp: replay_json = json.loads(await fp.read()) From 5d1d2f5c083ac10a7d28e5211dc172ecc41139cd Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 25 Feb 2020 20:26:48 +0200 Subject: [PATCH 2/2] Update stats cache time to 30 days --- blitzutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blitzutils.py b/blitzutils.py index c8c086d..8274e9e 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -437,7 +437,7 @@ class WG: URL_WG_ACCOUNT_ID = 'account/list/?fields=account_id%2Cnickname&application_id=' URL_WG_PLAYER_STATS = 'account/info/?application_id=' CACHE_DB_FILE = '.blitzutils_cache.sqlite3' - CACHE_GRACE_TIME = 2*7*24*3600 # 2 weeks cache + CACHE_GRACE_TIME = 30*24*3600 # 30 days cache # sql_create_player_stats_tbl = """CREATE TABLE IF NOT EXISTS player_stats ( # account_id INTEGER NOT NULL,