diff --git a/analyzeReplays.py b/analyze_wotb_replays.py similarity index 55% rename from analyzeReplays.py rename to analyze_wotb_replays.py index 23145c0..be8976a 100644 --- a/analyzeReplays.py +++ b/analyze_wotb_replays.py @@ -2,8 +2,8 @@ # Script Analyze WoT Blitz replays -import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole, re, logging, time, xmltodict, collections -import motor.motor_asyncio, configparser, ssl +import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole +import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections from datetime import datetime, timedelta, date import blitzutils as bu from blitzutils import WG @@ -11,9 +11,10 @@ from blitzutils import WoTinspector logging.getLogger("asyncio").setLevel(logging.DEBUG) -WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' -FILE_CONFIG = 'blitzstats.ini' -DB_C_TANK_STATS = 'WG_TankStats' +WG_APP_ID = '81381d3f45fa4aa75b78a7198eb216ad' +FILE_CONFIG = 'blitzstats.ini' +DB_C_TANK_STATS = 'WG_TankStats' +DB_C_PLAYER_STATS = 'WG_PlayerStats' wg = None wi = None @@ -21,6 +22,13 @@ REPLAY_N = 0 REPLAY_I = 0 STAT_TANK_BATTLE_MIN = 100 +## For different player stat functions (global stats, tank tier stats, etc) +# 1st function = for forming stat_id, 2nd for DB stats query, 3rd for WG API stats query +STAT_FUNC = { + 'tank_tier': [ 'get_stat_id_tank_tier', 'get_db_tank_tier_stats', 'get_wg_tank_tier_stats' ], + 'player': [ 'get_stat_id_player', 'get_db_player_stats', 'get_wg_player_stats' ] +} + replay_summary_flds = [ 'battle_result', 'battle_type', @@ -81,8 +89,8 @@ result_categories_default = [ 'tank_tier' ] -result_cat_header_frmt = '{:_<17s}' -result_cat_frmt = '{:>17s}' +RESULT_CAT_HEADER_FRMT = '{:_<17s}' +RESULT_CAT_FRMT = '{:>17s}' ## Syntax: Check how the replay JSON files look. The algorithm is counting/recording fields @@ -113,10 +121,10 @@ histogram_fields = { } -def defaultvalueZero(): +def def_value_zero(): return 0 -def defaultvalueBtlRec(): +def def_value_BattleRecord(): return BattleRecord() @@ -125,14 +133,14 @@ class PlayerHistogram(): self.field = field self.name = name self.fields = fields - self.catFactor = factor - self.catFormat = format + self.cat_factor = factor + self.cat_format = format self.ncat = len(self.fields) - 1 self.allies = [0] * self.ncat self.enemies = [0] * self.ncat self.total = [0] * self.ncat - def recordAlly(self, stat: float): + def record_ally(self, stat: float): for cat in range(0, self.ncat): if self.fields[cat] <= stat <= self.fields[cat+1]: self.allies[cat] += 1 @@ -140,7 +148,7 @@ class PlayerHistogram(): return True return False - def recordEnemy(self, stat: float): + def record_enemy(self, stat: float): for cat in range(0, self.ncat): if self.fields[cat] <= stat <= self.fields[cat+1]: self.enemies[cat] += 1 @@ -148,15 +156,15 @@ class PlayerHistogram(): return True return False - def getCatname(self, cat: int): + def get_category_name(self, cat: int): if cat < self.ncat - 1: - return self.formatCat(cat) + ' - ' + self.formatCat(cat+1) + return self.format_category(cat) + ' - ' + self.format_category(cat+1) else: - return self.formatCat(cat) + ' -' + return self.format_category(cat) + ' -' - def formatCat(self, cat: int): + def format_category(self, cat: int): val = self.fields[cat] - return self.catFormat.format(float(val * self.catFactor)) + return self.cat_format.format(float(val * self.cat_factor)) def print(self): N_enemies = 0 @@ -168,7 +176,7 @@ class PlayerHistogram(): print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL")) for cat in range(0, self.ncat): - print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(self.getCatname(cat), self.allies[cat], self.allies[cat]/N_allies*100, self.enemies[cat], self.enemies[cat]/N_enemies*100, self.allies[cat] + self.enemies[cat], (self.allies[cat] + self.enemies[cat])/N_total*100 )) + print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(self.get_category_name(cat), self.allies[cat], self.allies[cat]/N_allies*100, self.enemies[cat], self.enemies[cat]/N_enemies*100, self.allies[cat] + self.enemies[cat], (self.allies[cat] + self.enemies[cat])/N_total*100 )) return None @@ -178,7 +186,7 @@ class BattleRecord(): self.battles = 0 self.ratios_ready = False self.results_ready = False - self.results = collections.defaultdict(defaultvalueZero) + self.results = collections.defaultdict(def_value_zero) self.avg_fields = set(result_fields.keys()) self.avg_fields.remove('battles') self.avg_fields.difference_update(result_ratios.keys()) @@ -188,9 +196,9 @@ class BattleRecord(): self.ratio_fields.add(result_ratios[ratio][1]) except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) - def recordResult(self, result : dict) -> bool: + def record_result(self, result : dict) -> bool: try: for field in self.avg_fields | self.ratio_fields: if result[field] != None: @@ -198,10 +206,10 @@ class BattleRecord(): self.battles += 1 return True except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) return False - def calcRatios(self) -> bool: + def calc_ratios(self) -> bool: if self.ratios_ready: return True try: @@ -215,14 +223,14 @@ class BattleRecord(): self.ratios_ready = True return True except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) return False - def calcResults(self): + def calc_results(self): if self.results_ready == True: return True if not self.ratios_ready: - self.calcRatios() + self.calc_ratios() try: for field in self.avg_fields: self.results[field] = self.results[field] / max(self.battles,1) @@ -230,33 +238,33 @@ class BattleRecord(): self.results_ready = True return True except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: - bu.error(str(err)) + bu.error(exception=err) return False - def getResults(self): + def get_results(self): if not self.results_ready: - self.calcResults() + self.calc_results() results = [] try: for field in result_fields.keys(): results.append( result_fields[field][3].format(self.results[field]) ) return results except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: - bu.error(str(err)) + bu.error(exception=err) return None - def printResults(self): - print(' : '.join(self.getResults())) + def print_results(self): + print(' : '.join(self.get_results())) class BattleRecordCategory(): def __init__(self, cat_name : str): self.category_name = cat_name - self.category = collections.defaultdict(defaultvalueBtlRec) + self.category = collections.defaultdict(def_value_BattleRecord) if result_categories[self.category_name][1] == 'string': self.type = 'string' elif result_categories[self.category_name][1] == 'number': @@ -266,10 +274,10 @@ class BattleRecordCategory(): else: self.type = 'category' - def getSubCategories(self): + def get_sub_categories(self): return self.category.keys() - def recordResult(self, result: dict): + def record_result(self, result: dict): try: if self.type == 'total': cat = 'Total' @@ -279,56 +287,56 @@ class BattleRecordCategory(): cat = result[self.category_name] else: cat = result_categories[self.category_name][1][result[self.category_name]] - self.category[cat].recordResult(result) + self.category[cat].record_result(result) return True except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: bu.error(str(err)) return False - def getHeaders(self): + def get_headers(self): try: - headers = [ result_cat_header_frmt.format(result_categories[self.category_name][0]) ] + headers = [ RESULT_CAT_HEADER_FRMT.format(result_categories[self.category_name][0]) ] for field in result_fields.keys(): print_format = '{:^' + str(result_fields[field][2]) + 's}' headers.append(print_format.format(result_fields[field][0])) return headers except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: bu.error(str(err)) return None - def printResults(self): + def print_results(self): try: - print(' '.join(self.getHeaders())) - for row in self.getResults(): + print(' '.join(self.get_headers())) + for row in self.get_results(): print(' : '.join(row)) except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: bu.error(str(err)) return None - def getResults(self): + def get_results(self): try: results = [] - # results.append(self.getHeaders()) + # results.append(self.get_headers()) if self.type == 'number': for cat in sorted( [ int(s) for s in self.category.keys() ] ): cat = str(cat) - row = [ result_cat_frmt.format(cat) ] - row.extend(self.category[cat].getResults()) + row = [ RESULT_CAT_FRMT.format(cat) ] + row.extend(self.category[cat].get_results()) results.append(row) else: for cat in sorted(self.category.keys() , key=str.casefold): - row = [ result_cat_frmt.format(cat) ] - row.extend(self.category[cat].getResults()) + row = [ RESULT_CAT_FRMT.format(cat) ] + row.extend(self.category[cat].get_results()) results.append(row) return results except KeyError as err: - bu.error('Key ' + str(err) + ' not found') + bu.error('Key not found', err) return None ## main() ------------------------------------------------------------- @@ -361,12 +369,13 @@ async def main(argv): TASK_N = 7 - parser = argparse.ArgumentParser(description='ANalyze Blitz replay JSONs from WoTinspector.com') + parser = argparse.ArgumentParser(description='Analyze Blitz replay JSONs from WoTinspector.com') parser.add_argument('--output', default='plain', choices=['json', 'plain', 'db'], help='Select output mode: JSON, plain text or database') parser.add_argument('-id', dest='accountID', type=int, default=None, help='WG account_id to analyze') parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-x', '--extended', action='store_true', default=False, help='Print Extended stats') parser.add_argument('-s', '--stats', action='store_true', default=False, help='Print player stats (WR/battles)') + parser.add_argument('--stat_func', default='tank_tier', choices=STAT_FUNC.keys(), help='Select how to calculate for ally/enemy performance: tank-tier stats, global player stats') parser.add_argument('-u', '--url', dest= 'url', action='store_true', default=False, help='Print replay URLs') parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"') @@ -377,14 +386,14 @@ async def main(argv): parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"') args = parser.parse_args() - bu.setVerbose(args.verbose) - bu.setDebug(args.debug) - wg = WG(WG_appID, args.tankfile, args.mapfile) + bu.set_verbose(args.verbose) + bu.set_debug(args.debug) + wg = await WG.create(WG_APP_ID, args.tankfile, args.mapfile, True) wi = WoTinspector() - bu.setWaiter(100) + bu.set_progress_step(30) # Set the frequency of the progress dots. if args.account != None: - args.accountID = await wg.getAccountID(args.account) + args.accountID = await wg.get_account_id(args.account) bu.debug('WG account_id: ' + str(args.accountID)) #### Connect to MongoDB (TBD) @@ -396,7 +405,7 @@ async def main(argv): db = client[DB_NAME] bu.debug('Database connection initiated') except Exception as err: - bu.error("Could no initiate DB connection: Disabling DB" + str(err)) + bu.error("Could no initiate DB connection: Disabling DB", err) args.db = False pass else: @@ -406,11 +415,11 @@ async def main(argv): replayQ = asyncio.Queue() reader_tasks = [] # Make replay Queue - scanner_task = asyncio.create_task(mkReplayQ(replayQ, args.files)) + scanner_task = asyncio.create_task(mk_replayQ(replayQ, args.files)) bu.debug('Replay scanner started') # Start tasks to process the Queue for i in range(TASK_N): - reader_tasks.append(asyncio.create_task(replayReader(replayQ, i, args))) + reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args))) bu.debug('ReplayReader ' + str(i) + ' started') bu.debug('Waiting for the replay scanner to finish') @@ -429,53 +438,60 @@ async def main(argv): results.extend(res[0]) players.update(res[1]) - ## Remove battle_time from the player:tank data in case DB is not in use - if db == None: - players_tmp = set() - for player in players: - #bu.debug('playerID: ' + player + ' ==> ' + prunePlayerStatID(player)) - players_tmp.add(prunePlayerStatID(player)) - #bu.debug(str(len(players_tmp))) - players = players_tmp + # ## Remove battle_time from the player:tank data in case DB is not in use + # if db == None: + # players_tmp = set() + # for player in players: + # #bu.debug('playerID: ' + player + ' ==> ' + prune_stat_id(player)) + # players_tmp.add(prune_stat_id(player)) + # #bu.debug(str(len(players_tmp))) + # players = players_tmp - player_stats = await processPlayerStats(players, TASK_N, db) + (player_stats, stat_id_map) = await process_player_stats(players, TASK_N, args, db) bu.verbose('') - bu.debug('PLAYER STATS:\n' + str(player_stats)) - teamresults = calcTeamStats(results, player_stats) - processBattleStats(teamresults, args) + bu.debug('Number of player stats: ' + str(len(player_stats))) + teamresults = calc_team_stats(results, player_stats, stat_id_map, args) + process_battle_results(teamresults, args) if args.stats: - processPlayerDist(results, player_stats) + process_player_dist(results, player_stats, stat_id_map) + bu.debug('Finished. Cleaning up..................') except Exception as err: - bu.error(str(type(err)) + ' : ' + str(err)) + bu.error(exception=err) finally: ## Need to close the aiohttp.session since Python destructors do not support async methods... await wg.close() await wi.close() return None -def processPlayerDist(result_list: list, player_stats: dict): - +def process_player_dist(results: list, player_stats: dict, stat_id_map: dict): + """Process player distribution""" hist_stats = dict() - - for field in histogram_fields.keys(): - hist_stats[field] = PlayerHistogram(field, histogram_fields[field][0], histogram_fields[field][1], histogram_fields[field][2], histogram_fields[field][3] ) + try: + for field in histogram_fields: + hist_stats[field] = PlayerHistogram(field, histogram_fields[field][0], histogram_fields[field][1], histogram_fields[field][2], histogram_fields[field][3] ) - for result in result_list: - for ally in result['allies']: - for stat_field in hist_stats.keys(): - hist_stats[stat_field].recordAlly(player_stats[ally][stat_field]) - for enemy in result['enemies']: - for stat_field in hist_stats.keys(): - hist_stats[stat_field].recordEnemy(player_stats[enemy][stat_field]) + for result in results: + for player in result['allies'] | result['enemies']: # uniton of Sets + player_remapped = stat_id_map[player] + if player_remapped in player_stats: + if player in result['allies']: + for stat_field in hist_stats: + hist_stats[stat_field].record_ally(player_stats[player_remapped][stat_field]) + else: + for stat_field in hist_stats: + hist_stats[stat_field].record_enemy(player_stats[player_remapped][stat_field]) + + for stat_field in hist_stats: + hist_stats[stat_field].print() - for stat_field in hist_stats.keys(): - hist_stats[stat_field].print() - + except Exception as err: + bu.error(exception=err) + return None -def processBattleStats(results: dict, args : argparse.Namespace): - +def process_battle_results(results: dict, args : argparse.Namespace): + """Process replay battle results""" url = args.url urls = collections.OrderedDict() categories = {} @@ -490,16 +506,16 @@ def processBattleStats(results: dict, args : argparse.Namespace): max_title_len = 0 for result in results: for cat in cats: - categories[cat].recordResult(result) + categories[cat].record_result(result) if url: max_title_len = max(max_title_len, len(result['title'])) urls[result['title']] = result['url'] for cat in cats: - for sub_cat in categories[cat].getSubCategories(): - categories[cat].category[sub_cat].calcResults() + for sub_cat in categories[cat].get_sub_categories(): + categories[cat].category[sub_cat].calc_results() print('') - categories[cat].printResults() + categories[cat].print_results() if url: print('\nURLs to Battle replays:\n') for title in urls.keys(): @@ -507,173 +523,284 @@ def processBattleStats(results: dict, args : argparse.Namespace): return None -async def processPlayerStats(players, N_workers: int, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict: + +async def process_player_stats(players, N_workers: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict: + """Start stat_workers to retrieve and store players' stats""" ## Create queue of player-tank pairs to find stats for statsQ = asyncio.Queue() bu.debug('Create player stats queue: ' + str(len(players)) + ' players') + stat_id_map = {} + stat_ids = set() + + stat_id_map_func = globals()[STAT_FUNC[args.stat_func][0]] + for player in players: - #bu.debug(player) - await statsQ.put(player) + stat_id_map[player] = stat_id_map_func(player) + stat_ids.add(stat_id_map[player]) + + # create statsQ + for stat_id in stat_ids: + await statsQ.put(stat_id) + # Process player WR / battle stats stats_tasks = [] bu.debug('Starting player stats workers') for i in range(N_workers): bu.debug("Starting worker " + str(i)) - stats_tasks.append(asyncio.create_task(statWorker(statsQ, i, db))) - ## let the workers to process + stats_tasks.append(asyncio.create_task(stat_worker(statsQ, i, args, db))) + bu.debug('Waiting stats workers to finish') await statsQ.join() bu.debug('Cancelling stats workers') for task in stats_tasks: task.cancel() player_stats = {} - bu.debug('Gathering stats worker outputs') - for stats in await asyncio.gather(*stats_tasks): - player_stats = {**player_stats, **stats} - bu.debug('Returning player_stats') - return player_stats + stat_id_remap = {} -def calcTeamStats(result_list: list, player_stats : dict) -> list: + bu.debug('Gathering stats worker outputs') + for (stats, id_remap) in await asyncio.gather(*stats_tasks): + player_stats = {**player_stats, **stats} + stat_id_remap = {**stat_id_remap, **id_remap} + + ## DO REMAPPING + stat_id_map = remap_stat_id(stat_id_map, stat_id_remap) + + bu.debug('Returning player_stats') + return (player_stats, stat_id_map) + + +def remap_stat_id(stat_id_map: dict, stat_id_remap: dict): + bu.debug('Remapping stat_ids: stat_id_map: ' + str(len(stat_id_map)) + ' remap needed: ' + str(len(stat_id_remap))) + for stat_id in stat_id_map: + # bu.debug(stat_id) + if stat_id_map[stat_id] in stat_id_remap: + stat_id_map[stat_id] = stat_id_remap[stat_id_map[stat_id]] + return stat_id_map + + +def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, args : argparse.Namespace) -> list: + """Calculate team stats""" + return_list = [] + stat_types = player_stats[list(player_stats.keys())[0]].keys() bu.debug('stat_types: ' + ','.join(stat_types)) - for result in result_list: + + for result in results: try: - n_allies = collections.defaultdict(defaultvalueZero) - allies_stats = collections.defaultdict(defaultvalueZero) - bu.debug('Processing Allies') + n_allies = collections.defaultdict(def_value_zero) + allies_stats = collections.defaultdict(def_value_zero) + #bu.debug('Processing Allies') for ally in result['allies']: - # Player itself is not in 'allies': see readReplayJSON() - if ally not in player_stats: continue + # Player itself is not in 'allies': see read_replay_JSON() + ally_mapped = stat_id_map[ally] + if ally_mapped not in player_stats: + continue for stat in stat_types: - if player_stats[ally][stat] != None: - allies_stats[stat] += player_stats[ally][stat] + if player_stats[ally_mapped][stat] != None: + allies_stats[stat] += player_stats[ally_mapped][stat] n_allies[stat] += 1 - bu.debug('Processing Allies\' avg stats') + + #bu.debug('Processing Allies\' avg stats') for stat in stat_types: if n_allies[stat] > 0: result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat] - bu.debug('Processing Enemies') - n_enemies = collections.defaultdict(defaultvalueZero) - enemies_stats = collections.defaultdict(defaultvalueZero) + #bu.debug('Processing Enemies') + n_enemies = collections.defaultdict(def_value_zero) + enemies_stats = collections.defaultdict(def_value_zero) for enemy in result['enemies']: - if enemy not in player_stats: continue + enemy_mapped = stat_id_map[enemy] + if enemy_mapped not in player_stats: continue for stat in stat_types: - if player_stats[enemy][stat] != None: - enemies_stats[stat] += player_stats[enemy][stat] + if player_stats[enemy_mapped][stat] != None: + enemies_stats[stat] += player_stats[enemy_mapped][stat] n_enemies[stat] += 1 + + #bu.debug('Processing Enemies\' avg stats') for stat in stat_types: if n_enemies[stat] > 0: result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat] return_list.append(result) except KeyError as err: - bu.error('Key :' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: - bu.error(err) + bu.error(exception=err) return return_list -async def statWorker(queue : asyncio.Queue, workerID: int, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> list: + +async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> list: """Worker thread to find stats for player / tank pairs""" # global wg - stats = {} + stats = {} + stat_id_remap = {} + + stat_db_func = globals()[STAT_FUNC[args.stat_func][1]] + stat_wg_func = globals()[STAT_FUNC[args.stat_func][2]] + bu.debug("workedID: " + str(workerID) + ' started') try: while True: - item = await queue.get() + stat_id = await queue.get() try: - bu.printWaiter() - bu.debug(str(item)) - ## Add time stamp here - stat = item.split(':') - account_id = int(stat[0]) - tier = int(stat[1]) + bu.debug('Stat_id: ' + stat_id) + bu.print_progress() + #bu.debug(str(stat_id), workerID) + # ## Add time stamp here + # stat = stat_id.split(':') + # account_id = int(stat[0]) + # tier = int(stat[1]) - if len(stat) == 3: - battletime = int(stat[2]) - else: - battletime = None + # if len(stat) == 3: + # battletime = int(stat[2]) + # else: + # battletime = None + # bu.debug('account_id: ' + str(account_id) + ' tank_tier: ' + str(tier), workerID) - bu.debug('account_id: ' + str(account_id) + ' tank_tier: ' + str(tier), workerID) + # Analysing player performance based on their stats on the tier tanks they are playing - stats[item] = {} - stats[item] = await getDBplayerStats(db, account_id, tier, battletime) - bu.debug('getDBplayerStats returned: ' + str(stats[item]), workerID) - if stats[item] == None: - stats[item] = await getWGplayerStats(account_id, tier) - stats[prunePlayerStatID(item)] = stats[item] - if stats[item] == None: - del stats[item] - + stats[stat_id] = await stat_db_func(db, stat_id) + bu.debug('get_db_' + args.stat_func + '_stats returned: ' + str(stats[stat_id]), workerID) + + if (stats[stat_id] == None): + pruned_stat_id = prune_stat_id(stat_id) + if (pruned_stat_id not in stats): + stats[pruned_stat_id] = await stat_wg_func(pruned_stat_id) + stat_id_remap[stat_id] = pruned_stat_id + del stats[stat_id] + except KeyError as err: - bu.error('Key :' + str(err) + ' not found', workerID) + bu.error('Key not found', err, workerID) except Exception as err: - bu.error('Type: ' + str(type(err)) + ' : ' + str(err), workerID) + bu.error('Unexpected error', err, workerID) queue.task_done() except (asyncio.CancelledError, concurrent.futures._base.CancelledError): - bu.debug('Stats queue is empty', workerID) + bu.debug('Stats queue is empty', workerID) except Exception as err: - bu.error(str(err)) - return stats + bu.error(exception=err) + finally: + # remove empty stats + try: + keys_2_del = [] + for key in stats: + if stats[key] == None: + keys_2_del.append(key) + for key in keys_2_del: + del stats[key] + except KeyError as err: + bu.error('Error in pruning empty stats', err) + # bu.debug('Returning stats & exiting') + return (stats, stat_id_remap) - -async def getWGplayerStats(account_id: int, tier: int) -> dict: +## player stat functions: tank_tier +async def get_wg_tank_tier_stats(stat_id_str: str) -> dict: """Get player stats from WG. Returns WR per tier of tank_id""" try: - tier_tanks = wg.getTanksByTier(tier) + (account_id, tier) = str2ints(stat_id_str) + tier_tanks = wg.get_tanks_by_tier(tier) # 'battles' must always be there hist_stats = [ 'all.' + x for x in histogram_fields.keys() ] + hist_stats.append('tank_id') - playerStats = await wg.getPlayerTanksStats(account_id, tier_tanks ,hist_stats) - bu.debug('account_id: ' + str(account_id) + ' ' + str(playerStats)) + player_stats = await wg.get_player_tanks_stats(account_id, tier_tanks ,hist_stats) + #bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats)) - return await statsHelper(playerStats) + return await tank_stats_helper(player_stats) except KeyError as err: - bu.error('account_id: ' + str(account_id) + ' Key :' + str(err) + ' not found') + bu.error('account_id: ' + str(account_id) + ' Key not found', err) except Exception as err: - bu.error('Type: ' + str(type(err)) + ' : ' + str(err)) + bu.error(exception=err) return None -async def getDBplayerStats(db : motor.motor_asyncio.AsyncIOMotorDatabase, account_id : int, tier: int, battletime: int) -> dict: +async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_id_str: str) -> dict: """Get player stats from MongoDB (you are very unlikely to have it, unless you have set it up)""" if db == None: return None try: dbc = db[DB_C_TANK_STATS] - tier_tanks = wg.getTanksByTier(tier) + ( account_id, tier, battletime ) = str2ints(stat_id_str) + tier_tanks = wg.get_tanks_by_tier(tier) + time_buffer = 2*7*24*3600 #bu.debug('Tank_ids: ' + ','.join(map(str, tier_tanks))) - if battletime == None: - pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'tank_id' : {'$in': tier_tanks} } ]}}, - { '$sort': { 'last_battle_time': -1 }}, - { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, - { '$replaceRoot': { 'newRoot': '$doc' }}, - { '$project': { '_id': 0 }} ] - else: - pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, - { '$sort': { 'last_battle_time': 1 }}, - { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, - { '$replaceRoot': { 'newRoot': '$doc' }}, - { '$project': { '_id': 0 }} ] + # if battletime == None: + # pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'tank_id' : {'$in': tier_tanks} } ]}}, + # { '$sort': { 'last_battle_time': -1 }}, + # { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, + # { '$replaceRoot': { 'newRoot': '$doc' }}, + # { '$project': { '_id': 0 }} ] + # else: + pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, + { '$sort': { 'last_battle_time': -1 }}, + { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, + { '$replaceRoot': { 'newRoot': '$doc' }}, + { '$project': { '_id': 0 }} ] # cursor = dbc.aggregate(pipeline, allowDiskUse=True) cursor = dbc.aggregate(pipeline) - return await statsHelper(await cursor.to_list(1000)) + return await tank_stats_helper(await cursor.to_list(1000)) except Exception as err: - bu.error('account_id: ' + str(account_id) + ' Error :' + str(err)) + bu.error('account_id: ' + str(account_id) + ' Error', err) return None -async def statsHelper(stat_list: list): - """Helpher func got getDBplayerStats() and getWGplayerStats()""" +## player stat functions: player +async def get_db_player_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_id_str: str) -> dict: + """Get player stats from MongoDB (you are very unlikely to have it, unless you have set it up)""" + if db == None: + return None + try: + dbc = db[DB_C_TANK_STATS] ## not collecting player stats yet 2019-12-23 + ( account_id, battletime ) = str2ints(stat_id_str) + time_buffer = 2*7*24*3600 + + pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }} ]}}, + { '$sort': { 'last_battle_time': -1 }}, + { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, + { '$replaceRoot': { 'newRoot': '$doc' }}, + { '$project': { '_id': 0 }} ] + + # cursor = dbc.aggregate(pipeline, allowDiskUse=True) + cursor = dbc.aggregate(pipeline) + + return await tank_stats_helper(await cursor.to_list(1000)) + + except Exception as err: + bu.error('account_id: ' + str(account_id) + ' Error', err) + return None + + +async def get_wg_player_stats(stat_id_str: str) -> dict: + """Get player stats from WG. Returns WR per tier of tank_id""" + try: + account_id = int(stat_id_str) + + # 'battles' must always be there + hist_stats = [ 'all.' + x for x in histogram_fields.keys() ] + hist_stats.append('tank_id') + + player_stats = await wg.get_player_stats(account_id, hist_stats) + #bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats)) + + return await player_stats_helper(player_stats) + + except KeyError as err: + bu.error('account_id: ' + str(account_id) + ' Key not found', err) + except Exception as err: + bu.error(exception=err) + return None + + +async def tank_stats_helper(stat_list: list): + """Helpher func got get_db_tank_tier_stats() and get_wg_tank_tier_stats()""" try: if stat_list == None: return None @@ -682,13 +809,14 @@ async def statsHelper(stat_list: list): if 'battles' not in hist_fields: bu.error('\'battles\' must be defined in \'histogram_fields\'') sys.exit(1) + for field in hist_fields: stats[field] = 0 - for tankStat in stat_list: - tankStat = tankStat['all'] + for tank_stat in stat_list: + tank_stat = tank_stat['all'] for field in hist_fields: - stats[field] += max(0, tankStat[field]) + stats[field] += max(0, tank_stat[field]) if stats['battles'] == 0: return None @@ -699,11 +827,41 @@ async def statsHelper(stat_list: list): return stats except Exception as err: - bu.error(str(type(err)) + ' : ' + str(err)) + bu.error(exception=err) return None -async def mkReplayQ(queue : asyncio.Queue, files : list): +async def player_stats_helper(player_stats: dict): + """Extract relevant stats for player histogram from raw player stats""" + try: + if player_stats == None: + return None + stats = {} + hist_fields = histogram_fields.keys() + if 'battles' not in hist_fields: + bu.error('\'battles\' must be defined in \'histogram_fields\'') + sys.exit(1) + + for field in hist_fields: + stats[field] = 0 + + for field in hist_fields: + stats[field] += max(0, player_stats['statistics']['all'][field]) + + if stats['battles'] == 0: + return None + + for field in hist_fields: + if field == 'battles': continue + stats[field] = stats[field] / stats['battles'] + return stats + + except Exception as err: + bu.error(exception=err) + return None + + +async def mk_replayQ(queue : asyncio.Queue, files : list): """Create queue of replays to post""" p_replayfile = re.compile('.*\\.wotbreplay\\.json$') if files[0] == '-': @@ -715,32 +873,34 @@ async def mkReplayQ(queue : asyncio.Queue, files : list): break else: if (p_replayfile.match(line) != None): - await queue.put(await mkReaderQitem(line)) + await queue.put(await mk_readerQ_item(line)) else: for fn in files: # bu.debug('Filename: ' + fn) if fn.endswith('"'): fn = fn[:-1] if os.path.isfile(fn) and (p_replayfile.match(fn) != None): - await queue.put(await mkReaderQitem(fn)) + await queue.put(await mk_readerQ_item(fn)) bu.debug('File added to queue: ' + fn) elif os.path.isdir(fn): with os.scandir(fn) as dirEntry: for entry in dirEntry: if entry.is_file() and (p_replayfile.match(entry.name) != None): bu.debug(entry.name) - await queue.put(await mkReaderQitem(entry.path)) + await queue.put(await mk_readerQ_item(entry.path)) bu.debug('File added to queue: ' + entry.path) bu.debug('Finished: ' + str(queue.qsize()) + ' replays to process') return None -async def mkReaderQitem(filename : str) -> list: + +async def mk_readerQ_item(filename : str) -> list: """Make an item to replay queue""" global REPLAY_N REPLAY_N +=1 return [filename, REPLAY_N] -async def replayReader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace): + +async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace): """Async Worker to process the replay queue""" #global SKIPPED_N # account_id = args.accountID @@ -753,7 +913,7 @@ async def replayReader(queue: asyncio.Queue, readerID: int, args : argparse.Name replayID = item[1] try: - replay_json = await bu.readJSON(filename, wi.chkJSONreplay) + replay_json = await bu.open_JSON(filename, wi.chk_JSON_replay) if replay_json == None: bu.verbose('Replay[' + str(replayID) + ']: ' + filename + ' is invalid. Skipping.' ) #SKIPPED_N += 1 @@ -761,11 +921,11 @@ async def replayReader(queue: asyncio.Queue, readerID: int, args : argparse.Name continue ## Read the replay JSON - bu.debug('[' + str(readerID) + '] reading replay: ' + filename) - result = await readReplayJSON(replay_json, args) - bu.printWaiter() + bu.debug('reading replay: ' + filename, readerID) + result = await read_replay_JSON(replay_json, args) + bu.print_progress() if result == None: - bu.error('[' + str(readerID) + '] Error reading replay: ' + filename) + bu.error('Replay[' + str(replayID) + ']: Error reading: ' + filename, id=readerID) queue.task_done() continue @@ -777,19 +937,20 @@ async def replayReader(queue: asyncio.Queue, readerID: int, args : argparse.Name bu.debug('Marking task ' + str(replayID) + ' done') except Exception as err: - bu.error(str(err)) + bu.error(exception=err) queue.task_done() except (asyncio.CancelledError, concurrent.futures.CancelledError): - bu.debug('[' + str(readerID) + ']: ' + str(len(results)) + ' replays, ' + str(len(playerstanks)) + ' player/tanks') + bu.debug( str(len(results)) + ' replays, ' + str(len(playerstanks)) + ' player/tanks', readerID) return results, playerstanks return None -async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: + +async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict: """Parse replay JSON dict""" global REPLAY_I account_id = args.accountID url = args.url - db = args.db + #db = args.db result = {} try: if account_id == None: @@ -809,34 +970,28 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: # account_id looked for but not found in teams bu.debug('Replay ' + replay_json['data']['summary']['title'] + ' does not have account_id ' + str(account_id) + '. Skipping.') return None - if url: result['url'] = replay_json['data']['view_url'] - #bu.debug('1') for key in replay_summary_flds: result[key] = replay_json['data']['summary'][key] result['battle_start_timestamp'] = int(replay_json['data']['summary']['battle_start_timestamp']) - #bu.debug('2') result['allies'] = set() result['enemies'] = set() - btl_duration = 0 btl_tier = 0 for player in replay_json['data']['summary']['details']: - #bu.debug('3') btl_duration = max(btl_duration, player['time_alive']) - player_tank_tier = wg.getTankData(player['vehicle_descr'], 'tier') + player_tank_tier = wg.get_tank_data(player['vehicle_descr'], 'tier') btl_tier = max(btl_tier, player_tank_tier) - #bu.debug('4') + if player['dbid'] == account_id: # player itself is not put in results['allies'] tmp = {} tmp['account_id'] = account_id tmp['tank_id'] = player['vehicle_descr'] tmp['tank_tier'] = player_tank_tier - tmp['tank_name'] = wg.getTankData(tmp['tank_id'], 'name') + tmp['tank_name'] = wg.get_tank_data(tmp['tank_id'], 'name') tmp['squad_index'] = player['squad_index'] - for key in replay_details_flds: tmp[key] = player[key] if player['hitpoints_left'] == 0: @@ -845,62 +1000,74 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: else: tmp['survived'] = 1 tmp['destroyed'] = 0 - for key in tmp.keys(): result[key] = tmp[key] else: tmp_account_id = player['dbid'] tmp_tank_id = player['vehicle_descr'] - tmp_tank_tier = wg.getTankTier(tmp_tank_id) + # tmp_tank_tier = wg.get_tank_tier(tmp_tank_id) tmp_battletime = int(replay_json['data']['summary']['battle_start_timestamp']) if player['dbid'] in replay_json['data']['summary']['allies']: - result['allies'].add(getPlayerStatID(tmp_account_id, tmp_tank_tier, tmp_battletime, db)) + result['allies'].add(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) else: - result['enemies'].add(getPlayerStatID(tmp_account_id, tmp_tank_tier, tmp_battletime, db)) - - # platoon buddy removed from stats - if result['squad_index'] != None: - for ally in replay_json['data']['summary']['allies']: - if (ally['dbid'] != account_id) and (ally['squad_index'] == result['squad_index']): - # platoon buddy found - tmp_account_id = str(ally['dbid']) - tmp_tank_id = ally['vehicle_descr'] - tmp_tank_tier = str(wg.getTankTier(tmp_tank_id)) - tmp_battletime = str(int(replay_json['data']['summary']['battle_start_timestamp'])) - # platoon buddy removed from stats - result['allies'].remove(getPlayerStatID(tmp_account_id, tmp_tank_tier, tmp_battletime, db)) + result['enemies'].add(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) - #bu.debug('6') + # remove platoon buddy from stats + if result['squad_index'] != None: + for player in replay_json['data']['summary']['details']: + bu.debug(str(player)) + if (player['squad_index'] == result['squad_index']) and (player['dbid'] in replay_json['data']['summary']['allies']) and (player['dbid'] != account_id): + # platoon buddy found + tmp_account_id = player['dbid'] + tmp_tank_id = player['vehicle_descr'] + #tmp_tank_tier = str(wg.get_tank_tier(tmp_tank_id)) + tmp_battletime = int(replay_json['data']['summary']['battle_start_timestamp']) + # platoon buddy removed from stats + result['allies'].remove(get_stat_id(tmp_account_id, tmp_tank_id, tmp_battletime)) + result['time_alive%'] = result['time_alive'] / btl_duration result['battle_tier'] = btl_tier result['top_tier'] = 1 if (result['tank_tier'] == btl_tier) else 0 result['win'] = 1 if result['battle_result'] == 1 else 0 REPLAY_I += 1 result['battle_i'] = REPLAY_I - bu.debug(str(result)) return result except KeyError as err: - bu.error('Key :' + str(err) + ' not found') + bu.error('Key not found', err) except Exception as err: - bu.error(str(type(err)) + ' : ' + str(err)) + bu.error(exception=err) return None -def prunePlayerStatID(id: str) -> str: - stat = id.split(':') - #bu.debug('Stat: ' + id + ' ==> ' + ':'.join([stat[0], stat[1]])) - return ':'.join([stat[0], stat[1]]) -def getPlayerStatID(account_id: int, tier: int, battletime: int, db = False) -> str: - if db: - battle_date = date.fromtimestamp(battletime) - battle_date = battle_date - timedelta(days=battle_date.weekday()) - #bu.debug('account_id: ' + str(account_id) + ' tier: ' + str(tier) + ' battle_time: ' + str(int(time.mktime(battle_date.timetuple())))) - return ':'.join( [ str(account_id), str(tier), str(int(time.mktime(battle_date.timetuple()))) ] ) - else: - return ':'.join([ str(account_id), str(tier)]) +def str2ints(stat_id_str: str) -> list: + """Convert stat_id string e.g. 'account_id:tank_id' to list of ints""" + return [ int(x) for x in stat_id_str.split(':') ] + +def prune_stat_id(stat_id_str: str) -> str: + stat_id = stat_id_str.split(':') + del stat_id[-1] + return ':'.join(stat_id) + + +def get_stat_id(account_id: int, tank_id: int, battletime: int) -> str: + return ':'.join([ str(account_id), str(tank_id), str(battletime)]) + + +def get_stat_id_tank_tier(stat_id_str: str) -> str: + """Return stat_id = account_id:tank_tier""" + stat_id = stat_id_str.split(':') + tank_tier = wg.get_tank_tier(int(stat_id[1])) + return ':'.join([ stat_id[0], str(tank_tier), stat_id[2] ]) + + +def get_stat_id_player(stat_id_str: str) -> str: + """Return stat_id = account_id:tank_tier""" + stat_id = stat_id_str.split(':') + return ':'.join([ stat_id[0], stat_id[2] ]) + ### main() if __name__ == "__main__": diff --git a/blitzutils.py b/blitzutils.py index 42c3fb7..096ba9c 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -1,84 +1,81 @@ #!/usr/bin/python3.7 -import sys, os, json, time, base64, urllib, inspect, hashlib +import sys, os, json, time, base64, urllib, inspect, hashlib, re import asyncio, aiofiles, aiohttp, aiosqlite, lxml +from pathlib import Path from bs4 import BeautifulSoup MAX_RETRIES= 3 SLEEP = 3 -DEBUG = False -VERBOSE = False -SILENT = False -WAITER_N = 0 -WAITER_I = 0 +_debug = False +_verbose = False +_silent = False +_progress_N = 100 +_progress_i = 0 UMASK= os.umask(0) os.umask(UMASK) -# def chkJSONaccountList(account_list: list) -> bool: -# """"Check String for being a valid accountID list JSON file""" -# try: -# if int(account_list[0]) > 0: -# return True -# except: -# cls.debug("JSON check failed") -# return False -def setDebug(debug: bool): - global DEBUG, VERBOSE, SILENT +def set_debug(debug: bool): + global _debug, _verbose, _silent if debug != None: - DEBUG = debug - if DEBUG: - VERBOSE = True - SILENT = False + _debug = debug + if _debug: + _verbose = True + _silent = False -def setVerbose(verbose: bool): - global VERBOSE, SILENT + +def set_verbose(verbose: bool): + global _verbose, _silent if verbose != None: - VERBOSE = verbose - if VERBOSE: - SILENT = False + _verbose = verbose + if _verbose: + _silent = False -def setSilent(silent: bool): - global DEBUG, VERBOSE, SILENT + +def set_silent(silent: bool): + global _debug, _verbose, _silent if silent != None: - SILENT = silent - if SILENT: - VERBOSE = False - DEBUG = False + _silent = silent + if _silent: + _verbose = False + _debug = False -def verbose(msg = "", n = None): + +def verbose(msg = "", id = None): """Print a message""" - if VERBOSE: - if n == None: + if _verbose: + if id == None: print(msg) else: - print('[' + str(n) + ']: ' + msg) + print('[' + str(id) + ']: ' + msg) return None -def verbose_std(msg = "", n = None): + +def verbose_std(msg = "", id = None): """Print a message""" - if not SILENT: - if n == None: + if not _silent: + if id == None: print(msg) else: - print('[' + str(n) + ']: ' + msg) + print('[' + str(id) + ']: ' + msg) return None -def printWaiter(force = False): - global WAITER_N, WAITER_I - if not DEBUG and (not SILENT or force): - if WAITER_N > 0: - WAITER_I = (WAITER_I+1) % WAITER_N - if WAITER_I == 0: - print('.', end='', flush=True) - else: - print('.', end='', flush=True) -def setWaiter(n: int): - global WAITER_N - if n > -1: - WAITER_N = n +def print_progress(force = False): + """Print progress dots""" + global _progress_N, _progress_i + if not _silent and (force or not _debug): + _progress_i = (_progress_i + 1) % _progress_N + if _progress_i == 0: + print('.', end='', flush=True) + +def set_progress_step(n: int): + """Set the frquency of the progress dots. The bigger 'n', the fewer dots""" + global _progress_N + if n > 0: + _progress_N = n return @@ -86,18 +83,18 @@ def wait(sec : int): for i in range(0, sec): i=i ## to get rid of the warning... time.sleep(1) - printWaiter(True) + print_progress(True) print('', flush=True) -def printNewline(force = False): - if not DEBUG and (not SILENT or force): +def print_new_line(force = False): + if not _silent and (force or not _debug): print('', flush=True) def debug(msg = "", n = None): """print a conditional debug message""" - if DEBUG: + if _debug: curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) caller = calframe[1][3] @@ -108,22 +105,26 @@ def debug(msg = "", n = None): return None -def error(msg = "", n = None): +def error(msg = "", exception = None, id = None): """Print an error message""" curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) caller = calframe[1][3] - if n == None: - print('ERROR: ' + caller + '(): ' + msg) + exception_msg = '' + if (exception != None) and isinstance(exception, Exception): + exception_msg = ' : Exception: ' + str(type(exception)) + ' : ' + str(exception) + + if id == None: + print('ERROR: ' + caller + '(): ' + msg + exception_msg) else: - print('ERROR: ' + caller + '()' + '[' + str(n) + ']: ' + msg) + print('ERROR: ' + caller + '()' + '[' + str(id) + ']: ' + msg + exception_msg) return None def NOW() -> int: return int(time.time()) -async def readPlainList(filename: str) -> list(): +async def read_int_list(filename: str) -> list(): """Read file to a list and return list of integers in the input file""" input_list = [] @@ -135,11 +136,11 @@ async def readPlainList(filename: str) -> list(): except (ValueError, TypeError) as err: pass except Exception as err: - error('Unexpected error when reading file: ' + filename + ' : ' + str(type(err)) + ' : '+ str(err)) + error('Unexpected error when reading file: ' + filename, err) return input_list -async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool: +async def save_JSON(filename: str, json_data: dict, sort_keys = False) -> bool: """Save JSON data into file""" try: dirname = os.path.dirname(filename) @@ -149,37 +150,41 @@ async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool: await outfile.write(json.dumps(json_data, ensure_ascii=False, indent=4, sort_keys=sort_keys)) return True except Exception as err: - error(str(err)) + error('Error saving JSON', err) return False -async def readJSON(filename: str, chkJSONfunc = None): +async def open_JSON(filename: str, chk_JSON_func = None) -> dict: try: async with aiofiles.open(filename) as fp: json_data = json.loads(await fp.read()) - if (chkJSONfunc == None): + if (chk_JSON_func == None): debug("JSON file content not checked: " + filename) return json_data - elif chkJSONfunc(json_data): + elif chk_JSON_func(json_data): debug("JSON File is valid: " + filename) return json_data else: debug('JSON File has invalid content: ' + filename) except Exception as err: - error('Unexpected error when reading file: ' + filename + ' : ' + str(type(err)) + ' : '+ str(err)) + error('Unexpected error when reading file: ' + filename, err) return None -async def getUrlJSON(session: aiohttp.ClientSession, url: str, chkJSONfunc = None, max_tries = MAX_RETRIES) -> dict: + +async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = None, max_tries = MAX_RETRIES) -> dict: """Retrieve (GET) an URL and return JSON object""" try: - debug(url) + if session == None: + error('Session must be initialized first') + sys.exit(1) + #debug(url) ## To avoid excessive use of servers for retry in range(1,max_tries+1): async with session.get(url) as resp: if resp.status == 200: debug('HTTP request OK') json_resp = await resp.json() - if (chkJSONfunc == None) or chkJSONfunc(json_resp): + if (chk_JSON_func == None) or chk_JSON_func(json_resp): # debug("Received valid JSON: " + str(json_resp)) return json_resp else: @@ -190,22 +195,22 @@ async def getUrlJSON(session: aiohttp.ClientSession, url: str, chkJSONfunc = Non if json_resp != None: str_json = str(json_resp) else: - str_json = 'None' + str_json = 'None' ## Change to None ?? raise aiohttp.ClientError('Request failed: ' + str(resp.status) + ' JSON Response: ' + str_json ) debug('Retrying URL [' + str(retry) + '/' + str(max_tries) + ']: ' + url ) await asyncio.sleep(SLEEP) except aiohttp.ClientError as err: error("Could not retrieve URL: " + url) - error(str(err)) + error(exception=err) except asyncio.CancelledError as err: - error('Queue gets cancelled while still working.') - + error('Queue gets cancelled while still working.') except Exception as err: - error('Unexpected Exception: ' + str(err)) + error('Unexpected Exception', err) return None -def bldDictHierarcy(d : dict, key : str, value) -> dict: + +def bld_dict_hierarcy(d : dict, key : str, value) -> dict: """Build hierarcical dict based on multi-level key separated with """ try: key_hier = key.split('.') @@ -213,50 +218,93 @@ def bldDictHierarcy(d : dict, key : str, value) -> dict: if len(key_hier) == 0: d[sub_key] = value elif sub_key not in d: - d[sub_key] = bldDictHierarcy({}, key_hier, value) + d[sub_key] = bld_dict_hierarcy({}, '.'.join(key_hier), value) else: - d[sub_key] = bldDictHierarcy(d[sub_key], key_hier, value) + d[sub_key] = bld_dict_hierarcy(d[sub_key], '.'.join(key_hier), value) return d except KeyError as err: - error('Key not found: ' + str(err)) + error('Key not found', err) except Exception as err: - error(str(err)) + error('Unexpected Exception', err) return None + +## ----------------------------------------------------------- +#### Class StatsNotFound +## ----------------------------------------------------------- + +class CachedStatsNotFound(Exception): + pass + ## ----------------------------------------------------------- #### Class WG ## ----------------------------------------------------------- class WG: - URL_WG_clanInfo = 'clans/info/?application_id=' - #URL_WG_playerTankList = 'tanks/stats/?fields=tank_id%2Clast_battle_time&application_id=' - URL_WG_playerTankList = 'tanks/stats/?fields=account_id%2Ctank_id%2Clast_battle_time%2Cbattle_life_time%2Call&application_id=' - URL_WG_playerTankStats = 'tanks/stats/?application_id=' - URL_WG_accountID = 'account/list/?fields=account_id%2Cnickname&application_id=' - URL_WG_playerStats = 'account/info/?application_id=' - SQLITE_STATS_CACHE = '.stats_cache.db' + URL_WG_CLAN_INFO = 'clans/info/?application_id=' + #URL_WG_PLAYER_TANK_LIST = 'tanks/stats/?fields=tank_id%2Clast_battle_time&application_id=' + #URL_WG_PLAYER_TANK_LIST = 'tanks/stats/?fields=account_id%2Ctank_id%2Clast_battle_time%2Cbattle_life_time%2Call&application_id=' + URL_WG_PLAYER_TANK_STATS = 'tanks/stats/?application_id=' + URL_WG_ACCOUNT_ID = 'account/list/?fields=account_id%2Cnickname&application_id=' + URL_WG_PLAYER_STATS = 'account/info/?application_id=' + CACHE_DB_FILE = '.blitzutils_cache.sqlite3' + CACHE_GRACE_TIME = 2*7*24*3600 # 2 weeks cache - sql_create_player_stats_tbl = """CREATE TABLE IF NOT EXISTS player_stats ( - account_id INTEGER NOT NULL, - date INTEGER NOT NULL, - stat TEXT, - value FLOAT - ); """ + # sql_create_player_stats_tbl = """CREATE TABLE IF NOT EXISTS player_stats ( + # account_id INTEGER NOT NULL, + # date INTEGER NOT NULL, + # stat TEXT, + # value FLOAT + # ); """ - sql_create_player_tank_stats_tbl = """CREATE TABLE IF NOT EXISTS player_tank_stats ( - account_id INTEGER NOT NULL, - tank_id INTEGER DEFAULT NULL, - date INTEGER NOT NULL, - stat TEXT, - value FLOAT - ); """ + # sql_create_player_tank_stats_tbl = """CREATE TABLE IF NOT EXISTS player_tank_stats ( + # account_id INTEGER NOT NULL, + # tank_id INTEGER DEFAULT NULL, + # date INTEGER NOT NULL, + # stat TEXT, + # value FLOAT + # ); """ - sql_select_player_stats = """SELECT value FROM player_stats ORDERBY date ASC - WHERE account_id = {} AND stat = {} - AND date >= {} LIMIT 1;""" + # sql_select_player_stats = """SELECT value FROM player_stats ORDERBY date ASC + # WHERE account_id = {} AND stat = {} + # AND date >= {} LIMIT 1;""" + + SQL_TANK_STATS_TBL = 'tank_stats' + + SQL_TANK_STATS_CREATE_TBL = 'CREATE TABLE IF NOT EXISTS ' + SQL_TANK_STATS_TBL + \ + """ ( account_id INTEGER NOT NULL, + tank_id INTEGER NOT NULL, + update_time INTEGER NOT NULL, + stats TEXT, + PRIMARY KEY (account_id, tank_id) )""" + + SQL_TANK_STATS_COUNT = 'SELECT COUNT(*) FROM ' + SQL_TANK_STATS_TBL + + SQL_TANK_STATS_UPDATE = 'REPLACE INTO ' + SQL_TANK_STATS_TBL + '(account_id, tank_id, update_time, stats) VALUES(?,?,?,?)' + + SQL_PLAYER_STATS_TBL = 'player_stats' + + SQL_PLAYER_STATS_CREATE_TBL = 'CREATE TABLE IF NOT EXISTS ' + SQL_PLAYER_STATS_TBL + \ + """ ( account_id INTEGER PRIMARY KEY, + update_time INTEGER NOT NULL, + stats TEXT)""" + + SQL_PLAYER_STATS_COUNT = 'SELECT COUNT(*) FROM ' + SQL_PLAYER_STATS_TBL + + SQL_PLAYER_STATS_UPDATE = 'REPLACE INTO ' + SQL_PLAYER_STATS_TBL + '(account_id, update_time, stats) VALUES(?,?,?)' + + SQL_PLAYER_STATS_CACHED = 'SELECT * FROM ' + SQL_PLAYER_STATS_TBL + ' WHERE account_id = ? AND update_time > ?' + + SQL_TABLES = [ SQL_PLAYER_STATS_TBL, SQL_TANK_STATS_TBL ] + + SQL_CHECK_TABLE_EXITS = """SELECT name FROM sqlite_master WHERE type='table' AND name=?""" + + SQL_PRUNE_CACHE = """DELETE from {} WHERE update_time < {}""" + +## Default data. Please use the latest maps.json maps = { "Random": "Random map", @@ -294,8 +342,8 @@ class WG: tanks = None tanks_by_tier = None - nations = [ 'ussr', 'germany', 'usa', 'china', 'france', 'uk', 'japan', 'other', 'european'] - nation_id = { + NATIONS = [ 'ussr', 'germany', 'usa', 'china', 'france', 'uk', 'japan', 'other', 'european'] + NATION_ID = { 'ussr' : 0, 'germany' : 1, 'usa' : 2, @@ -307,33 +355,33 @@ class WG: 'european' : 8 } - tank_type = [ 'lightTank', 'mediumTank', 'heavyTank', 'AT-SPG' ] - tank_type_id = { + TANK_TYPE = [ 'lightTank', 'mediumTank', 'heavyTank', 'AT-SPG' ] + TANK_TYPE_ID = { 'lightTank' : 0, 'mediumTank' : 1, 'heavyTank' : 2, 'AT-SPG' : 3 } - URL_WG_server = { + URL_WG_SERVER = { 'eu' : 'https://api.wotblitz.eu/wotb/', 'ru' : 'https://api.wotblitz.ru/wotb/', 'na' : 'https://api.wotblitz.com/wotb/', 'asia' : 'https://api.wotblitz.asia/wotb/' } - accountID_server= { + ACCOUNT_ID_SERVER= { 'ru' : range(0, int(5e8)), 'eu' : range(int(5e8), int(10e8)), 'na' : range(int(1e9),int(2e9)), 'asia' : range(int(2e9),int(4e9)) } - def __init__(self, WG_appID = None, tankopedia_fn = None, maps_fn = None): + def __init__(self, WG_app_id = None, tankopedia_fn = None, maps_fn = None, stats_cache = False): - self.WG_appID = WG_appID - self.loadTanks(tankopedia_fn) - WG.tanks =self.tanks + self.WG_app_id = WG_app_id + self.load_tanks(tankopedia_fn) + WG.tanks = self.tanks if (maps_fn != None): if os.path.exists(maps_fn) and os.path.isfile(maps_fn): @@ -344,7 +392,7 @@ class WG: error('Could not read maps file: ' + maps_fn + '\n' + str(err)) else: verbose('Could not find maps file: ' + maps_fn) - if self.WG_appID != None: + if self.WG_app_id != None: self.session = aiohttp.ClientSession() debug('WG aiohttp session initiated') else: @@ -354,177 +402,225 @@ class WG: # cache TBD self.cache = None self.statsQ = None - self.statCacheTask = None - #if os.path.isfile(self.SQLITE_STATS_CACHE): - if False: - try: - self.cache = aiosqlite.connect('SQLITE_STATS_CACHE') - self.cache.execute(self.sql_create_player_stats_tbl) - self.cache.execute(self.sql_create_player_tank_stats_tbl) - self.cache.commit() - self.statsQ = asyncio.Queue() - self.statCacheTask = asyncio.create_task(self.statsSaver()) + self.stat_saver_task = None + + @classmethod + async def create(cls, WG_app_id = None, tankopedia_fn = None, maps_fn = None, stats_cache = False): + """Separete Constuctor method to handle async calls required to initialize the object""" + self = WG(WG_app_id, tankopedia_fn , maps_fn ) + + if stats_cache: + try: + self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE) + ## Player Tank stats cache table + await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL) + + await self.cache.commit() + + async with self.cache.execute(WG.SQL_TANK_STATS_COUNT) as cursor: + debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' cached player tank stat records' ) + + self.statsQ = asyncio.Queue() + self.stat_saver_task = asyncio.create_task(self.stat_saver()) except Exception as err: - error(str(err)) - + error(exception=err) + sys.exit(1) + return self + async def close(self): + # close stats queue if self.statsQ != None: + debug('WG.close(): Waiting for statsQ to finish') await self.statsQ.join() - self.statCacheTask.cancel() - await asyncio.gather(*self.statCacheTask) + debug('WG.close(): statsQ finished') + self.stat_saver_task.cancel() + debug('statsCacheTask cancelled') + await self.stat_saver_task + + + + # close cacheDB + if self.cache != None: + # prune old cache records + await self.cleanup_cache() + await self.cache.commit() + await self.cache.close() if self.session != None: await self.session.close() + return - - ## Class methods ------------------------------ - + ## Class methods ---------------------------------------------------------- @classmethod - def getServer(cls, accountID: int) -> str: + def get_server(cls, account_id: int) -> str: """Get Realm/server of an account based on account ID""" - if accountID > 1e9: - if accountID > 2e9: + if account_id > 1e9: + if account_id > 2e9: return 'asia' return 'na' else: - if accountID < 5e8: + if account_id < 5e8: return 'ru' return 'eu' return None - @classmethod - def updateMaps(cls, mapdata: dict): - """Update maps data""" - cls.maps = mapdata - return None @classmethod - def getMap(cls, mapStr: str) -> str: + def update_maps(cls, map_data: dict): + """Update maps data""" + cls.maps = map_data + return None + + + @classmethod + def get_map(cls, map_str: str) -> str: """Return map name from short map string in replays""" try: - return cls.maps[mapStr] + return cls.maps[map_str] except: - error('Map ' + mapStr + ' not found') + error('Map ' + map_str + ' not found') return None + @classmethod - def getMapUserStrs(cls) -> str: + def get_map_user_strs(cls) -> str: return cls.maps.keys() + @classmethod - def getTankUserStrs(cls) -> str: + def get_tank_user_strs(cls) -> str: return cls.tanks["userStr"].keys() - @classmethod - def chkJSONcontent(cls, json_obj, check = None) -> bool: - try: - if (check == 'player') and (not cls.chkJSONplayerStats(json_obj)): - debug('Checking player JSON failed.') - return False - elif (check == 'tank') and (not cls.chkJSONtank(json_obj)): - debug('Checking tank JSON failed.') - return False - # elif (check == 'clan') and (not chkJSONclan(json_obj)): - # debug('Checking clan JSON failed.') - # return False - elif (check == 'tankList') and (not cls.chkJSONtankList(json_obj)): - debug('Checking tank list JSON failed.') - return False - # elif (check == 'accountlist') and (not cls.chkJSONaccountList(json_obj)): - # cls.debug('Checking account list JSON failed.') - # return False - except (TypeError, ValueError) as err: - debug(str(err)) - return False - return True @classmethod - def chkJSONstatus(cls, json_resp: dict) -> bool: + def chk_JSON(cls, json_obj, check = None) -> bool: try: - if json_resp['status'] == 'ok': + if (check == None): + # nothing to check + return True + if (check == 'tank_stats'): + if cls.chk_JSON_tank_stats(json_obj): + return True + else: + debug('Checking tank list JSON failed.') + return False + elif (check == 'player_stats'): + if cls.chk_JSON_player_stats(json_obj): + return True + else: + debug('Checking player JSON failed.') + return False + elif (check == 'tankopedia'): + if cls.chk_JSON_tankopedia(json_obj): + return True + else: + debug('Checking tank JSON failed.') + return False + elif (check == 'account_id'): + if cls.chk_JSON_get_account_id(json_obj): + return True + else: + debug('Checking account_id JSON failed.') + return False + except (TypeError, ValueError) as err: + debug(str(err)) + return False + + + @classmethod + def chk_JSON_status(cls, json_resp: dict) -> bool: + try: + if (json_resp['status'] == 'ok') and ('data' in json_resp): return True elif json_resp['status'] == 'error': - #error(str(json_resp['error']['code']) + ' : ' + json_resp['error']['message'] ) + if ('error' in json_resp): + error_msg = 'Received an error' + if ('message' in json_resp['error']): + error_msg = error_msg + ': ' + json_resp['error']['message'] + if ('value' in json_resp['error']): + error_msg = error_msg + ' Value: ' + json_resp['error']['value'] + debug(error_msg) return False else: - debug('Unknown status-code: ' + json_resp['status']) + error('Unknown status-code: ' + json_resp['status']) return False except KeyError as err: - error('No field found in JSON data: ' + str(err)) - except: - error("JSON format error") + error('No field found in JSON data', err) + except Exception as err: + error("JSON format error", err) return False @classmethod - def chkJSONgetAccountID(cls, json_resp: dict) -> bool: + def chk_JSON_get_account_id(cls, json_resp: dict) -> bool: try: - if cls.chkJSONstatus(json_resp): + if cls.chk_JSON_status(json_resp): if (json_resp['meta']['count'] > 0): return True except KeyError as err: - error('Key :' + str(err) + ' not found') + error('Key not found', err) except Exception as err: - error(str(err)) + error(exception=err) return False # @classmethod # def chkJSONplayer(cls, json_resp: dict) -> bool: # """"Check String for being a valid Player JSON file""" # try: - # if cls.chkJSONstatus(json_resp): + # if cls.chk_JSON_status(json_resp): # if int(json_resp[0]['account_id']) > 0: # return True # except KeyError as err: - # error('Key :' + str(err) + ' not found') + # error('Key not found', err) # except: # debug("JSON check failed") # return False @classmethod - def chkJSONtank(cls, json_resp: dict) -> bool: - """"Check String for being a valid Tank JSON file""" + def chk_JSON_tankopedia(cls, json_resp: dict) -> bool: + """"Check String for being a valid Tankopedia JSON file""" try: - if cls.chkJSONstatus(json_resp): + if cls.chk_JSON_status(json_resp): if int(json_resp[0]['tank_id']) > 0: return True except KeyError as err: - error('Key :' + str(err) + ' not found') + error('Key not found', err) except: debug("JSON check failed") return False @classmethod - def chkJSONplayerStats(cls, json_resp: dict) -> bool: + def chk_JSON_player_stats(cls, json_resp: dict) -> bool: """"Check String for being a valid Tank JSON file""" try: - if cls.chkJSONstatus(json_resp): + if cls.chk_JSON_status(json_resp): for acc in json_resp['data']: if json_resp['data'][acc] != None: return True except KeyError as err: - error('Key :' + str(err) + ' not found') + error('Key not found', err) except: debug("JSON check failed") return False @classmethod - def chkJSONtankList(cls, json_resp: dict) -> bool: + def chk_JSON_tank_stats(cls, json_resp: dict) -> bool: """"Check String for being a valid Tank list JSON file""" try: - accountID = next(iter(json_resp['data'])) - if (json_resp['data'][accountID] != None) and (int(json_resp['data'][accountID][0]['tank_id']) > 0): - debug('JSON tank list check OK') - return True + if cls.chk_JSON_status(json_resp): + if ('data' in json_resp) and (len(json_resp['data']) > 0): + debug('JSON tank list check OK') + return True except Exception as err: - error('JSON check FAILED: ' + str(type(err)) + ' : ' + str(err) + ' : ' + str(json_resp) ) + error('JSON check FAILED: ' + str(json_resp) ) + error(exception=err) return False ## Methods -------------------------------------------------- - def loadTanks(self, tankopedia_fn: str): + def load_tanks(self, tankopedia_fn: str): """Load tanks from tankopedia JSON""" if tankopedia_fn == None: return False @@ -539,165 +635,160 @@ class WG: self.tanks_by_tier[str(tank['tier'])].append(tank['tank_id']) return True except Exception as err: - error('Could not read tankopedia: ' + tankopedia_fn + ' : ' + str(err)) + error('Could not read tankopedia: ' + tankopedia_fn, err) return False - def getTanksByTier(self, tier: int) -> list(): + def get_tanks_by_tier(self, tier: int) -> list(): """Returns tank_ids by tier""" - return self.tanks_by_tier[str(tier)] - - def getUrlClanInfo(self, server: str, clanID: int) -> str: - return self.URL_WG_server[server] + self.URL_WG_clanInfo + self.WG_appID + '&clan_id=' + str(clanID) - - def getUrlPlayerTankList(self, accountID: int) -> str: - server = self.getServer(accountID) - return self.URL_WG_server[server] + self.URL_WG_playerTankList + self.WG_appID + '&account_id=' + str(accountID) - - def getUrlPlayerTankStats(self, accountID: int, tankID: int, fields: list) -> str: - server = self.getServer(accountID) - url = self.URL_WG_server[server] + self.URL_WG_playerTankStats + self.WG_appID + '&account_id=' + str(accountID) + '&tank_id=' + str(tankID) - if (fields != None) and (len(fields) > 0): - field_str = '%2C' + '%2C'.join(fields) - else: - field_str = '' - return url + '&fields=tank_id' + field_str - - def getUrlPlayerTanksStats(self, accountID: int, tankIDs: list, fields: list) -> str: - server = self.getServer(accountID) - tank_ids = '%2C'.join([ str(x) for x in tankIDs]) - url = self.URL_WG_server[server] + self.URL_WG_playerTankStats + self.WG_appID + '&account_id=' + str(accountID) + '&tank_id=' + tank_ids - if (fields != None) and (len(fields) > 0): - field_str = '%2C' + '%2C'.join(fields) - else: - field_str = '' - return url + '&fields=tank_id' + field_str - - - def getUrlPlayerStats(self, accountID, fields) -> str: - server = self.getServer(accountID) - return self.URL_WG_server[server] + self.URL_WG_playerStats + self.WG_appID + '&account_id=' + str(accountID) + '&fields=' + '%2C'.join(fields) - - def getUrlAccountID(self, nickname, server) -> int: try: - return self.URL_WG_server[server] + self.URL_WG_accountID + self.WG_appID + '&search=' + urllib.parse.quote(nickname) + return self.tanks_by_tier[str(tier)] + except KeyError as err: + error('Invalid tier', err) + return None + + def get_url_clan_info(self, server: str, clan_id: int) -> str: + try: + return self.URL_WG_SERVER[server] + self.URL_WG_CLAN_INFO + self.WG_app_id + '&clan_id=' + str(clan_id) except Exception as err: - print('ERROR: getUrlAccountID(): ' + str(err)) - return None + if (server == None) or (server.lower() not in WG.ACCOUNT_ID_SERVER.keys()): + error('No server name or invalid server name given: ' + server if (server != None) else '') + error('Available servers: ' + ', '.join(WG.ACCOUNT_ID_SERVER.keys())) + error(exception=err) + return None + + + def get_url_player_tank_list(self, account_id: int) -> str: + return self.get_url_player_tanks_stats(account_id, fields='tank_id') + + + def get_url_player_tanks_stats(self, account_id: int, tank_ids = [], fields = []) -> str: + server = self.get_server(account_id) + + if (tank_ids != None) and (len(tank_ids) > 0): + tank_id_str= '&tank_id=' + '%2C'.join([ str(x) for x in tank_ids]) + else: + # emtpy tank-id list returns all the player's tanks + tank_id_str = '' + + if (fields != None) and (len(fields) > 0): + field_str = '&fields=' + '%2C'.join(fields) + else: + # return all the fields + field_str = '' + + return self.URL_WG_SERVER[server] + self.URL_WG_PLAYER_TANK_STATS + self.WG_app_id + '&account_id=' + str(account_id) + tank_id_str + field_str + + + def get_url_player_stats(self, account_id, fields) -> str: + try: + server = self.get_server(account_id) + if (fields != None) and (len(fields) > 0): + field_str = '&fields=' + '%2C'.join(fields) + else: + # return all the fields + field_str = '' + + return self.URL_WG_SERVER[server] + self.URL_WG_PLAYER_STATS + self.WG_app_id + '&account_id=' + str(account_id) + field_str + except Exception as err: + if (server == None): + error('Invalid account_id') + error(exception=err) + return None + + + def get_url_account_id(self, nickname, server) -> int: + try: + return self.URL_WG_SERVER[server] + self.URL_WG_ACCOUNT_ID + self.WG_app_id + '&search=' + urllib.parse.quote(nickname) + except Exception as err: + if nickname == None or len(nickname) == 0: + error('No nickname given') + if (server == None) or (server.lower() not in WG.ACCOUNT_ID_SERVER.keys()): + error('No server name or invalid server name given: ' + server if (server != None) else '') + error('Available servers: ' + ', '.join(WG.ACCOUNT_ID_SERVER.keys())) + error(exception=err) + return None - async def getAccountID(self, nickname: str) -> int: + + async def get_account_id(self, nickname: str) -> int: """Get WG account_id for a nickname""" try: + nick = None + server = None nick, server = nickname.split('@') debug(nick + ' @ '+ server) server = server.lower() if nick == None or server == None: raise ValueError('Invalid nickname given: ' + nickname) - url = self.getUrlAccountID(nick, server) - json_data = await getUrlJSON(self.session, url) + url = self.get_url_account_id(nick, server) + json_data = await get_url_JSON(self.session, url, self.chk_JSON_status) for res in json_data['data']: if res['nickname'].lower() == nick.lower(): return res['account_id'] - raise ValueError('No WG account_id found: ' + nickname) + error('No WG account_id found: ' + nickname) except Exception as err: - error(str(err)) - return None + error(exception=err) + return None - async def getPlayerTanksStats(self, accountID: int, tankIDs : list, fields: list) -> dict: + + async def get_player_tanks_stats(self, account_id: int, tank_ids = [], fields = []) -> dict: """Get player's stats (WR, # of battles) in a tank""" # debug('Started') - if self.session == None: - error('Session must be initialized first') - sys.exit(1) + try: - debug('AccountID: ' + str(accountID) + ' TankID: ' + ','.join([ str(id) for id in tankIDs])) - url = self.getUrlPlayerTanksStats(accountID, tankIDs, fields) - json_data = await getUrlJSON(self.session, url, self.chkJSONtankList) + #debug('account_id: ' + str(account_id) + ' TankID: ' + ','.join([ str(id) for id in tank_ids])) + stats = None + + # try cached stats first: + stats = await self.get_cached_tank_stats(account_id, tank_ids, fields) + if stats != None: + return stats + + # Cached stats not found, fetching new ones + url = self.get_url_player_tanks_stats(account_id, tank_ids, fields) + json_data = await get_url_JSON(self.session, url, self.chk_JSON_status) if json_data != None: - debug('JSON Response received: ' + str(json_data)) - return json_data['data'][str(accountID)] + #debug('JSON Response received: ' + str(json_data)) + stats = json_data['data'][str(account_id)] + await self.save_stats('tank_stats', [account_id, tank_ids], stats) + return stats except Exception as err: - error('Type: ' + str(type(err)) + ' : ' + str(err)) + error(exception=err) return None - async def getPlayerTankStats(self, accountID: int, tankID : int, fields: list) -> dict: - """Get player's stats (WR, # of battles) in a tank""" - # debug('Started') - if self.session == None: - error('Session must be initialized first') - sys.exit(1) - try: - debug('AccountID: ' + str(accountID) + ' TankID: ' + str(tankID)) - url = self.getUrlPlayerTankStats(accountID, tankID, fields) - json_data = await getUrlJSON(self.session, url, self.chkJSONtankList) - if json_data != None: - debug('JSON Response received: ' + str(json_data)) - return json_data['data'][str(accountID)][0] - except Exception as err: - error(err) - return None - - async def getPlayerStats(self, accountID: int, fields: list, battle_time = None) -> dict: + + async def get_player_stats(self, account_id: int, fields = []) -> dict: """Get player's global stats """ - # debug('Started') - fields_missing = fields - cached_stats = None - if self.cache != None: - cached_stats, fields_missing = await self.getCachedPlayerStats(accountID, fields) + try: + #debug('account_id: ' + str(account_id) ) + stats = None - if fields_missing != None: - if self.session == None: - error('Session must be initialized first') - sys.exit(1) - try: - debug('AccountID: ' + str(accountID) + ' Fields: ' + ', '.join(fields_missing)) - url = self.getUrlPlayerStats(accountID, fields_missing) - json_data = await getUrlJSON(self.session, url, self.chkJSONplayerStats) - if json_data != None: - debug('JSON Response received: ' + str(json_data)) - stats = json_data['data'][str(accountID)] - if self.statsQ != None: - self.statsQ.put([accountID, 'player', stats]) - return self.mergePlayerStats(stats, cached_stats) - except Exception as err: - error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) - else: - return cached_stats + # try cached stats first: + stats = await self.get_cached_player_stats(account_id,fields) + # stats found unless CachedStatsNotFound exception is raised + return stats + + except CachedStatsNotFound as err: + # No cached stats found, need to retrieve + debug(str(err)) + pass + + try: + # Cached stats not found, fetching new ones + url = self.get_url_player_stats(account_id, fields) + json_data = await get_url_JSON(self.session, url, self.chk_JSON_status) + if json_data != None: + #debug('JSON Response received: ' + str(json_data)) + stats = json_data['data'][str(account_id)] + await self.save_stats('player_stats', [account_id], stats) + return stats + except Exception as err: + error(exception=err) return None - async def getCachedPlayerStats(self, accountID, fields): - if self.cache == None: - error('No stats cache available') - return None, fields - try: - res_tmp = {} - missing = [] - for field in fields: - curr = self.cache.execute(self.sql_select_player_stats.format(accountID, field, NOW() )) - res_tmp[field] = curr.fetchone() - if res_tmp[field] == None: - res_tmp.pop(field, None) - missing.append(field) - else: - res_tmp[field] = res_tmp[field][0] - res = {} - for field in res_tmp.keys(): - res = bldDictHierarcy(res, field, res_tmp[field]) - - if len(missing) == 0: - missing = None - if len(res.keys()) == 0: - res = None - return res, missing - except KeyError as err: - error('Key not found: ' + str(err)) - except Exception as err: - error(str(err)) - return None, fields - - def mergePlayerStats(self, stats1: dict, stats2: dict) -> dict: + def merge_player_stats(self, stats1: dict, stats2: dict) -> dict: try: if stats2 == None: return stats1 for keyA in stats2: @@ -708,44 +799,197 @@ class WG: stats1[keyA][keyB] = stats2[keyA][keyB] return stats1 except KeyError as err: - error('Key not found: ' + str(err)) + error('Key not found', err) return None - def getTankData(self, tank_id: int, field: str): + def get_tank_data(self, tank_id: int, field: str): if self.tanks == None: return None try: return self.tanks['data'][str(tank_id)][field] except KeyError as err: - error('Key not found: ' + str(err)) + error('Key not found', err) return None - def getTankTier(self, tank_id: int): - return self.getTankData(tank_id, 'tier') + def get_tank_tier(self, tank_id: int): + return self.get_tank_data(tank_id, 'tier') - async def statsSaver(self): + + async def save_stats(self, statsType: str, key: list, stats: list): + """Save stats to a async queue to be saved by the stat_saver -task""" + if self.statsQ == None: + return False + else: + await self.statsQ.put([ statsType, key, stats, NOW() ]) + return True + + async def stat_saver(self): + """Async task for saving stats into cache in background""" + + if self.statsQ == None: + error('No statsQ defined') + return None while True: - stats = await self.statsQ.get() try: - return stats ## FIX - - + stats = await self.statsQ.get() + + stats_type = stats[0] + key = stats[1] + stats_data = stats[2] + update_time = stats[3] + + if stats_type == 'tank_stats': + await self.store_tank_stats(key, stats_data, update_time) + elif stats_type == 'player_stats': + await self.store_player_stats(key, stats_data, update_time) + else: + error('Function to saves stats type \'' + stats_type + '\' is not implemented yet') + + except (asyncio.CancelledError): + # this is an eternal loop that will wait until cancelled + return None except Exception as err: - error(str(err)) + error(exception=err) + self.statsQ.task_done() return None + + async def cleanup_cache(self, grace_time = CACHE_GRACE_TIME): + """Clean old cache records""" + if self.cache == None: + debug('No active cache') + return None + for table in WG.SQL_TABLES: + async with self.cache.execute(WG.SQL_CHECK_TABLE_EXITS, (table,)) as cursor: + if (await cursor.fetchone()) != None: + debug('Pruning cache table: ' + table) + await self.cache.execute(WG.SQL_PRUNE_CACHE.format(table, NOW() - grace_time)) + await self.cache.commit() + return None + + + async def store_tank_stats(self, key: list , stats_data: list, update_time: int): + """Save tank stats into cache""" + try: + account_id = key[0] + tank_ids = set(key[1]) + if stats_data != None: + for stat in stats_data: + tank_id = stat['tank_id'] + await self.cache.execute(WG.SQL_TANK_STATS_UPDATE, (account_id, tank_id, update_time, json.dumps(stat))) + tank_ids.remove(tank_id) + # no stats found => Add None to mark that + for tank_id in tank_ids: + await self.cache.execute(WG.SQL_TANK_STATS_UPDATE, (account_id, tank_id, update_time, None)) + await self.cache.commit() + debug('Cached tank stats saved for account_id: ' + str(account_id) ) + return True + except Exception as err: + error(exception=err) + return False + + + async def store_player_stats(self, key: list , stats_data: list, update_time: int): + """Save player stats into cache""" + try: + account_id = key[0] + if stats_data != None: + await self.cache.execute(WG.SQL_PLAYER_STATS_UPDATE, (account_id, update_time, json.dumps(stats_data))) + else: + await self.cache.execute(WG.SQL_PLAYER_STATS_UPDATE, (account_id, update_time, None)) + await self.cache.commit() + debug('Cached player stats saved for account_id: ' + str(account_id) ) + return True + except Exception as err: + error(exception=err) + return False + + + async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ): + try: + # test for cacheDB existence + debug('Trying cached stats first') + if self.cache == None: + debug('No cache DB') + return None + + stats = [] + if len(tank_ids) > 0: + sql_query = 'SELECT * FROM ' + WG.SQL_TANK_STATS_TBL + ' WHERE account_id = ? AND update_time > ? AND tank_id IN (' + ','.join([str(x) for x in tank_ids]) + ')' + else: + sql_query = 'SELECT * FROM ' + WG.SQL_TANK_STATS_TBL + ' WHERE account_id = ? AND update_time > ?' + + async with self.cache.execute(sql_query, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor: + tank_ids = set(tank_ids) + async for row in cursor: + #debug('account_id: ' + str(account_id) + ': 1') + if row[3] == None: + # None/null stats found in cache + # i.e. stats have been requested, but not returned from WG API + tank_ids.remove(row[1]) + continue + #debug('account_id: ' + str(account_id) + ': 2') + stats.append(json.loads(row[3])) + # debug('account_id: ' + str(account_id) + ': 3') + tank_ids.remove(row[1]) + + # return stats ONLY if ALL the requested stats were found in cache + if tank_ids == set(): + debug('Cached stats found: ' + str(account_id)) + return stats + + except Exception as err: + error(exception=err) + debug('No cached stats found') + return None + + + async def get_cached_player_stats(self, account_id, fields): + try: + # test for cacheDB existence + debug('Trying cached stats first') + if self.cache == None: + #debug('No cache DB') + raise CachedStatsNotFound('No cache DB in use') + + async with self.cache.execute(WG.SQL_PLAYER_STATS_CACHED, (account_id, NOW() - WG.CACHE_GRACE_TIME) ) as cursor: + row = await cursor.fetchone() + #debug('account_id: ' + str(account_id) + ': 1') + if row == None: + # no cached stats found, marked with an empty array + #debug('No cached stats found') + raise CachedStatsNotFound('No cached stats found') + + debug('Cached stats found') + if row[3] == None: + # None/null stats found in cache + # i.e. stats have been requested, but not returned from WG API + return None + else: + # Return proper stats + return json.loads(row[3]) + except CachedStatsNotFound as err: + debug(str(err)) + raise + except Exception as err: + error('Error trying to look for cached stats', err) + return None + + + ## ----------------------------------------------------------- #### Class WoTinspector ## ----------------------------------------------------------- class WoTinspector: - URL_WI = 'https://replays.wotinspector.com' - URL_REPLAYS = URL_WI + '/en/sort/ut/page/' + URL_WI = 'https://replays.wotinspector.com' + URL_REPLAY_LIST = URL_WI + '/en/sort/ut/page/' URL_REPLAY_DL = URL_WI + '/en/download/' URL_REPLAY_UL = 'https://api.wotinspector.com/replay/upload?' URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key=' + URL_TANK_DB ="https://wotinspector.com/static/armorinspector/tank_db_blitz.js" REPLAY_N = 1 @@ -754,20 +998,59 @@ class WoTinspector: async def close(self): if self.session != None: + debug('Closing aiohttp session') await self.session.close() - async def getReplayJSON(self, replay_id: str): - json_resp = await getUrlJSON(self.session, self.URL_REPLAY_INFO + replay_id, None) + async def get_tankopedia(self, filename = 'tanks.json'): + """Retrieve Tankpedia from WoTinspector.com""" + + async with self.session.get(self.URL_TANK_DB) as r: + if r.status == 200: + WI_tank_db=await r.text() + WI_tank_db = WI_tank_db.split("\n") + else: + print('Error: Could not get valid HTTPS response. HTTP: ' + str(r.status) ) + sys.exit(1) + tanks = {} + n = 0 + p = re.compile('\\s*(\\d+):\\s{"en":"([^"]+)",.*?"tier":(\\d+), "type":(\\d), "premium":(\\d).*') + for line in WI_tank_db[1:-1]: + try: + m = p.match(line) + tank = {} + tank['tank_id'] = int(m.group(1)) + tank['name'] = m.group(2) + tank['tier'] = int(m.group(3)) + tank['type'] = WG.TANK_TYPE[int(m.group(4))] + tank['is_premium'] = (int(m.group(5)) == 1) + tanks[str(m.group(1))] = tank + n += 1 + except Exception as err: + error(exception=err) + + tankopedia = {} + tankopedia['status'] = "ok" + tankopedia['meta'] = {"count" : n} + tankopedia['data'] = tanks + + verbose_std("Tankopedia has " + str(n) + " tanks in: " + filename) + with open(filename,'w') as outfile: + outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False)) + return None + + + async def get_replay_JSON(self, replay_id: str): + json_resp = await get_url_JSON(self.session, self.URL_REPLAY_INFO + replay_id, None) try: - if self.chkJSONreplay(json_resp): + if self.chk_JSON_replay(json_resp): return json_resp else: return None except Exception as err: - error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) + error('Unexpected Exception', err) return None - async def postReplay(self, data, filename = 'Replay', account_id = 0, title = 'Replay', priv = False, N = None): + async def post_replay(self, data, filename = 'Replay', account_id = 0, title = 'Replay', priv = False, N = None): try: N = N if N != None else self.REPLAY_N msg_str = 'Replay[' + str(N) + ']: ' @@ -778,7 +1061,7 @@ class WoTinspector: replay_id = hash.hexdigest() ## Testing if the replay has already been posted - json_resp = await self.getReplayJSON(replay_id) + json_resp = await self.get_replay_JSON(replay_id) if json_resp != None: debug(msg_str + 'Already uploaded: ' + title) return json_resp @@ -796,7 +1079,7 @@ class WoTinspector: headers ={'Content-type': 'application/x-www-form-urlencoded'} payload = { 'file' : (filename, base64.b64encode(data)) } except Exception as err: - error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) + error(msg_str + 'Unexpected Exception', err) return None for retry in range(MAX_RETRIES): @@ -820,18 +1103,18 @@ class WoTinspector: else: debug(msg_str + 'Got HTTP/' + str(resp.status)) except Exception as err: - error(str(err)) + error(exception=err) await asyncio.sleep(SLEEP) error(msg_str + ' Could not post replay: ' + title) return None @classmethod - def getUrlReplayListing(cls, page : int): - return cls.URL_REPLAYS + str(page) + '?vt=#filters' + def get_url_replay_listing(cls, page : int): + return cls.URL_REPLAY_LIST + str(page) + '?vt=#filters' @classmethod - def getReplayLinks(cls, doc: str): + def get_replay_links(cls, doc: str): """Get replay download links from WoTinspector.com replay listing page""" try: soup = BeautifulSoup(doc, 'lxml') @@ -843,22 +1126,22 @@ class WoTinspector: replay_links.add(link) debug('Adding replay link:' + link) except Exception as err: - debug(str(err)) + error(exception=err) return replay_links @classmethod - def getReplayID(cls, url): + def get_replay_id(cls, url): return url.rsplit('/', 1)[-1] @classmethod - def chkJSONreplay(cls, json_resp): + def chk_JSON_replay(cls, json_resp): """"Check String for being a valid JSON file""" try: if ('status' in json_resp) and json_resp['status'] == 'ok' and ('data' in json_resp) and json_resp['data'] != None: debug("JSON check OK") return True except KeyError as err: - error('Key :' + str(err) + ' not found') + error('Key not found', err) except: debug("JSON check failed: " + str(json_resp)) return False @@ -868,9 +1151,10 @@ class BlitzStars: URL_BlitzStars = 'https://www.blitzstars.com' URL_playerStats = URL_BlitzStars + '/api/playerstats' URL_playerTankStats = URL_BlitzStars + '/api/tanks' + URL_playersTankStats= URL_BlitzStars + '/api/top/forjylpah?' URL_tankAverages = URL_BlitzStars + '/tankaverages.json' URL_activeplayers = URL_BlitzStars + '/api/playerstats/activeinlast30days' - URL_playersTankStats= URL_BlitzStars + '/api/top/forjylpah?' + def __init__(self): self.session = aiohttp.ClientSession() @@ -884,16 +1168,16 @@ class BlitzStars: return cls.URL_tankAverages @classmethod - def getUrlPlayerStats(cls, accountID: int): - return cls.URL_playerStats + '/' + str(accountID) + def get_url_player_stats(cls, account_id: int): + return cls.URL_playerStats + '/' + str(account_id) @classmethod - def getUrlPlayerTankStats(cls, accountID: int, tankID: int): - return cls.URL_playerTankStats + '/' + str(accountID) + '/' + str(tankID) + def getUrlPlayerTankStats(cls, account_id: int, tank_id: int): + return cls.URL_playerTankStats + '/' + str(account_id) + '/' + str(tank_id) @classmethod - def getUrlPlayersTankStats(cls, accountID: int): - return cls.URL_playersTankStats + '&accountId=' + str(accountID) + def getUrlPlayersTankStats(cls, account_id: int): + return cls.URL_playersTankStats + '&accountId=' + str(account_id) @classmethod def getUrlActivePlayers(cls): @@ -908,7 +1192,7 @@ class BlitzStars: debug("JSON check OK") return True except KeyError as err: - error('Key :' + str(err) + ' not found') + error('Key not found', err) except: debug("JSON check failed") return False diff --git a/extractTankopedia.py b/extract_tankopedia.py similarity index 81% rename from extractTankopedia.py rename to extract_tankopedia.py index f1fc4b0..1331dd3 100644 --- a/extractTankopedia.py +++ b/extract_tankopedia.py @@ -21,7 +21,7 @@ async def main(argv): os.chdir(os.path.dirname(sys.argv[0])) parser = argparse.ArgumentParser(description='Extract Tankopedia data from Blitz game files') - parser.add_argument('blitzAppBase', type=str, metavar="BLITZAPP_FOLDER", default=".", help='Base dir of the Blitz App files') + parser.add_argument('blitz_app_base', type=str, metavar="BLITZAPP_FOLDER", default=".", help='Base dir of the Blitz App files') parser.add_argument('tanks', type=str, default='tanks.json', nargs='?', metavar="TANKS_FILE", help='File to write Tankopedia') parser.add_argument('maps', type=str, default='maps.json', nargs='?', metavar='MAPS_FILE', help='File to write map names') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') @@ -29,19 +29,19 @@ async def main(argv): args = parser.parse_args() - bu.setVerbose(args.verbose) - bu.setDebug(args.debug) + bu.set_verbose(args.verbose) + bu.set_debug(args.debug) wg = WG() tasks = [] - for nation in wg.nations: - tasks.append(asyncio.create_task(extractTanks(args.blitzAppBase, nation))) + for nation in wg.NATIONS: + tasks.append(asyncio.create_task(extract_tanks(args.blitz_app_base, nation))) tanklist = [] for tanklist_tmp in await asyncio.gather(*tasks): tanklist.extend(tanklist_tmp) - tank_strs, map_strs = await readUserStrs(args.blitzAppBase) + tank_strs, map_strs = await read_user_strs(args.blitz_app_base) json_data = None userStrs = {} @@ -56,7 +56,7 @@ async def main(argv): bu.error('Unexpected error when reading file: ' + args.tanks + ' : ' + str(err)) async with aiofiles.open(args.tanks, 'w', encoding="utf8") as outfile: - new_tanks, new_userStrs = await convertTankNames(tanklist, tank_strs) + new_tanks, new_userStrs = await convert_tank_names(tanklist, tank_strs) # merge old and new tankopedia tanks.update(new_tanks) userStrs.update(new_userStrs) @@ -65,6 +65,7 @@ async def main(argv): tankopedia['meta'] = { "count": len(tanks) } tankopedia['data'] = tanks tankopedia['userStr'] = userStrs + bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks') await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False)) if args.maps != None: @@ -78,14 +79,15 @@ async def main(argv): # merge old and new map data maps.update(map_strs) async with aiofiles.open(args.maps, 'w', encoding="utf8") as outfile: + bu.verbose_std('New maps file \'' + args.maps + '\' contains ' + str(len(maps)) + ' maps') await outfile.write(json.dumps(maps, ensure_ascii=False, indent=4, sort_keys=True)) return None -async def extractTanks(blitzAppBase : str, nation: str): +async def extract_tanks(blitz_app_base : str, nation: str): tanks = [] - list_xml_file = blitzAppBase + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE + list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE if not os.path.isfile(list_xml_file): print('ERROR: cannot open ' + list_xml_file) return None @@ -96,24 +98,24 @@ async def extractTanks(blitzAppBase : str, nation: str): for data in tankList['root'].keys(): tank_xml = tankList['root'][data] tank = {} - tank['tank_id'] = await getTankID(nation, int(tank_xml['id'])) + tank['tank_id'] = await get_tank_id(nation, int(tank_xml['id'])) tank['userStr'] = tank_xml['userString'] tank['nation'] = nation tank['tier'] = int(tank_xml['level']) #debug(tank_xml['price']) tank['is_premium'] = issubclass(type(tank_xml['price']), dict) - tank['type'] = await getTankType(tank_xml['tags']) + tank['type'] = await get_tank_type(tank_xml['tags']) tanks.append(tank) except Exception as err: bu.error(err) sys.exit(2) return tanks -async def readUserStrs(blitzAppBase : str) -> dict: +async def read_user_strs(blitz_app_base : str) -> dict: """Read user strings to convert map and tank names""" tank_strs = {} map_strs = {} - filename = blitzAppBase + BLITZAPP_STRINGS + filename = blitz_app_base + BLITZAPP_STRINGS bu.debug('Opening file: ' + filename + ' for reading UserStrings') try: async with aiofiles.open(filename, 'r', encoding="utf8") as f: @@ -135,7 +137,7 @@ async def readUserStrs(blitzAppBase : str) -> dict: return tank_strs, map_strs -async def convertTankNames(tanklist : list, tank_strs: dict) -> dict: +async def convert_tank_names(tanklist : list, tank_strs: dict) -> dict: """Convert tank names for Tankopedia""" tankopedia = {} userStrs = {} @@ -162,12 +164,12 @@ async def convertTankNames(tanklist : list, tank_strs: dict) -> dict: return tankopedia_sorted, userStrs_sorted -async def getTankID(nation: str, tankID : int) -> int: - return (tankID << 8) + (wg.nation_id[nation] << 4) + 1 +async def get_tank_id(nation: str, tank_id : int) -> int: + return (tank_id << 8) + (wg.NATION_ID[nation] << 4) + 1 -async def getTankType(tagstr : str): +async def get_tank_type(tagstr : str): tags = tagstr.split(' ') - for t_type in wg.tank_type: + for t_type in wg.TANK_TYPE: if tags[0] == t_type: return t_type return None diff --git a/getTankopedia.py b/get_tankopedia.py similarity index 87% rename from getTankopedia.py rename to get_tankopedia.py index 05f08d0..ba68873 100644 --- a/getTankopedia.py +++ b/get_tankopedia.py @@ -4,6 +4,9 @@ import sys, argparse, json, os, inspect, aiohttp, asyncio, re, logging from pathlib import Path +import blitzutils as bu +from blitzutils import WG +from blitzutils import WoTinspector logging.getLogger("asyncio").setLevel(logging.DEBUG) @@ -47,8 +50,7 @@ async def get_tankopedia(filename: str): return None -def main(argv): - global VERBOSE, DEBUG +async def main(argv): # set the directory for the script os.chdir(os.path.dirname(sys.argv[0])) @@ -58,12 +60,16 @@ def main(argv): parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Verbose mode') args = parser.parse_args() - VERBOSE = args.verbose - DEBUG = args.debug - if DEBUG: VERBOSE = True + bu.set_verbose(args.verbose) + bu.set_debug(args.debug) + + wi = WoTinspector() - asyncio.run(get_tankopedia(args.outfile)) + await wi.get_tankopedia(args.outfile) + + await wi.close() +### main() if __name__ == "__main__": - main(sys.argv[1:]) + asyncio.run(main(sys.argv[1:]), debug=False) diff --git a/wotbreplay_uploader.py b/upload_wotb_replays.py similarity index 93% rename from wotbreplay_uploader.py rename to upload_wotb_replays.py index df814ae..395ebc1 100644 --- a/wotbreplay_uploader.py +++ b/upload_wotb_replays.py @@ -70,28 +70,30 @@ async def main(argv): parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"') args = parser.parse_args() - bu.setVerbose(args.verbose) - bu.setDebug(args.debug) - if args.silent: - bu.setVerbose(False) + bu.set_verbose(args.verbose) + bu.set_debug(args.debug) + bu.set_silent(args.silent) wg = WG(WG_appID, args.tankopedia, args.mapfile) wi = WoTinspector() - #### Connect to MongoDB (TBD) + #### Connect to MongoDB (TBD) + client = None + db = None if args.db: try: client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) db = client[DB_NAME] - except: - db = None + bu.debug('Database connection initiated') + except Exception as err: + bu.error("Could no initiate DB connection: Disabling DB", err) + args.db = False pass else: - db = None - + bu.debug('No DB in use') if args.account != None: - args.accountID = await wg.getAccountID(args.account) + args.accountID = await wg.get_account_id(args.account) bu.debug('WG account_id: ' + str(args.accountID)) if args.accountID == None: @@ -194,7 +196,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto async with aiofiles.open(replay_json_fn) as fp: replay_json = json.loads(await fp.read()) #if replay_json['status'] == 'ok': - if wi.chkJSONreplay(replay_json): + if wi.chk_JSON_replay(replay_json): bu.verbose(msg_str + title + ' has already been posted. Skipping.' ) SKIPPED_N += 1 queue.task_done() @@ -211,9 +213,9 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto async with aiofiles.open(filename,'rb') as fp: filename = os.path.basename(filename) bu.debug(msg_str + 'File: ' + filename) - json_resp = await wi.postReplay(await fp.read(), filename, account_id, title, priv, N) + json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N) if json_resp != None: - if (await bu.saveJSON(replay_json_fn,json_resp)): + if (await bu.save_JSON(replay_json_fn,json_resp)): bu.debug(msg_str + 'Replay saved OK: ' + filename ) else: bu.error(msg_str + 'Error saving replay: ' + filename) @@ -234,7 +236,7 @@ async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: di return False dbc = db[DB_C_REPLAYS] replay_link = replay['data']['download_url'] - replay_id = wi.getReplayID(replay_link) + replay_id = wi.get_replay_id(replay_link) replay['_id'] = replay_id await dbc.insert_one(replay) bu.debug('Replay added to database') @@ -254,8 +256,8 @@ def getTitle(replayfile: str, title: str, i : int) -> str: try: filename = os.path.basename(replayfile) bu.debug(filename) - map_usrStrs = wg.getMapUserStrs() - tank_userStrs = wg.getTankUserStrs() + map_usrStrs = wg.get_map_user_strs() + tank_userStrs = wg.get_tank_user_strs() #p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') # update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah