From c588805f22b69dcc8c4b3056ea4c7d5ee9b8f657 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Wed, 18 Dec 2019 00:19:21 +0200 Subject: [PATCH] Added support for player population analysis Please ignore the "database support". It is for my custom MongoDB setup. --- analyzeReplays.py | 401 +++++++++++++++++++++++++++++++++-------- blitzutils.py | 57 +++--- wotbreplay_uploader.py | 23 ++- 3 files changed, 368 insertions(+), 113 deletions(-) diff --git a/analyzeReplays.py b/analyzeReplays.py index fc0159d..23145c0 100644 --- a/analyzeReplays.py +++ b/analyzeReplays.py @@ -3,13 +3,18 @@ # Script Analyze WoT Blitz replays import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole, re, logging, time, xmltodict, collections +import motor.motor_asyncio, configparser, ssl +from datetime import datetime, timedelta, date import blitzutils as bu from blitzutils import WG from blitzutils import WoTinspector logging.getLogger("asyncio").setLevel(logging.DEBUG) -WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' +WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' +FILE_CONFIG = 'blitzstats.ini' +DB_C_TANK_STATS = 'WG_TankStats' + wg = None wi = None REPLAY_N = 0 @@ -79,6 +84,8 @@ result_categories_default = [ result_cat_header_frmt = '{:_<17s}' result_cat_frmt = '{:>17s}' + +## Syntax: Check how the replay JSON files look. The algorithm is counting/recording fields result_fields = { 'battles' : [ 'Battles', 'Number of battles', 8, '{:^8.0f}' ], 'win' : [ 'WR', 'Win rate', 6, '{:6.1%}' ], @@ -91,22 +98,20 @@ result_fields = { 'survived' : [ 'Surv%', 'Survival rate', 6, '{:6.1%}' ], 'time_alive%' : [ 'T alive%', 'Percentage of time being alive in a battle', 8, '{:8.0%}' ], 'top_tier' : [ 'Top tier', 'Share of games as top tier', 8, '{:8.0%}' ], - 'allies_win_rate' : [ 'Allies WR', 'Average WR of allies: WR in the tank played counted if more than ' + str(STAT_TANK_BATTLE_MIN) + ' in the tank', 9, '{:9.2%}' ], - 'enemies_win_rate' : [ 'Enemies WR', 'Average WR of enemies: WR in the tank played counted if more than ' + str(STAT_TANK_BATTLE_MIN) + ' in the tank', 10, '{:10.2%}' ], + 'allies_wins' : [ 'Allies WR', 'Average WR of allies at the tier of their tank', 9, '{:9.2%}' ], + 'enemies_wins' : [ 'Enemies WR', 'Average WR of enemies at the tier of their tank', 10, '{:10.2%}' ], 'allies_battles' : [ 'Allies Btls', 'Average number battles of the allies', 11, '{:11.0f}' ], - 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ] - + 'enemies_battles' : [ 'Enemies Btls', 'Average number battles of the enemies', 12, '{:12.0f}' ] } -battle_stats_fields = { - 'WR' : 'Win rate', - 'Battles' : 'Battles' +## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY) +## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ] +histogram_fields = { + 'wins' : [ 'Win rate', [0, .35, .40, .45, .5, .55, .60, .65, 1], 100, '{:.0f}%' ], + 'damage_dealt' : [ 'Avg. Dmg.', [0, .25e3, .5e3, 1e3, 1.5e3, 2e3,2.5e3,3e3,100e3], 1, '{:.0f}' ], + 'battles' : [ 'Battles', [0, 1e3, 3e3, 5e3, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k'] # battles is a mandatory stat to include } -stats_buckets = {} -stats_buckets['WR'] = [0, .35, .40, .45, .5, .55, .60, .65, 1] -stats_buckets['Battles'] = [0, 1e3, 3e3, 5e3, 10e3, 15e3, 25e3, 50e3, 5e9] - def defaultvalueZero(): return 0 @@ -114,6 +119,59 @@ def defaultvalueZero(): def defaultvalueBtlRec(): return BattleRecord() + +class PlayerHistogram(): + def __init__(self, field: str, name: str, fields : list, factor: float, format: str ): + self.field = field + self.name = name + self.fields = fields + self.catFactor = factor + self.catFormat = format + self.ncat = len(self.fields) - 1 + self.allies = [0] * self.ncat + self.enemies = [0] * self.ncat + self.total = [0] * self.ncat + + def recordAlly(self, stat: float): + for cat in range(0, self.ncat): + if self.fields[cat] <= stat <= self.fields[cat+1]: + self.allies[cat] += 1 + self.total[cat] += 1 + return True + return False + + def recordEnemy(self, stat: float): + for cat in range(0, self.ncat): + if self.fields[cat] <= stat <= self.fields[cat+1]: + self.enemies[cat] += 1 + self.total[cat] += 1 + return True + return False + + def getCatname(self, cat: int): + if cat < self.ncat - 1: + return self.formatCat(cat) + ' - ' + self.formatCat(cat+1) + else: + return self.formatCat(cat) + ' -' + + def formatCat(self, cat: int): + val = self.fields[cat] + return self.catFormat.format(float(val * self.catFactor)) + + def print(self): + N_enemies = 0 + N_allies = 0 + for cat in range(0, self.ncat): + N_enemies += self.enemies[cat] + N_allies += self.allies[cat] + N_total = N_allies + N_enemies + + print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL")) + for cat in range(0, self.ncat): + print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(self.getCatname(cat), self.allies[cat], self.allies[cat]/N_allies*100, self.enemies[cat], self.enemies[cat]/N_enemies*100, self.allies[cat] + self.enemies[cat], (self.allies[cat] + self.enemies[cat])/N_total*100 )) + return None + + class BattleRecord(): def __init__(self): try: @@ -194,6 +252,7 @@ class BattleRecord(): def printResults(self): print(' : '.join(self.getResults())) + class BattleRecordCategory(): def __init__(self, cat_name : str): self.category_name = cat_name @@ -276,7 +335,30 @@ class BattleRecordCategory(): async def main(argv): global wg, wi - + # set the directory for the script + os.chdir(os.path.dirname(sys.argv[0])) + + ## Read config + config = configparser.ConfigParser() + config.read(FILE_CONFIG) + configDB = config['DATABASE'] + DB_SERVER = configDB.get('db_server', 'localhost') + DB_PORT = configDB.getint('db_port', 27017) + DB_SSL = configDB.getboolean('db_ssl', False) + DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) + DB_AUTH = configDB.get('db_auth', 'admin') + DB_NAME = configDB.get('db_name', 'BlitzStats') + DB_USER = configDB.get('db_user', 'mongouser') + DB_PASSWD = configDB.get('db_password', "PASSWORD") + DB_CERT = configDB.get('db_ssl_cert_file', None) + DB_CA = configDB.get('db_ssl_ca_file', None) + + bu.debug('DB_SERVER: ' + DB_SERVER) + bu.debug('DB_PORT: ' + str(DB_PORT)) + bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") + bu.debug('DB_AUTH: ' + DB_AUTH) + bu.debug('DB_NAME: ' + DB_NAME) + TASK_N = 7 parser = argparse.ArgumentParser(description='ANalyze Blitz replay JSONs from WoTinspector.com') @@ -289,6 +371,7 @@ async def main(argv): parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"') parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT') + parser.add_argument('--db', action='store_true', default=False, help='Use DB - You are unlikely to have it') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"') @@ -298,14 +381,29 @@ async def main(argv): bu.setDebug(args.debug) wg = WG(WG_appID, args.tankfile, args.mapfile) wi = WoTinspector() + bu.setWaiter(100) if args.account != None: args.accountID = await wg.getAccountID(args.account) bu.debug('WG account_id: ' + str(args.accountID)) + #### Connect to MongoDB (TBD) + client = None + db = None + if args.db: + try: + client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) + db = client[DB_NAME] + bu.debug('Database connection initiated') + except Exception as err: + bu.error("Could no initiate DB connection: Disabling DB" + str(err)) + args.db = False + pass + else: + bu.debug('No DB in use') + try: - replayQ = asyncio.Queue() - + replayQ = asyncio.Queue() reader_tasks = [] # Make replay Queue scanner_task = asyncio.create_task(mkReplayQ(replayQ, args.files)) @@ -326,17 +424,27 @@ async def main(argv): task.cancel() await asyncio.sleep(0.1) results = [] - playerstanks = set() + players = set() for res in await asyncio.gather(*reader_tasks): results.extend(res[0]) - playerstanks.update(res[1]) + players.update(res[1]) - player_stats = await processTankStats(playerstanks, TASK_N) + ## Remove battle_time from the player:tank data in case DB is not in use + if db == None: + players_tmp = set() + for player in players: + #bu.debug('playerID: ' + player + ' ==> ' + prunePlayerStatID(player)) + players_tmp.add(prunePlayerStatID(player)) + #bu.debug(str(len(players_tmp))) + players = players_tmp + + player_stats = await processPlayerStats(players, TASK_N, db) bu.verbose('') - results = calcTeamStats(results, player_stats, args.accountID) - processStats(results, args) + bu.debug('PLAYER STATS:\n' + str(player_stats)) + teamresults = calcTeamStats(results, player_stats) + processBattleStats(teamresults, args) if args.stats: - processPlayerStats(player_stats) + processPlayerDist(results, player_stats) except Exception as err: bu.error(str(type(err)) + ' : ' + str(err)) finally: @@ -345,16 +453,31 @@ async def main(argv): await wi.close() return None -def processPlayerStats(player_stats: dict): - ## TBD +def processPlayerDist(result_list: list, player_stats: dict): + hist_stats = dict() + + for field in histogram_fields.keys(): + hist_stats[field] = PlayerHistogram(field, histogram_fields[field][0], histogram_fields[field][1], histogram_fields[field][2], histogram_fields[field][3] ) + + for result in result_list: + for ally in result['allies']: + for stat_field in hist_stats.keys(): + hist_stats[stat_field].recordAlly(player_stats[ally][stat_field]) + for enemy in result['enemies']: + for stat_field in hist_stats.keys(): + hist_stats[stat_field].recordEnemy(player_stats[enemy][stat_field]) + + for stat_field in hist_stats.keys(): + hist_stats[stat_field].print() + return None -def processStats(results: dict, args : argparse.Namespace): +def processBattleStats(results: dict, args : argparse.Namespace): - url = args.url - urls = collections.OrderedDict() + url = args.url + urls = collections.OrderedDict() categories = {} cats = result_categories_default @@ -384,20 +507,21 @@ def processStats(results: dict, args : argparse.Namespace): return None -async def processTankStats(playerstanks, N_workers: int) -> dict: +async def processPlayerStats(players, N_workers: int, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict: ## Create queue of player-tank pairs to find stats for statsQ = asyncio.Queue() - bu.debug('Create player/tank stats queue: ' + str(len(playerstanks)) + ' player/tanks') - for playertank in playerstanks: - await statsQ.put(playertank) + bu.debug('Create player stats queue: ' + str(len(players)) + ' players') + for player in players: + #bu.debug(player) + await statsQ.put(player) # Process player WR / battle stats stats_tasks = [] - bu.debug('Starting player/tank stats queue workers') + bu.debug('Starting player stats workers') for i in range(N_workers): bu.debug("Starting worker " + str(i)) - stats_tasks.append(asyncio.create_task(statWorker(statsQ, i))) + stats_tasks.append(asyncio.create_task(statWorker(statsQ, i, db))) ## let the workers to process - bu.debug('Waiting stats workers to finnish') + bu.debug('Waiting stats workers to finish') await statsQ.join() bu.debug('Cancelling stats workers') for task in stats_tasks: @@ -409,32 +533,39 @@ async def processTankStats(playerstanks, N_workers: int) -> dict: bu.debug('Returning player_stats') return player_stats -def calcTeamStats(result_list: list, player_stats : dict, account_id: int) -> list: +def calcTeamStats(result_list: list, player_stats : dict) -> list: return_list = [] stat_types = player_stats[list(player_stats.keys())[0]].keys() + bu.debug('stat_types: ' + ','.join(stat_types)) for result in result_list: try: n_allies = collections.defaultdict(defaultvalueZero) allies_stats = collections.defaultdict(defaultvalueZero) + bu.debug('Processing Allies') for ally in result['allies']: # Player itself is not in 'allies': see readReplayJSON() + if ally not in player_stats: continue for stat in stat_types: if player_stats[ally][stat] != None: allies_stats[stat] += player_stats[ally][stat] n_allies[stat] += 1 - + bu.debug('Processing Allies\' avg stats') for stat in stat_types: - result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat] - + if n_allies[stat] > 0: + result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat] + + bu.debug('Processing Enemies') n_enemies = collections.defaultdict(defaultvalueZero) enemies_stats = collections.defaultdict(defaultvalueZero) for enemy in result['enemies']: + if enemy not in player_stats: continue for stat in stat_types: if player_stats[enemy][stat] != None: enemies_stats[stat] += player_stats[enemy][stat] n_enemies[stat] += 1 for stat in stat_types: - result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat] + if n_enemies[stat] > 0: + result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat] return_list.append(result) except KeyError as err: @@ -444,48 +575,134 @@ def calcTeamStats(result_list: list, player_stats : dict, account_id: int) -> l return return_list -async def statWorker(queue : asyncio.Queue, workerID: int) -> list: +async def statWorker(queue : asyncio.Queue, workerID: int, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> list: """Worker thread to find stats for player / tank pairs""" # global wg stats = {} bu.debug("workedID: " + str(workerID) + ' started') try: - i = 0 while True: item = await queue.get() try: - i = (i+1) % 10 - if i == 0 : - bu.printWaiter() + bu.printWaiter() + bu.debug(str(item)) ## Add time stamp here - acc, tank = item.split(':') - account_id = int(acc) - bu.debug('[' +str(workerID) + '] AccountID: ' + acc + ' TankID: ' + tank) + stat = item.split(':') + account_id = int(stat[0]) + tier = int(stat[1]) - playerStat = await wg.getPlayerStats(account_id, ['statistics.all.battles', 'statistics.all.wins']) - bu.debug('[' +str(workerID) + '] ' + str(playerStat)) - - stats[item] = {} - stats[item]['win_rate'] = None - stats[item]['battles'] = None - if (playerStat != None): - playerStat = playerStat['statistics']['all'] - battles = playerStat['battles'] - stats[item]['battles'] = max(battles, 1) - stats[item]['win_rate'] = min(playerStat['wins'] / battles, 1) # To cope with broken stats in WG DB + if len(stat) == 3: + battletime = int(stat[2]) + else: + battletime = None + bu.debug('account_id: ' + str(account_id) + ' tank_tier: ' + str(tier), workerID) + + stats[item] = {} + stats[item] = await getDBplayerStats(db, account_id, tier, battletime) + bu.debug('getDBplayerStats returned: ' + str(stats[item]), workerID) + if stats[item] == None: + stats[item] = await getWGplayerStats(account_id, tier) + stats[prunePlayerStatID(item)] = stats[item] + if stats[item] == None: + del stats[item] + except KeyError as err: - bu.error('[' +str(workerID) + '] Key :' + str(err) + ' not found') + bu.error('Key :' + str(err) + ' not found', workerID) except Exception as err: - bu.error('[' +str(workerID) + '] ' + str(type(err)) + ' : '+ str(err)) + bu.error('Type: ' + str(type(err)) + ' : ' + str(err), workerID) queue.task_done() except (asyncio.CancelledError, concurrent.futures._base.CancelledError): - bu.debug('Stats queue[' + str(workerID) + '] is empty') + bu.debug('Stats queue is empty', workerID) except Exception as err: bu.error(str(err)) return stats + +async def getWGplayerStats(account_id: int, tier: int) -> dict: + """Get player stats from WG. Returns WR per tier of tank_id""" + try: + tier_tanks = wg.getTanksByTier(tier) + + # 'battles' must always be there + hist_stats = [ 'all.' + x for x in histogram_fields.keys() ] + + playerStats = await wg.getPlayerTanksStats(account_id, tier_tanks ,hist_stats) + bu.debug('account_id: ' + str(account_id) + ' ' + str(playerStats)) + + return await statsHelper(playerStats) + + except KeyError as err: + bu.error('account_id: ' + str(account_id) + ' Key :' + str(err) + ' not found') + except Exception as err: + bu.error('Type: ' + str(type(err)) + ' : ' + str(err)) + return None + + +async def getDBplayerStats(db : motor.motor_asyncio.AsyncIOMotorDatabase, account_id : int, tier: int, battletime: int) -> dict: + """Get player stats from MongoDB (you are very unlikely to have it, unless you have set it up)""" + if db == None: + return None + try: + dbc = db[DB_C_TANK_STATS] + tier_tanks = wg.getTanksByTier(tier) + #bu.debug('Tank_ids: ' + ','.join(map(str, tier_tanks))) + + if battletime == None: + pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'tank_id' : {'$in': tier_tanks} } ]}}, + { '$sort': { 'last_battle_time': -1 }}, + { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, + { '$replaceRoot': { 'newRoot': '$doc' }}, + { '$project': { '_id': 0 }} ] + else: + pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, + { '$sort': { 'last_battle_time': 1 }}, + { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, + { '$replaceRoot': { 'newRoot': '$doc' }}, + { '$project': { '_id': 0 }} ] + + # cursor = dbc.aggregate(pipeline, allowDiskUse=True) + cursor = dbc.aggregate(pipeline) + + return await statsHelper(await cursor.to_list(1000)) + + except Exception as err: + bu.error('account_id: ' + str(account_id) + ' Error :' + str(err)) + return None + + +async def statsHelper(stat_list: list): + """Helpher func got getDBplayerStats() and getWGplayerStats()""" + try: + if stat_list == None: + return None + stats = {} + hist_fields = histogram_fields.keys() + if 'battles' not in hist_fields: + bu.error('\'battles\' must be defined in \'histogram_fields\'') + sys.exit(1) + for field in hist_fields: + stats[field] = 0 + + for tankStat in stat_list: + tankStat = tankStat['all'] + for field in hist_fields: + stats[field] += max(0, tankStat[field]) + + if stats['battles'] == 0: + return None + + for field in hist_fields: + if field == 'battles': continue + stats[field] = stats[field] / stats['battles'] + return stats + + except Exception as err: + bu.error(str(type(err)) + ' : ' + str(err)) + return None + + async def mkReplayQ(queue : asyncio.Queue, files : list): """Create queue of replays to post""" p_replayfile = re.compile('.*\\.wotbreplay\\.json$') @@ -526,7 +743,7 @@ async def mkReaderQitem(filename : str) -> list: async def replayReader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace): """Async Worker to process the replay queue""" #global SKIPPED_N - account_id = args.accountID + # account_id = args.accountID results = [] playerstanks = set() try: @@ -552,9 +769,9 @@ async def replayReader(queue: asyncio.Queue, readerID: int, args : argparse.Name queue.task_done() continue - if (account_id != None): - playerstanks.update(set(result['allies'])) - playerstanks.update(set(result['enemies'])) + # if (account_id != None): + playerstanks.update(set(result['allies'])) + playerstanks.update(set(result['enemies'])) results.append(result) bu.debug('Marking task ' + str(replayID) + ' done') @@ -572,6 +789,7 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: global REPLAY_I account_id = args.accountID url = args.url + db = args.db result = {} try: if account_id == None: @@ -601,7 +819,7 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: #bu.debug('2') result['allies'] = set() result['enemies'] = set() - + btl_duration = 0 btl_tier = 0 for player in replay_json['data']['summary']['details']: @@ -611,7 +829,14 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: btl_tier = max(btl_tier, player_tank_tier) #bu.debug('4') if player['dbid'] == account_id: + # player itself is not put in results['allies'] tmp = {} + tmp['account_id'] = account_id + tmp['tank_id'] = player['vehicle_descr'] + tmp['tank_tier'] = player_tank_tier + tmp['tank_name'] = wg.getTankData(tmp['tank_id'], 'name') + tmp['squad_index'] = player['squad_index'] + for key in replay_details_flds: tmp[key] = player[key] if player['hitpoints_left'] == 0: @@ -621,20 +846,32 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: tmp['survived'] = 1 tmp['destroyed'] = 0 - tmp['account_id'] = account_id - tmp['tank_id'] = player['vehicle_descr'] - tmp['tank_tier'] = player_tank_tier - tmp['tank_name'] = wg.getTankData(tmp['tank_id'], 'name') - for key in tmp.keys(): result[key] = tmp[key] - elif player['dbid'] in replay_json['data']['summary']['allies']: - result['allies'].add(':'.join([ str(player['dbid']), str(player['vehicle_descr'])])) - #result['allies'].add(':'.join([ str(player['dbid']), str(player['vehicle_descr']), str(replay_json['data']['summary']['battle_start_timestamp']) ])) else: - result['enemies'].add(':'.join([ str(player['dbid']), str(player['vehicle_descr'])])) - #result['enemies'].add(':'.join([ str(player['dbid']), str(player['vehicle_descr']), str(replay_json['data']['summary']['battle_start_timestamp']) ])) + tmp_account_id = player['dbid'] + tmp_tank_id = player['vehicle_descr'] + tmp_tank_tier = wg.getTankTier(tmp_tank_id) + tmp_battletime = int(replay_json['data']['summary']['battle_start_timestamp']) + + if player['dbid'] in replay_json['data']['summary']['allies']: + result['allies'].add(getPlayerStatID(tmp_account_id, tmp_tank_tier, tmp_battletime, db)) + else: + result['enemies'].add(getPlayerStatID(tmp_account_id, tmp_tank_tier, tmp_battletime, db)) + + # platoon buddy removed from stats + if result['squad_index'] != None: + for ally in replay_json['data']['summary']['allies']: + if (ally['dbid'] != account_id) and (ally['squad_index'] == result['squad_index']): + # platoon buddy found + tmp_account_id = str(ally['dbid']) + tmp_tank_id = ally['vehicle_descr'] + tmp_tank_tier = str(wg.getTankTier(tmp_tank_id)) + tmp_battletime = str(int(replay_json['data']['summary']['battle_start_timestamp'])) + # platoon buddy removed from stats + result['allies'].remove(getPlayerStatID(tmp_account_id, tmp_tank_tier, tmp_battletime, db)) + #bu.debug('6') result['time_alive%'] = result['time_alive'] / btl_duration result['battle_tier'] = btl_tier @@ -651,6 +888,20 @@ async def readReplayJSON(replay_json: dict, args : argparse.Namespace) -> dict: bu.error(str(type(err)) + ' : ' + str(err)) return None +def prunePlayerStatID(id: str) -> str: + stat = id.split(':') + #bu.debug('Stat: ' + id + ' ==> ' + ':'.join([stat[0], stat[1]])) + return ':'.join([stat[0], stat[1]]) + +def getPlayerStatID(account_id: int, tier: int, battletime: int, db = False) -> str: + if db: + battle_date = date.fromtimestamp(battletime) + battle_date = battle_date - timedelta(days=battle_date.weekday()) + #bu.debug('account_id: ' + str(account_id) + ' tier: ' + str(tier) + ' battle_time: ' + str(int(time.mktime(battle_date.timetuple())))) + return ':'.join( [ str(account_id), str(tier), str(int(time.mktime(battle_date.timetuple()))) ] ) + else: + return ':'.join([ str(account_id), str(tier)]) + ### main() if __name__ == "__main__": asyncio.run(main(sys.argv[1:]), debug=False) diff --git a/blitzutils.py b/blitzutils.py index aff64e8..42c3fb7 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -332,17 +332,9 @@ class WG: def __init__(self, WG_appID = None, tankopedia_fn = None, maps_fn = None): self.WG_appID = WG_appID + self.loadTanks(tankopedia_fn) + WG.tanks =self.tanks - if (tankopedia_fn != None): - if os.path.exists(tankopedia_fn) and os.path.isfile(tankopedia_fn): - try: - with open(tankopedia_fn, 'rt', encoding='utf8') as f: - WG.tanks = json.loads(f.read()) - debug(str(WG.tanks["meta"]["count"]) + " tanks") - except Exception as err: - error('Could not read tankopedia: ' + tankopedia_fn + '\n' + str(err)) - else: - verbose('Could not find Tankopedia file: ' + tankopedia_fn) if (maps_fn != None): if os.path.exists(maps_fn) and os.path.isfile(maps_fn): try: @@ -358,7 +350,8 @@ class WG: else: self.session = None debug('WG aiohttp session NOT initiated') - + + # cache TBD self.cache = None self.statsQ = None self.statCacheTask = None @@ -529,25 +522,28 @@ class WG: error('JSON check FAILED: ' + str(type(err)) + ' : ' + str(err) + ' : ' + str(json_resp) ) return False + ## Methods -------------------------------------------------- - async def loadTanks(self, tankopedia_fn: str): + def loadTanks(self, tankopedia_fn: str): """Load tanks from tankopedia JSON""" - if tankopedia_fn != None: - try: - async with aiofiles.open(tankopedia_fn, 'rt', encoding='utf8') as f: - self.tanks = json.loads(await f.read()) - self.tanks_by_tier = dict() - for tier in range(1,11): - self.tanks_by_tier[str(tier)]= list() - for tank in self.tanks['data']: - self.tanks_by_tier[str(tank['tier'])].append(tank['tank_id']) - return True - except Exception as err: - error('Could not read tankopedia: ' + tankopedia_fn + '\n' + str(err)) - return False + if tankopedia_fn == None: + return False + + try: + with open(tankopedia_fn, 'rt', encoding='utf8') as f: + self.tanks = json.loads(f.read()) + self.tanks_by_tier = dict() + for tier in range(1,11): + self.tanks_by_tier[str(tier)] = list() + for tank in self.tanks['data'].values(): + self.tanks_by_tier[str(tank['tier'])].append(tank['tank_id']) + return True + except Exception as err: + error('Could not read tankopedia: ' + tankopedia_fn + ' : ' + str(err)) + return False - async def getTanksByTier(self, tier: int) -> list(): + def getTanksByTier(self, tier: int) -> list(): """Returns tank_ids by tier""" return self.tanks_by_tier[str(tier)] @@ -569,7 +565,7 @@ class WG: def getUrlPlayerTanksStats(self, accountID: int, tankIDs: list, fields: list) -> str: server = self.getServer(accountID) - tank_ids = '%2C'.join(str(x) for x in tankIDs) + tank_ids = '%2C'.join([ str(x) for x in tankIDs]) url = self.URL_WG_server[server] + self.URL_WG_playerTankStats + self.WG_appID + '&account_id=' + str(accountID) + '&tank_id=' + tank_ids if (fields != None) and (len(fields) > 0): field_str = '%2C' + '%2C'.join(fields) @@ -615,14 +611,14 @@ class WG: error('Session must be initialized first') sys.exit(1) try: - debug('AccountID: ' + str(accountID) + ' TankID: ' + ','.join(tankIDs)) + debug('AccountID: ' + str(accountID) + ' TankID: ' + ','.join([ str(id) for id in tankIDs])) url = self.getUrlPlayerTanksStats(accountID, tankIDs, fields) json_data = await getUrlJSON(self.session, url, self.chkJSONtankList) if json_data != None: debug('JSON Response received: ' + str(json_data)) return json_data['data'][str(accountID)] except Exception as err: - error(err) + error('Type: ' + str(type(err)) + ' : ' + str(err)) return None async def getPlayerTankStats(self, accountID: int, tankID : int, fields: list) -> dict: @@ -724,6 +720,9 @@ class WG: except KeyError as err: error('Key not found: ' + str(err)) return None + + def getTankTier(self, tank_id: int): + return self.getTankData(tank_id, 'tier') async def statsSaver(self): while True: diff --git a/wotbreplay_uploader.py b/wotbreplay_uploader.py index 043b2d6..df814ae 100644 --- a/wotbreplay_uploader.py +++ b/wotbreplay_uploader.py @@ -54,14 +54,6 @@ async def main(argv): # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ WG_ID = configGeneral.getint('wg_id', 0) - #### Connect to MongoDB (TBD) - try: - client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) - db = client[DB_NAME] - except: - db = None - pass - parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data') parser.add_argument('--output', default='single', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database') parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id') @@ -71,6 +63,7 @@ async def main(argv): parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') + parser.add_argument('--db', action='store_true', default=False, help='Use DB - You are unlikely to have it') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=True, help='Verbose mode') parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') @@ -85,6 +78,18 @@ async def main(argv): wg = WG(WG_appID, args.tankopedia, args.mapfile) wi = WoTinspector() + #### Connect to MongoDB (TBD) + if args.db: + try: + client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) + db = client[DB_NAME] + except: + db = None + pass + else: + db = None + + if args.account != None: args.accountID = await wg.getAccountID(args.account) bu.debug('WG account_id: ' + str(args.accountID)) @@ -234,7 +239,7 @@ async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: di await dbc.insert_one(replay) bu.debug('Replay added to database') except pymongo.errors.DuplicateKeyError as err: - bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : _id=' + replay_id) + bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) return False except Exception as err: bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err))