From e4be7bf38119295f7c1644d771c3fece0cca36f1 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Mon, 26 Oct 2020 00:03:04 +0200 Subject: [PATCH 1/2] _filler_simple(): not tested yet --- blitzutils.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index 3012dac..92b7441 100755 --- a/blitzutils.py +++ b/blitzutils.py @@ -35,7 +35,7 @@ os.umask(UMASK) class ThrottledClientSession(aiohttp.ClientSession): """Rate-throttled client session class inherited from aiohttp.ClientSession)""" - MIN_SLEEP = 0.1 + #MIN_SLEEP = 0.1 def __init__(self, rate_limit: float = None, *args,**kwargs) -> None: super().__init__(*args,**kwargs) @@ -50,17 +50,16 @@ class ThrottledClientSession(aiohttp.ClientSession): if rate_limit <= 0: raise ValueError('rate_limit must be positive') self.rate_limit = rate_limit - #(increment, sleep) = self._get_rate_increment() self._queue = asyncio.Queue(min(2, int(rate_limit)+1)) - # self._fillerTask = asyncio.create_task(self._filler(rate_limit)) self._fillerTask = asyncio.create_task(self._filler()) def _get_sleep(self) -> list: - if self.rate_limit != None: - return max(1/self.rate_limit, self.MIN_SLEEP) - return None + if self.rate_limit == None: + return None + else: + return 1/self.rate_limit def get_rate(self) -> float: @@ -108,14 +107,28 @@ class ThrottledClientSession(aiohttp.ClientSession): await super().close() - # async def _filler(self, rate_limit: float = 1): + async def _filler_simple(self): + """Filler task to fill the leaky bucket algo""" + try: + if self._queue == None: + return + sleep = self._get_sleep() + while True: + self._queue.put(None) + await asyncio.sleep(sleep) + except asyncio.CancelledError: + debug('Cancelled') + except Exception as err: + error(exception=err) + + async def _filler(self): """Filler task to fill the leaky bucket algo""" try: if self._queue == None: return sleep = self._get_sleep() - debug('SLEEP: ' + str(sleep)) + # debug('SLEEP: ' + str(sleep)) updated_at = time.monotonic() fraction = 0 extra_increment = 0 @@ -139,6 +152,8 @@ class ThrottledClientSession(aiohttp.ClientSession): error(exception=err) + + async def _request(self, *args,**kwargs): """Throttled _request()""" if self._queue != None: From 2d8a19c60087a2fee4206c41a6f510541bc8618c Mon Sep 17 00:00:00 2001 From: Jylpah Date: Mon, 26 Oct 2020 00:08:44 +0200 Subject: [PATCH 2/2] Categories by player performance: WR, avg dmg, battles --- analyze_wotb_replays.py | 219 +++++++++++++++++++++++++++++++--------- 1 file changed, 172 insertions(+), 47 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 7922a44..0ebc147 100755 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -78,8 +78,8 @@ replay_details_flds = [ ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY) ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ] histogram_fields = { - 'wins' : [ 'Win rate', [0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:.0f}%' ], - 'damage_dealt' : [ 'Avg. Dmg.', [0, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ], + 'wins' : [ 'Win rate', [0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:2.0f}%' ], + 'damage_dealt' : [ 'Avg. Dmg.', [0, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:3.0f}' ], 'battles' : [ 'Battles', [0, 1000, 2500, 5000, 7000, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k'] # battles is a mandatory stat to include } @@ -120,19 +120,22 @@ class BattleRecordCategory(): _BATTLE_MODES = mk_battle_modes.__func__(_battle_modes) _result_categories = { - 'total' : [ 'TOTAL', 'total' ], - 'battle_result' : [ 'Result', [ 'Loss', 'Win', 'Draw']], - 'battle_type' : [ 'Battle Type', ['Encounter', 'Supremacy']], - 'tank_tier' : [ 'Tank Tier', 'number' ], - 'top_tier' : [ 'Tier', ['Bottom tier', 'Top tier']], - 'room_type' : [ 'Battle Mode', _BATTLE_MODES ], - 'mastery_badge' : [ 'Battle Medal', ['-', '3rd Class', '2nd Class', '1st Class', 'Mastery' ]], - 'team_result' : [ 'Team Result', 'string' ], - 'map_name' : [ 'Map', 'string' ], - 'tank_name' : [ 'Tank', 'string' ], - 'player_name' : [ 'Player', 'string'], - 'protagonist' : [ 'account_id', 'number'], - 'battle_i' : [ 'Battle #', 'number'] + 'total' : [ 'TOTAL', 'total' ], + 'battle_result' : [ 'Result', 'category', [ 'Loss', 'Win', 'Draw']], + 'battle_type' : [ 'Battle Type', 'category', ['Encounter', 'Supremacy']], + 'tank_tier' : [ 'Tank Tier', 'number' ], + 'top_tier' : [ 'Tier', 'category', ['Bottom tier', 'Top tier']], + 'room_type' : [ 'Battle Mode', 'category', _BATTLE_MODES ], + 'mastery_badge' : [ 'Battle Medal', 'category', ['-', '3rd Class', '2nd Class', '1st Class', 'Mastery' ]], + 'team_result' : [ 'Team Result', 'string' ], + 'map_name' : [ 'Map', 'string' ], + 'tank_name' : [ 'Tank', 'string' ], + 'player_name' : [ 'Player', 'string' ], + 'protagonist' : [ 'account_id', 'number' ], + 'battle_i' : [ 'Battle #', 'number' ], + 'player_wins' : [ 'Player WR', 'bucket', [ 0, .35, .45, .50, .55, .65], '%'], + 'player_battles' : [ 'Player Battles', 'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3]], + 'player_damage_dealt' : [ 'Player Avg Dmg', 'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500]] } _result_categories_default = [ @@ -143,16 +146,17 @@ class BattleRecordCategory(): 'top_tier' ] - _result_categories_extra = [ - 'mastery_badge', - 'room_type', - 'tank_name', - 'map_name', - 'team_result', - 'battle_i', - 'player_name', - 'protagonist' - ] + # _result_categories_extra = [ + # 'mastery_badge', + # 'room_type', + # 'tank_name', + # 'map_name', + # 'team_result', + # 'battle_i', + # 'player_wins', + # 'player_name', + # 'protagonist' + # ] total_battles = 0 @@ -186,11 +190,13 @@ class BattleRecordCategory(): @classmethod def get_extra_categories(cls) -> list: - return cls._result_categories_extra + #return cls._result_categories_extra + return list( set(cls._result_categories.keys() ) - set(cls.get_default_categories() ) ) def __init__(self, cat_name : str): self.category_name = cat_name + self.bucket_labels = list() self.categories = collections.defaultdict(def_value_BattleRecord) if self._result_categories[self.category_name][1] == 'string': self.type = 'string' @@ -198,6 +204,13 @@ class BattleRecordCategory(): self.type = 'number' elif self._result_categories[self.category_name][1] == 'total': self.type = 'total' + elif self._result_categories[self.category_name][1] == 'bucket': + self.type = 'bucket' + self.bucket_format = self.get_bucket_label_format() + self.buckets = self.get_bucket_breaks() + for start in range(len(self.buckets)): + self.bucket_labels.append(self.mk_bucket_label(start, self.buckets)) + else: self.type = 'category' @@ -220,7 +233,96 @@ class BattleRecordCategory(): return self._result_categories[self.category_name][0] - def record_result(self, result: dict): + def get_category(self, id): + """Get category of 'category' type of BattleRecordCategory""" + try: + return self._result_categories[self.category_name][2][id] + except Exception as err: + bu.error(exception=err) + return None + + + def get_bucket_breaks(self) -> list: + """Return bucket breaks""" + try: + return sorted(self._result_categories[self.category_name][2]) + except Exception as err: + bu.error(exception=err) + return None + + + def get_bucket_label_format(self) -> str: + """Return bucket label type, if any""" + try: + if len(self._result_categories[self.category_name]) > 3: + return self._result_categories[self.category_name][3] + return '' + except Exception as err: + bu.error(exception=err) + return None + + + def get_bucket(self, result: dict) -> str: + try: + ndx = self.find_bucket(result[self.category_name], self.buckets) + return self.bucket_labels[ndx] + except Exception as err: + bu.error('Category: ' + self.category_name) + bu.error('Unknown error', exception=err) + return None + + + def find_bucket(self, value, buckets): + """Find bucket for value""" + max = len(buckets) + if max <= 1: + return 0 + ndx = int(max//2) + if value > buckets[ndx]: + return ndx + self.find_bucket(value, buckets[ndx:]) + elif value < buckets[ndx]: + return self.find_bucket(value, buckets[:ndx]) + else: + return ndx + + + def mk_bucket_label(self, ndx: int, buckets: list) -> str: + try: + low = buckets[ndx] + if ndx >= len(buckets)-1: + high = None + else: + high = buckets[min(ndx+1, len(buckets)-1)] + + prefix = '' + if self.bucket_format == '%': + frmt = '{:2.0f}' + if low != None: + low *= 100 + if high != None: + high *= 100 + prefix = '%' + elif self.bucket_format == 'ratio': + frmt = '{:.2f}' + else: + frmt = '{:.0f}' + + label = frmt.format(low) + ' - ' + if high != None: + label = label + frmt.format(high) + else: + label = label + ' ' + label = label + prefix + return label + except KeyError as err: + bu.error('KeyError: ndx=' + str(ndx) + ' buckets=' + str(buckets), exception=err) + except Exception as err: + bu.error('Unknown error', exception=err) + return None + + + def get_sub_category(self, result: dict) -> str: + """Get category""" try: cat = None if self.type == 'total': @@ -229,8 +331,24 @@ class BattleRecordCategory(): cat = str(result[self.category_name]) elif self.type == 'string': cat = result[self.category_name] - else: - cat = self._result_categories[self.category_name][1][result[self.category_name]] + elif self.type == 'category': + cat = self.get_category(result[self.category_name]) + #cat = self._result_categories[self.category_name][2][result[self.category_name]] + elif self.type == 'bucket': + cat = self.get_bucket(result) + return cat + except KeyError as err: + bu.error('Category: ', str(cat)) + bu.error('Key not found', exception=err) + except Exception as err: + bu.error('Category: ', str(cat)) + bu.error('Unknown error', exception=err) + return None + + + def record_result(self, result: dict): + try: + cat = self.get_sub_category(result) self.categories[cat].record_result(result) self.total_battles += 1 return True @@ -275,6 +393,11 @@ class BattleRecordCategory(): row = [ self.RESULT_CAT_FRMT.format(cat) ] row.extend(self.categories[cat].get_results()) results.append(row) + if self.type == 'bucket': + for cat in [ c for c in self.bucket_labels if c in self.categories.keys()]: + row = [ self.RESULT_CAT_FRMT.format(cat) ] + row.extend(self.categories[cat].get_results()) + results.append(row) else: for cat in sorted(self.categories.keys() , key=str.casefold): row = [ self.RESULT_CAT_FRMT.format(cat) ] @@ -611,6 +734,7 @@ class PlayerHistogram(): else: return self.format_category(cat) + ' -' + def format_category(self, cat: int): val = self.fields[cat] return self.cat_format.format(float(val * self.cat_factor)) @@ -652,10 +776,11 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser): ## main() ------------------------------------------------------------- +OPT_MODE_DEFAULT = None OPT_MODE_TEAM = 'team' OPT_MODE_EXTENDED = 'extended' -OPT_MODE_DEFAULT = None -OPT_MODES = [ None, OPT_MODE_TEAM, OPT_MODE_EXTENDED ] +OPT_MODE_HELP = 'help' +OPT_MODES = [ None, OPT_MODE_TEAM, OPT_MODE_EXTENDED, OPT_MODE_HELP ] async def main(argv): global wg, wi, WG_APP_ID @@ -748,7 +873,7 @@ async def main(argv): parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze') parser.add_argument('-a', '--account', type=str, default=WG_ACCOUNT, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER') parser.add_argument('--mode', default=OPT_MODE, choices=OPT_MODES, help='Select stats mode. Options: ' + ', '.join(OPT_MODES[1:])) - parser.add_argument('--extra', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print extra categories: ' + ', '.join( cat + '=' + BattleRecordCategory._result_categories[cat][0] for cat in BattleRecordCategory._result_categories_extra)) + parser.add_argument('--extra', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print extra categories: ' + ', '.join( cat + '=' + BattleRecordCategory._result_categories[cat][0] for cat in BattleRecordCategory.get_extra_categories())) parser.add_argument('--only_extra', action='store_true', default=False, help='Print only the extra categories') parser.add_argument('--hist', action='store_true', default=OPT_HIST, help='Print player histograms: ' + ', '.join( histogram_fields[k][0] for k in histogram_fields)) parser.add_argument('--stat_func', default=OPT_STAT_FUNC, choices=STAT_FUNC.keys(), help='Select how to calculate for ally/enemy performance: tank-tier stats, global player stats') @@ -807,10 +932,10 @@ async def main(argv): bu.debug('No DB in use') # TBD - # if args.filter_help: - # await help_filters(db) - # sys.exit(0) - + if args.mode == OPT_MODE_HELP: + await help_filters(db) + sys.exit(0) + # rebase file arguments due to moving the working directory to the script location args.files = bu.rebase_file_args(current_dir, args.files) @@ -884,11 +1009,11 @@ async def help_filters(db : motor.motor_asyncio.AsyncIOMotorDatabase = None): print('data.summary.KEYS:') for s in summary: print("\t" + s) - print(print('data.summary.details.[].KEYS:')) + print(print('data.summary.details.[].keys:')) for d in details: print("\t" + d) - except: - pass + except Exception as err: + bu.error(exception=err) def set_histogram_buckets(json: dict): @@ -1054,9 +1179,8 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg n_players = len(result['allies']) + len(result['enemies']) n_allies = collections.defaultdict(def_value_zero) allies_stats = collections.defaultdict(def_value_zero) + #bu.debug('Processing Allies') - - for ally in result['allies']: # Player itself is not in 'allies': see read_replay_JSON() ally_mapped = stat_id_map[ally] @@ -1081,7 +1205,7 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg enemies_stats[stat] += player_stats[enemy_mapped][stat] n_enemies[stat] += 1 - #bu.debug('Processing avg stats') + #bu.debug('Processing player's own stats') player_mapped = stat_id_map[result['player']] if player_mapped not in player_stats: missing_stats += 1 @@ -1297,7 +1421,7 @@ async def tank_stats_helper(stat_list: list): hist_fields = histogram_fields.keys() if 'battles' not in hist_fields: bu.error('\'battles\' must be defined in \'histogram_fields\'') - sys.exit(1) + return None for field in hist_fields: stats[field] = 0 @@ -1329,13 +1453,14 @@ async def player_stats_helper(player_stats: dict): hist_fields = histogram_fields.keys() if 'battles' not in hist_fields: bu.error('\'battles\' must be defined in \'histogram_fields\'') - sys.exit(1) + return None - for field in hist_fields: - stats[field] = 0 + # NEEDED 2020-10-25? + #for field in hist_fields: + # stats[field] = 0 for field in hist_fields: - stats[field] += max(0, player_stats['statistics']['all'][field]) + stats[field] = max(0, player_stats['statistics']['all'][field]) if stats['battles'] == 0: return None