From 6651d77ec3bf046f0bfcfa76ee15e324c0bab144 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 24 Nov 2020 15:18:09 +0200 Subject: [PATCH 1/5] "tank" option for player performance. not ready. --- analyze_wotb_replays.py | 72 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index cefc11a..78b2b5e 100755 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -33,6 +33,7 @@ RE_SRC_IS_DB = re.compile(r'^DB:') # 1st function = for forming stat_id, 2nd for DB stats query, 3rd for WG API stats query STAT_FUNC = { 'tank_tier': [ 'get_stat_id_tank_tier', 'get_db_tank_tier_stats', 'get_wg_tank_tier_stats' ], + 'tank': [ 'get_stat_id_tank', 'get_db_tank_stats', 'get_wg_tank_stats' ], 'player': [ 'get_stat_id_player', 'get_db_player_stats', 'get_wg_player_stats' ] } @@ -1448,6 +1449,58 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name # bu.debug('Returning stats & exiting') return (stats, stat_id_remap) + +## player stat functions: tank +async def get_wg_tank_stats(stat_id_str: str, cache_only = False) -> dict: + """Get player stats from WG. Returns WR per tier of tank_id""" + try: + (account_id, tank_id) = str2ints(stat_id_str) + + # 'battles' must always be there + hist_stats = [ 'all.' + x for x in histogram_fields.keys() ] + hist_stats.append('tank_id') + + player_stats = await wg.get_player_tank_stats(account_id, [ tank_id ], hist_stats, cache_only = cache_only) + #bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats)) + + return await tank_stats_helper(player_stats) + + except KeyError as err: + bu.error('account_id: ' + str(account_id) + ' tank_id:' + str(tank_id) +' : Key not found', err) + except Exception as err: + bu.error(exception=err) + return None + + +async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_id_str: str) -> dict: + """Get player stats from MongoDB (you are very unlikely to have it, unless you have set it up)""" + if db == None: + return None + try: + dbc = db[DB_C_TANK_STATS] + ( account_id, tank_id, battletime ) = str2ints(stat_id_str) + + time_buffer = 2*7*24*3600 + + FIX ## find() rather than aggregate! + cursor = dbc.find({ '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : tank_id } ]}).sort('last_battle_time',-1).limit(1) + + # pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : tank_id } ]}}, + # { '$sort': { 'last_battle_time': -1 }}, + # { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, + # { '$replaceRoot': { 'newRoot': '$doc' }}, + # { '$project': { '_id': 0 }} ] + + # cursor = dbc.aggregate(pipeline, allowDiskUse=True) + #cursor = dbc.aggregate(pipeline) + + return await tank_stats_helper(await cursor.to_list(10)) + + except Exception as err: + bu.error('account_id: ' + str(account_id) + ' Error', err) + return None + + ## player stat functions: tank_tier async def get_wg_tank_tier_stats(stat_id_str: str, cache_only = False) -> dict: """Get player stats from WG. Returns WR per tier of tank_id""" @@ -1890,17 +1943,30 @@ def get_stat_id(account_id: int, tank_id: int, battletime: int) -> str: def get_stat_id_tank_tier(stat_id_str: str) -> str: - """Return stat_id = account_id:tank_tier""" + """Return stat_id = account_id:tank_tier:battletime""" try: stat_id = str2ints(stat_id_str) - tank_tier = wg.get_tank_tier(stat_id[1]) + account_id = stat_id[0] + tank_tier = wg.get_tank_tier(stat_id[1]) battle_time = (stat_id[2] // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET - return ':'.join(map(str, [stat_id[0],tank_tier, battle_time ])) + return ':'.join(map(str, [account_id, tank_tier, battle_time ])) except Exception as err: bu.error('Stats_id: ' + stat_id_str, exception=err) return None +def get_stat_id_tank(stat_id_str: str) -> str: + """Return stat_id = account_id:tank_id:battletime""" + try: + stat_id = str2ints(stat_id_str) + account_id = stat_id[0] + tank_id = stat_id[1] + battle_time = (stat_id[2] // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET + return ':'.join(map(str, [account_id, tank_id, battle_time ])) + except Exception as err: + bu.error('Stats_id: ' + stat_id_str, exception=err) + return None + def get_stat_id_player(stat_id_str: str) -> str: """Return stat_id = account_id:battletime""" try: From b8d9f9125e681d8b88a7ea7efabdbe8a90a4b7de Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 24 Nov 2020 22:49:03 +0200 Subject: [PATCH 2/5] --mode help: improvements, --stat_func=tank --- analyze_wotb_replays.py | 101 +++++++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 78b2b5e..3cf8885 100755 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -126,15 +126,15 @@ class BattleCategorizationList(): _categorizations = { 'total' : [ 'TOTAL', 'total' ], 'battle_result' : [ 'Result', 'category', [ 'Loss', 'Win', 'Draw']], - 'battle_type' : [ 'Battle Type', 'category', ['Encounter', 'Supremacy']], + 'battle_type' : [ 'Battle Type', 'category', [ 'Encounter', 'Supremacy']], 'tank_tier' : [ 'Tank Tier', 'number' ], - 'top_tier' : [ 'Tier', 'category', ['Bottom tier', 'Top tier']], + 'top_tier' : [ 'Tier', 'category', [ 'Bottom tier', 'Top tier']], 'room_type' : [ 'Battle Mode', 'category', _BATTLE_MODES ], 'mastery_badge' : [ 'Battle Medal', 'category', ['-', '3rd Class', '2nd Class', '1st Class', 'Mastery' ]], 'team_result' : [ 'Team Result', 'string' ], 'player_wins' : [ 'Player WR', 'bucket', [ 0, .35, .45, .50, .55, .65], '%' ], - 'player_battles' : [ 'Player Battles', 'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ], - 'player_damage_dealt' : [ 'Player Avg Dmg', 'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ], + 'player_battles' : [ 'Player Battles','bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ], + 'player_damage_dealt' : [ 'Player Avg Dmg','bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ], 'map_name' : [ 'Map', 'string' ], 'tank_name' : [ 'Tank', 'string' ], 'player_name' : [ 'Player', 'string' ], @@ -207,7 +207,35 @@ class BattleCategorizationList(): except Exception as err: bu.error(exception=err) return None - + + + @classmethod + def help(cls): + for cat in cls.get_categorizations_all(): + try: + ctitle = cls.get_categorization_title(cat) + ctype = cls.get_categorization_type(cat) + cparams = cls.get_categorization_params(cat) + + if ctype == 'number': + cat_tmp = BattleNumberCategorization(cat, ctitle) + elif ctype == 'string': + cat_tmp = BattleStringCategorization(cat, ctitle) + elif ctype == 'category': + cat_tmp = BattleClassCategorization(cat, ctitle, cparams[0]) + elif ctype == 'bucket': + cat_tmp = BattleBucketCategorization(cat, ctitle, cparams[0], cparams[1]) + else: + cat_tmp = BattleTotals(ctitle) + + cat_tmp.help() + + except KeyError as err: + bu.error('BattleCategorizationList(): Key not found: Category: ' + cat, exception=err) + except Exception as err: + bu.error('BattleCategorizationList()', exception=err) + + def __init__(self, args : argparse.Namespace): self.urls = collections.OrderedDict() @@ -329,6 +357,12 @@ class BattleCategorization(): """Get category. Must be implemented in the child classes""" return 'NONE' + + def help(self): + """Help""" + FORMAT = '\t{:<15s}\t{}. Options:' + print(FORMAT.format(self.category_key, self.title), end=' ') + def record_result(self, result: dict): try: @@ -393,6 +427,10 @@ class BattleTotals(BattleCategorization): def get_categories(self) -> str: return [ self.TOTAL ] + def help(self): + super().help() + print('-') + class BattleClassCategorization(BattleCategorization): """Class for categorization by fixed classes with ID (int)""" @@ -402,6 +440,14 @@ class BattleClassCategorization(BattleCategorization): self.category_labels = classes + def help(self): + super().help() + labels = list() + for ndx in range(0, len(self.category_labels)): + labels.append(str(ndx) + '=' + self.category_labels[ndx]) + print(', '.join(labels)) + + def get_category(self, result: dict) -> str: """Get category""" cat_id = -1 @@ -442,6 +488,11 @@ class BattleStringCategorization(BattleCategorization): bu.error('Category: ' + self.category_key, exception=err) return None + + def help(self): + super().help() + print(self.title + ' name') + class BattleNumberCategorization(BattleCategorization): """Class for categorization by number""" @@ -466,6 +517,11 @@ class BattleNumberCategorization(BattleCategorization): return None + def help(self): + super().help() + print('INTEGER') + + class BattleBucketCategorization(BattleCategorization): """Class for categorization by buckets based on value""" @@ -478,6 +534,14 @@ class BattleBucketCategorization(BattleCategorization): except Exception as err: bu.error('BattleBucketCategorization()', exception=err) + + def help(self): + super().help() + labels = list() + for ndx in range(0, len(self.category_labels)): + labels.append(str(ndx) + '=' + self.category_labels[ndx]) + print(', '.join(labels)) + def get_category(self, result: dict) -> str: """Get category""" cat_id = -1 @@ -1127,7 +1191,7 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa # Filter usage print('\n-------------------------------------------------------------------') - print('| Filter usage |') + print('| DB Filter usage |') print('-------------------------------------------------------------------') print("Syntax: --filters '{ \"replay.param\": VALUE, \"replay.param2\": { \"$MONGO_OPERATOR\" : VALUE }, ... }'") @@ -1146,6 +1210,12 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa except Exception as err: bu.error(exception=err) + print('\n-------------------------------------------------------------------') + print('| Filter usage |') + print('-------------------------------------------------------------------') + BattleCategorizationList.help() + + ## move the class PlayerHistogram? def set_histogram_buckets(json: dict): global histogram_fields @@ -1422,8 +1492,10 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name # no DB stats found, trying WG AP if (stats[stat_id] == None): stats[pruned_stat_id] = await stat_wg_func(pruned_stat_id) - stat_id_remap[stat_id] = pruned_stat_id + stat_id_remap[stat_id] = pruned_stat_id del stats[stat_id] + bu.debug('get_wg_' + args.stat_func + '_stats returned: account_id=' + str(get_account_id_f_stat_id(pruned_stat_id)) + ': ' + str(stats[pruned_stat_id]), workerID) + except KeyError as err: bu.error('Key not found', err, id=workerID) @@ -1479,11 +1551,18 @@ async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_ try: dbc = db[DB_C_TANK_STATS] ( account_id, tank_id, battletime ) = str2ints(stat_id_str) - time_buffer = 2*7*24*3600 + + hist_stats = [ 'all.' + x for x in histogram_fields.keys() ] + hist_stats.append('tank_id') + hist_stats.append('account_id') + hist_stats.append('last_battle_time') + project = dict() + for stat in hist_stats: + project[stat] = True + project['_id'] = False - FIX ## find() rather than aggregate! - cursor = dbc.find({ '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : tank_id } ]}).sort('last_battle_time',-1).limit(1) + cursor = dbc.find({ '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }}, { 'tank_id' : tank_id } ] }, projection=project).sort('last_battle_time',-1).limit(1) # pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : tank_id } ]}}, # { '$sort': { 'last_battle_time': -1 }}, @@ -1494,7 +1573,7 @@ async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_ # cursor = dbc.aggregate(pipeline, allowDiskUse=True) #cursor = dbc.aggregate(pipeline) - return await tank_stats_helper(await cursor.to_list(10)) + return await tank_stats_helper(await cursor.to_list(1)) except Exception as err: bu.error('account_id: ' + str(account_id) + ' Error', err) From f975720a5365ecef39bbb230d7fb7d322d7a6ec4 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 24 Nov 2020 22:51:57 +0200 Subject: [PATCH 3/5] Fix Alpenstadt --- maps.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maps.json b/maps.json index 0ff3ea9..a857de6 100644 --- a/maps.json +++ b/maps.json @@ -24,7 +24,7 @@ "karelia": "Rockfield", "karieri": "Copperfield", "lake": "Mirage", - "lumber": "Horrorstadt", + "lumber": "Alpenstadt", "lumber_01": "Alpenstadt - Town", "malinovka": "Winter Malinovka", "medvedkovo": "Dead Rail", @@ -51,4 +51,4 @@ "skit": "Naval Frontier", "test": "World of Ducks", "tutorial": "Proving Grounds" -} \ No newline at end of file +} From 9c8dc8d005236279217892c4e37d89fdb9ef76cf Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 24 Nov 2020 22:57:48 +0200 Subject: [PATCH 4/5] Update analyze_wotb_replays.py --- analyze_wotb_replays.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 3cf8885..309865e 100755 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -1611,7 +1611,7 @@ async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, dbc = db[DB_C_TANK_STATS] ( account_id, tier, battletime ) = str2ints(stat_id_str) tier_tanks = wg.get_tanks_by_tier(tier) - time_buffer = 2*7*24*3600 + time_buffer = 8*7*24*3600 #bu.debug('Tank_ids: ' + ','.join(map(str, tier_tanks))) # if battletime == None: @@ -1621,7 +1621,7 @@ async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, # { '$replaceRoot': { 'newRoot': '$doc' }}, # { '$project': { '_id': 0 }} ] # else: - pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, + pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, { '$sort': { 'last_battle_time': -1 }}, { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, { '$replaceRoot': { 'newRoot': '$doc' }}, From 5ad93c84df9d476b05a2bc4b56d67fc907752dc6 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 24 Nov 2020 22:58:38 +0200 Subject: [PATCH 5/5] reverse get_db_tank_tier_stats() changes --- analyze_wotb_replays.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index 309865e..3cf8885 100755 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -1611,7 +1611,7 @@ async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, dbc = db[DB_C_TANK_STATS] ( account_id, tier, battletime ) = str2ints(stat_id_str) tier_tanks = wg.get_tanks_by_tier(tier) - time_buffer = 8*7*24*3600 + time_buffer = 2*7*24*3600 #bu.debug('Tank_ids: ' + ','.join(map(str, tier_tanks))) # if battletime == None: @@ -1621,7 +1621,7 @@ async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, # { '$replaceRoot': { 'newRoot': '$doc' }}, # { '$project': { '_id': 0 }} ] # else: - pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, + pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, { '$sort': { 'last_battle_time': -1 }}, { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, { '$replaceRoot': { 'newRoot': '$doc' }},