GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-21 10:17:05 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #17 from Jylpah/stable-filters

Browse Source 
--filters support
This commit is contained in:
Jylpah authored and GitHub committed 2020-11-25 23:25:16 +02:00
2 parents 6636b8cf99 14680cb549
commit 9d6f6da7d5
1 file changed
+163 -25
Download Patch File Download Diff File Expand all files Collapse all files
analyze_wotb_replays.py
+163 -25
View File
@@ -235,21 +235,26 @@ class BattleCategorizationList():
except Exception as err:
bu.error('BattleCategorizationList()', exception=err)
def __init__(self, args : argparse.Namespace):
self.urls = collections.OrderedDict()
self.url_title_max_len = 0
@classmethod
def get_categorizations(cls, args : argparse.Namespace):
extra_cats = args.extra
only_extra = args.only_extra
if only_extra:
cats = list()
else:
cats = self.get_categorizations_default()
cats = cls.get_categorizations_default()
if extra_cats != None:
cats = cats + extra_cats
return cats
def __init__(self, cats: list):
self.urls = collections.OrderedDict()
self.url_title_max_len = 0
cats = list(set(cats)) # remove duplicates
# ordering
@@ -277,7 +282,15 @@ class BattleCategorizationList():
bu.error('BattleCategorizationList(): Key not found: Category: ' + cat, exception=err)
except Exception as err:
bu.error('BattleCategorizationList()', exception=err)
def get_categorization(self, cat: str):
"""Return a BattleCategorization for a category"""
try:
return self.categorizations[cat]
except Exception as err:
bu.error(exception=err)
return None
def record_result(self, result: dict):
try:
@@ -305,6 +318,7 @@ class BattleCategorizationList():
bu.error(exception=err)
return None
def print_results(self):
try:
for cat in self.categorizations_list:
@@ -325,6 +339,26 @@ class BattleCategorizationList():
bu.error(exception=err)
def get_filter_categories(self, cat: str, filters: list) -> dict:
try:
return self.categorizations[cat].get_filter_categories(filters)
except Exception as err:
bu.error(exception=err)
return None
def get_categories(self, result: dict) -> dict:
"""Get categories for each categorization in the CategorizationList() for a battle result"""
try:
res = dict()
for cat in self.categorizations_list:
res[cat] = self.categorizations[cat].get_category(result)
return res
except Exception as err:
bu.error(exception=err)
return None
def get_results_json(self):
try:
res = dict()
@@ -357,7 +391,13 @@ class BattleCategorization():
"""Get category. Must be implemented in the child classes"""
return 'NONE'
def get_filter_categories(self, filters: list) -> list:
if type(filters) != list:
filters = [ filters ]
return [ x for x in filters if x in self.get_categories() ]
def help(self):
"""Help"""
FORMAT = '\t{:<15s}\t{}. Options:'
@@ -424,9 +464,11 @@ class BattleTotals(BattleCategorization):
def get_category(self, result: dict) -> str:
return self.TOTAL
def get_categories(self) -> str:
return [ self.TOTAL ]
def help(self):
super().help()
print('-')
@@ -460,7 +502,24 @@ class BattleClassCategorization(BattleCategorization):
bu.error('Category: ' + str(self.category_key) + ' ID: ' + str(cat_id), exception=err)
return None
def get_filter_categories(self, filters: list) -> list:
try:
# bu.debug(str(filters), force=True)
if type(filters) != list:
filters = [ filters ]
bu.debug(str(self.category_labels))
res = list()
for ndx in filters:
if ndx in range(0,len(self.category_labels)):
res.append(self.category_labels[ndx])
return res
#return [ self.category_labels[int(x)] for x in filters if x in range(0, len(self.get_categories())) ]
except Exception as err:
bu.error('Category: ' + str(self.category_key), exception=err)
return None
def get_categories(self) -> list:
try:
cats = list()
@@ -488,6 +547,14 @@ class BattleStringCategorization(BattleCategorization):
bu.error('Category: ' + self.category_key, exception=err)
return None
def get_filter_categories(self, filters: list) -> list:
try:
return filters
except Exception as err:
bu.error('Category: ' + str(self.category_key), exception=err)
return None
def help(self):
super().help()
@@ -517,6 +584,13 @@ class BattleNumberCategorization(BattleCategorization):
return None
def get_filter_categories(self, filters: list) -> list:
try:
return filters
except Exception as err:
bu.error('Category: ' + str(self.category_key), exception=err)
return None
def help(self):
super().help()
print('INTEGER')
@@ -542,16 +616,19 @@ class BattleBucketCategorization(BattleCategorization):
labels.append(str(ndx) + '=' + self.category_labels[ndx])
print(', '.join(labels))
def get_category(self, result: dict) -> str:
"""Get category"""
cat_id = -1
value = None
try:
cat_id = self.find_bucket(result[self.category_key], self.breaks)
value = result[self.category_key]
cat_id = self.find_bucket(value, self.breaks)
return self.category_labels[cat_id]
except KeyError as err:
bu.error('Category: ' + str(self.category_key) + ' cat ID: ' + str(cat_id), exception=err)
bu.error('Category: ' + str(self.category_key) + ' value: ' + str(value) + ' cat ID: ' + str(cat_id), exception=err)
except Exception as err:
bu.error('Category: ' + str(self.category_key) + ' cat ID: ' + str(cat_id), exception=err)
bu.error('Category: ' + str(self.category_key) + ' value: ' + str(value) + ' cat ID: ' + str(cat_id), exception=err)
return None
@@ -569,6 +646,20 @@ class BattleBucketCategorization(BattleCategorization):
return None
def get_filter_categories(self, filters: list) -> list:
try:
if type(filters) != list:
filters = [ filters ]
res = list()
for ndx in filters:
if ndx in range(0,len(self.category_labels)):
res.append(self.category_labels[ndx])
return res
except Exception as err:
bu.error('Category: ' + str(self.category_key), exception=err)
return None
def find_bucket(self, value, bucket_breaks: list):
"""Find bucket for value"""
try:
@@ -586,7 +677,7 @@ class BattleBucketCategorization(BattleCategorization):
bu.error('Key not found ndx=' + str(ndx) + ', value=' + str(value), exception = err)
except Exception as err:
bu.error('ndx=' + str(ndx) + ', value=' + str(value), exception = err)
return None
return -2
def mk_category_labels(self, bucket_breaks: list, bucket_format: str) -> list:
@@ -1056,7 +1147,8 @@ async def main(argv):
parser.add_argument('-j', '--json', action='store_true', default=False, help='Export data in JSON')
parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
parser.add_argument('--filters', type=str, default=None, help='Filters for DB based analyses. MongoDB find() filter JSON format. see --help_filters')
parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,20] }, { "player_wins" : 5 }]')
parser.add_argument('--filters_db', type=str, default=None, help='[only for DB setup] Filter replays in DB based on categories. Filters given in MongoDB JSON format. See --mode help')
parser.add_argument('--min', type=int, default=None, help='Only select replays from players with minimum number of replays in the dataset')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
@@ -1193,7 +1285,9 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa
print('\n-------------------------------------------------------------------')
print('| DB Filter usage |')
print('-------------------------------------------------------------------')
print('')
print('You are unlikely to have DB setup required for these filters. This is "Jylpah\'s special"')
print('')
print("Syntax: --filters '{ \"replay.param\": VALUE, \"replay.param2\": { \"$MONGO_OPERATOR\" : VALUE }, ... }'")
print("\tExample: --filters '{ \"data.summary.protagonist\": ACCOUNT_ID, \"data.summary.battle_start_timestamp\": { \"$gt\" : 1602853200 } }'")
if db != None:
@@ -1211,8 +1305,11 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa
bu.error(exception=err)
print('\n-------------------------------------------------------------------')
print('| Filter usage |')
print('| Filters example: --filter \'{ "room_type" :1 }\' |')
print('-------------------------------------------------------------------')
print('')
print('\tThese filters let you filter only battles matching the filter. The filter has been proper JSON dict.')
print('')
BattleCategorizationList.help()
@@ -1298,9 +1395,11 @@ def process_battle_results(results: dict, args : argparse.Namespace):
"""Process replay battle results"""
try:
url = args.url
blt_cat_list = BattleCategorizationList(args)
cats = BattleCategorizationList.get_categorizations(args)
blt_cat_list = BattleCategorizationList(cats)
if args.filters != None:
results = filter_replays(results, args.filters)
for result in results:
blt_cat_list.record_result(result)
if url:
@@ -1313,6 +1412,45 @@ def process_battle_results(results: dict, args : argparse.Namespace):
return blt_cat_list
def filter_replays(results: list, filter_json : str) -> bool:
"""Filter replays based on battle category filters"""
try:
filters = json.loads(filter_json)
bu.debug(str(filters))
for filter in filters:
if type(filters[filter]) != list:
filters[filter] = [ filters[filter]]
if type(filters) != dict:
bu.error('invalid --filter arguments: ' + str(filters))
sys.exit(1)
filter_cats = BattleCategorizationList(filters.keys())
for cat in filters:
categorization = filter_cats.get_categorization(cat)
filters[cat] = categorization.get_filter_categories(filters[cat])
bu.verbose('filter: ' + cat + ': ' + ','.join(filters[cat]))
res = list()
for result in results:
cats = filter_cats.get_categories(result)
ret = False
for cat in cats:
if cats[cat] not in filters[cat]:
ret = False
break
ret = True
if ret:
res.append(result)
return res
except Exception as err:
bu.error(exception=err)
return None
async def process_player_stats(players, N_workers: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict:
"""Start stat_workers to retrieve and store players' stats"""
## Create queue of player-tank pairs to find stats for
@@ -1381,7 +1519,7 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg
## Bug here??
#stat_types = player_stats[list(player_stats.keys())[0]].keys()
stat_types = list()
#stat_types = list()
stat_types = BattleCategory.get_fields_team()
for result in results:
@@ -1423,16 +1561,16 @@ def calc_team_stats(results: list, player_stats : dict, stat_id_map : dict, arg
else:
for stat in stat_types:
if player_stats[player_mapped][stat] != None:
result['player_' + str(stat)] = player_stats[player_mapped][stat]
result['player_' + stat] = player_stats[player_mapped][stat]
for stat in stat_types:
if n_allies[stat] > 0:
result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat]
result['allies_' + stat] = allies_stats[stat] / n_allies[stat]
else:
bu.debug('No allies stats for: ' + str(result))
if n_enemies[stat] > 0:
result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat]
result['enemies_' + stat] = enemies_stats[stat] / n_enemies[stat]
else:
bu.debug('No enemies stats for: ' + str(result))
@@ -1751,9 +1889,9 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
try:
dbc = db[DB_C_REPLAYS]
cursor = None
if args.filters != None:
bu.debug(str(args.filters))
filters = json.loads(args.filters)
if args.filters_db != None:
bu.debug(str(args.filters_db))
filters = json.loads(args.filters_db)
bu.debug(json.dumps(filters, indent=2))
cursor = dbc.find(filters)
else:
@@ -1992,7 +2130,7 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
bu.debug(str(result))
return result
except KeyError as err:
bu.error('Key not found', err)
bu.warning('Invalid replay. Key not found', err)
except Exception as err:
bu.error(exception=err)
return None