Merge pull request #34 from Jylpah/stable-filters-update2

Fix --filters and improve performance when filtering

analyze_wotb_replays.py

@@ -29,6 +29,8 @@ STAT_TANK_BATTLE_MIN = 100
 BATTLE_TIME_BUCKET = 3600*24*14
 
 OPT_EXPORT_REPLAY_DIR = 'export_replays'
+OPT_EXPORT_CSV_FILE	 = 'export.csv'
+OPT_EXPORT_JSON_FILE = 'export.json'
 
 RE_SRC_IS_DB = re.compile(r'^DB:')
 
@@ -205,7 +207,11 @@ class BattleCategorizationList():
 		'battle_duration'		: [ 'Battle Duration', 	'bucket', [ x*60 for x in range(0,8) ], 'int' ],
 		'distance_travelled'	: [ 'Distance Driven',	'bucket', [ x*250 for x in range(0,20) ], 'int' ],
 		'allies_wins'			: [ 'Allies WR', 		'bucket', [ 0, .35, .45, .50, .55, .65], '%' ],
+		'allies_battles'		: [ 'Player Battles',	'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ],
+		'allies_damage_dealt'	: [ 'Player Avg Dmg',	'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ],
 		'enemies_wins'			: [ 'Enemies WR', 		'bucket', [ 0, .35, .45, .50, .55, .65], '%' ],
+		'enemies_battles'		: [ 'Player Battles',	'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ],
+		'enemies_damage_dealt'	: [ 'Player Avg Dmg',	'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ],
 		'player_name'			: [ 'Player', 			'string', 25 ],
 		'protagonist'			: [ 'account_id', 		'number' ],
 		'tank_name'				: [ 'Tank', 			'string', 25],
@@ -222,10 +228,41 @@ class BattleCategorizationList():
 		'top_tier'
 		]
 
+	## Player stats based categorizations
+	_categorizations_stats = [ group + '_' + measure for group in [ 'player', 'allies', 'enemies'] for measure in ['wins', 'battles', 'damage_dealt' ]]
+
+	
+	@classmethod
+	def get_categorizations_all(cls, cats: list = None) -> list:
+		try:
+			if cats == None:
+				return list(cls._categorizations.keys())
+			else:
+				return list(set(cls._categorizations.keys()) & set(cats)) 
+		except Exception as err:
+			bu.error(exception=err)
+		return None	
+
 
 	@classmethod
-	def get_categorizations_all(cls):
-		return cls._categorizations.keys()
+	def get_categorizations_stats(cls, cats=None, stats_cats=None):
+		try:
+			if cats == None:
+				return cls._categorizations_stats
+			else:
+				if stats_cats == None:
+					return cats
+				player_stat_filters = set(cls._categorizations_stats)
+				cats = set(cls.get_categorizations_all(cats))
+				if stats_cats:
+					cats = cats & player_stat_filters
+				else:
+					cats = cats - player_stat_filters
+				bu.debug(', '.join(list(cats)))
+				return list(cats)
+		except Exception as err:
+			bu.error(exception=err)
+		return None		
 
 
 	@classmethod
@@ -360,6 +397,10 @@ class BattleCategorizationList():
 		return BattleCategorizationListIterator(self)
 
 
+	def len(self):
+		return len(self.categorizations)
+
+
 	def get_categorization(self, cat: str):	
 		"""Return a BattleCategorization for a category"""
 		try:
@@ -368,6 +409,7 @@ class BattleCategorizationList():
 			bu.error(exception=err)
 		return None
 	
+
 	def record_result(self, result: dict):
 		try:
 			for cat in self.categorizations_list:
@@ -1378,7 +1420,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 ## main() -------------------------------------------------------------
 
 async def main(argv):
-	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT 
+	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT, OPT_EXPORT_CSV_FILE, OPT_EXPORT_JSON_FILE 
 	# set the directory for the script
 	current_dir = os.getcwd()
 	os.chdir(os.path.dirname(sys.argv[0]))
@@ -1391,8 +1433,8 @@ async def main(argv):
 	OPT_HIST			= False
 	OPT_STAT_FUNC		= StatFunc.get_default()
 	OPT_WORKERS_N 		= 10
-	OPT_EXPORT_CSV_FILE	 = 'export.csv'
-	OPT_EXPORT_JSON_FILE = 'export.json'
+	# OPT_EXPORT_CSV_FILE	 = 'export.csv'
+	# OPT_EXPORT_JSON_FILE = 'export.json'
 
 	#WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					 	# China is not supported since WG API stats are not available there
@@ -1548,98 +1590,78 @@ async def main(argv):
 			args.account_id = await wg.get_account_id(args.account)
 			bu.debug('WG  account_id: ' + str(args.account_id))
 
-		try:
-			BattleCategory.set_fields(args.mode)
-			replayQ  = asyncio.Queue(maxsize=1000)			
-			reader_tasks = []
-			# Make replay Queue
+		# try:
+		BattleCategory.set_fields(args.mode)
+		replayQ  = asyncio.Queue(maxsize=1000)			
+		reader_tasks = []
+		# Make replay Queue
 
-			scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
-			bu.set_counter('Reading replays', 10)
-			# Start tasks to process the Queue
-			for i in range(OPT_WORKERS_N):
-				reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
-				bu.debug('ReplayReader ' + str(i) + ' started')
+		scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
+		bu.set_counter('Reading replays', 10)
+		# Start tasks to process the Queue
+		for i in range(OPT_WORKERS_N):
+			reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
+			bu.debug('ReplayReader ' + str(i) + ' started')
 
-			bu.debug('Waiting for the replay scanner to finish')
-			await asyncio.wait([scanner_task])
-			bu.finish_progress_bar()
-			# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
-			await replayQ.join()
-			await asyncio.sleep(0.1)
-			bu.debug('Replays read. Cancelling Readers and analyzing results')
-			for task in reader_tasks:
-				task.cancel()
-				await asyncio.sleep(0.1)	
-			results = []
-			players = set()
-			replays	= dict()
-			for res in await asyncio.gather(*reader_tasks):
-				results.extend(res[0])
-				players.update(res[1])
-				replays.update(res[2])
+		bu.debug('Waiting for the replay scanner to finish')
+		await asyncio.wait([scanner_task])
+		bu.finish_progress_bar()
+		# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
+		await replayQ.join()
+		await asyncio.sleep(0.1)
+		bu.debug('Replays read. Cancelling Readers and analyzing results')
+		for task in reader_tasks:
+			task.cancel()
+			await asyncio.sleep(0.1)	
+		results = []
+		players = set()
+		replays	= dict()
+		for res in await asyncio.gather(*reader_tasks):
+			results.extend(res[0])
+			players.update(res[1])
+			replays.update(res[2])
+		
+		if len(players) == 0:
+			raise Exception("No players found to fetch stats for. No replays found?")
+
+		if args.min != None:
+			results = filter_min_replays_by_player(results, args.min)
 			
-			if len(players) == 0:
-				raise Exception("No players found to fetch stats for. No replays found?")
+		# Filter based on non-stats filters
+		results = filter_results(results, args.filters, stats_filters=False)	
+		players = get_players(results)
+		
+		(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
+	
+		# Filter based on stats filters
+		results = filter_results(results, args.filters, stats_filters=True)
+		bu.debug(str(len(replays)) + ' replays to process')
+		replays = filter_replays(replays, results)				
+		
+		bu.debug('Number of player stats: ' + str(len(player_stats)))
+		teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
 
-			if args.min != None:
-				results = filter_min_replays_by_player(results, args.min)
-				players = get_players(results)
-				replays = filter_replays(replays, results)
+		blt_cat_list = process_battle_results(teamresults, args)
+		bu.verbose_std('WR = ' + StatFunc.get_title())
+		blt_cat_list.print_results()
 
-				
+		histograms = None
+		if args.hist:
+			histograms = process_player_dist(results, player_stats, stat_id_map)
+			print_player_dist(histograms)			
+		
+		if args.json:
+			await export_json(args, blt_cat_list, histograms)
+		
+		if args.csv:
+			await export_csv(args, blt_cat_list, histograms)	
+		
+		if args.replays:
+			await export_replays(replays)
 
-			(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
-			bu.debug('Number of player stats: ' + str(len(player_stats)))
-			teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
-
-			blt_cat_list = process_battle_results(teamresults, args)
-			bu.verbose_std('WR = ' + StatFunc.get_title())
-			blt_cat_list.print_results()
-
-			histograms = None
-			if args.hist:
-				histograms = process_player_dist(results, player_stats, stat_id_map)
-				print_player_dist(histograms)			
-			
-			if args.json:
-				res = blt_cat_list.get_results_json()
-				if histograms != None:
-					res_hist = dict()
-					for histogram in histograms.keys():
-						res_hist[histogram] = histograms[histogram].get_results()
-					res['histograms'] = res_hist
-				if args.outfile == '-':
-					export_file = OPT_EXPORT_JSON_FILE
-				else:
-					export_file = args.outfile
-				await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
-				bu.verbose_std('Results exported to: ' + export_file)
-			
-			if args.csv:
-				rows = blt_cat_list.get_results_list()
-				if args.hist:
-					rows.append('')
-					rows.append(['Player Histograms'])
-					for histogram in histograms.values():
-						rows.append('')
-						rows.extend(histogram.get_results_list())
-				if args.outfile == '-':
-					export_file = OPT_EXPORT_CSV_FILE
-				else:
-					export_file = args.outfile
-				with open(export_file, 'w') as csvfile:  
-					# creating a csv writer object  
-					csvwriter = csv.writer(csvfile)        					
-					csvwriter.writerows(rows)
-					bu.verbose_std('Results exported to: ' + export_file)			
-			
-			if args.replays:
-				await export_replays(replays)
-
-			bu.debug('Finished. Cleaning up..................')
-		except Exception as err:
-			bu.error(exception=err)
+		bu.debug('Finished. Cleaning up..................')
+		# except Exception as err:
+		# 	bu.error(exception=err)
 	except UserWarning as err:
 		bu.warning(str(err))
 	except asyncio.CancelledError:
@@ -1655,6 +1677,48 @@ async def main(argv):
 	return None
 
 
+async def export_csv(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
+	"""Export results into a CSV file"""
+	try:
+		rows = blt_cat_list.get_results_list()
+		if args.hist:
+			rows.append('')
+			rows.append(['Player Histograms'])
+			for histogram in histograms.values():
+				rows.append('')
+				rows.extend(histogram.get_results_list())
+		if args.outfile == '-':
+			export_file = OPT_EXPORT_CSV_FILE
+		else:
+			export_file = args.outfile
+		with open(export_file, 'w') as csvfile:  
+			# creating a csv writer object  
+			csvwriter = csv.writer(csvfile)        					
+			csvwriter.writerows(rows)
+			bu.verbose_std('Results exported to: ' + export_file)		
+	except Exception as err:
+		bu.error(exception=err)
+
+
+async def export_json(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
+	"""Export results into a JSON file"""
+	try:
+		res = blt_cat_list.get_results_json()
+		if histograms != None:
+			res_hist = dict()
+			for histogram in histograms.keys():
+				res_hist[histogram] = histograms[histogram].get_results()
+			res['histograms'] = res_hist
+		if args.outfile == '-':
+			export_file = OPT_EXPORT_JSON_FILE
+		else:
+			export_file = args.outfile
+		await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
+		bu.verbose_std('Results exported to: ' + export_file)
+	except Exception as err:
+		bu.error(exception=err)
+
+
 async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, parser: argparse.ArgumentParser = None):
 	"""Help for --filters"""
 	if parser != None:
@@ -1808,6 +1872,7 @@ def print_player_dist(histograms: dict):
 		print('')
 		hist.print()
 
+
 def process_battle_results(results: dict, args : argparse.Namespace):
 	"""Process replay battle results""" 
 	try:
@@ -1815,8 +1880,7 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 		cats 		= BattleCategorizationList.get_categorizations(args)
 		blt_cat_list= None
 		
-		if args.filters != None:
-			results = filter_results(results, args.filters)
+		# remove if? 
 		if len(results) > 0:
 			blt_cat_list = BattleCategorizationList(cats)
 			for result in results:
@@ -1827,7 +1891,7 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 			blt_cat_list.calc_results()
 		else:
 			warning_txt = 'No replays to process '
-			if args.filter != None: 
+			if args.filters != None: 
 				warning_txt = warning_txt + 'after filtering'
 			bu.warning(warning_txt)
 
@@ -1836,21 +1900,28 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 	return blt_cat_list
 
 
-def filter_results(results: list, filter_json : str) -> bool:
+def filter_results(results: list, filters_json : str, stats_filters=None) -> bool:
 	"""Filter replays based on battle category filters"""
 	try:
-		filters = json.loads(filter_json)
+		if filters_json == None: 
+			return results
+		filters = json.loads(filters_json)
 		bu.debug(str(filters))
 		if type(filters) != dict:
 			bu.error('invalid --filter arguments: ' + str(filter))
 			sys.exit(1)
 		
-		filter_cats = BattleCategorizationList(filters.keys())
+		filter_cats = BattleCategorizationList.get_categorizations_stats(filters.keys(), stats_filters)
+		filter_cats = BattleCategorizationList(filter_cats)
+		if filter_cats.len() == 0:
+			bu.debug('Nothing to filter')
+			return results
+
 		bu.verbose('Filters applied:')
 		for category in filter_cats:
 			cat = category.category_key
 			categorization = filter_cats.get_categorization(cat)
-			bu.debug('Category: ' + cat + ' filters type: ' + str(type(filters)) + ' filters: ' + str(filters))
+			bu.debug('Category: ' + cat +  ' filters: ' + str(filters))
 			filters[cat] = categorization.get_filter_categories(filters[cat])
 			bu.verbose('  ' + cat + ' = ' + ','.join(filters[cat]))		
 		bu.verbose('')
@@ -1870,8 +1941,12 @@ def filter_results(results: list, filter_json : str) -> bool:
 					res.append(result)
 			except Exception as err:
 				bu.error(exception=err)
+		if len(res) == 0:
+			raise UserWarning('No replays left after filtering')
 		bu.verbose_std('Replays after filtering: ' + str(len(res)))	
 		return res
+	except UserWarning:
+		raise
 	except Exception as err:
 		bu.error(exception=err)
 	return None
@@ -2424,7 +2499,7 @@ async def mk_readerQ_item(replay_json, filename : str = None, _id: str = None) -
 	try:
 		if wi.chk_JSON_replay(replay_json): 
 			if not '_id' in replay_json:
-				if _id  != None:			
+				if _id  == None:			
 					_id = wi.read_replay_id(replay_json)
 				replay_json['_id'] = _id
 		else:

tanks.json

@@ -1,7 +1,7 @@
 {
     "status": "ok",
     "meta": {
-        "count": 517
+        "count": 520
     },
     "data": {
         "1": {
@@ -1958,7 +1958,7 @@
         },
         "8833": {
             "is_premium": true,
-            "name": "Blaze",
+            "name": "Spark",
             "nation": "european",
             "tank_id": 8833,
             "tier": 6,
@@ -2892,6 +2892,14 @@
             "tier": 10,
             "type": "heavyTank"
         },
+        "19009": {
+            "is_premium": true,
+            "name": "AMXmas",
+            "nation": "france",
+            "tank_id": 19009,
+            "tier": 6,
+            "type": "lightTank"
+        },
         "19025": {
             "is_premium": true,
             "name": "Defender Mk. 1",
@@ -3420,6 +3428,14 @@
             "tier": 6,
             "type": "mediumTank"
         },
+        "24321": {
+            "is_premium": true,
+            "name": "T-100 LT",
+            "nation": "ussr",
+            "tank_id": 24321,
+            "tier": 10,
+            "type": "lightTank"
+        },
         "24337": {
             "is_premium": true,
             "name": "M48A2 Räumpanzer",
@@ -3428,6 +3444,14 @@
             "tier": 8,
             "type": "mediumTank"
         },
+        "24577": {
+            "is_premium": true,
+            "name": "Object 268 Version 4",
+            "nation": "ussr",
+            "tank_id": 24577,
+            "tier": 10,
+            "type": "AT-SPG"
+        },
         "24609": {
             "is_premium": false,
             "name": "Concept 1B",
@@ -4244,6 +4268,7 @@
         "AMX_AC_Mle1948": "AMX AC mle. 48",
         "AMX_AC_Mle1948_short": "AMX AC 48",
         "AMX_B_270": "AMX B 270",
+        "AMX_B_270BP": "AMXmas B 270",
         "AMX_M4_1945": "AMX M4 mle. 45",
         "AMX_M4_1945_short": "AMX M4 45",
         "AMX_dracula": "Dracula",
@@ -4511,10 +4536,10 @@
         "Cz07_TVP_46": "TVP VTU Koncept",
         "Cz07_TVP_46_short": "TVP VTU",
         "Cz08_T-25": "Škoda T 25",
-        "Cz08_T-25BP": "Blaze",
-        "Cz08_T-25BP_Custom": "Blaze Extreme",
-        "Cz08_T-25BP_Custom_short": "Blaze Extreme",
-        "Cz08_T-25BP_short": "Blaze",
+        "Cz08_T-25BP": "Spark",
+        "Cz08_T-25BP_Custom": "Spark Extreme",
+        "Cz08_T-25BP_Custom_short": "Spark Extreme",
+        "Cz08_T-25BP_short": "Spark",
         "Cz08_T-25_short": "Škoda T 25",
         "Cz13_T-27": "Škoda T 27",
         "Cz13_T-27_Custom": "Škoda T 27 Rampant",
@@ -4563,6 +4588,8 @@
         "E50_Ausf_M": "E 50 Ausf. M",
         "E50_Ausf_M_Custom": "E 50 M Roaring",
         "E50_Ausf_M_Custom_short": "E 50 M Roaring",
+        "E50_Ausf_M_Custom_short_v2": "E 50 M Monster",
+        "E50_Ausf_M_Custom_v2": "E 50 Ausf. M Monster",
         "E50_Ausf_M_short": "E 50 M",
         "ELC_AMX": "AMX ELC bis",
         "ELC_AMX_short": "AMX ELC bis",
@@ -4620,6 +4647,10 @@
         "FCM_50t_Custom": "FCM 50 t Twister",
         "FCM_50t_Custom_short": "FCM 50 t Twister",
         "FIAT_RA1050I_C_A": "FIAT RA1050I C.A.",
+        "F_AMX-12BP": "AMXmas",
+        "F_AMX-12BP_Custom": "AMXmas Avalanche",
+        "F_AMX-12BP_Custom_short": "AMXmas Avalanche",
+        "F_AMX-12BP_short": "AMXmas",
         "Ferdinand": "Ferdinand",
         "Ferdinandstandardketten": "Ferdinand",
         "Ferdinandturm": "Ferdinand",
@@ -5477,7 +5508,7 @@
         "Pl13_53TP_Markowskiego": "53TP Markowskiego",
         "Pl13_53TP_Markowskiego_short": "53TP Markowskiego",
         "Pl15_60TP": "60TP Lewandowskiego",
-        "Pl15_60TP_short": "60TP",
+        "Pl15_60TP_short": "60TP Lewandowskiego",
         "Porsche_16_Cyl_Y1": "Porsche 16 Cyl. Y",
         "Porsche_16_Cyl_Y2": "Porsche Typ 212 NL",
         "Porsche_Deutz_Typ180_2": "Porsche Deutz Typ 180/2",
@@ -5691,6 +5722,8 @@
         "R120_T22SR_short": "T-22 medium",
         "R127_T44_100_P": "T-44-100",
         "R127_T44_100_P_short": "T-44-100",
+        "R132_T100LT": "T-100 LT",
+        "R132_T100LT_short": "T-100 LT",
         "R134_Object_252U": "Object 252U",
         "R134_Object_252U_short": "Obj. 252U",
         "R140_M4A2_Loza": "Loza's M4-A2 Sherman",
@@ -5699,6 +5732,8 @@
         "R146_STG_Custom": "STG Guard",
         "R146_STG_Custom_short": "STG Guard",
         "R146_STG_short": "STG",
+        "R149_Object_268_4": "Object 268 Version 4",
+        "R149_Object_268_4_short": "Object 268/4",
         "R154_T-34E-43": "T-34 shielded",
         "R154_T-34E-43_short": "T-34 shielded",
         "R158_LT_432": "LT-432",
@@ -5719,6 +5754,8 @@
         "R98_T44_85": "T-44-85",
         "R98_T44_85_Custom": "T-44-85 Bravery",
         "R98_T44_85_Custom_short": "T-44-85 Bravery",
+        "R98_T44_85_Custom_short_v2": "T-44-85 Space",
+        "R98_T44_85_Custom_v2": "T-44-85 Space",
         "R98_T44_85_short": "T-44-85",
         "RX_20_1_chassis": "RX-20-1",
         "RX_20_1_turret": "RX-20-1",
@@ -6121,6 +6158,7 @@
         "Type_61": "Type 61",
         "Type_61_short": "Type 61",
         "UBR-240B": "AP Shell",
+        "UTD_20": "UTD-20",
         "V-11": "V-11",
         "V-11-IS-3": "Object Defender",
         "V-12": "V-12",
@@ -6250,6 +6288,7 @@
         "VVSS_T80": "VVSS T80",
         "VVSS_WE210": "VVSS WE210",
         "V_12CN": "V-12CN",
+        "V_12_5F": "B-12-5B",
         "V_23TG": "V 23 Smasher",
         "V_2_34": "V-2-34",
         "V_2_34M": "V-2-34M",