Add updates 8.5 - 8.8

Fix --filters and improve performance when filtering

README.md

@@ -1,5 +1,5 @@
 # blitz-tools
-Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or higher. I am using 3.8 myself so you may have to change the Python interpreter version in the scripts' first line. 
+Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or later. 
 
 ### Replays
 * [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader. 
@@ -13,46 +13,48 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
 * [blitzutils.py](blitzutils.py): A utils class for various Blitz related to functions. *Other scripts use this*, so you need to place it to the same folder with the other scripts
 
 ### Data
-* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py]](extractTankopedia.py]). Contains only limited information required by the scripts. 
-* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py]](extract_tankopedia.py]). 
+* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py](extractTankopedia.py). Contains only limited information required by the scripts. 
+* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py](extract_tankopedia.py). 
 
 ### Quick instructions
 
-1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
-2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
+1. Install [Python 3.8](https://www.python.org/downloads/) or later to your computer/laptop
+2. Install the required Python modules (see [requirements.txt](requirements.txt): motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
 ```
-# Remember to run the command with a Python interpreter >= 3.7
-python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress 
+python -m pip install -r requirements.txt
 ```
-3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. **The recommened way** is to install the scipts via `git clone`: [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for git. This makes it easy to receive the updates via `git pull`. 
-
-4. Upload your replays. 
+3. Clone the repository. [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for `git`. This makes it easy to receive the updates via `git pull`. 
 ```
-# Windows, run directly from Command Prompt or Powershell
-# Check the options
+git clone https://github.com/Jylpah/blitz-tools.git
+```
+4. Upload your replays with `upload_wotb_replays.py` 
+```
+## Check the options
+# Windows Command Prompt or Powershell
 python.exe .\upload_wotb_replays.py --help
 
-# Syntax
-python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
+# Linux or Windows Subsystem for Linux (WSL)
+./upload_wotb_replays.py --help
+
+# Upload replays (Windows)
+python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay  DIR_WITH_REPLAYS
 
 # example
 python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
-
-# Linux
-./upload_wotb_replays.py --help
 ```
-5. Analyze replays. 
+5. Analyze replays with `analyze_wotb_replays.py`
 ```
-# Windows, run directly from Command Prompt or Powershell
+## Check options
+# Windows Command Prompt or Powershell
 python.exe .\analyze_wotb_replays.py --help
 
-# Syntax
-python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
+# Syntax (Windows)
+python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json [DIR_WITH_REPLAYS ...]
 
-# Example
+# Example (Windows)
 python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
 
-# Linux
+# Linux or Windows Subsystem for Linux (WSL)
 ./analyze_wotb_replays.py --help
 ```
 

analyze_wotb_replays.py

@@ -29,6 +29,8 @@ STAT_TANK_BATTLE_MIN = 100
 BATTLE_TIME_BUCKET = 3600*24*14
 
 OPT_EXPORT_REPLAY_DIR = 'export_replays'
+OPT_EXPORT_CSV_FILE	 = 'export.csv'
+OPT_EXPORT_JSON_FILE = 'export.json'
 
 RE_SRC_IS_DB = re.compile(r'^DB:')
 
@@ -100,7 +102,7 @@ replay_summary_flds = [
 	'protagonist',
 	'player_name',
 	'title', 
-	'mastery_badge'
+	'mastery_badge'	
 ]
 
 replay_details_flds = [
@@ -167,6 +169,7 @@ class BattleCategorizationList():
 		'Regular'				: 1,
 		'Training Room'			: 2,
 		'Tournament'			: 4,
+		'Quick Tournament'		: 5,  # maybe? 
 		'Rating'				: 7,
 		'Mad Games'				: 8,
 		'Realistic Battles'		: 22,
@@ -184,6 +187,7 @@ class BattleCategorizationList():
 		'battle_type'			: [ 'Battle Type', 		'category', 	[ 'Encounter', 'Supremacy']],
 		'room_type'				: [ 'Battle Mode', 		'category', 	_BATTLE_MODES ],
 		'top_tier'				: [ 'Tier', 			'category', 	[ 'Bottom tier', 'Top tier']],
+		'in_platoon'			: [ 'Platoon', 			'category', 	[ 'Solo', 'Platoon']],
 		'tank_tier'				: [ 'Tank Tier', 		'number' ],
 		'is_premium'			: [ 'Tank Type', 		'category', 	[ 'Tech tree', 'Premium']],
 		'tank_type'				: [ 'Tank Class', 		'category', 	WG.TANK_TYPE_STR],
@@ -203,7 +207,11 @@ class BattleCategorizationList():
 		'battle_duration'		: [ 'Battle Duration', 	'bucket', [ x*60 for x in range(0,8) ], 'int' ],
 		'distance_travelled'	: [ 'Distance Driven',	'bucket', [ x*250 for x in range(0,20) ], 'int' ],
 		'allies_wins'			: [ 'Allies WR', 		'bucket', [ 0, .35, .45, .50, .55, .65], '%' ],
+		'allies_battles'		: [ 'Player Battles',	'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ],
+		'allies_damage_dealt'	: [ 'Player Avg Dmg',	'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ],
 		'enemies_wins'			: [ 'Enemies WR', 		'bucket', [ 0, .35, .45, .50, .55, .65], '%' ],
+		'enemies_battles'		: [ 'Player Battles',	'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ],
+		'enemies_damage_dealt'	: [ 'Player Avg Dmg',	'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ],
 		'player_name'			: [ 'Player', 			'string', 25 ],
 		'protagonist'			: [ 'account_id', 		'number' ],
 		'tank_name'				: [ 'Tank', 			'string', 25],
@@ -220,10 +228,41 @@ class BattleCategorizationList():
 		'top_tier'
 		]
 
+	## Player stats based categorizations
+	_categorizations_stats = [ group + '_' + measure for group in [ 'player', 'allies', 'enemies'] for measure in ['wins', 'battles', 'damage_dealt' ]]
+
+	
+	@classmethod
+	def get_categorizations_all(cls, cats: list = None) -> list:
+		try:
+			if cats == None:
+				return list(cls._categorizations.keys())
+			else:
+				return list(set(cls._categorizations.keys()) & set(cats)) 
+		except Exception as err:
+			bu.error(exception=err)
+		return None	
+
 
 	@classmethod
-	def get_categorizations_all(cls):
-		return cls._categorizations.keys()
+	def get_categorizations_stats(cls, cats=None, stats_cats=None):
+		try:
+			if cats == None:
+				return cls._categorizations_stats
+			else:
+				if stats_cats == None:
+					return cats
+				player_stat_filters = set(cls._categorizations_stats)
+				cats = set(cls.get_categorizations_all(cats))
+				if stats_cats:
+					cats = cats & player_stat_filters
+				else:
+					cats = cats - player_stat_filters
+				bu.debug(', '.join(list(cats)))
+				return list(cats)
+		except Exception as err:
+			bu.error(exception=err)
+		return None		
 
 
 	@classmethod
@@ -323,8 +362,7 @@ class BattleCategorizationList():
 
 	def __init__(self, cats: list):
 		self.urls 				= collections.OrderedDict()
-		self.url_title_max_len 	= 0
-		
+		self.url_title_max_len 	= 0		
 		
 		cats = list(set(cats))  	# remove duplicates
 		# ordering		
@@ -354,6 +392,15 @@ class BattleCategorizationList():
 				bu.error('BattleCategorizationList()', exception=err) 
 
 
+	def __iter__(self):
+		"""Returns the Iterator object"""
+		return BattleCategorizationListIterator(self)
+
+
+	def len(self):
+		return len(self.categorizations)
+
+
 	def get_categorization(self, cat: str):	
 		"""Return a BattleCategorization for a category"""
 		try:
@@ -362,6 +409,7 @@ class BattleCategorizationList():
 			bu.error(exception=err)
 		return None
 	
+
 	def record_result(self, result: dict):
 		try:
 			for cat in self.categorizations_list:
@@ -411,7 +459,7 @@ class BattleCategorizationList():
 
 	def get_filter_categories(self, cat: str, filters: list) -> dict:
 		try:
-			return self.categorizations[cat].get_filter_categories(filters)
+			return self.categorizations[str(cat)].get_filter_categories(filters)
 		except Exception as err:
 			bu.error(exception=err)
 		return None
@@ -457,6 +505,20 @@ class BattleCategorizationList():
 		return None
 
 
+class BattleCategorizationListIterator():
+	def __init__(self, btl_cats: BattleCategorizationList) -> None:
+		self._btl_cats = btl_cats
+		self._ndx = 0
+
+	def __next__(self):
+		if self._ndx < len(self._btl_cats.categorizations_list):
+			cat = self._btl_cats.categorizations_list[self._ndx]
+			res = self._btl_cats.categorizations[cat]
+			self._ndx += 1
+			return res
+		raise StopIteration
+
+
 class BattleCategorization():
 
 	RESULT_CAT_HEADER_LEN = 15
@@ -491,10 +553,13 @@ class BattleCategorization():
 
 
 	def get_filter_categories(self, filters: list) -> list:
-		if type(filters) != list:
-			filters = [ filters ]
-		return [ x for x in filters if x in self.get_categories() ]
-
+		try:
+			if type(filters) != list:
+				filters = [ filters ]
+			return [ x for x in filters if x in self.get_categories() ]
+		except Exception as err:
+			bu.error('Category: ' + str(self.category_key), exception=err)
+		return None
 
 	def help(self): 
 		"""Help"""
@@ -713,7 +778,10 @@ class BattleStringCategorization(BattleCategorization):
 
 	def get_filter_categories(self, filters: list) -> list:
 		try:
-			return filters
+			if type(filters) != list:
+				filters = [ filters ]
+			return [ x for x in filters if x in self.get_categories() ]
+			# return filters
 		except Exception as err:
 			bu.error('Category: ' + str(self.category_key), exception=err)
 		return None
@@ -749,11 +817,14 @@ class BattleNumberCategorization(BattleCategorization):
 
 	def get_filter_categories(self, filters: list) -> list:
 		try:
+			if type(filters) != list:
+				filters = [ filters ]
 			return list(map(str,filters))
 		except Exception as err:
 			bu.error('Category: ' + str(self.category_key), exception=err)
 		return None
 
+
 	def help(self):
 		super().help()
 		print('INTEGER')
@@ -894,15 +965,23 @@ class BattleCategory():
 		'battles%'			: [ '% Battles', 'Share of Battles',								9, '{:9.0%}' ],
 		'win'				: [ 'WR', 'Win rate', 												6, '{:6.1%}' ],
 		'damage_made'		: [ 'DPB', 'Average Damage', 										5, '{:5.0f}' ],
+		'damage_received'	: [ 'DPBr', 'Average Damage  Received', 							5, '{:5.0f}' ],
 		'enemies_destroyed'	: [ 'KPB', 'Kills / Battle', 										4, '{:4.2f}' ],
 		'DR'				: [ 'DR', 'Damage Ratio', 											5, '{:5.1f}' ],
 		'KDR'				: [ 'KDR', 'Kills / Death', 										4, '{:4.1f}' ],
 		'enemies_spotted'	: [ 'Spot', 'Enemies spotted per battle', 							4, '{:4.1f}' ],
 		'hit_rate'			: [ 'Hit rate', 'Shots hit / all shots made', 						8, '{:8.1%}' ],
+		'shots_made'		: [ 'Shots', 'Shots made',			 								5, '{:5.1f}' ],
+		'hits_received'		: [ 'Hits R.', 'Hits reveived',		 								7, '{:7.1f}' ],
+		'hits_bounced'		: [ 'Bounced R.', 'Hits & bounced reveived',						10, '{:10.1f}' ],
+		'in_platoon'		: [ 'Plat%', 'Platoon rate',										6, '{:6.1%}' ],
 		'pen_rate'			: [ 'Pen rate', 'Shots pen / shots hit', 							8, '{:8.1%}' ],
+		'pen_rate_received'	: [ 'Penned%', 'Received hits pen / hits received',					7, '{:7.1%}' ],
+		'splash_rate'		: [ 'Splash%', 'HE splash shots / shots hit',						7, '{:7.1%}' ],
+		'splash_rate_received'	: [ 'Splashed%', 'HE splash shots received / hits hit',			9, '{:9.1%}' ],		
 		'survived'			: [ 'Surv%', 'Survival rate', 										6, '{:6.1%}' ],
 		'time_alive'		: [ 'T alive', 'Time being alive in a battle in secs', 				7, '{:7.0f}' ],
-		'alive'				: [ 'Share live', 'Percentage of time being alive in a battle', 		8, '{:8.0%}' ],
+		'alive'				: [ 'Share live', 'Percentage of time being alive in a battle', 	8, '{:8.0%}' ],
 		'battle_duration'	: [ 'Duration', 'Battle duration', 									8, '{:8.0f}' ], 
 		'distance_travelled': [ 'Distance', 'Distance travelled',								8, '{:8.0f}' ], 
 		'top_tier'			: [ 'Top tier', 'Share of games as top tier', 						8, '{:8.0%}' ],
@@ -912,9 +991,9 @@ class BattleCategory():
 		'player_battles'	: [ 'Player Btls', 'Average number battles of the player', 			11, '{:11.0f}' ],
 		'allies_battles'	: [ 'Allies Btls', 'Average number battles of the allies', 			11, '{:11.0f}' ],
 		'enemies_battles'	: [ 'Enemies Btls', 'Average number battles of the enemies', 		12, '{:12.0f}' ],
-		'player_damage_dealt'	: [ 'Player Avg Dmg', 'Player Average damage', 					11, '{:11.0f}' ],
-		'allies_damage_dealt'	: [ 'Allies Avg Dmg', 'Average damage of the allies', 			11, '{:11.0f}' ],
-		'enemies_damage_dealt'	: [ 'Enemies Avg Dmg', 'Average damage of the enemies', 		12, '{:12.0f}' ],
+		'player_damage_dealt'	: [ 'Player Avg Dmg', 'Player Average damage', 					14, '{:14.0f}' ],
+		'allies_damage_dealt'	: [ 'Allies Avg Dmg', 'Average damage of the allies', 			14, '{:14.0f}' ],
+		'enemies_damage_dealt'	: [ 'Enemies Avg Dmg', 'Average damage of the enemies', 		15, '{:14.0f}' ],
 		MISSING_STATS		: [ 'No stats', 'Players without stats avail', 						8, '{:8.1%}']		
 	}
 
@@ -923,13 +1002,15 @@ class BattleCategory():
 	_result_fields_modes = {
 		'default'		: [ 'battles',	'win','damage_made', 'enemies_destroyed', 'enemies_spotted', 
 							'top_tier', 'DR', 'survived', 'allies_wins', 'enemies_wins'],
-		'team'			: [ 'battles',	'win','player_wins','allies_wins','enemies_wins'],
-		'team_extended'	: [ 'battles',	'battles%',	'win','player_wins','allies_wins','enemies_wins', 
+		'team'			: [ 'battles',	'win', 'in_platoon', 'player_wins', 'allies_wins','enemies_wins'],
+		'team_extended'	: [ 'battles',	'battles%',	'win', 'in_platoon', 'player_wins', 'allies_wins', 'enemies_wins', 
 							'player_damage_dealt', 'allies_damage_dealt', 'enemies_damage_dealt', 
 							'player_battles','allies_battles','enemies_battles' ],
-		'extended'		: [ 'battles',	'battles%',	'win','damage_made', 'enemies_destroyed', 'enemies_spotted', 
+		'extended'		: [ 'battles',	'battles%',	'win', 'in_platoon', 'damage_made', 'enemies_destroyed', 'enemies_spotted', 
 							'top_tier', 'DR', 'KDR', 'hit_rate', 'pen_rate', 'survived', 'alive', 
-							'player_wins', 'allies_wins', 'enemies_wins', MISSING_STATS ], 
+							'player_wins', 'allies_wins', 'enemies_wins', MISSING_STATS ],
+		'dmg_received'	: [ 'battles',	'win','damage_made', 'damage_received', 'enemies_destroyed', 'top_tier', 'DR', 
+							'hits_received', 'hits_bounced', 'pen_rate_received', 'splash_rate_received'], 
 		'all'			: [ cat for cat in _result_fields.keys() ]
 	}
 
@@ -942,11 +1023,14 @@ class BattleCategory():
 	]
 
 	_result_ratios = {
-		'KDR'				: [ 'enemies_destroyed', 'destroyed' ],
-		'DR'				: [ 'damage_made', 'damage_received' ],
-		'hit_rate'			: [ 'shots_hit', 'shots_made' ],
-		'pen_rate'			: [ 'shots_pen', 'shots_hit' ],
-		'dmg_block_rate' 	: [ 'damage_blocked', 'damage_received' ]
+		'KDR'					: [ 'enemies_destroyed', 'destroyed' ],
+		'DR'					: [ 'damage_made', 'damage_received' ],
+		'hit_rate'				: [ 'shots_hit', 'shots_made' ],
+		'splash_rate'			: [ 'shots_splash', 'shots_hit' ],
+		'splash_rate_received'	: [ 'hits_splash', 'hits_received' ],
+		'pen_rate'				: [ 'shots_pen', 'shots_hit' ],
+		'pen_rate_received'		: [ 'hits_pen', 'hits_received' ],		
+		'dmg_block_rate' 		: [ 'damage_blocked', 'damage_received' ]
 	}
 
 	# _extended_stats = False
@@ -1016,12 +1100,21 @@ class BattleCategory():
 
 
 	@classmethod
-	def get_field_name(cls, field: str):
+	def get_field_name(cls, field: str) -> str:
 		try:
 			return cls._result_fields[field][0]
 		except Exception as err:
 			bu.error(exception=err)
 
+	
+	@classmethod
+	def get_field_description(cls, field: str) -> str:
+		try:
+			return cls._result_fields[field][1]
+		except Exception as err:
+			bu.error(exception=err)
+
+	
 	@classmethod
 	def get_field_width(cls, field: str):
 		try:
@@ -1083,6 +1176,17 @@ class BattleCategory():
 		return None
 
 
+	@classmethod
+	def help(cls):
+		try:
+			for field in cls.get_result_fields_all():
+				print('{:24s} : {}'.format(field, cls.get_field_description(field)))
+		except Exception as err:
+			bu.error(exception=err) 
+		return None
+
+
+
 	def __init__(self):
 		try:
 			
@@ -1316,7 +1420,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 ## main() -------------------------------------------------------------
 
 async def main(argv):
-	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT 
+	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT, OPT_EXPORT_CSV_FILE, OPT_EXPORT_JSON_FILE 
 	# set the directory for the script
 	current_dir = os.getcwd()
 	os.chdir(os.path.dirname(sys.argv[0]))
@@ -1329,8 +1433,8 @@ async def main(argv):
 	OPT_HIST			= False
 	OPT_STAT_FUNC		= StatFunc.get_default()
 	OPT_WORKERS_N 		= 10
-	OPT_EXPORT_CSV_FILE	 = 'export.csv'
-	OPT_EXPORT_JSON_FILE = 'export.json'
+	# OPT_EXPORT_CSV_FILE	 = 'export.csv'
+	# OPT_EXPORT_JSON_FILE = 'export.json'
 
 	#WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					 	# China is not supported since WG API stats are not available there
@@ -1424,7 +1528,7 @@ async def main(argv):
 		parser.add_argument('--replays', action='store_true', default=False, help='Export replay info in JSON format')
 		parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
 		parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
-		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,10] }, { "player_wins" : 5 }]')
+		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tank_tier" : [8,9,10] }, { "player_wins" : 5 }]')
 		parser.add_argument('--filters_db', type=str, default=None, help='[only for DB setup] Filter replays in DB based on categories. Filters given in MongoDB JSON format. See --mode help')
 		parser.add_argument('--min', type=int, default=None, help='Only select replays from players with minimum number of replays in the dataset')
 		parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
@@ -1486,96 +1590,78 @@ async def main(argv):
 			args.account_id = await wg.get_account_id(args.account)
 			bu.debug('WG  account_id: ' + str(args.account_id))
 
-		try:
-			BattleCategory.set_fields(args.mode)
-			replayQ  = asyncio.Queue(maxsize=1000)			
-			reader_tasks = []
-			# Make replay Queue
+		# try:
+		BattleCategory.set_fields(args.mode)
+		replayQ  = asyncio.Queue(maxsize=1000)			
+		reader_tasks = []
+		# Make replay Queue
 
-			scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
-			bu.set_counter('Reading replays', 10)
-			# Start tasks to process the Queue
-			for i in range(OPT_WORKERS_N):
-				reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
-				bu.debug('ReplayReader ' + str(i) + ' started')
+		scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
+		bu.set_counter('Reading replays', 10)
+		# Start tasks to process the Queue
+		for i in range(OPT_WORKERS_N):
+			reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
+			bu.debug('ReplayReader ' + str(i) + ' started')
 
-			bu.debug('Waiting for the replay scanner to finish')
-			await asyncio.wait([scanner_task])
-			bu.finish_progress_bar()
-			# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
-			await replayQ.join()
-			await asyncio.sleep(0.1)
-			bu.debug('Replays read. Cancelling Readers and analyzing results')
-			for task in reader_tasks:
-				task.cancel()
-				await asyncio.sleep(0.1)	
-			results = []
-			players = set()
-			replays	= dict()
-			for res in await asyncio.gather(*reader_tasks):
-				results.extend(res[0])
-				players.update(res[1])
-				replays.update(res[2])
+		bu.debug('Waiting for the replay scanner to finish')
+		await asyncio.wait([scanner_task])
+		bu.finish_progress_bar()
+		# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
+		await replayQ.join()
+		await asyncio.sleep(0.1)
+		bu.debug('Replays read. Cancelling Readers and analyzing results')
+		for task in reader_tasks:
+			task.cancel()
+			await asyncio.sleep(0.1)	
+		results = []
+		players = set()
+		replays	= dict()
+		for res in await asyncio.gather(*reader_tasks):
+			results.extend(res[0])
+			players.update(res[1])
+			replays.update(res[2])
+		
+		if len(players) == 0:
+			raise Exception("No players found to fetch stats for. No replays found?")
+
+		if args.min != None:
+			results = filter_min_replays_by_player(results, args.min)
 			
-			if len(players) == 0:
-				raise Exception("No players found to fetch stats for. No replays found?")
+		# Filter based on non-stats filters
+		results = filter_results(results, args.filters, stats_filters=False)	
+		players = get_players(results)
+		
+		(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
+	
+		# Filter based on stats filters
+		results = filter_results(results, args.filters, stats_filters=True)
+		bu.debug(str(len(replays)) + ' replays to process')
+		replays = filter_replays(replays, results)				
+		
+		bu.debug('Number of player stats: ' + str(len(player_stats)))
+		teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
 
-			if args.min != None:
-				results = filter_min_replays_by_player(results, args.min)
-				players = get_players(results)
-				replays = filter_replays(replays, results)
+		blt_cat_list = process_battle_results(teamresults, args)
+		bu.verbose_std('WR = ' + StatFunc.get_title())
+		blt_cat_list.print_results()
 
-			(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
-			bu.debug('Number of player stats: ' + str(len(player_stats)))
-			teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
+		histograms = None
+		if args.hist:
+			histograms = process_player_dist(results, player_stats, stat_id_map)
+			print_player_dist(histograms)			
+		
+		if args.json:
+			await export_json(args, blt_cat_list, histograms)
+		
+		if args.csv:
+			await export_csv(args, blt_cat_list, histograms)	
+		
+		if args.replays:
+			await export_replays(replays)
 
-			blt_cat_list = process_battle_results(teamresults, args)
-			bu.verbose_std('WR = ' + StatFunc.get_title())
-			blt_cat_list.print_results()
-
-			histograms = None
-			if args.hist:
-				histograms = process_player_dist(results, player_stats, stat_id_map)
-				print_player_dist(histograms)			
-			
-			if args.json:
-				res = blt_cat_list.get_results_json()
-				if histograms != None:
-					res_hist = dict()
-					for histogram in histograms.keys():
-						res_hist[histogram] = histograms[histogram].get_results()
-					res['histograms'] = res_hist
-				if args.outfile == '-':
-					export_file = OPT_EXPORT_JSON_FILE
-				else:
-					export_file = args.outfile
-				await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
-				bu.verbose_std('Results exported to: ' + export_file)
-			
-			if args.csv:
-				rows = blt_cat_list.get_results_list()
-				if args.hist:
-					rows.append('')
-					rows.append(['Player Histograms'])
-					for histogram in histograms.values():
-						rows.append('')
-						rows.extend(histogram.get_results_list())
-				if args.outfile == '-':
-					export_file = OPT_EXPORT_CSV_FILE
-				else:
-					export_file = args.outfile
-				with open(export_file, 'w') as csvfile:  
-					# creating a csv writer object  
-					csvwriter = csv.writer(csvfile)        					
-					csvwriter.writerows(rows)
-					bu.verbose_std('Results exported to: ' + export_file)			
-			
-			if args.replays:
-				await export_replays(replays)
-
-			bu.debug('Finished. Cleaning up..................')
-		except Exception as err:
-			bu.error(exception=err)
+		bu.debug('Finished. Cleaning up..................')
+		# except Exception as err:
+		# 	bu.error(exception=err)
 	except UserWarning as err:
 		bu.warning(str(err))
 	except asyncio.CancelledError:
@@ -1591,17 +1677,81 @@ async def main(argv):
 	return None
 
 
+async def export_csv(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
+	"""Export results into a CSV file"""
+	try:
+		rows = blt_cat_list.get_results_list()
+		if args.hist:
+			rows.append('')
+			rows.append(['Player Histograms'])
+			for histogram in histograms.values():
+				rows.append('')
+				rows.extend(histogram.get_results_list())
+		if args.outfile == '-':
+			export_file = OPT_EXPORT_CSV_FILE
+		else:
+			export_file = args.outfile
+		with open(export_file, 'w') as csvfile:  
+			# creating a csv writer object  
+			csvwriter = csv.writer(csvfile)        					
+			csvwriter.writerows(rows)
+			bu.verbose_std('Results exported to: ' + export_file)		
+	except Exception as err:
+		bu.error(exception=err)
+
+
+async def export_json(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
+	"""Export results into a JSON file"""
+	try:
+		res = blt_cat_list.get_results_json()
+		if histograms != None:
+			res_hist = dict()
+			for histogram in histograms.keys():
+				res_hist[histogram] = histograms[histogram].get_results()
+			res['histograms'] = res_hist
+		if args.outfile == '-':
+			export_file = OPT_EXPORT_JSON_FILE
+		else:
+			export_file = args.outfile
+		await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
+		bu.verbose_std('Results exported to: ' + export_file)
+	except Exception as err:
+		bu.error(exception=err)
+
+
 async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, parser: argparse.ArgumentParser = None):
 	"""Help for --filters"""
 	if parser != None:
 		parser.print_help()
 	# Result category options
 
+	# Results fields
+	print('\n-------------------------------------------------------------------')
+	print('| Result fields                                                   |')
+	print('-------------------------------------------------------------------\n')
+	
+	BattleCategory.help()
+
+	print('\n-------------------------------------------------------------------')
+	print('| Filters                                                         |')
+	print('-------------------------------------------------------------------\n')
+	print("\t--filters '{ \"categorization\" : category }'")
+	print('\tThese filters let you filter only battles matching the filter. The filter has been proper JSON dict.')
+	print("\tExample: --filter '{ \"room_type\" :1 }'")
+	print('')
+	BattleCategorizationList.help()
+	print('\n-------------------------------------------------------------------')
+	print('| Rounding errors                                                 |')
+	print('-------------------------------------------------------------------\n')
+	print('Yes, the numbers are rounded UP because reasons.')
+	print('If interested why please read this: ')
+	print('\thttps://realpython.com/python-rounding/#pythons-built-in-round-function')
+	print('No, I am not planning to fix these for now.')	
+
 	# Filter usage
 	print('\n-------------------------------------------------------------------')
 	print('| DB Filter usage                                                 |')
-	print('-------------------------------------------------------------------')
-	print('')
+	print('-------------------------------------------------------------------\n')
 	print('You are unlikely to have DB setup required for these filters. This is "Jylpah\'s special"')
 	print('')
 	print("Syntax: --filters_db '{ \"replay.param\": VALUE, \"replay.param2\": { \"$MONGO_OPERATOR\" : VALUE }, ... }'")
@@ -1620,24 +1770,6 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa
 		except Exception as err:
 			bu.error(exception=err)
 
-	print('\n-------------------------------------------------------------------')
-	print('| Filters                                                         |')
-	print('-------------------------------------------------------------------')
-	print('')
-	print("\t--filters '{ \"categorization\" : category }'")
-	print('\tThese filters let you filter only battles matching the filter. The filter has been proper JSON dict.')
-	print("\tExample: --filter '{ \"room_type\" :1 }'")
-	print('')
-	BattleCategorizationList.help()
-	print('\n-------------------------------------------------------------------')
-	print('| Rounding errors                                                 |')
-	print('-------------------------------------------------------------------')
-	print('')
-	print('Yes, the numbers are rounded UP because reasons.')
-	print('If interested why please read this: ')
-	print('\thttps://realpython.com/python-rounding/#pythons-built-in-round-function')
-	print('No, I am not planning to fix these for now.')	
-
 
 async def export_replays(replays: dict):
 	"""Export replays"""
@@ -1740,15 +1872,15 @@ def print_player_dist(histograms: dict):
 		print('')
 		hist.print()
 
+
 def process_battle_results(results: dict, args : argparse.Namespace):
 	"""Process replay battle results""" 
 	try:
 		url 		= args.url
 		cats 		= BattleCategorizationList.get_categorizations(args)
+		blt_cat_list= None
 		
-		
-		if args.filters != None:
-			results = filter_results(results, args.filters)
+		# remove if? 
 		if len(results) > 0:
 			blt_cat_list = BattleCategorizationList(cats)
 			for result in results:
@@ -1759,7 +1891,7 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 			blt_cat_list.calc_results()
 		else:
 			warning_txt = 'No replays to process '
-			if args.filter != None: 
+			if args.filters != None: 
 				warning_txt = warning_txt + 'after filtering'
 			bu.warning(warning_txt)
 
@@ -1768,25 +1900,32 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 	return blt_cat_list
 
 
-def filter_results(results: list, filter_json : str) -> bool:
+def filter_results(results: list, filters_json : str, stats_filters=None) -> bool:
 	"""Filter replays based on battle category filters"""
 	try:
-		filters = json.loads(filter_json)
+		if filters_json == None: 
+			return results
+		filters = json.loads(filters_json)
 		bu.debug(str(filters))
-		for filter in filters:
-			if type(filters[filter]) != list:
-				filters[filter] = [ filters[filter]]
-
 		if type(filters) != dict:
-			bu.error('invalid --filter arguments: ' + str(filters))
+			bu.error('invalid --filter arguments: ' + str(filter))
 			sys.exit(1)
 		
-		filter_cats = BattleCategorizationList(filters.keys())
-		for cat in filters:
+		filter_cats = BattleCategorizationList.get_categorizations_stats(filters.keys(), stats_filters)
+		filter_cats = BattleCategorizationList(filter_cats)
+		if filter_cats.len() == 0:
+			bu.debug('Nothing to filter')
+			return results
+
+		bu.verbose('Filters applied:')
+		for category in filter_cats:
+			cat = category.category_key
 			categorization = filter_cats.get_categorization(cat)
+			bu.debug('Category: ' + cat +  ' filters: ' + str(filters))
 			filters[cat] = categorization.get_filter_categories(filters[cat])
-			bu.verbose('Filter: ' + cat + ': ' + ','.join(filters[cat]))		
-		
+			bu.verbose('  ' + cat + ' = ' + ','.join(filters[cat]))		
+		bu.verbose('')
+
 		res = list()
 		for result in results:
 			try:
@@ -1802,8 +1941,12 @@ def filter_results(results: list, filter_json : str) -> bool:
 					res.append(result)
 			except Exception as err:
 				bu.error(exception=err)
+		if len(res) == 0:
+			raise UserWarning('No replays left after filtering')
 		bu.verbose_std('Replays after filtering: ' + str(len(res)))	
 		return res
+	except UserWarning:
+		raise
 	except Exception as err:
 		bu.error(exception=err)
 	return None
@@ -2346,7 +2489,6 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 								Nreplays += 1
 			except Exception as err:
 				bu.error(exception=err)					
-	bu.verbose('Finished scanning replays: ' + str(Nreplays)  + ' replays to process') 
 	return Nreplays
 
 
@@ -2357,7 +2499,7 @@ async def mk_readerQ_item(replay_json, filename : str = None, _id: str = None) -
 	try:
 		if wi.chk_JSON_replay(replay_json): 
 			if not '_id' in replay_json:
-				if _id  != None:			
+				if _id  == None:			
 					_id = wi.read_replay_id(replay_json)
 				replay_json['_id'] = _id
 		else:
@@ -2483,20 +2625,20 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
 			player_tank_tier = wg.get_tank_data(player['vehicle_descr'], 'tier')
 			btl_tier = max(btl_tier, player_tank_tier)
 
-			if (protagonist != None) and (player['dbid'] == protagonist):
-				protagonist_tank = player['vehicle_descr']
-
 			if player['dbid'] == account_id:
 				# player itself is not put in results['allies']
 				tmp = dict()
 				tmp['account_id'] 	= account_id
-				tmp['tank_id'] 		= player['vehicle_descr']
+				tank_id 			= player['vehicle_descr']
+				tmp['tank_id'] 		= tank_id
 				tmp['tank_tier'] 	= player_tank_tier
 				tmp['tank_name']	= wg.get_tank_name(tmp['tank_id'])
 				tmp['tank_type']	= wg.get_tank_type_id(tmp['tank_id'])
 				tmp['tank_nation']	= wg.get_tank_nation_id(tmp['tank_id'])
 				tmp['is_premium']	= int(wg.is_premium(tmp['tank_id']))
-				tmp['squad_index'] 	= player['squad_index']
+				platoon_ndx 		= player['squad_index']
+				tmp['squad_index'] 	= platoon_ndx
+				tmp['in_platoon']	= 0 if platoon_ndx == None else 1
 				for key in replay_details_flds:
 					tmp[key] = player[key]
 				if player['hitpoints_left'] == 0:
@@ -2529,7 +2671,7 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
 
 		result['battle_duration'] = btl_duration
 		## Rather use 'player' than incomprehensible 'protagonist'...		
-		result['player'] = get_stat_id(protagonist, protagonist_tank, result['battle_start_timestamp'])
+		result['player'] = get_stat_id(account_id, tank_id, result['battle_start_timestamp'])
 		bu.debug('Player stats_id: ' + result['player'])
 		# remove platoon buddy from stats 			
 		if result['squad_index'] != None:

blitzutils.py

@@ -643,6 +643,14 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
     raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
 
 
+def sort_dict(d: dict, number: bool = False) -> dict:
+    """Sort a dict by keys"""
+    if number:
+        return dict(sorted(d.items(), key=lambda item: int(item[0])))
+    else:
+        return dict(sorted(d.items(), key=lambda item: item[0]))
+
+
 # -----------------------------------------------------------
 # Class SlowBar 
 # -----------------------------------------------------------

extract_tankopedia.py

@@ -79,8 +79,8 @@ async def main(argv):
         tankopedia = collections.OrderedDict()
         tankopedia['status'] = 'ok'
         tankopedia['meta'] = { "count":  len(tanks) }
-        tankopedia['data'] = tanks
-        tankopedia['userStr'] = userStrs
+        tankopedia['data'] = bu.sort_dict(tanks, number=True)
+        tankopedia['userStr'] = bu.sort_dict(userStrs)
         bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
         bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')        
         await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))

get_tankopedia.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3.7
+#!/usr/bin/env python3
 
 # Get Tankpedia from WoTinspector.com
 

maps.json

@@ -18,6 +18,7 @@
     "fort": "Fort Despair",
     "grossberg": "Dynasty's Pearl",
     "himmelsdorf": "Himmelsdorf",
+    "holland": "Molendijk",
     "holmeisk": "Wasteland",
     "iceworld": "Everfrost",
     "idle": "Yukon",
@@ -43,6 +44,7 @@
     "neptune_02": "Normandy - Beach",
     "north": "North",
     "ordeal": "Trial by Fire",
+    "plant": "Ghost Factory",
     "pliego": "Castilla",
     "port": "Port Bay",
     "rift": "Hellas",

releases.md

@@ -2,6 +2,15 @@
 
 | Release | Date     | Notes       | Inaccurate? |
 |:-------:|:--------:|-------------|-----|
+| 8.8 | 2022-02-16 | Molendijk map |  |
+| 8.7 | 2022-01-19 | Soviet Light Tanks |  |
+| 8.6 | 2021-12-15 | Joining Light and Medium Tank classes in MM |  |
+| 8.5 | 2021-11-17 | New national crew voices, economy update |  |
+| 8.4 | 2021-10-20 | Polish heavy tanks |  |
+| 8.3 | 2021-09-16 | FV4005, FV4022, VK 91.01 (P) and WZ-121 legendary camos, -5 deg gun depression to WZ-120 |  |
+| 8.2 | 2021-08-18 | Ghost Factory map, XP/credits nerf for inactive players, new store UI |  |
+| 8.1 | 2021-07-15 | Armor highlighting updated |  |
+| 8.0 | 2021-06-08 | American experimental heavies (Yoh-line), PBR camos |  |
 | 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |  |
 | 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables  |  |
 | 7.7 | 2021-02-24 | Czechoslovakian medium tanks  |  |

upload_wotb_replays.py

@@ -195,7 +195,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 						bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
 					else:
 						os.remove(replay_json_fn)
-						bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
+						bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID)
 					SKIPPED_N += 1						
 					queue.task_done()						
 					continue
@@ -216,13 +216,13 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 							if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
 								bu.verbose_std(msg_str + title + ' posted')
 						else:
-							bu.warning(msg_str +'Replay file is not valid: ' + filename)
+							bu.warning(msg_str +'Replay file is not valid/complete: ' + filename)
 							ERROR_N += 1
 					else:
 						bu.error(msg_str + 'Error saving replay: ' + filename)
 						ERROR_N += 1
 				else:
-					bu.error(msg_str + 'Replay file is not valid: ' + filename)
+					bu.error(msg_str + 'Replay file is not valid/complete: ' + filename)
 					ERROR_N += 1	
 		except Exception as err:
 			bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
@@ -286,14 +286,14 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
 					# bu.debug('Replay file\'s metadata: opened')
 					try:
 						metadata_json = json.load(meta)
-						#player = metadata_json['playerName']
-						tank = wg.tank_str2name(metadata_json['playerVehicleName'])
+						#player = metadata_json['playerName']						
+						tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
 						map_name = wg.get_map(metadata_json['mapName'])
-						bu.debug('Tank: ' + tank +  ' Map: ' + map_name)
+						bu.debug('Tank: ' + tank_name if tank_name != None else "NOT FOUND IN tanks.json" +  ' Map: ' + map_name)
 					except Exception as err:
 						bu.error(exception = err)
-			if (tank != None) and  (map_name != None):
-				title = tank + ' @ ' + map_name
+			if (tank_name != None) and  (map_name != None):
+				title = tank_name + ' @ ' + map_name
 			else:
 				title = re.sub('\\.wotbreplay$', '', filename)
 		except Exception as err: