read_replay_JSON(): Start of same-tier support

.gitignore

@@ -1,15 +0,0 @@
-__pycache__/
-BlitzApp
-tmp/
-ssl/
-*.ini
-*.swp
-*.code-workspace
-*.sqlite3
-.vscode/
-*.sqlite3-journal
-.gitattributes
-export*
-*.csv
-!/stats/*.csv
-!*.py

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "pyutils"]
-	path = pyutils
-	url = https://github.com/Jylpah/pyutils.git

README.md

@@ -1,5 +1,5 @@
 # blitz-tools
-Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or later. 
+Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or higher. I am using 3.8 myself so you may have to change the Python interpreter version in the scripts' first line. 
 
 ### Replays
 * [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader. 
@@ -13,48 +13,46 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
 * [blitzutils.py](blitzutils.py): A utils class for various Blitz related to functions. *Other scripts use this*, so you need to place it to the same folder with the other scripts
 
 ### Data
-* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py](extractTankopedia.py). Contains only limited information required by the scripts. 
-* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py](extract_tankopedia.py). 
+* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py]](extractTankopedia.py]). Contains only limited information required by the scripts. 
+* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py]](extract_tankopedia.py]). 
 
 ### Quick instructions
 
-1. Install [Python 3.8](https://www.python.org/downloads/) or later to your computer/laptop
-2. Install the required Python modules (see [requirements.txt](requirements.txt): motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
+1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
+2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
 ```
-python -m pip install -r requirements.txt
+# Remember to run the command with a Python interpreter >= 3.7
+python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress 
 ```
-3. Clone the repository. [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for `git`. This makes it easy to receive the updates via `git pull`. 
+3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. **The recommened way** is to install the scipts via `git clone`: [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for git. This makes it easy to receive the updates via `git pull`. 
+
+4. Upload your replays. 
 ```
-git clone https://github.com/Jylpah/blitz-tools.git
-```
-4. Upload your replays with `upload_wotb_replays.py` 
-```
-## Check the options
-# Windows Command Prompt or Powershell
+# Windows, run directly from Command Prompt or Powershell
+# Check the options
 python.exe .\upload_wotb_replays.py --help
 
-# Linux or Windows Subsystem for Linux (WSL)
-./upload_wotb_replays.py --help
-
-# Upload replays (Windows)
-python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay  DIR_WITH_REPLAYS
+# Syntax
+python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
 
 # example
 python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
+
+# Linux
+./upload_wotb_replays.py --help
 ```
-5. Analyze replays with `analyze_wotb_replays.py`
+5. Analyze replays. 
 ```
-## Check options
-# Windows Command Prompt or Powershell
+# Windows, run directly from Command Prompt or Powershell
 python.exe .\analyze_wotb_replays.py --help
 
-# Syntax (Windows)
-python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json [DIR_WITH_REPLAYS ...]
+# Syntax
+python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
 
-# Example (Windows)
+# Example
 python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
 
-# Linux or Windows Subsystem for Linux (WSL)
+# Linux
 ./analyze_wotb_replays.py --help
 ```
 

analyze_wotb_replays.py

@@ -28,10 +28,6 @@ REPLAY_I = 0
 STAT_TANK_BATTLE_MIN = 100
 BATTLE_TIME_BUCKET = 3600*24*14
 
-OPT_EXPORT_REPLAY_DIR = 'export_replays'
-OPT_EXPORT_CSV_FILE	 = 'export.csv'
-OPT_EXPORT_JSON_FILE = 'export.json'
-
 RE_SRC_IS_DB = re.compile(r'^DB:')
 
 class StatFunc:
@@ -169,7 +165,6 @@ class BattleCategorizationList():
 		'Regular'				: 1,
 		'Training Room'			: 2,
 		'Tournament'			: 4,
-		'Quick Tournament'		: 5,  # maybe? 
 		'Rating'				: 7,
 		'Mad Games'				: 8,
 		'Realistic Battles'		: 22,
@@ -187,7 +182,6 @@ class BattleCategorizationList():
 		'battle_type'			: [ 'Battle Type', 		'category', 	[ 'Encounter', 'Supremacy']],
 		'room_type'				: [ 'Battle Mode', 		'category', 	_BATTLE_MODES ],
 		'top_tier'				: [ 'Tier', 			'category', 	[ 'Bottom tier', 'Top tier']],
-		'in_platoon'			: [ 'Platoon', 			'category', 	[ 'Solo', 'Platoon']],
 		'tank_tier'				: [ 'Tank Tier', 		'number' ],
 		'is_premium'			: [ 'Tank Type', 		'category', 	[ 'Tech tree', 'Premium']],
 		'tank_type'				: [ 'Tank Class', 		'category', 	WG.TANK_TYPE_STR],
@@ -207,11 +201,7 @@ class BattleCategorizationList():
 		'battle_duration'		: [ 'Battle Duration', 	'bucket', [ x*60 for x in range(0,8) ], 'int' ],
 		'distance_travelled'	: [ 'Distance Driven',	'bucket', [ x*250 for x in range(0,20) ], 'int' ],
 		'allies_wins'			: [ 'Allies WR', 		'bucket', [ 0, .35, .45, .50, .55, .65], '%' ],
-		'allies_battles'		: [ 'Player Battles',	'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ],
-		'allies_damage_dealt'	: [ 'Player Avg Dmg',	'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ],
 		'enemies_wins'			: [ 'Enemies WR', 		'bucket', [ 0, .35, .45, .50, .55, .65], '%' ],
-		'enemies_battles'		: [ 'Player Battles',	'bucket', [ 0, 500, 1000, 2500, 5e3, 10e3, 15e3, 25e3], 'int' ],
-		'enemies_damage_dealt'	: [ 'Player Avg Dmg',	'bucket', [ 0, 500, 1000, 1250, 1500, 1750, 2000, 2500], 'int' ],
 		'player_name'			: [ 'Player', 			'string', 25 ],
 		'protagonist'			: [ 'account_id', 		'number' ],
 		'tank_name'				: [ 'Tank', 			'string', 25],
@@ -228,41 +218,10 @@ class BattleCategorizationList():
 		'top_tier'
 		]
 
-	## Player stats based categorizations
-	_categorizations_stats = [ group + '_' + measure for group in [ 'player', 'allies', 'enemies'] for measure in ['wins', 'battles', 'damage_dealt' ]]
-
 
 	@classmethod
-	def get_categorizations_all(cls, cats: list = None) -> list:
-		try:
-			if cats == None:
-				return list(cls._categorizations.keys())
-			else:
-				return list(set(cls._categorizations.keys()) & set(cats)) 
-		except Exception as err:
-			bu.error(exception=err)
-		return None	
-
-
-	@classmethod
-	def get_categorizations_stats(cls, cats=None, stats_cats=None):
-		try:
-			if cats == None:
-				return cls._categorizations_stats
-			else:
-				if stats_cats == None:
-					return cats
-				player_stat_filters = set(cls._categorizations_stats)
-				cats = set(cls.get_categorizations_all(cats))
-				if stats_cats:
-					cats = cats & player_stat_filters
-				else:
-					cats = cats - player_stat_filters
-				bu.debug(', '.join(list(cats)))
-				return list(cats)
-		except Exception as err:
-			bu.error(exception=err)
-		return None		
+	def get_categorizations_all(cls):
+		return cls._categorizations.keys()
 
 
 	@classmethod
@@ -364,6 +323,7 @@ class BattleCategorizationList():
 		self.urls 				= collections.OrderedDict()
 		self.url_title_max_len 	= 0
 		
+		
 		cats = list(set(cats))  	# remove duplicates
 		# ordering		
 		self.categorizations_list = [ cat for cat in self._categorizations if cat in cats ]
@@ -392,15 +352,6 @@ class BattleCategorizationList():
 				bu.error('BattleCategorizationList()', exception=err) 
 
 
-	def __iter__(self):
-		"""Returns the Iterator object"""
-		return BattleCategorizationListIterator(self)
-
-
-	def len(self):
-		return len(self.categorizations)
-
-
 	def get_categorization(self, cat: str):	
 		"""Return a BattleCategorization for a category"""
 		try:
@@ -409,7 +360,6 @@ class BattleCategorizationList():
 			bu.error(exception=err)
 		return None
 	
-
 	def record_result(self, result: dict):
 		try:
 			for cat in self.categorizations_list:
@@ -459,7 +409,7 @@ class BattleCategorizationList():
 
 	def get_filter_categories(self, cat: str, filters: list) -> dict:
 		try:
-			return self.categorizations[str(cat)].get_filter_categories(filters)
+			return self.categorizations[cat].get_filter_categories(filters)
 		except Exception as err:
 			bu.error(exception=err)
 		return None
@@ -505,20 +455,6 @@ class BattleCategorizationList():
 		return None
 
 
-class BattleCategorizationListIterator():
-	def __init__(self, btl_cats: BattleCategorizationList) -> None:
-		self._btl_cats = btl_cats
-		self._ndx = 0
-
-	def __next__(self):
-		if self._ndx < len(self._btl_cats.categorizations_list):
-			cat = self._btl_cats.categorizations_list[self._ndx]
-			res = self._btl_cats.categorizations[cat]
-			self._ndx += 1
-			return res
-		raise StopIteration
-
-
 class BattleCategorization():
 
 	RESULT_CAT_HEADER_LEN = 15
@@ -553,13 +489,10 @@ class BattleCategorization():
 
 
 	def get_filter_categories(self, filters: list) -> list:
-		try:
 		if type(filters) != list:
 			filters = [ filters ]
 		return [ x for x in filters if x in self.get_categories() ]
-		except Exception as err:
-			bu.error('Category: ' + str(self.category_key), exception=err)
-		return None
+
 
 	def help(self): 
 		"""Help"""
@@ -778,10 +711,7 @@ class BattleStringCategorization(BattleCategorization):
 
 	def get_filter_categories(self, filters: list) -> list:
 		try:
-			if type(filters) != list:
-				filters = [ filters ]
-			return [ x for x in filters if x in self.get_categories() ]
-			# return filters
+			return filters
 		except Exception as err:
 			bu.error('Category: ' + str(self.category_key), exception=err)
 		return None
@@ -817,14 +747,11 @@ class BattleNumberCategorization(BattleCategorization):
 
 	def get_filter_categories(self, filters: list) -> list:
 		try:
-			if type(filters) != list:
-				filters = [ filters ]
 			return list(map(str,filters))
 		except Exception as err:
 			bu.error('Category: ' + str(self.category_key), exception=err)
 		return None
 
-
 	def help(self):
 		super().help()
 		print('INTEGER')
@@ -965,20 +892,12 @@ class BattleCategory():
 		'battles%'			: [ '% Battles', 'Share of Battles',								9, '{:9.0%}' ],
 		'win'				: [ 'WR', 'Win rate', 												6, '{:6.1%}' ],
 		'damage_made'		: [ 'DPB', 'Average Damage', 										5, '{:5.0f}' ],
-		'damage_received'	: [ 'DPBr', 'Average Damage  Received', 							5, '{:5.0f}' ],
 		'enemies_destroyed'	: [ 'KPB', 'Kills / Battle', 										4, '{:4.2f}' ],
 		'DR'				: [ 'DR', 'Damage Ratio', 											5, '{:5.1f}' ],
 		'KDR'				: [ 'KDR', 'Kills / Death', 										4, '{:4.1f}' ],
 		'enemies_spotted'	: [ 'Spot', 'Enemies spotted per battle', 							4, '{:4.1f}' ],
 		'hit_rate'			: [ 'Hit rate', 'Shots hit / all shots made', 						8, '{:8.1%}' ],
-		'shots_made'		: [ 'Shots', 'Shots made',			 								5, '{:5.1f}' ],
-		'hits_received'		: [ 'Hits R.', 'Hits reveived',		 								7, '{:7.1f}' ],
-		'hits_bounced'		: [ 'Bounced R.', 'Hits & bounced reveived',						10, '{:10.1f}' ],
-		'in_platoon'		: [ 'Plat%', 'Platoon rate',										6, '{:6.1%}' ],
 		'pen_rate'			: [ 'Pen rate', 'Shots pen / shots hit', 							8, '{:8.1%}' ],
-		'pen_rate_received'	: [ 'Penned%', 'Received hits pen / hits received',					7, '{:7.1%}' ],
-		'splash_rate'		: [ 'Splash%', 'HE splash shots / shots hit',						7, '{:7.1%}' ],
-		'splash_rate_received'	: [ 'Splashed%', 'HE splash shots received / hits hit',			9, '{:9.1%}' ],		
 		'survived'			: [ 'Surv%', 'Survival rate', 										6, '{:6.1%}' ],
 		'time_alive'		: [ 'T alive', 'Time being alive in a battle in secs', 				7, '{:7.0f}' ],
 		'alive'				: [ 'Share live', 'Percentage of time being alive in a battle', 	8, '{:8.0%}' ],
@@ -991,9 +910,9 @@ class BattleCategory():
 		'player_battles'	: [ 'Player Btls', 'Average number battles of the player', 			11, '{:11.0f}' ],
 		'allies_battles'	: [ 'Allies Btls', 'Average number battles of the allies', 			11, '{:11.0f}' ],
 		'enemies_battles'	: [ 'Enemies Btls', 'Average number battles of the enemies', 		12, '{:12.0f}' ],
-		'player_damage_dealt'	: [ 'Player Avg Dmg', 'Player Average damage', 					14, '{:14.0f}' ],
-		'allies_damage_dealt'	: [ 'Allies Avg Dmg', 'Average damage of the allies', 			14, '{:14.0f}' ],
-		'enemies_damage_dealt'	: [ 'Enemies Avg Dmg', 'Average damage of the enemies', 		15, '{:14.0f}' ],
+		'player_damage_dealt'	: [ 'Player Avg Dmg', 'Player Average damage', 					11, '{:11.0f}' ],
+		'allies_damage_dealt'	: [ 'Allies Avg Dmg', 'Average damage of the allies', 			11, '{:11.0f}' ],
+		'enemies_damage_dealt'	: [ 'Enemies Avg Dmg', 'Average damage of the enemies', 		12, '{:12.0f}' ],
 		MISSING_STATS		: [ 'No stats', 'Players without stats avail', 						8, '{:8.1%}']		
 	}
 
@@ -1002,15 +921,13 @@ class BattleCategory():
 	_result_fields_modes = {
 		'default'		: [ 'battles',	'win','damage_made', 'enemies_destroyed', 'enemies_spotted', 
 							'top_tier', 'DR', 'survived', 'allies_wins', 'enemies_wins'],
-		'team'			: [ 'battles',	'win', 'in_platoon', 'player_wins', 'allies_wins','enemies_wins'],
-		'team_extended'	: [ 'battles',	'battles%',	'win', 'in_platoon', 'player_wins', 'allies_wins', 'enemies_wins', 
+		'team'			: [ 'battles',	'win','player_wins','allies_wins','enemies_wins'],
+		'team_extended'	: [ 'battles',	'battles%',	'win','player_wins','allies_wins','enemies_wins', 
 							'player_damage_dealt', 'allies_damage_dealt', 'enemies_damage_dealt', 
 							'player_battles','allies_battles','enemies_battles' ],
-		'extended'		: [ 'battles',	'battles%',	'win', 'in_platoon', 'damage_made', 'enemies_destroyed', 'enemies_spotted', 
+		'extended'		: [ 'battles',	'battles%',	'win','damage_made', 'enemies_destroyed', 'enemies_spotted', 
 							'top_tier', 'DR', 'KDR', 'hit_rate', 'pen_rate', 'survived', 'alive', 
 							'player_wins', 'allies_wins', 'enemies_wins', MISSING_STATS ], 
-		'dmg_received'	: [ 'battles',	'win','damage_made', 'damage_received', 'enemies_destroyed', 'top_tier', 'DR', 
-							'hits_received', 'hits_bounced', 'pen_rate_received', 'splash_rate_received'], 
 		'all'			: [ cat for cat in _result_fields.keys() ]
 	}
 
@@ -1026,10 +943,7 @@ class BattleCategory():
 		'KDR'				: [ 'enemies_destroyed', 'destroyed' ],
 		'DR'				: [ 'damage_made', 'damage_received' ],
 		'hit_rate'			: [ 'shots_hit', 'shots_made' ],
-		'splash_rate'			: [ 'shots_splash', 'shots_hit' ],
-		'splash_rate_received'	: [ 'hits_splash', 'hits_received' ],
 		'pen_rate'			: [ 'shots_pen', 'shots_hit' ],
-		'pen_rate_received'		: [ 'hits_pen', 'hits_received' ],		
 		'dmg_block_rate' 	: [ 'damage_blocked', 'damage_received' ]
 	}
 
@@ -1100,21 +1014,12 @@ class BattleCategory():
 
 
 	@classmethod
-	def get_field_name(cls, field: str) -> str:
+	def get_field_name(cls, field: str):
 		try:
 			return cls._result_fields[field][0]
 		except Exception as err:
 			bu.error(exception=err)
 
-	
-	@classmethod
-	def get_field_description(cls, field: str) -> str:
-		try:
-			return cls._result_fields[field][1]
-		except Exception as err:
-			bu.error(exception=err)
-
-	
 	@classmethod
 	def get_field_width(cls, field: str):
 		try:
@@ -1176,17 +1081,6 @@ class BattleCategory():
 		return None
 
 
-	@classmethod
-	def help(cls):
-		try:
-			for field in cls.get_result_fields_all():
-				print('{:24s} : {}'.format(field, cls.get_field_description(field)))
-		except Exception as err:
-			bu.error(exception=err) 
-		return None
-
-
-
 	def __init__(self):
 		try:
 			
@@ -1408,6 +1302,7 @@ class PlayerHistogram():
 		return None
 
 
+
 class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 	def exit(self, status=0, message=None):
 		if status:
@@ -1420,7 +1315,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 ## main() -------------------------------------------------------------
 
 async def main(argv):
-	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT, OPT_EXPORT_CSV_FILE, OPT_EXPORT_JSON_FILE 
+	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT 
 	# set the directory for the script
 	current_dir = os.getcwd()
 	os.chdir(os.path.dirname(sys.argv[0]))
@@ -1433,8 +1328,8 @@ async def main(argv):
 	OPT_HIST			= False
 	OPT_STAT_FUNC		= StatFunc.get_default()
 	OPT_WORKERS_N 		= 10
-	# OPT_EXPORT_CSV_FILE	 = 'export.csv'
-	# OPT_EXPORT_JSON_FILE = 'export.json'
+	OPT_EXPORT_CSV_FILE	 = 'export.csv'
+	OPT_EXPORT_JSON_FILE = 'export.json'
 
 	#WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					 	# China is not supported since WG API stats are not available there
@@ -1446,8 +1341,8 @@ async def main(argv):
 	## VERY unlikely you have a DB set up
 	DB_SERVER 	= 'localhost'
 	DB_PORT 	= 27017
-	DB_TLS		= False
-	DB_CERT_REQ = False
+	DB_SSL		= False
+	DB_CERT_REQ = ssl.CERT_NONE
 	DB_AUTH 	= 'admin'
 	DB_NAME 	= 'BlitzStats'
 	DB_USER		= 'mongouser'
@@ -1499,14 +1394,14 @@ async def main(argv):
 					OPT_DB		= configDB.getboolean('opt_DB', OPT_DB)
 					DB_SERVER 	= configDB.get('db_server', DB_SERVER)
 					DB_PORT 	= configDB.getint('db_port', DB_PORT)
-					DB_TLS		= configDB.getboolean('db_tls', DB_TLS)
-					DB_CERT_REQ = configDB.getboolean('db_tls_req', DB_CERT_REQ)
+					DB_SSL		= configDB.getboolean('db_ssl', DB_SSL)
+					DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ)
 					DB_AUTH 	= configDB.get('db_auth', DB_AUTH)
 					DB_NAME 	= configDB.get('db_name', DB_NAME)
 					DB_USER		= configDB.get('db_user', DB_USER)
 					DB_PASSWD 	= configDB.get('db_password', DB_PASSWD)
-					DB_CERT 	= configDB.get('db_tls_cert_file', DB_CERT)
-					DB_CA 		= configDB.get('db_tls_ca_file', DB_CA)
+					DB_CERT 	= configDB.get('db_ssl_cert_file', DB_CERT)
+					DB_CA 		= configDB.get('db_ssl_ca_file', DB_CA)
 			except (KeyError, configparser.NoSectionError)  as err:
 				bu.error(exception=err)
 
@@ -1525,10 +1420,9 @@ async def main(argv):
 		parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"')
 		parser.add_argument('--json', action='store_true', default=False, help='Export data in JSON format')
 		parser.add_argument('--csv', action='store_true', default=False, help='Export data in CSV format')
-		parser.add_argument('--replays', action='store_true', default=False, help='Export replay info in JSON format')
 		parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
 		parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
-		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tank_tier" : [8,9,10] }, { "player_wins" : 5 }]')
+		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,10] }, { "player_wins" : 5 }]')
 		parser.add_argument('--filters_db', type=str, default=None, help='[only for DB setup] Filter replays in DB based on categories. Filters given in MongoDB JSON format. See --mode help')
 		parser.add_argument('--min', type=int, default=None, help='Only select replays from players with minimum number of replays in the dataset')
 		parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
@@ -1547,12 +1441,12 @@ async def main(argv):
 		# res_categories = BattleCategorization.get_categorizations(OPT_CATEGORIZATIONS, args)
 
 		bu.set_log_level(args.silent, args.verbose, args.debug)
-		# bu.set_progress_step(250)  						# Set the frequency of the progress dots. 
+		bu.set_progress_step(250)  						# Set the frequency of the progress dots. 
 
 		#### Connect to MongoDB. You are unlikely to have this set up... 
 		bu.debug('DB_SERVER: ' + DB_SERVER)
 		bu.debug('DB_PORT: ' + str(DB_PORT))
-		bu.debug('DB_TLS: ' + "True" if DB_TLS else "False")
+		bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
 		bu.debug('DB_AUTH: ' + DB_AUTH)
 		bu.debug('DB_NAME: ' + DB_NAME)
 		
@@ -1560,7 +1454,7 @@ async def main(argv):
 		db = None
 		if args.db:
 			try:
-				client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, tls=DB_TLS, tlsAllowInvalidCertificates=DB_CERT_REQ, tlsCertificateKeyFile=DB_CERT, tlsCAFile=DB_CA)
+				client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
 				db = client[DB_NAME]
 				bu.debug('Database connection initiated')
 			except Exception as err: 
@@ -1590,14 +1484,14 @@ async def main(argv):
 			args.account_id = await wg.get_account_id(args.account)
 			bu.debug('WG  account_id: ' + str(args.account_id))
 
-		# try:
+		try:
 			BattleCategory.set_fields(args.mode)
 			replayQ  = asyncio.Queue(maxsize=1000)			
 			reader_tasks = []
 			# Make replay Queue
 
 			scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
-		bu.set_counter('Reading replays', 10)
+			bu.debug('Replay scanner started')
 			# Start tasks to process the Queue
 			for i in range(OPT_WORKERS_N):
 				reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
@@ -1605,7 +1499,7 @@ async def main(argv):
 
 			bu.debug('Waiting for the replay scanner to finish')
 			await asyncio.wait([scanner_task])
-		bu.finish_progress_bar()
+			
 			# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
 			await replayQ.join()
 			await asyncio.sleep(0.1)
@@ -1615,29 +1509,17 @@ async def main(argv):
 				await asyncio.sleep(0.1)	
 			results = []
 			players = set()
-		replays	= dict()
 			for res in await asyncio.gather(*reader_tasks):
 				results.extend(res[0])
 				players.update(res[1])
-			replays.update(res[2])
-		
 			if len(players) == 0:
 				raise Exception("No players found to fetch stats for. No replays found?")
 
 			if args.min != None:
 				results = filter_min_replays_by_player(results, args.min)
-			
-		# Filter based on non-stats filters
-		results = filter_results(results, args.filters, stats_filters=False)	
 				players = get_players(results)
 
 			(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
-	
-		# Filter based on stats filters
-		results = filter_results(results, args.filters, stats_filters=True)
-		bu.debug(str(len(replays)) + ' replays to process')
-		replays = filter_replays(replays, results)				
-		
 			bu.debug('Number of player stats: ' + str(len(player_stats)))
 			teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
 
@@ -1651,35 +1533,20 @@ async def main(argv):
 				print_player_dist(histograms)			
 			
 			if args.json:
-			await export_json(args, blt_cat_list, histograms)
+				res = blt_cat_list.get_results_json()
+				if histograms != None:
+					res_hist = dict()
+					for histogram in histograms.keys():
+						res_hist[histogram] = histograms[histogram].get_results()
+					res['histograms'] = res_hist
+				if args.outfile == '-':
+					export_file = OPT_EXPORT_JSON_FILE
+				else:
+					export_file = args.outfile
+				await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
+				bu.verbose_std('Results exported to: ' + export_file)
 			
 			if args.csv:
-			await export_csv(args, blt_cat_list, histograms)	
-		
-		if args.replays:
-			await export_replays(replays)
-
-		bu.debug('Finished. Cleaning up..................')
-		# except Exception as err:
-		# 	bu.error(exception=err)
-	except UserWarning as err:
-		bu.warning(str(err))
-	except asyncio.CancelledError:
-		pass
-	except Exception as err:
-		bu.error(exception=err)
-	finally:
-		## Need to close the aiohttp.session since Python destructors do not support async methods... 
-		if wg != None: 
-			await wg.close()
-		if wi != None: 
-			await wi.close()		
-	return None
-
-
-async def export_csv(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
-	"""Export results into a CSV file"""
-	try:
 				rows = blt_cat_list.get_results_list()
 				if args.hist:
 					rows.append('')
@@ -1696,27 +1563,23 @@ async def export_csv(args : argparse.Namespace, blt_cat_list: BattleCategorizati
 					csvwriter = csv.writer(csvfile)        					
 					csvwriter.writerows(rows)
 					bu.verbose_std('Results exported to: ' + export_file)			
+			
+			bu.debug('Finished. Cleaning up..................')
 		except Exception as err:
 			bu.error(exception=err)
-
-
-async def export_json(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
-	"""Export results into a JSON file"""
-	try:
-		res = blt_cat_list.get_results_json()
-		if histograms != None:
-			res_hist = dict()
-			for histogram in histograms.keys():
-				res_hist[histogram] = histograms[histogram].get_results()
-			res['histograms'] = res_hist
-		if args.outfile == '-':
-			export_file = OPT_EXPORT_JSON_FILE
-		else:
-			export_file = args.outfile
-		await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
-		bu.verbose_std('Results exported to: ' + export_file)
+	except UserWarning as err:
+		bu.warning(str(err))
+	except asyncio.CancelledError:
+		pass
 	except Exception as err:
 		bu.error(exception=err)
+	finally:
+		## Need to close the aiohttp.session since Python destructors do not support async methods... 
+		if wg != None: 
+			await wg.close()
+		if wi != None: 
+			await wi.close()		
+	return None
 
 
 async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, parser: argparse.ArgumentParser = None):
@@ -1725,33 +1588,11 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa
 		parser.print_help()
 	# Result category options
 
-	# Results fields
-	print('\n-------------------------------------------------------------------')
-	print('| Result fields                                                   |')
-	print('-------------------------------------------------------------------\n')
-	
-	BattleCategory.help()
-
-	print('\n-------------------------------------------------------------------')
-	print('| Filters                                                         |')
-	print('-------------------------------------------------------------------\n')
-	print("\t--filters '{ \"categorization\" : category }'")
-	print('\tThese filters let you filter only battles matching the filter. The filter has been proper JSON dict.')
-	print("\tExample: --filter '{ \"room_type\" :1 }'")
-	print('')
-	BattleCategorizationList.help()
-	print('\n-------------------------------------------------------------------')
-	print('| Rounding errors                                                 |')
-	print('-------------------------------------------------------------------\n')
-	print('Yes, the numbers are rounded UP because reasons.')
-	print('If interested why please read this: ')
-	print('\thttps://realpython.com/python-rounding/#pythons-built-in-round-function')
-	print('No, I am not planning to fix these for now.')	
-
 	# Filter usage
 	print('\n-------------------------------------------------------------------')
 	print('| DB Filter usage                                                 |')
-	print('-------------------------------------------------------------------\n')
+	print('-------------------------------------------------------------------')
+	print('')
 	print('You are unlikely to have DB setup required for these filters. This is "Jylpah\'s special"')
 	print('')
 	print("Syntax: --filters_db '{ \"replay.param\": VALUE, \"replay.param2\": { \"$MONGO_OPERATOR\" : VALUE }, ... }'")
@@ -1770,24 +1611,23 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa
 		except Exception as err:
 			bu.error(exception=err)
 
-
-async def export_replays(replays: dict):
-	"""Export replays"""
-	try:
-		replay_dir = OPT_EXPORT_REPLAY_DIR +  '_' + bu.get_date_str()
-		bu.verbose_std('Exporting replays to ' + replay_dir)
-		if not os.path.isdir(replay_dir):
-			os.mkdir(replay_dir)
-
-		i = 0
-		for replay in replays.values():
-			filename = wg.get_replay_filename(replay)
-			async with aiofiles.open(os.path.join(replay_dir, filename), 'w', encoding="utf8") as f:
-				await f.write(json.dumps(replay, indent=4))
-			i += 1
-		bu.verbose_std( str(i) + ' replays exported')
-	except Exception as err:
-		bu.error(exception=err)
+	print('\n-------------------------------------------------------------------')
+	print('| Filters                                                         |')
+	print('-------------------------------------------------------------------')
+	print('')
+	print("\t--filters '{ \"categorization\" : category }'")
+	print('\tThese filters let you filter only battles matching the filter. The filter has been proper JSON dict.')
+	print("\tExample: --filter '{ \"room_type\" :1 }'")
+	print('')
+	BattleCategorizationList.help()
+	print('\n-------------------------------------------------------------------')
+	print('| Rounding errors                                                 |')
+	print('-------------------------------------------------------------------')
+	print('')
+	print('Yes, the numbers are rounded UP because reasons.')
+	print('If interested why please read this: ')
+	print('\thttps://realpython.com/python-rounding/#pythons-built-in-round-function')
+	print('No, I am not planning to fix these for now.')	
 
 
 ## move the class PlayerHistogram?
@@ -1829,7 +1669,7 @@ def filter_min_replays_by_player(results: list, min_replays: int) -> list:
 		for res in results:
 			if res['protagonist'] in players_enough_replays:
 				res_ret.append(res)
-		bu.verbose_std('Replays after filtering: ' + str(len(res_ret)))	
+
 		return res_ret
 
 	except Exception as err:
@@ -1872,15 +1712,15 @@ def print_player_dist(histograms: dict):
 		print('')
 		hist.print()
 
-
 def process_battle_results(results: dict, args : argparse.Namespace):
 	"""Process replay battle results""" 
 	try:
 		url 		= args.url
 		cats 		= BattleCategorizationList.get_categorizations(args)
-		blt_cat_list= None
 		
-		# remove if? 
+		
+		if args.filters != None:
+			results = filter_replays(results, args.filters)
 		if len(results) > 0:
 			blt_cat_list = BattleCategorizationList(cats)
 			for result in results:
@@ -1891,7 +1731,7 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 			blt_cat_list.calc_results()
 		else:
 			warning_txt = 'No replays to process '
-			if args.filters != None: 
+			if args.filter != None: 
 				warning_txt = warning_txt + 'after filtering'
 			bu.warning(warning_txt)
 
@@ -1900,31 +1740,24 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 	return blt_cat_list
 
 
-def filter_results(results: list, filters_json : str, stats_filters=None) -> bool:
+def filter_replays(results: list, filter_json : str) -> bool:
 	"""Filter replays based on battle category filters"""
 	try:
-		if filters_json == None: 
-			return results
-		filters = json.loads(filters_json)
+		filters = json.loads(filter_json)
 		bu.debug(str(filters))
+		for filter in filters:
+			if type(filters[filter]) != list:
+				filters[filter] = [ filters[filter]]
+
 		if type(filters) != dict:
-			bu.error('invalid --filter arguments: ' + str(filter))
+			bu.error('invalid --filter arguments: ' + str(filters))
 			sys.exit(1)
 		
-		filter_cats = BattleCategorizationList.get_categorizations_stats(filters.keys(), stats_filters)
-		filter_cats = BattleCategorizationList(filter_cats)
-		if filter_cats.len() == 0:
-			bu.debug('Nothing to filter')
-			return results
-
-		bu.verbose('Filters applied:')
-		for category in filter_cats:
-			cat = category.category_key
+		filter_cats = BattleCategorizationList(filters.keys())
+		for cat in filters:
 			categorization = filter_cats.get_categorization(cat)
-			bu.debug('Category: ' + cat +  ' filters: ' + str(filters))
 			filters[cat] = categorization.get_filter_categories(filters[cat])
-			bu.verbose('  ' + cat + ' = ' + ','.join(filters[cat]))		
-		bu.verbose('')
+			bu.verbose('Filter: ' + cat + ': ' + ','.join(filters[cat]))		
 		
 		res = list()
 		for result in results:
@@ -1941,31 +1774,13 @@ def filter_results(results: list, filters_json : str, stats_filters=None) -> boo
 					res.append(result)
 			except Exception as err:
 				bu.error(exception=err)
-		if len(res) == 0:
-			raise UserWarning('No replays left after filtering')
-		bu.verbose_std('Replays after filtering: ' + str(len(res)))	
+		bu.verbose('Replays after filtering: ' + str(len(res)))	
 		return res
-	except UserWarning:
-		raise
 	except Exception as err:
 		bu.error(exception=err)
 	return None
 
 
-def filter_replays(replays_in: dict, results: list) -> list:
-	"""Filter source replays based on (filtered) results"""
-	replays = dict()
-	for res in results:
-		try:
-			# bu.debug('res[_id] : ' + res['_id'], force=True)			
-			_id = res['_id']
-			replays[_id] = replays_in[_id]
-		except Exception as err:
-			bu.error(exception=err)	
-	return replays
-	
-	
-
 async def process_player_stats(players, N_workers: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict:
 	"""Start stat_workers to retrieve and store players' stats"""
 	## Create queue of player-tank pairs to find stats for
@@ -2119,19 +1934,23 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name
 			try:
 				bu.debug('Stat_id: ' + stat_id, id=workerID)
 				bu.print_progress()
-				stats_tmp = None
+				# Analysing player performance based on their stats on the tier tanks they are playing 
 
 				# Try cache first
+				if (stat_id not in stats):
+					stats_tmp = None
 					pruned_stat_id = prune_stat_id(stat_id)
 					if (pruned_stat_id not in stats):
 						stats_tmp = await stat_wg_func(pruned_stat_id, cache_only = True)
 					else:
 						stat_id_remap[stat_id] = pruned_stat_id
+						queue.task_done()
 						continue
 
 					if (stats_tmp != None):
 						stats[pruned_stat_id] = stats_tmp
 						stat_id_remap[stat_id] = pruned_stat_id
+						queue.task_done()
 						continue
 					
 					# try DB
@@ -2145,11 +1964,11 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name
 						del stats[stat_id]
 						bu.debug('get_wg_' + args.stat_func + '_stats returned: account_id=' + str(get_account_id_f_stat_id(pruned_stat_id)) + ': ' + str(stats[pruned_stat_id]), workerID)
 
+					
 			except KeyError as err:
 				bu.error('Key not found', err, id=workerID)
 			except Exception as err:
 				bu.error('Unexpected error', err, id=workerID)
-			finally:
 			queue.task_done()
 
 	except (asyncio.CancelledError, concurrent.futures.CancelledError):
@@ -2184,7 +2003,7 @@ async def get_wg_tank_stats(stat_id_str: str, cache_only = False) -> dict:
 		player_stats = await wg.get_player_tank_stats(account_id, [ tank_id ], hist_stats, cache_only = cache_only)
 		#bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats))
 
-		return tank_stats_helper(player_stats)
+		return await tank_stats_helper(player_stats)
 
 	except KeyError as err:
 		bu.error('account_id: ' + str(account_id) + ' tank_id:' + str(tank_id) +' : Key not found', err)
@@ -2210,8 +2029,8 @@ async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_
 		for stat in hist_stats:			
 			project[stat] = True
 		project['_id'] = False
-		bu.debug('find(): tank_id={} account_id={} last_battle_time>={}'.format(tank_id, account_id, battletime - time_buffer))
-		cursor = dbc.find({ '$and': [ { 'tank_id' : tank_id }, { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }} ] }, projection=project).sort('last_battle_time',-1).limit(1)
+		
+		cursor = dbc.find({ '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }}, { 'tank_id' : tank_id } ] }, projection=project).sort('last_battle_time',-1).limit(1)
 
 		# pipeline = 	[ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : tank_id } ]}}, 
 		# 		{ '$sort': { 'last_battle_time': -1 }}, 
@@ -2222,11 +2041,7 @@ async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_
 		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
 		#cursor = dbc.aggregate(pipeline)
 
-		stats = await cursor.to_list(1)
-		# store cache
-		# await wg.save_tank_stats(account_id, [tank_id], stats)
-		
-		return tank_stats_helper(stats) 	
+		return await tank_stats_helper(await cursor.to_list(1)) 
 				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
@@ -2247,7 +2062,7 @@ async def get_wg_tank_tier_stats(stat_id_str: str, cache_only = False) -> dict:
 		player_stats = await wg.get_player_tank_stats(account_id, tier_tanks, hist_stats, cache_only = cache_only)
 		#bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats))
 
-		return tank_stats_helper(player_stats)
+		return await tank_stats_helper(player_stats)
 
 	except KeyError as err:
 		bu.error('account_id: ' + str(account_id) + ' Key not found', err)
@@ -2270,7 +2085,7 @@ async def get_wg_tier_x_stats(stat_id_str: str, cache_only = False) -> dict:
 		player_stats = await wg.get_player_tank_stats(account_id, tier_tanks, hist_stats, cache_only = cache_only)
 		#bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats))
 
-		return tank_stats_helper(player_stats)
+		return await tank_stats_helper(player_stats)
 
 	except KeyError as err:
 		bu.error('account_id: ' + str(account_id) + ' Key not found', err)
@@ -2298,9 +2113,7 @@ async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase,
 		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
 		cursor = dbc.aggregate(pipeline)
 
-		stats = await cursor.to_list(1000)
-		# await wg.save_tank_stats(account_id, tier_tanks, stats)
-		return tank_stats_helper(stats) 
+		return await tank_stats_helper(await cursor.to_list(1000)) 
 				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
@@ -2312,8 +2125,22 @@ async def get_db_tier_x_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, sta
 	if db == None:
 		return None
 	try:
+		dbc = db[DB_C_TANK_STATS]
 		( account_id, battletime ) = str2ints(stat_id_str)
-		return await get_db_tank_tier_stats(db,get_stat_id(account_id, 10, battletime) )
+		tier_tanks = wg.get_tanks_by_tier(10)
+		time_buffer = 2*7*24*3600
+
+		pipeline = 	[ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, 
+				{ '$sort': { 'last_battle_time': -1 }}, 
+				{ '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, 
+				{ '$replaceRoot': { 'newRoot': '$doc' }}, 
+				{ '$project': { '_id': 0 }} ]
+
+		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
+		cursor = dbc.aggregate(pipeline)
+
+		return await tank_stats_helper(await cursor.to_list(1000)) 
+				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
 	return None
@@ -2331,16 +2158,14 @@ async def get_db_player_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, sta
 		
 		pipeline = 	[ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }} ]}}, 
 				{ '$sort': { 'last_battle_time': -1 }}, 
-				{ '$group': { '_id': { 'tank_id':'$tank_id'}, 'doc': { '$first': '$$ROOT' }}}, 
+				{ '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, 
 				{ '$replaceRoot': { 'newRoot': '$doc' }}, 
 				{ '$project': { '_id': 0 }} ]
 
 		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
 		cursor = dbc.aggregate(pipeline)
 
-		stats = await cursor.to_list(1000)
-		# await wg.save_tank_stats(account_id, [], stats)
-		return tank_stats_helper(stats) 
+		return await tank_stats_helper(await cursor.to_list(1000)) 
 				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
@@ -2368,17 +2193,20 @@ async def get_wg_player_stats(stat_id_str: str, cache_only = False) -> dict:
 	return None
 
 
-def tank_stats_helper(stat_list: list):
+async def tank_stats_helper(stat_list: list):
 	"""Helpher func got get_db_tank_tier_stats() and get_wg_tank_tier_stats()"""
 	try:
 		if stat_list == None: 
 			return None
-		stats = collections.defaultdict(def_value_zero)
+		stats = dict()
 		hist_fields = histogram_fields.keys()
 		if 'battles' not in hist_fields:
 			bu.error('\'battles\' must be defined in \'histogram_fields\'')
 			return None
 
+		for field in hist_fields:
+			stats[field] = 0
+		
 		for tank_stat in stat_list: 
 			tank_stat = tank_stat['all']
 			for field in hist_fields:
@@ -2448,7 +2276,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 			async for replay_json in cursor:
 				_id = replay_json['_id']
 				#del(replay_json['_id'])
-				await queue.put(await mk_readerQ_item(replay_json, _id=_id))
+				await queue.put(await mk_readerQ_item(replay_json, 'DB: _id = ' + _id))
 				Nreplays += 1
 			bu.debug('All the matching replays have been read from the DB')
 		except Exception as err:
@@ -2464,7 +2292,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 				
 				if (p_replayfile.match(line) != None):
 					replay_json = await bu.open_JSON(line, wi.chk_JSON_replay)						
-					await queue.put(await mk_readerQ_item(replay_json, filename=line))
+					await queue.put(await mk_readerQ_item(replay_json, line))
 			except Exception as err:
 				bu.error(exception=err)
 	else:
@@ -2475,7 +2303,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 					fn = fn[:-1] 
 				if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
 					replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay)
-					await queue.put(await mk_readerQ_item(replay_json, filename=fn))
+					await queue.put(await mk_readerQ_item(replay_json, fn))
 					bu.debug('File added to queue: ' + fn)
 					Nreplays += 1
 				elif os.path.isdir(fn):
@@ -2484,35 +2312,34 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 							if entry.is_file() and (p_replayfile.match(entry.name) != None): 
 								bu.debug(entry.name)
 								replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay)
-								await queue.put(await mk_readerQ_item(replay_json, filename=entry.name))
+								await queue.put(await mk_readerQ_item(replay_json, entry.name))
 								bu.debug('File added to queue: ' + entry.path)
 								Nreplays += 1
 			except Exception as err:
 				bu.error(exception=err)					
+	bu.verbose('Finished scanning replays: ' + str(Nreplays)  + ' replays to process') 
 	return Nreplays
 
 
-async def mk_readerQ_item(replay_json, filename : str = None, _id: str = None) -> list:
+async def mk_readerQ_item(replay_json, filename : str = None) -> list:
 	"""Make an item to replay queue"""
 	global REPLAY_N
 	REPLAY_N +=1
 	try:
 		if wi.chk_JSON_replay(replay_json): 
 			if not '_id' in replay_json:
-				if _id  == None:			
-					_id = wi.read_replay_id(replay_json)
-				replay_json['_id'] = _id
+				id = wi.read_replay_id(replay_json)
+				replay_json['_id'] = id	
 		else:
 			replay_json = None # mark error
 	except Exception as err:
 		bu.error(exception=err)
-	
-	if filename != None:
-		return [replay_json, REPLAY_N, os.path.basename(filename) ]
-	elif _id != None: 
-		return [replay_json, REPLAY_N, 'DB: ' + _id ]
+	if filename == None:
+		return [replay_json, REPLAY_N, '' ]
+	elif RE_SRC_IS_DB.match(filename) != None:
+		return [replay_json, REPLAY_N, filename ]
 	else:
-		return None
+		return [replay_json, REPLAY_N, os.path.basename(filename) ]
 
 
 async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
@@ -2520,7 +2347,6 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 	#global SKIPPED_N
 	results = []
 	playerstanks = set()
-	replays		 = dict()
 	try:
 		while True:
 			item = await queue.get()
@@ -2547,10 +2373,8 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 				playerstanks.update(set(result['allies']))
 				playerstanks.update(set(result['enemies']))	
 				playerstanks.update(set([result['player']]))
-				_id = wi.read_replay_id(replay_json)
-				replays[_id] = replay_json
-				results.append(result)
 				
+				results.append(result)
 				bu.debug('Marking replay [' + str(replayID) + '] done', id=readerID)
 						
 			except Exception as err:
@@ -2558,7 +2382,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 			queue.task_done()
 	except (asyncio.CancelledError, concurrent.futures.CancelledError):		
 		bu.debug( str(len(results)) + ' replays, ' + str(len(playerstanks)) + ' player/tanks', readerID)
-		return results, playerstanks, replays
+		return results, playerstanks
 	return None
 
 
@@ -2619,26 +2443,28 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
 		result['enemies_survived']  = 0	# for steamroller stats
 		btl_duration = 0
 		btl_tier 	 = 0
+		btl_tier_low = 10
 		protagonist_tank  = None
 		for player in replay_summary['details']:
 			btl_duration = max(btl_duration, player['time_alive'])
 			player_tank_tier = wg.get_tank_data(player['vehicle_descr'], 'tier')
 			btl_tier 		 = max(btl_tier, player_tank_tier)
+			btl_tier_low     = min(btl_tier_low, player_tank_tier)
+
+			if (protagonist != None) and (player['dbid'] == protagonist):
+				protagonist_tank = player['vehicle_descr']
 
 			if player['dbid'] == account_id:
 				# player itself is not put in results['allies']
 				tmp = dict()
 				tmp['account_id'] 	= account_id
-				tank_id 			= player['vehicle_descr']
-				tmp['tank_id'] 		= tank_id
+				tmp['tank_id'] 		= player['vehicle_descr']
 				tmp['tank_tier'] 	= player_tank_tier
 				tmp['tank_name']	= wg.get_tank_name(tmp['tank_id'])
 				tmp['tank_type']	= wg.get_tank_type_id(tmp['tank_id'])
 				tmp['tank_nation']	= wg.get_tank_nation_id(tmp['tank_id'])
 				tmp['is_premium']	= int(wg.is_premium(tmp['tank_id']))
-				platoon_ndx 		= player['squad_index']
-				tmp['squad_index'] 	= platoon_ndx
-				tmp['in_platoon']	= 0 if platoon_ndx == None else 1
+				tmp['squad_index'] 	= player['squad_index']
 				for key in replay_details_flds:
 					tmp[key] = player[key]
 				if player['hitpoints_left'] == 0:
@@ -2671,7 +2497,7 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
 
 		result['battle_duration'] = btl_duration
 		## Rather use 'player' than incomprehensible 'protagonist'...		
-		result['player'] = get_stat_id(account_id, tank_id, result['battle_start_timestamp'])
+		result['player'] = get_stat_id(protagonist, protagonist_tank, result['battle_start_timestamp'])
 		bu.debug('Player stats_id: ' + result['player'])
 		# remove platoon buddy from stats 			
 		if result['squad_index'] != None:
@@ -2716,8 +2542,8 @@ def prune_stat_id(stat_id_str: str) -> str:
 	return ':'.join(stat_id)
 
 
-def get_stat_id(account_id: int, tank_id: int, battle_time: int) -> str:
-	return ':'.join(map(str, [account_id, tank_id, battle_time ]))
+def get_stat_id(account_id: int, tank_id: int, battletime: int) -> str:
+	return ':'.join([ str(account_id), str(tank_id), str(battletime)])
 
 
 def get_stat_id_tank_tier(stat_id_str: str) -> str:
@@ -2727,7 +2553,7 @@ def get_stat_id_tank_tier(stat_id_str: str) -> str:
 		account_id	= stat_id[0]
 		tank_tier 	= wg.get_tank_tier(stat_id[1])
 		battle_time = (stat_id[2] // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET
-		return get_stat_id(account_id, tank_tier, battle_time)
+		return ':'.join(map(str, [account_id, tank_tier, battle_time ]))
 	except Exception as err:
 		bu.error('Stats_id: ' + stat_id_str, exception=err)
 	return None
@@ -2740,7 +2566,7 @@ def get_stat_id_tank(stat_id_str: str) -> str:
 		account_id	= stat_id[0]
 		tank_id 	= stat_id[1]
 		battle_time = (stat_id[2] // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET
-		return get_stat_id(account_id, tank_id, battle_time)		
+		return ':'.join(map(str, [account_id, tank_id, battle_time ]))
 	except Exception as err:
 		bu.error('Stats_id: ' + stat_id_str, exception=err)
 	return None

blitzutils.py

@@ -22,11 +22,6 @@ _log_level = NORMAL
 LOG = False
 LOGGER = None
 
-MODE_TANK_STATS         = 'tank_stats'
-MODE_PLAYER_STATS       = 'player_stats'
-MODE_PLAYER_ACHIEVEMENTS= 'player_achievements'
-MODE_TANKOPEDIA			= 'tankopedia'
-
 # Progress display
 _progress_N = 100
 _progress_i = 0
@@ -423,14 +418,12 @@ def set_progress_bar(heading: str, max_value: int, step: int = None, slow: bool
     return
 
 
-def set_counter(heading: str, step = None):
+def set_counter(heading: str):
     global _progress_obj, _progress_i
     _progress_i = 0
     if _progress_obj != None:
         finish_progress_bar()
-    _progress_obj = Counter(heading + ': ')
-    if step != None:
-        set_progress_step(step)
+    _progress_obj = Counter(heading)
     return 
 
 
@@ -486,16 +479,6 @@ def NOW() -> int:
     return int(time.time())
 
 
-def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
-    """Return YYYYMMDD_HHmm date string"""
-    try:
-        ts = datetime.fromtimestamp(timestamp)
-        return ts.strftime(date_format)
-    except Exception as err:
-        error(exception=err)
-    return None
-
-
 def rebase_file_args(current_dir, files):
     """REbase file command line params after moving working dir to the script's dir""" 
     if isinstance(files, list):    
@@ -643,14 +626,6 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
     raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
 
 
-def sort_dict(d: dict, number: bool = False) -> dict:
-    """Sort a dict by keys"""
-    if number:
-        return dict(sorted(d.items(), key=lambda item: int(item[0])))
-    else:
-        return dict(sorted(d.items(), key=lambda item: item[0]))
-
-
 # -----------------------------------------------------------
 # Class SlowBar 
 # -----------------------------------------------------------
@@ -674,6 +649,8 @@ class StatsNotFound(Exception):
     pass
 
 
+
+
 # -----------------------------------------------------------
 # Class WG 
 # -----------------------------------------------------------
@@ -709,7 +686,7 @@ class WG:
     #                                 WHERE account_id = {} AND stat = {} 
     #                                 AND date >= {} LIMIT 1;"""
           
-    # TANK_STATS
+    
     SQL_TANK_STATS_TBL          = 'tank_stats'
 
     SQL_TANK_STATS_CREATE_TBL   = 'CREATE TABLE IF NOT EXISTS ' + SQL_TANK_STATS_TBL + \
@@ -723,7 +700,6 @@ class WG:
 
     SQL_TANK_STATS_UPDATE       = 'REPLACE INTO ' + SQL_TANK_STATS_TBL + '(account_id, tank_id, update_time, stats) VALUES(?,?,?,?)'
 
-    # PLAYER_STATS
     SQL_PLAYER_STATS_TBL        = 'player_stats'
 
     SQL_PLAYER_STATS_CREATE_TBL = 'CREATE TABLE IF NOT EXISTS ' + SQL_PLAYER_STATS_TBL + \
@@ -737,7 +713,6 @@ class WG:
 
     SQL_PLAYER_STATS_CACHED     = 'SELECT * FROM ' +  SQL_PLAYER_STATS_TBL + ' WHERE account_id = ? AND update_time > ?'
 
-    # PLAYER_ACHIEVEMENTS
     SQL_PLAYER_ACHIEVEMENTS_TBL  = 'player_achievements'
 
     SQL_PLAYER_ACHIEVEMENTS_CREATE_TBL = 'CREATE TABLE IF NOT EXISTS ' + SQL_PLAYER_ACHIEVEMENTS_TBL + \
@@ -745,12 +720,11 @@ class WG:
                                     update_time INTEGER NOT NULL, 
                                     stats TEXT)"""
 
-    SQL_PLAYER_ACHIEVEMENTS_COUNT      = 'SELECT COUNT(*) FROM ' + SQL_PLAYER_ACHIEVEMENTS_TBL
-    
     SQL_PLAYER_ACHIEVEMENTS_CACHED     = 'SELECT * FROM ' +  SQL_PLAYER_ACHIEVEMENTS_TBL + ' WHERE account_id = ? AND update_time > ?'
 
     SQL_PLAYER_ACHIEVEMENTS_UPDATE     = 'REPLACE INTO ' + SQL_PLAYER_ACHIEVEMENTS_TBL + '(account_id, update_time, stats) VALUES(?,?,?)'
 
+    SQL_PLAYER_ACHIEVEMENTS_COUNT      = 'SELECT COUNT(*) FROM ' + SQL_PLAYER_ACHIEVEMENTS_TBL
 
     SQL_TABLES                  = [ SQL_PLAYER_STATS_TBL, SQL_TANK_STATS_TBL, SQL_PLAYER_ACHIEVEMENTS_TBL ]
 
@@ -934,18 +908,33 @@ class WG:
 
 
     @classmethod
-    def get_map(self, map_str: str) -> str:
+    def get_map(cls, map_str: str) -> str:
         """Return map name from short map string in replays"""
         try:
-            return self.maps[map_str]
+            return cls.maps[map_str]
         except:
             error('Map ' + map_str + ' not found')
         return None
     
 
     @classmethod
-    def get_tank_user_strs(self) -> str:
-        return self.tanks["userStr"].keys()
+    def get_map_user_strs(cls) -> str:
+        return cls.maps.keys()
+
+
+    @classmethod
+    def tank_str2name(cls, tank_str: str) -> str:
+        """Return tank name from short tank string in replays"""
+        try:
+            return cls.tanks["userStr"][tank_str]
+        except:
+            error('Tank ' + tank_str + ' not found')
+        return tank_str
+
+
+    @classmethod
+    def get_tank_user_strs(cls) -> str:
+        return cls.tanks["userStr"].keys()
 
 
     @classmethod
@@ -1082,14 +1071,10 @@ class WG:
         """Load tanks from tankopedia JSON"""
         if tankopedia_fn == None:
             return False 
+
         try:
             with open(tankopedia_fn, 'rt', encoding='utf8') as f:
                 self.tanks = json.loads(f.read())
-                self.tanks['tankStr'] = dict()
-                p = re.compile('.+_short$')
-                for usr_str in self.tanks['userStr']:
-                    if p.match(usr_str) == None:
-                        self.tanks['tankStr'][self.tanks['userStr'][usr_str]] = usr_str
                 self.tanks_by_tier = dict()
                 for tier in range(1,11):
                     self.tanks_by_tier[str(tier)] = list()
@@ -1101,43 +1086,6 @@ class WG:
         return False     
      
 
-    def get_replay_filename(self, replay: dict):
-        try:
-            summary = replay['data']['summary']
-            timestamp   = int(summary['battle_start_timestamp'])
-            timestamp    = get_date_str(timestamp, date_format='%Y%m%d_%H%M')
-            player      = summary['player_name']
-            arena_id    = int(summary['arena_unique_id'])
-            vehicle     = self.name2tank_str(summary['vehicle'])
-            vehicle.replace(' ','_')
-
-            return '{:s}_{:s}_{:s}_{:d}.wotbreplay.json'.format(timestamp, player, vehicle, arena_id)
-        except Exception as err:
-            error('Unexpected Exception', err) 
-            return None 
-
-    def get_map_user_strs(self) -> str:
-        return self.maps.keys()
-
-
-    def tank_str2name(self, tank_str: str) -> str:
-        """Return tank name from short tank string in replays"""
-        try:
-            return self.tanks["userStr"][tank_str]
-        except:
-            error('Tank ' + tank_str + ' not found')
-        return tank_str
-
-    
-    def name2tank_str(self, name: str) -> str:
-        """Return shorttank string from tank string in replays"""
-        try:
-            return self.tanks["tankStr"][name]
-        except:
-            debug('Tank ' + name + ' not found')
-        return name
-
-
     def get_tanks_by_tier(self, tier: int) -> list():
         """Returns tank_ids by tier"""
         try:
@@ -1313,7 +1261,7 @@ class WG:
                 # debug('JSON Response received: ' + str(json_data))
                 stats = json_data['data'][str(account_id)]
                 if cache:
-                    await self.save_tank_stats(account_id, tank_ids, stats)
+                    await self.put_2_statsQ('tank_stats', [account_id, tank_ids], stats)
                 return stats
         except Exception as err:
             error(exception=err)
@@ -1346,7 +1294,7 @@ class WG:
                 # debug('JSON Response received: ' + str(json_data))
                 stats = json_data['data'][str(account_id)]
                 if cache:
-                    await self.save_player_stats(account_id, stats)
+                    await self.put_2_statsQ('player_stats', [account_id], stats)
                 return stats
         except Exception as err:
             error(exception=err)
@@ -1385,7 +1333,7 @@ class WG:
                 for account_id in json_data['data'].keys():
                     stats[account_id] = json_data['data'][account_id]
                     if cache:
-                        await self.save_player_achievements(int(account_id), json_data['data'][account_id])
+                        await self.put_2_statsQ('player_achievements', [int(account_id)], json_data['data'][account_id])
                 return stats
         except Exception as err:
             error(exception=err)
@@ -1492,7 +1440,7 @@ class WG:
         return None
 
 
-    async def save_stats(self, statsType: str, key: list, stats: list):
+    async def put_2_statsQ(self, statsType: str, key: list, stats: list):
         """Save stats to a async queue to be saved by the stat_saver -task"""
         if self.statsQ == None:
             return False
@@ -1501,30 +1449,6 @@ class WG:
             return True
 
 
-    async def save_player_stats(self, account_id: int, stats: dict):
-        try:
-            await self.save_stats(MODE_PLAYER_STATS, [int(account_id)], stats)
-        except Exception as err:
-            error(exception=err)
-        return None
-
-
-    async def save_player_achievements(self, account_id: int, stats: dict):
-        try:
-            await self.save_stats(MODE_PLAYER_ACHIEVEMENTS, [int(account_id)], stats)
-        except Exception as err:
-            error(exception=err)
-        return None
-
-
-    async def save_tank_stats(self, account_id: int, tank_ids: list, stats: dict):
-        try:
-            await self.save_stats(MODE_TANK_STATS, [int(account_id), tank_ids], stats)
-        except Exception as err:
-            error(exception=err)
-        return None
-
-
     async def stat_saver(self): 
         """Async task for saving stats into cache in background"""
 
@@ -1540,13 +1464,8 @@ class WG:
 
             await self.cache.commit()
             
-            if is_debug():
-                async with self.cache.execute(WG.SQL_PLAYER_STATS_COUNT) as cursor:
-                    debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' player stat records' )
-                async with self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_COUNT) as cursor:
-                    debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' player achievement records' )
             async with self.cache.execute(WG.SQL_TANK_STATS_COUNT) as cursor:
-                    debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' player tank stat records' )
+                debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' cached player tank stat records' )
         except Exception as err:
             error(exception=err)
             sys.exit(1)
@@ -1555,20 +1474,19 @@ class WG:
             try:
                 stats = await self.statsQ.get()
             
-                stat_type  = stats[0]
+                stats_type  = stats[0]
                 key         = stats[1]
                 stats_data  = stats[2]
                 update_time = stats[3]
 
-                #debug('stat_type: ' + stat_type + ', key: ' + str(key))
-                if stat_type == MODE_TANK_STATS:
+                if stats_type == 'tank_stats':
                     await self.store_tank_stats(key, stats_data, update_time)
-                elif stat_type == MODE_PLAYER_STATS:
+                elif stats_type == 'player_stats':
                     await self.store_player_stats(key, stats_data, update_time)
-                elif stat_type == MODE_PLAYER_ACHIEVEMENTS:
+                elif stats_type == 'player_achievements':
                     await self.store_player_achievements(key, stats_data, update_time)
                 else: 
-                    error('Function to saves stats type \'' + stat_type + '\' is not implemented yet')
+                    error('Function to saves stats type \'' + stats_type + '\' is not implemented yet')
             
             except asyncio.CancelledError:
                 # this is an eternal loop that will wait until cancelled	
@@ -1603,7 +1521,7 @@ class WG:
                 for stat in stats_data:
                     tank_id = stat['tank_id']
                     await self.cache.execute(WG.SQL_TANK_STATS_UPDATE, (account_id, tank_id, update_time, json.dumps(stat)))
-                    tank_ids.discard(tank_id)
+                    tank_ids.remove(tank_id)
             # no stats found => Add None to mark that
             for tank_id in tank_ids:
                 await self.cache.execute(WG.SQL_TANK_STATS_UPDATE, (account_id, tank_id, update_time, None))
@@ -1875,8 +1793,6 @@ class WoTinspector:
             except Exception as err:
                 debug(exception=err, id=N)
             await asyncio.sleep(SLEEP)
-            _id  = self.read_replay_id(json_resp)
-            json_resp['_id']  =  _id
             
         debug(' Could not post replay: ' + title, id=N)
         return json_resp

extract_tankopedia.py

@@ -3,7 +3,6 @@
 
 # Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
 
-from os import path
 import sys, argparse, json, os, inspect, aiohttp, asyncio, aiofiles, re, logging, time, xmltodict, collections, configparser
 import blitzutils as bu
 from blitzutils import WG
@@ -12,11 +11,9 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG)
 
 FILE_CONFIG 	= 'blitzstats.ini'
 
-# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
-# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
-BLITZAPP_STRINGS='Data/Strings/en.yaml'
-BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
-BLITZAPP_VEHICLE_FILE='list.xml'
+BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
+BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
+BLITZAPP_VEHICLE_FILE='/list.xml'
 
 wg = None
 
@@ -82,8 +79,8 @@ async def main(argv):
         tankopedia = collections.OrderedDict()
         tankopedia['status'] = 'ok'
         tankopedia['meta'] = { "count":  len(tanks) }
-        tankopedia['data'] = bu.sort_dict(tanks, number=True)
-        tankopedia['userStr'] = bu.sort_dict(userStrs)
+        tankopedia['data'] = tanks
+        tankopedia['userStr'] = userStrs
         bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
         bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')        
         await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
@@ -107,11 +104,8 @@ async def main(argv):
 async def extract_tanks(blitz_app_base : str, nation: str):
 
     tanks = []
-    # WG has changed the location of Data directory - at least in steam client
-    if path.isdir(path.join(blitz_app_base, 'assets')):
-        blitz_app_base = path.join(blitz_app_base, 'assets')
-    list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
-    if not path.isfile(list_xml_file): 
+    list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
+    if not os.path.isfile(list_xml_file): 
         print('ERROR: cannot open ' + list_xml_file)
         return None
     bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
@@ -121,12 +115,13 @@ async def extract_tanks(blitz_app_base : str, nation: str):
             for data in tankList['root'].keys():
                 tank_xml = tankList['root'][data]
                 tank = dict()
-                tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
-                tank['nation']  = nation
                 tank['tank_id'] = await get_tank_id(nation, int(tank_xml['id']))
-                tank['tier']    = int(tank_xml['level'])
-                tank['type'] = await get_tank_type(tank_xml['tags'])
                 tank['userStr'] = tank_xml['userString']
+                tank['nation']  = nation
+                tank['tier']    = int(tank_xml['level'])
+                #debug(tank_xml['price'])
+                tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
+                tank['type'] = await get_tank_type(tank_xml['tags'])
                 #bu.debug('Reading tank string: ' + tank['userStr'], force=True)
                 tanks.append(tank)
         except Exception as err:
@@ -138,7 +133,7 @@ async def read_user_strs(blitz_app_base : str) -> dict:
     """Read user strings to convert map and tank names"""
     tank_strs = {}
     map_strs = {}
-    filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
+    filename = blitz_app_base + BLITZAPP_STRINGS
     bu.debug('Opening file: ' + filename + ' for reading UserStrings')
     try:
         async with aiofiles.open(filename, 'r', encoding="utf8") as f:
@@ -170,10 +165,7 @@ async def convert_tank_names(tanklist : list, tank_strs: dict) -> dict:
             tank['name'] = tank_strs[tank['userStr']]
             #userStrs[tank['userStr'].split(':')[1]] = tank['name']
             tank.pop('userStr', None)
-            tank_tmp = collections.OrderedDict()
-            for key in sorted(tank.keys()):
-                tank_tmp[key] = tank[key]
-            tankopedia[str(tank['tank_id'])] = tank_tmp
+            tankopedia[str(tank['tank_id'])] = tank
 
         for tank_str in tank_strs:
             skip = False

get_tankopedia.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/local/bin/python3.7
 
 # Get Tankpedia from WoTinspector.com
 

maps.json

@@ -18,7 +18,6 @@
     "fort": "Fort Despair",
     "grossberg": "Dynasty's Pearl",
     "himmelsdorf": "Himmelsdorf",
-    "holland": "Molendijk",
     "holmeisk": "Wasteland",
     "iceworld": "Everfrost",
     "idle": "Yukon",
@@ -44,15 +43,11 @@
     "neptune_02": "Normandy - Beach",
     "north": "North",
     "ordeal": "Trial by Fire",
-    "plant": "Ghost Factory",
     "pliego": "Castilla",
     "port": "Port Bay",
     "rift": "Hellas",
     "rock": "Mayan Ruins",
     "rudniki": "Mines",
-    "rudniki_01": "Mines - Hill",
-    "rudniki_02": "Mines - Village",
-    "rudniki_03": "Mines - Center",
     "savanna": "Oasis Palms",
     "skit": "Naval Frontier",
     "test": "World of Ducks",

releases.md

@@ -1,96 +1,81 @@
 # WoT Blitz releases 
 
-| Release | Date     | Notes       |
-|:-------:|:--------:|-------------|
-| 9.1 | 2022-07-14 | Releases updated via DLC, graphics updates for Object 140, Patton 46 |
-| 9.0 | 2022-06-08 | Japanies heavies, Castilla graphics update, new garage |
-| 8.10 | 2022-04-28 | New battle results screen, new "Operation" mechanics |
-| 8.9 | 2022-03-23 | "Free" camo resupply, VK 72.01 supplies |
-| 8.8 | 2022-02-16 | Molendijk map |
-| 8.7 | 2022-01-19 | Soviet Light Tanks |
-| 8.6 | 2021-12-15 | Joining Light and Medium Tank classes in MM |
-| 8.5 | 2021-11-17 | New national crew voices, economy update |
-| 8.4 | 2021-10-20 | Polish heavy tanks |
-| 8.3 | 2021-09-16 | FV4005, FV4022, VK 91.01 (P) and WZ-121 legendary camos, -5 deg gun depression to WZ-120 |
-| 8.2 | 2021-08-18 | Ghost Factory map, XP/credits nerf for inactive players, new store UI |
-| 8.1 | 2021-07-15 | Armor highlighting updated |
-| 8.0 | 2021-06-08 | American experimental heavies (Yoh-line), PBR camos |
-| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |
-| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables |
-| 7.7 | 2021-02-24 | Czechoslovakian medium tanks |
-| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera |
-| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |
-| 7.4 | 2020-11-12 | TD buffs, Clan missions |
-| 7.3 | 2020-10-08 | Swedish HT line |
-| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles |
-| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff  |
-| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes |
-| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes |
-| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff |
-| 6.8 | 2020-02-19 | ATGM nerf, Hellas map |
-| 6.7 | 2020-01-15 | ATGM US Lights |
-| 6.6 | 2019-12-18 | T-62A + Leo buffs |
-| 6.5 | 2019-11-21 | New UI |
-| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... |
-| 6.3 | 2019-09-18 | European line |
-| 6.2 | 2019-08-14 | New MM |
-| 6.1 | 2019-07-10 | Autoloader buff |
-| 6.0 | 2019-06-05 | UK TD line #2 | 
-| 5.10 | 2019-04-24 | new missions | 
-| 5.9 | 2019-03-27 | New shell tracers, minor fixes |
-| 5.8 | 2019-02-21 | Chinese TD line |
-| 5.7 | 2019-01-16 | Avatars |
-| 5.6 | 2018-12-05 | Normandy map |
-| 5.5 | 2018-11-13 | Low-tier clean-up :-( |
-| 5.4 | 2018-10-10 | MM "rigging" adjusted |
-| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added | 
-| 5.2 | 2018-08-22 | Faust map |
-| 5.1 | 2018-07-31 | New MM: vehicle class matching |
-| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts removed ! |
-| 4.10 | 2018-05-24 | Alpenstadt map | 
-| 4.9 | 2018-04-25 | Chinese heavy tanks |
-| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator |
-| 4.7 | 2018-02-28 | New minimap |
-| 4.6 | 2018-01-31 | Chinese nation |
-| 4.5 | 2017-12-19 | Dynasty's Pearl map |
-| 4.4 | 2017-11-09 | New British tanks | 
-| 4.3 | 2017-10-10 | Level 7 screw skills |
-| 4.2 | 2017-09-12 | French LTs, Naval Frontier map |
-| 4.1 | 2017-08-16 | New markers |
-| 4.0 | 2017-07-13 | Rating battles (date uncertain) | 
-| 3.10 | 2017-06-15 | Fixes (date uncertain) |
-| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins (date uncertain) |
-| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system (date uncertain) |
-| 3.7 | 2017-03-22 | Balance changes, Tournaments (date uncertain) |
-| 3.6 | 2017-02-16 | Vehicle Enrichment (date uncertain) |
-| 3.5 | 2017-01-13 | Boosters, Dead rail update (date uncertain) |
-| 3.4 | 2016-11-28 | Canyon map, USA T57 line |
-| 3.3 | 2016-11-01 | Updated Mines map |
-| 3.2 | 2016-10-04 | Copperfield update |
-| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map (date uncertain) |
-| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update (date uncertain) |
-| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs |
-| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 |
-| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker (date uncertain) |
-| 2.8 | 2016-04-04 | New Russian TD line, new consumables system (date uncertain) |
-| 2.7 | 2016-03-07 | Mirage map (date uncertain) |
-| 2.6 | 2016-02-10 | German LTs, shooting through obstacles (date uncertain) |
-| 2.5 | 2016-01-11 | Camouflages, Clan supply (date uncertain) |
-| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version (date uncertain) |
-| 2.3 | 2015-11-12 | Canal map (date uncertain) |
-| 2.2 | 2015-10-05 | Desert Sands map update (date uncertain) |
-| 2.1 | 2015-09-07 | UK TDs, Clan Chat (date uncertain) |
-| 2.0 | 2015-08-05 | Clans, Oasis Palms updated |
-| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo (date uncertain) |
-| 1.10 | 2015-06-11 | Port Bay map (date uncertain) |
-| 1.9 | 2015-05-13 | New German medium tanks (Leo) (date uncertain) |
-| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players  (date uncertain) |
-| 1.7 | 2015-02-16 | US Turreted TDs (date uncertain) |
-| 1.6 | 2015-01-19 | Castilla map, UK HT line (date uncertain) |
-| 1.5 | 2014-12-01 | UK Nation (date uncertain) |
-| 1.4 | 2014-10-29 | German HT line (Maus) (date uncertain) |
-| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps (date uncertain) |
-| 1.2 | 2014-08-20 | Oasis Palms map (date uncertain) |
-| 1.1.1 | 2014-07-23 | Fixes :-) |
-| 1.0 | 2014-06-26 | Worldwide launch | 
-| 0.9 | 2014-05-05 | Nordic soft launch |
+| Release | Date     | Notes       | Inaccurate? |
+|:-------:|:--------:|-------------|-----|
+| 7.7 | 2021-02-24 | Czechoslovakian medium tanks  |  |
+| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera  |  |
+| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |  |
+| 7.4 | 2020-11-12 | TD buffs, Clan missions |  |
+| 7.3 | 2020-10-08 | Swedish HT line |  |
+| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles |  |
+| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff  |  |
+| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes |  |
+| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes |  |
+| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff |  |
+| 6.8 | 2020-02-19 | ATGM nerf, Hellas map |  |
+| 6.7 | 2020-01-15 | ATGM US Lights |  | 
+| 6.6 | 2019-12-18 | T-62A + Leo buffs |  | 
+| 6.5 | 2019-11-21 | New UI |  | 
+| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... |  | 
+| 6.3 | 2019-09-18 | European line |  | 
+| 6.2 | 2019-08-14 | New MM |  | 
+| 6.1 | 2019-07-10 | Autoloader buff |  | 
+| 6.0 | 2019-06-05 | UK TD line #2 |   | 
+| 5.10 | 2019-04-24 | new missions |   | 
+| 5.9 | 2019-03-27 | New shell tracers, minor fixes |  | 
+| 5.8 | 2019-02-21 | Chinese TD line |  | 
+| 5.7 | 2019-01-16 | Avatars |  | 
+| 5.6 | 2018-12-05 | Normandy map |   | 
+| 5.5 | 2018-11-13 | Low-tier clean-up |  | 
+| 5.4 | 2018-10-10 | MM "rigging" adjusted |   | 
+| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added |   | 
+| 5.2 | 2018-08-22 | Faust map |  | 
+| 5.1 | 2018-07-31 | New MM: vehicle class matching |   | 
+| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts killed |   | 
+| 4.10 | 2018-05-24 | Alpenstadt map |   | 
+| 4.9 | 2018-04-25 | Chinese heavy tanks |  | 
+| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator |   | 
+| 4.7 | 2018-02-28 | New minimap |   | 
+| 4.6 | 2018-01-31 | Chinese nation |   | 
+| 4.5 | 2017-12-19 | Dynasty's Pearl map |  | 
+| 4.4 | 2017-11-09 | New British tanks |   | 
+| 4.3 | 2017-10-10 | Level 7 screw skills |   | 
+| 4.2 | 2017-09-12 | French LTs, Naval Frontier map |   | 
+| 4.1 | 2017-08-16 | New markers |   | 
+| 4.0 | 2017-07-13 | Rating battles |  Y | 
+| 3.10 | 2017-06-15 | Fixes | Y | 
+| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins | Y | 
+| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system | Y |
+| 3.7 | 2017-03-22 | Balance changes, Tournaments | Y |
+| 3.6 | 2017-02-16 | Vehicle Enrichment | Y |
+| 3.5 | 2017-01-13 | Boosters, Dead rail update | Y |
+| 3.4 | 2016-11-28 | Canyon map, USA T57 line |  |
+| 3.3 | 2016-11-01 | Updated Mines map |  |
+| 3.2 | 2016-10-04 | Copperfield update |  |
+| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map | Y |
+| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update | Y |
+| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs | |
+| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 |  |
+| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker | Y |
+| 2.8 | 2016-04-04 | New Russian TD line, new consumables system | Y |
+| 2.7 | 2016-03-07 | Mirage map | Y |
+| 2.6 | 2016-02-10 | German LTs, shooting through obstacles | Y |
+| 2.5 | 2016-01-11 | Camouflages, Clan supply | Y |
+| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version | Y |
+| 2.3 | 2015-11-12 | Canal map | Y |
+| 2.2 | 2015-10-05 | Desert Sands map update | Y |
+| 2.1 | 2015-09-07 | UK TDs, Clan Chat | Y |
+| 2.0 | 2015-08-05 | Clans, Oasis Palms updated | |
+| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo | Y |
+| 1.10 | 2015-06-11 | Port Bay map | Y |
+| 1.9 | 2015-05-13 | New German medium tanks (Leo) | Y |
+| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players  | Y |
+| 1.7 | 2015-02-16 | US Turreted TDs | Y |
+| 1.6 | 2015-01-19 | Castilla map, UK HT line | Y |
+| 1.5 | 2014-12-01 | UK Nation | Y |
+| 1.4 | 2014-10-29 | German HT line (Maus) | Y |
+| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps | Y |
+| 1.2 | 2014-08-20 | Oasis Palms map | Y |
+| 1.1.1 | 2014-07-23 | Fixes :-) |  |
+| 1.0 | 2014-06-26 | Worldwide launch |  | 
+| 0.9 | 2014-05-05 | Nordic soft launch |   |

upload_wotb_replays.py

@@ -195,7 +195,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 						bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
 					else:
 						os.remove(replay_json_fn)
-						bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID)
+						bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
 					SKIPPED_N += 1						
 					queue.task_done()						
 					continue
@@ -216,13 +216,13 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 							if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
 								bu.verbose_std(msg_str + title + ' posted')
 						else:
-							bu.warning(msg_str +'Replay file is not valid/complete: ' + filename)
+							bu.warning(msg_str +'Replay file is not valid: ' + filename)
 							ERROR_N += 1
 					else:
 						bu.error(msg_str + 'Error saving replay: ' + filename)
 						ERROR_N += 1
 				else:
-					bu.error(msg_str + 'Replay file is not valid/complete: ' + filename)
+					bu.error(msg_str + 'Replay file is not valid: ' + filename)
 					ERROR_N += 1	
 		except Exception as err:
 			bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
@@ -287,13 +287,13 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
 					try:
 						metadata_json = json.load(meta)
 						#player = metadata_json['playerName']
-						tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
+						tank = wg.tank_str2name(metadata_json['playerVehicleName'])
 						map_name = wg.get_map(metadata_json['mapName'])
-						bu.debug('Tank: ' + tank_name if tank_name != None else "NOT FOUND IN tanks.json" +  ' Map: ' + map_name)
+						bu.debug('Tank: ' + tank +  ' Map: ' + map_name)
 					except Exception as err:
 						bu.error(exception = err)
-			if (tank_name != None) and  (map_name != None):
-				title = tank_name + ' @ ' + map_name
+			if (tank != None) and  (map_name != None):
+				title = tank + ' @ ' + map_name
 			else:
 				title = re.sub('\\.wotbreplay$', '', filename)
 		except Exception as err: