read_replay_JSON(): Start of same-tier support

analyze_wotb_replays.py

@@ -28,8 +28,6 @@ REPLAY_I = 0
 STAT_TANK_BATTLE_MIN = 100
 BATTLE_TIME_BUCKET = 3600*24*14
 
-OPT_EXPORT_REPLAY_DIR = 'export_replays'
-
 RE_SRC_IS_DB = re.compile(r'^DB:')
 
 class StatFunc:
@@ -902,7 +900,7 @@ class BattleCategory():
 		'pen_rate'			: [ 'Pen rate', 'Shots pen / shots hit', 							8, '{:8.1%}' ],
 		'survived'			: [ 'Surv%', 'Survival rate', 										6, '{:6.1%}' ],
 		'time_alive'		: [ 'T alive', 'Time being alive in a battle in secs', 				7, '{:7.0f}' ],
-		'alive'				: [ 'Share live', 'Percentage of time being alive in a battle', 		8, '{:8.0%}' ],
+		'alive'				: [ 'Share live', 'Percentage of time being alive in a battle', 	8, '{:8.0%}' ],
 		'battle_duration'	: [ 'Duration', 'Battle duration', 									8, '{:8.0f}' ], 
 		'distance_travelled': [ 'Distance', 'Distance travelled',								8, '{:8.0f}' ], 
 		'top_tier'			: [ 'Top tier', 'Share of games as top tier', 						8, '{:8.0%}' ],
@@ -1304,6 +1302,7 @@ class PlayerHistogram():
 		return None
 
 
+
 class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 	def exit(self, status=0, message=None):
 		if status:
@@ -1359,14 +1358,14 @@ async def main(argv):
 
 			try:
 				if 'ANALYZER' in config.sections():
-					configAnalyzer		= config['ANALYZER']
+					configAnalyzer	= config['ANALYZER']
-					OPT_MODE_DEFAULT	= configAnalyzer.getboolean('mode', OPT_MODE_DEFAULT)
+					OPT_MODE_DEFAULT= configAnalyzer.getboolean('mode', OPT_MODE_DEFAULT)
-					OPT_HIST			= configAnalyzer.getboolean('histograms', OPT_HIST)
+					OPT_HIST		= configAnalyzer.getboolean('histograms', OPT_HIST)
-					OPT_STAT_FUNC		= configAnalyzer.get('stat_func', fallback=OPT_STAT_FUNC)
+					OPT_STAT_FUNC	= configAnalyzer.get('stat_func', fallback=OPT_STAT_FUNC)
-					OPT_WORKERS_N 		= configAnalyzer.getint('workers', OPT_WORKERS_N)
+					OPT_WORKERS_N 	= configAnalyzer.getint('workers', OPT_WORKERS_N)
-					OPT_EXPORT_CSV_FILE = configAnalyzer.get('csv_file', OPT_EXPORT_CSV_FILE)
+					OPT_EXPORT_CSV_FILE  = configAnalyzer.get('csv_file', OPT_EXPORT_CSV_FILE)
-					OPT_EXPORT_JSON_FILE= configAnalyzer.get('json_file', OPT_EXPORT_JSON_FILE)
+					OPT_EXPORT_JSON_FILE = configAnalyzer.get('json_file', OPT_EXPORT_JSON_FILE)
-					res_categorizations = configAnalyzer.get('categorizations', None)
+					res_categorizations  = configAnalyzer.get('categorizations', None)
 					if res_categorizations != None:
 						res_categorizations.replace(' ','')
 						BattleCategorizationList.set_categorizations_default(res_categorizations.split(','))
@@ -1421,7 +1420,6 @@ async def main(argv):
 		parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"')
 		parser.add_argument('--json', action='store_true', default=False, help='Export data in JSON format')
 		parser.add_argument('--csv', action='store_true', default=False, help='Export data in CSV format')
-		parser.add_argument('--replays', action='store_true', default=False, help='Export replay info in JSON format')
 		parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
 		parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
 		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,10] }, { "player_wins" : 5 }]')
@@ -1443,7 +1441,7 @@ async def main(argv):
 		# res_categories = BattleCategorization.get_categorizations(OPT_CATEGORIZATIONS, args)
 
 		bu.set_log_level(args.silent, args.verbose, args.debug)
-		# bu.set_progress_step(250)  						# Set the frequency of the progress dots. 
+		bu.set_progress_step(250)  						# Set the frequency of the progress dots. 
 
 		#### Connect to MongoDB. You are unlikely to have this set up... 
 		bu.debug('DB_SERVER: ' + DB_SERVER)
@@ -1493,7 +1491,7 @@ async def main(argv):
 			# Make replay Queue
 
 			scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
-			bu.set_counter('Reading replays', 10)
+			bu.debug('Replay scanner started')
 			# Start tasks to process the Queue
 			for i in range(OPT_WORKERS_N):
 				reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
@@ -1501,7 +1499,7 @@ async def main(argv):
 
 			bu.debug('Waiting for the replay scanner to finish')
 			await asyncio.wait([scanner_task])
-			bu.finish_progress_bar()
+			
 			# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
 			await replayQ.join()
 			await asyncio.sleep(0.1)
@@ -1511,19 +1509,15 @@ async def main(argv):
 				await asyncio.sleep(0.1)	
 			results = []
 			players = set()
-			replays	= dict()
 			for res in await asyncio.gather(*reader_tasks):
 				results.extend(res[0])
 				players.update(res[1])
-				replays.update(res[2])
-			
 			if len(players) == 0:
 				raise Exception("No players found to fetch stats for. No replays found?")
 
 			if args.min != None:
 				results = filter_min_replays_by_player(results, args.min)
 				players = get_players(results)
-				replays = filter_replays(replays, results)
 
 			(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
 			bu.debug('Number of player stats: ' + str(len(player_stats)))
@@ -1570,9 +1564,6 @@ async def main(argv):
 					csvwriter.writerows(rows)
 					bu.verbose_std('Results exported to: ' + export_file)			
 			
-			if args.replays:
-				await export_replays(replays)
-
 			bu.debug('Finished. Cleaning up..................')
 		except Exception as err:
 			bu.error(exception=err)
@@ -1639,25 +1630,6 @@ async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, pa
 	print('No, I am not planning to fix these for now.')	
 
 
-async def export_replays(replays: dict):
-	"""Export replays"""
-	try:
-		replay_dir = OPT_EXPORT_REPLAY_DIR +  '_' + bu.get_date_str()
-		bu.verbose_std('Exporting replays to ' + replay_dir)
-		if not os.path.isdir(replay_dir):
-			os.mkdir(replay_dir)
-
-		i = 0
-		for replay in replays.values():
-			filename = wg.get_replay_filename(replay)
-			async with aiofiles.open(os.path.join(replay_dir, filename), 'w', encoding="utf8") as f:
-				await f.write(json.dumps(replay, indent=4))
-			i += 1
-		bu.verbose_std( str(i) + ' replays exported')
-	except Exception as err:
-		bu.error(exception=err)
-
-
 ## move the class PlayerHistogram?
 def set_histogram_buckets(json: dict):
 	global histogram_fields
@@ -1697,7 +1669,7 @@ def filter_min_replays_by_player(results: list, min_replays: int) -> list:
 		for res in results:
 			if res['protagonist'] in players_enough_replays:
 				res_ret.append(res)
-		bu.verbose_std('Replays after filtering: ' + str(len(res_ret)))	
+
 		return res_ret
 
 	except Exception as err:
@@ -1748,7 +1720,7 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 		
 		
 		if args.filters != None:
-			results = filter_results(results, args.filters)
+			results = filter_replays(results, args.filters)
 		if len(results) > 0:
 			blt_cat_list = BattleCategorizationList(cats)
 			for result in results:
@@ -1768,7 +1740,7 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 	return blt_cat_list
 
 
-def filter_results(results: list, filter_json : str) -> bool:
+def filter_replays(results: list, filter_json : str) -> bool:
 	"""Filter replays based on battle category filters"""
 	try:
 		filters = json.loads(filter_json)
@@ -1802,27 +1774,13 @@ def filter_results(results: list, filter_json : str) -> bool:
 					res.append(result)
 			except Exception as err:
 				bu.error(exception=err)
-		bu.verbose_std('Replays after filtering: ' + str(len(res)))	
+		bu.verbose('Replays after filtering: ' + str(len(res)))	
 		return res
 	except Exception as err:
 		bu.error(exception=err)
 	return None
 
 
-def filter_replays(replays_in: dict, results: list) -> list:
-	"""Filter source replays based on (filtered) results"""
-	replays = dict()
-	for res in results:
-		try:
-			# bu.debug('res[_id] : ' + res['_id'], force=True)			
-			_id = res['_id']
-			replays[_id] = replays_in[_id]
-		except Exception as err:
-			bu.error(exception=err)	
-	return replays
-	
-	
-
 async def process_player_stats(players, N_workers: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict:
 	"""Start stat_workers to retrieve and store players' stats"""
 	## Create queue of player-tank pairs to find stats for
@@ -1976,38 +1934,42 @@ async def stat_worker(queue : asyncio.Queue, workerID: int, args : argparse.Name
 			try:
 				bu.debug('Stat_id: ' + stat_id, id=workerID)
 				bu.print_progress()
-				stats_tmp = None
+				# Analysing player performance based on their stats on the tier tanks they are playing 
-	
+
 				# Try cache first
-				pruned_stat_id = prune_stat_id(stat_id)
+				if (stat_id not in stats):
-				if (pruned_stat_id not in stats):
+					stats_tmp = None
-					stats_tmp = await stat_wg_func(pruned_stat_id, cache_only = True)
+					pruned_stat_id = prune_stat_id(stat_id)
-				else:
+					if (pruned_stat_id not in stats):
-					stat_id_remap[stat_id] = pruned_stat_id
+						stats_tmp = await stat_wg_func(pruned_stat_id, cache_only = True)
-					continue
+					else:
+						stat_id_remap[stat_id] = pruned_stat_id
+						queue.task_done()
+						continue
 
-				if (stats_tmp != None):
+					if (stats_tmp != None):
-					stats[pruned_stat_id] = stats_tmp
+						stats[pruned_stat_id] = stats_tmp
-					stat_id_remap[stat_id] = pruned_stat_id					
+						stat_id_remap[stat_id] = pruned_stat_id
-					continue
+						queue.task_done()
-				
+						continue
-				# try DB
+					
-				stats[stat_id] = await stat_db_func(db, stat_id)				
+					# try DB
-				bu.debug('get_db_' + args.stat_func + '_stats returned: account_id=' + str(get_account_id_f_stat_id(stat_id)) + ': ' + str(stats[stat_id]), workerID)
+					stats[stat_id] = await stat_db_func(db, stat_id)				
+					bu.debug('get_db_' + args.stat_func + '_stats returned: account_id=' + str(get_account_id_f_stat_id(stat_id)) + ': ' + str(stats[stat_id]), workerID)
 
-				# no DB stats found, trying WG AP
+					# no DB stats found, trying WG AP
-				if (stats[stat_id] == None):							
+					if (stats[stat_id] == None):							
-					stats[pruned_stat_id] = await stat_wg_func(pruned_stat_id)
+						stats[pruned_stat_id] = await stat_wg_func(pruned_stat_id)
-					stat_id_remap[stat_id] = pruned_stat_id						
+						stat_id_remap[stat_id] = pruned_stat_id						
-					del stats[stat_id]
+						del stats[stat_id]
-					bu.debug('get_wg_' + args.stat_func + '_stats returned: account_id=' + str(get_account_id_f_stat_id(pruned_stat_id)) + ': ' + str(stats[pruned_stat_id]), workerID)
+						bu.debug('get_wg_' + args.stat_func + '_stats returned: account_id=' + str(get_account_id_f_stat_id(pruned_stat_id)) + ': ' + str(stats[pruned_stat_id]), workerID)
 
+					
 			except KeyError as err:
 				bu.error('Key not found', err, id=workerID)
 			except Exception as err:
 				bu.error('Unexpected error', err, id=workerID)
-			finally:
+			queue.task_done()
-				queue.task_done()
 
 	except (asyncio.CancelledError, concurrent.futures.CancelledError):
 		bu.debug('Stats queue is empty', id=workerID)		
@@ -2041,7 +2003,7 @@ async def get_wg_tank_stats(stat_id_str: str, cache_only = False) -> dict:
 		player_stats = await wg.get_player_tank_stats(account_id, [ tank_id ], hist_stats, cache_only = cache_only)
 		#bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats))
 
-		return tank_stats_helper(player_stats)
+		return await tank_stats_helper(player_stats)
 
 	except KeyError as err:
 		bu.error('account_id: ' + str(account_id) + ' tank_id:' + str(tank_id) +' : Key not found', err)
@@ -2067,8 +2029,8 @@ async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_
 		for stat in hist_stats:			
 			project[stat] = True
 		project['_id'] = False
-		bu.debug('find(): tank_id={} account_id={} last_battle_time>={}'.format(tank_id, account_id, battletime - time_buffer))
+		
-		cursor = dbc.find({ '$and': [ { 'tank_id' : tank_id }, { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }} ] }, projection=project).sort('last_battle_time',-1).limit(1)
+		cursor = dbc.find({ '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$gte': battletime - time_buffer }}, { 'tank_id' : tank_id } ] }, projection=project).sort('last_battle_time',-1).limit(1)
 
 		# pipeline = 	[ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : tank_id } ]}}, 
 		# 		{ '$sort': { 'last_battle_time': -1 }}, 
@@ -2079,11 +2041,7 @@ async def get_db_tank_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_
 		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
 		#cursor = dbc.aggregate(pipeline)
 
-		stats = await cursor.to_list(1)
+		return await tank_stats_helper(await cursor.to_list(1)) 
-		# store cache
-		# await wg.save_tank_stats(account_id, [tank_id], stats)
-		
-		return tank_stats_helper(stats) 	
 				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
@@ -2104,7 +2062,7 @@ async def get_wg_tank_tier_stats(stat_id_str: str, cache_only = False) -> dict:
 		player_stats = await wg.get_player_tank_stats(account_id, tier_tanks, hist_stats, cache_only = cache_only)
 		#bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats))
 
-		return tank_stats_helper(player_stats)
+		return await tank_stats_helper(player_stats)
 
 	except KeyError as err:
 		bu.error('account_id: ' + str(account_id) + ' Key not found', err)
@@ -2127,7 +2085,7 @@ async def get_wg_tier_x_stats(stat_id_str: str, cache_only = False) -> dict:
 		player_stats = await wg.get_player_tank_stats(account_id, tier_tanks, hist_stats, cache_only = cache_only)
 		#bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats))
 
-		return tank_stats_helper(player_stats)
+		return await tank_stats_helper(player_stats)
 
 	except KeyError as err:
 		bu.error('account_id: ' + str(account_id) + ' Key not found', err)
@@ -2155,10 +2113,8 @@ async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase,
 		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
 		cursor = dbc.aggregate(pipeline)
 
-		stats = await cursor.to_list(1000)
+		return await tank_stats_helper(await cursor.to_list(1000)) 
-		# await wg.save_tank_stats(account_id, tier_tanks, stats)
+				
-		return tank_stats_helper(stats) 
-		
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
 	return None
@@ -2168,9 +2124,23 @@ async def get_db_tier_x_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, sta
 	"""Get player stats from MongoDB (you are very unlikely to have it, unless you have set it up)"""
 	if db == None:
 		return None
-	try:		
+	try:
+		dbc = db[DB_C_TANK_STATS]
 		( account_id, battletime ) = str2ints(stat_id_str)
-		return await get_db_tank_tier_stats(db,get_stat_id(account_id, 10, battletime) )
+		tier_tanks = wg.get_tanks_by_tier(10)
+		time_buffer = 2*7*24*3600
+
+		pipeline = 	[ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}}, 
+				{ '$sort': { 'last_battle_time': -1 }}, 
+				{ '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, 
+				{ '$replaceRoot': { 'newRoot': '$doc' }}, 
+				{ '$project': { '_id': 0 }} ]
+
+		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
+		cursor = dbc.aggregate(pipeline)
+
+		return await tank_stats_helper(await cursor.to_list(1000)) 
+				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
 	return None
@@ -2188,16 +2158,14 @@ async def get_db_player_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, sta
 		
 		pipeline = 	[ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }} ]}}, 
 				{ '$sort': { 'last_battle_time': -1 }}, 
-				{ '$group': { '_id': { 'tank_id':'$tank_id'}, 'doc': { '$first': '$$ROOT' }}}, 
+				{ '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}}, 
 				{ '$replaceRoot': { 'newRoot': '$doc' }}, 
 				{ '$project': { '_id': 0 }} ]
 
 		# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
 		cursor = dbc.aggregate(pipeline)
 
-		stats = await cursor.to_list(1000)
+		return await tank_stats_helper(await cursor.to_list(1000)) 
-		# await wg.save_tank_stats(account_id, [], stats)
-		return tank_stats_helper(stats) 
 				
 	except Exception as err:
 		bu.error('account_id: ' + str(account_id) + ' Error', err)
@@ -2225,17 +2193,20 @@ async def get_wg_player_stats(stat_id_str: str, cache_only = False) -> dict:
 	return None
 
 
-def tank_stats_helper(stat_list: list):
+async def tank_stats_helper(stat_list: list):
 	"""Helpher func got get_db_tank_tier_stats() and get_wg_tank_tier_stats()"""
 	try:
 		if stat_list == None: 
 			return None
-		stats = collections.defaultdict(def_value_zero)
+		stats = dict()
 		hist_fields = histogram_fields.keys()
 		if 'battles' not in hist_fields:
 			bu.error('\'battles\' must be defined in \'histogram_fields\'')
 			return None
 
+		for field in hist_fields:
+			stats[field] = 0
+		
 		for tank_stat in stat_list: 
 			tank_stat = tank_stat['all']
 			for field in hist_fields:
@@ -2305,7 +2276,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 			async for replay_json in cursor:
 				_id = replay_json['_id']
 				#del(replay_json['_id'])
-				await queue.put(await mk_readerQ_item(replay_json, _id=_id))
+				await queue.put(await mk_readerQ_item(replay_json, 'DB: _id = ' + _id))
 				Nreplays += 1
 			bu.debug('All the matching replays have been read from the DB')
 		except Exception as err:
@@ -2321,7 +2292,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 				
 				if (p_replayfile.match(line) != None):
 					replay_json = await bu.open_JSON(line, wi.chk_JSON_replay)						
-					await queue.put(await mk_readerQ_item(replay_json, filename=line))
+					await queue.put(await mk_readerQ_item(replay_json, line))
 			except Exception as err:
 				bu.error(exception=err)
 	else:
@@ -2332,7 +2303,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 					fn = fn[:-1] 
 				if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
 					replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay)
-					await queue.put(await mk_readerQ_item(replay_json, filename=fn))
+					await queue.put(await mk_readerQ_item(replay_json, fn))
 					bu.debug('File added to queue: ' + fn)
 					Nreplays += 1
 				elif os.path.isdir(fn):
@@ -2341,7 +2312,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 							if entry.is_file() and (p_replayfile.match(entry.name) != None): 
 								bu.debug(entry.name)
 								replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay)
-								await queue.put(await mk_readerQ_item(replay_json, filename=entry.name))
+								await queue.put(await mk_readerQ_item(replay_json, entry.name))
 								bu.debug('File added to queue: ' + entry.path)
 								Nreplays += 1
 			except Exception as err:
@@ -2350,27 +2321,25 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 	return Nreplays
 
 
-async def mk_readerQ_item(replay_json, filename : str = None, _id: str = None) -> list:
+async def mk_readerQ_item(replay_json, filename : str = None) -> list:
 	"""Make an item to replay queue"""
 	global REPLAY_N
 	REPLAY_N +=1
 	try:
 		if wi.chk_JSON_replay(replay_json): 
 			if not '_id' in replay_json:
-				if _id  != None:			
+				id = wi.read_replay_id(replay_json)
-					_id = wi.read_replay_id(replay_json)
+				replay_json['_id'] = id	
-				replay_json['_id'] = _id
 		else:
 			replay_json = None # mark error
 	except Exception as err:
 		bu.error(exception=err)
-	
+	if filename == None:
-	if filename != None:
+		return [replay_json, REPLAY_N, '' ]
-		return [replay_json, REPLAY_N, os.path.basename(filename) ]
+	elif RE_SRC_IS_DB.match(filename) != None:
-	elif _id != None: 
+		return [replay_json, REPLAY_N, filename ]
-		return [replay_json, REPLAY_N, 'DB: ' + _id ]
 	else:
-		return None
+		return [replay_json, REPLAY_N, os.path.basename(filename) ]
 
 
 async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
@@ -2378,7 +2347,6 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 	#global SKIPPED_N
 	results = []
 	playerstanks = set()
-	replays		 = dict()
 	try:
 		while True:
 			item = await queue.get()
@@ -2405,10 +2373,8 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 				playerstanks.update(set(result['allies']))
 				playerstanks.update(set(result['enemies']))	
 				playerstanks.update(set([result['player']]))
-				_id = wi.read_replay_id(replay_json)
+				
-				replays[_id] = replay_json
 				results.append(result)
-
 				bu.debug('Marking replay [' + str(replayID) + '] done', id=readerID)
 						
 			except Exception as err:
@@ -2416,7 +2382,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 			queue.task_done()
 	except (asyncio.CancelledError, concurrent.futures.CancelledError):		
 		bu.debug( str(len(results)) + ' replays, ' + str(len(playerstanks)) + ' player/tanks', readerID)
-		return results, playerstanks, replays
+		return results, playerstanks
 	return None
 
 
@@ -2476,12 +2442,14 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
 		result['allies_survived']  = 0 	# for steamroller stats
 		result['enemies_survived']  = 0	# for steamroller stats
 		btl_duration = 0
-		btl_tier = 0
+		btl_tier 	 = 0
+		btl_tier_low = 10
 		protagonist_tank  = None
 		for player in replay_summary['details']:
 			btl_duration = max(btl_duration, player['time_alive'])
 			player_tank_tier = wg.get_tank_data(player['vehicle_descr'], 'tier')
-			btl_tier = max(btl_tier, player_tank_tier)
+			btl_tier 		 = max(btl_tier, player_tank_tier)
+			btl_tier_low     = min(btl_tier_low, player_tank_tier)
 
 			if (protagonist != None) and (player['dbid'] == protagonist):
 				protagonist_tank = player['vehicle_descr']
@@ -2574,8 +2542,8 @@ def prune_stat_id(stat_id_str: str) -> str:
 	return ':'.join(stat_id)
 
 
-def get_stat_id(account_id: int, tank_id: int, battle_time: int) -> str:
+def get_stat_id(account_id: int, tank_id: int, battletime: int) -> str:
-	return ':'.join(map(str, [account_id, tank_id, battle_time ]))
+	return ':'.join([ str(account_id), str(tank_id), str(battletime)])
 
 
 def get_stat_id_tank_tier(stat_id_str: str) -> str:
@@ -2585,7 +2553,7 @@ def get_stat_id_tank_tier(stat_id_str: str) -> str:
 		account_id	= stat_id[0]
 		tank_tier 	= wg.get_tank_tier(stat_id[1])
 		battle_time = (stat_id[2] // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET
-		return get_stat_id(account_id, tank_tier, battle_time)
+		return ':'.join(map(str, [account_id, tank_tier, battle_time ]))
 	except Exception as err:
 		bu.error('Stats_id: ' + stat_id_str, exception=err)
 	return None
@@ -2598,7 +2566,7 @@ def get_stat_id_tank(stat_id_str: str) -> str:
 		account_id	= stat_id[0]
 		tank_id 	= stat_id[1]
 		battle_time = (stat_id[2] // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET
-		return get_stat_id(account_id, tank_id, battle_time)		
+		return ':'.join(map(str, [account_id, tank_id, battle_time ]))
 	except Exception as err:
 		bu.error('Stats_id: ' + stat_id_str, exception=err)
 	return None

blitzutils.py

@@ -22,11 +22,6 @@ _log_level = NORMAL
 LOG = False
 LOGGER = None
 
-MODE_TANK_STATS         = 'tank_stats'
-MODE_PLAYER_STATS       = 'player_stats'
-MODE_PLAYER_ACHIEVEMENTS= 'player_achievements'
-MODE_TANKOPEDIA			= 'tankopedia'
-
 # Progress display
 _progress_N = 100
 _progress_i = 0
@@ -423,14 +418,12 @@ def set_progress_bar(heading: str, max_value: int, step: int = None, slow: bool
     return
 
 
-def set_counter(heading: str, step = None):
+def set_counter(heading: str):
     global _progress_obj, _progress_i
     _progress_i = 0
     if _progress_obj != None:
         finish_progress_bar()
-    _progress_obj = Counter(heading + ': ')
+    _progress_obj = Counter(heading)
-    if step != None:
-        set_progress_step(step)
     return 
 
 
@@ -486,16 +479,6 @@ def NOW() -> int:
     return int(time.time())
 
 
-def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
-    """Return YYYYMMDD_HHmm date string"""
-    try:
-        ts = datetime.fromtimestamp(timestamp)
-        return ts.strftime(date_format)
-    except Exception as err:
-        error(exception=err)
-    return None
-
-
 def rebase_file_args(current_dir, files):
     """REbase file command line params after moving working dir to the script's dir""" 
     if isinstance(files, list):    
@@ -666,6 +649,8 @@ class StatsNotFound(Exception):
     pass
 
 
+
+
 # -----------------------------------------------------------
 # Class WG 
 # -----------------------------------------------------------
@@ -701,7 +686,7 @@ class WG:
     #                                 WHERE account_id = {} AND stat = {} 
     #                                 AND date >= {} LIMIT 1;"""
           
-    # TANK_STATS
+    
     SQL_TANK_STATS_TBL          = 'tank_stats'
 
     SQL_TANK_STATS_CREATE_TBL   = 'CREATE TABLE IF NOT EXISTS ' + SQL_TANK_STATS_TBL + \
@@ -715,7 +700,6 @@ class WG:
 
     SQL_TANK_STATS_UPDATE       = 'REPLACE INTO ' + SQL_TANK_STATS_TBL + '(account_id, tank_id, update_time, stats) VALUES(?,?,?,?)'
 
-    # PLAYER_STATS
     SQL_PLAYER_STATS_TBL        = 'player_stats'
 
     SQL_PLAYER_STATS_CREATE_TBL = 'CREATE TABLE IF NOT EXISTS ' + SQL_PLAYER_STATS_TBL + \
@@ -729,7 +713,6 @@ class WG:
 
     SQL_PLAYER_STATS_CACHED     = 'SELECT * FROM ' +  SQL_PLAYER_STATS_TBL + ' WHERE account_id = ? AND update_time > ?'
 
-    # PLAYER_ACHIEVEMENTS
     SQL_PLAYER_ACHIEVEMENTS_TBL  = 'player_achievements'
 
     SQL_PLAYER_ACHIEVEMENTS_CREATE_TBL = 'CREATE TABLE IF NOT EXISTS ' + SQL_PLAYER_ACHIEVEMENTS_TBL + \
@@ -737,13 +720,12 @@ class WG:
                                     update_time INTEGER NOT NULL, 
                                     stats TEXT)"""
 
-    SQL_PLAYER_ACHIEVEMENTS_COUNT      = 'SELECT COUNT(*) FROM ' + SQL_PLAYER_ACHIEVEMENTS_TBL
-    
     SQL_PLAYER_ACHIEVEMENTS_CACHED     = 'SELECT * FROM ' +  SQL_PLAYER_ACHIEVEMENTS_TBL + ' WHERE account_id = ? AND update_time > ?'
 
     SQL_PLAYER_ACHIEVEMENTS_UPDATE     = 'REPLACE INTO ' + SQL_PLAYER_ACHIEVEMENTS_TBL + '(account_id, update_time, stats) VALUES(?,?,?)'
 
-    
+    SQL_PLAYER_ACHIEVEMENTS_COUNT      = 'SELECT COUNT(*) FROM ' + SQL_PLAYER_ACHIEVEMENTS_TBL
+
     SQL_TABLES                  = [ SQL_PLAYER_STATS_TBL, SQL_TANK_STATS_TBL, SQL_PLAYER_ACHIEVEMENTS_TBL ]
 
     SQL_CHECK_TABLE_EXITS       = """SELECT name FROM sqlite_master WHERE type='table' AND name=?"""
@@ -926,18 +908,33 @@ class WG:
 
 
     @classmethod
-    def get_map(self, map_str: str) -> str:
+    def get_map(cls, map_str: str) -> str:
         """Return map name from short map string in replays"""
         try:
-            return self.maps[map_str]
+            return cls.maps[map_str]
         except:
             error('Map ' + map_str + ' not found')
         return None
     
 
     @classmethod
-    def get_tank_user_strs(self) -> str:
+    def get_map_user_strs(cls) -> str:
-        return self.tanks["userStr"].keys()
+        return cls.maps.keys()
+
+
+    @classmethod
+    def tank_str2name(cls, tank_str: str) -> str:
+        """Return tank name from short tank string in replays"""
+        try:
+            return cls.tanks["userStr"][tank_str]
+        except:
+            error('Tank ' + tank_str + ' not found')
+        return tank_str
+
+
+    @classmethod
+    def get_tank_user_strs(cls) -> str:
+        return cls.tanks["userStr"].keys()
 
 
     @classmethod
@@ -1073,15 +1070,11 @@ class WG:
     def load_tanks(self, tankopedia_fn: str):
         """Load tanks from tankopedia JSON"""
         if tankopedia_fn == None:
-            return False
+            return False 
+
         try:
             with open(tankopedia_fn, 'rt', encoding='utf8') as f:
                 self.tanks = json.loads(f.read())
-                self.tanks['tankStr'] = dict()
-                p = re.compile('.+_short$')
-                for usr_str in self.tanks['userStr']:
-                    if p.match(usr_str) == None:
-                        self.tanks['tankStr'][self.tanks['userStr'][usr_str]] = usr_str
                 self.tanks_by_tier = dict()
                 for tier in range(1,11):
                     self.tanks_by_tier[str(tier)] = list()
@@ -1091,44 +1084,7 @@ class WG:
         except Exception as err:
             error('Could not read tankopedia: ' + tankopedia_fn, err) 
         return False     
-    
+     
-
-    def get_replay_filename(self, replay: dict):
-        try:
-            summary = replay['data']['summary']
-            timestamp   = int(summary['battle_start_timestamp'])
-            timestamp    = get_date_str(timestamp, date_format='%Y%m%d_%H%M')
-            player      = summary['player_name']
-            arena_id    = int(summary['arena_unique_id'])
-            vehicle     = self.name2tank_str(summary['vehicle'])
-            vehicle.replace(' ','_')
-
-            return '{:s}_{:s}_{:s}_{:d}.wotbreplay.json'.format(timestamp, player, vehicle, arena_id)
-        except Exception as err:
-            error('Unexpected Exception', err) 
-            return None 
-
-    def get_map_user_strs(self) -> str:
-        return self.maps.keys()
-
-
-    def tank_str2name(self, tank_str: str) -> str:
-        """Return tank name from short tank string in replays"""
-        try:
-            return self.tanks["userStr"][tank_str]
-        except:
-            error('Tank ' + tank_str + ' not found')
-        return tank_str
-
-    
-    def name2tank_str(self, name: str) -> str:
-        """Return shorttank string from tank string in replays"""
-        try:
-            return self.tanks["tankStr"][name]
-        except:
-            debug('Tank ' + name + ' not found')
-        return name
-
 
     def get_tanks_by_tier(self, tier: int) -> list():
         """Returns tank_ids by tier"""
@@ -1305,7 +1261,7 @@ class WG:
                 # debug('JSON Response received: ' + str(json_data))
                 stats = json_data['data'][str(account_id)]
                 if cache:
-                    await self.save_tank_stats(account_id, tank_ids, stats)
+                    await self.put_2_statsQ('tank_stats', [account_id, tank_ids], stats)
                 return stats
         except Exception as err:
             error(exception=err)
@@ -1338,7 +1294,7 @@ class WG:
                 # debug('JSON Response received: ' + str(json_data))
                 stats = json_data['data'][str(account_id)]
                 if cache:
-                    await self.save_player_stats(account_id, stats)
+                    await self.put_2_statsQ('player_stats', [account_id], stats)
                 return stats
         except Exception as err:
             error(exception=err)
@@ -1377,7 +1333,7 @@ class WG:
                 for account_id in json_data['data'].keys():
                     stats[account_id] = json_data['data'][account_id]
                     if cache:
-                        await self.save_player_achievements(int(account_id), json_data['data'][account_id])
+                        await self.put_2_statsQ('player_achievements', [int(account_id)], json_data['data'][account_id])
                 return stats
         except Exception as err:
             error(exception=err)
@@ -1484,7 +1440,7 @@ class WG:
         return None
 
 
-    async def save_stats(self, statsType: str, key: list, stats: list):
+    async def put_2_statsQ(self, statsType: str, key: list, stats: list):
         """Save stats to a async queue to be saved by the stat_saver -task"""
         if self.statsQ == None:
             return False
@@ -1493,30 +1449,6 @@ class WG:
             return True
 
 
-    async def save_player_stats(self, account_id: int, stats: dict):
-        try:
-            await self.save_stats(MODE_PLAYER_STATS, [int(account_id)], stats)
-        except Exception as err:
-            error(exception=err)
-        return None
-
-
-    async def save_player_achievements(self, account_id: int, stats: dict):
-        try:
-            await self.save_stats(MODE_PLAYER_ACHIEVEMENTS, [int(account_id)], stats)
-        except Exception as err:
-            error(exception=err)
-        return None
-
-
-    async def save_tank_stats(self, account_id: int, tank_ids: list, stats: dict):
-        try:
-            await self.save_stats(MODE_TANK_STATS, [int(account_id), tank_ids], stats)
-        except Exception as err:
-            error(exception=err)
-        return None
-
-
     async def stat_saver(self): 
         """Async task for saving stats into cache in background"""
 
@@ -1532,13 +1464,8 @@ class WG:
 
             await self.cache.commit()
             
-            if is_debug():
+            async with self.cache.execute(WG.SQL_TANK_STATS_COUNT) as cursor:
-                async with self.cache.execute(WG.SQL_PLAYER_STATS_COUNT) as cursor:
+                debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' cached player tank stat records' )
-                    debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' player stat records' )
-                async with self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_COUNT) as cursor:
-                    debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' player achievement records' )
-                async with self.cache.execute(WG.SQL_TANK_STATS_COUNT) as cursor:
-                    debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' player tank stat records' )
         except Exception as err:
             error(exception=err)
             sys.exit(1)
@@ -1547,20 +1474,19 @@ class WG:
             try:
                 stats = await self.statsQ.get()
             
-                stat_type  = stats[0]
+                stats_type  = stats[0]
                 key         = stats[1]
                 stats_data  = stats[2]
                 update_time = stats[3]
 
-                #debug('stat_type: ' + stat_type + ', key: ' + str(key))
+                if stats_type == 'tank_stats':
-                if stat_type == MODE_TANK_STATS:
                     await self.store_tank_stats(key, stats_data, update_time)
-                elif stat_type == MODE_PLAYER_STATS:
+                elif stats_type == 'player_stats':
                     await self.store_player_stats(key, stats_data, update_time)
-                elif stat_type == MODE_PLAYER_ACHIEVEMENTS:
+                elif stats_type == 'player_achievements':
                     await self.store_player_achievements(key, stats_data, update_time)
                 else: 
-                    error('Function to saves stats type \'' + stat_type + '\' is not implemented yet')
+                    error('Function to saves stats type \'' + stats_type + '\' is not implemented yet')
             
             except asyncio.CancelledError:
                 # this is an eternal loop that will wait until cancelled	
@@ -1595,7 +1521,7 @@ class WG:
                 for stat in stats_data:
                     tank_id = stat['tank_id']
                     await self.cache.execute(WG.SQL_TANK_STATS_UPDATE, (account_id, tank_id, update_time, json.dumps(stat)))
-                    tank_ids.discard(tank_id)
+                    tank_ids.remove(tank_id)
             # no stats found => Add None to mark that
             for tank_id in tank_ids:
                 await self.cache.execute(WG.SQL_TANK_STATS_UPDATE, (account_id, tank_id, update_time, None))
@@ -1867,8 +1793,6 @@ class WoTinspector:
             except Exception as err:
                 debug(exception=err, id=N)
             await asyncio.sleep(SLEEP)
-            _id  = self.read_replay_id(json_resp)
-            json_resp['_id']  =  _id
             
         debug(' Could not post replay: ' + title, id=N)
         return json_resp

extract_tankopedia.py

@@ -115,12 +115,13 @@ async def extract_tanks(blitz_app_base : str, nation: str):
             for data in tankList['root'].keys():
                 tank_xml = tankList['root'][data]
                 tank = dict()
-                tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
-                tank['nation']  = nation
                 tank['tank_id'] = await get_tank_id(nation, int(tank_xml['id']))
-                tank['tier']    = int(tank_xml['level'])
-                tank['type'] = await get_tank_type(tank_xml['tags'])
                 tank['userStr'] = tank_xml['userString']
+                tank['nation']  = nation
+                tank['tier']    = int(tank_xml['level'])
+                #debug(tank_xml['price'])
+                tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
+                tank['type'] = await get_tank_type(tank_xml['tags'])
                 #bu.debug('Reading tank string: ' + tank['userStr'], force=True)
                 tanks.append(tank)
         except Exception as err:
@@ -164,10 +165,7 @@ async def convert_tank_names(tanklist : list, tank_strs: dict) -> dict:
             tank['name'] = tank_strs[tank['userStr']]
             #userStrs[tank['userStr'].split(':')[1]] = tank['name']
             tank.pop('userStr', None)
-            tank_tmp = collections.OrderedDict()
+            tankopedia[str(tank['tank_id'])] = tank
-            for key in sorted(tank.keys()):
-                tank_tmp[key] = tank[key]
-            tankopedia[str(tank['tank_id'])] = tank_tmp
 
         for tank_str in tank_strs:
             skip = False

maps.json

@@ -48,11 +48,8 @@
     "rift": "Hellas",
     "rock": "Mayan Ruins",
     "rudniki": "Mines",
-    "rudniki_01": "Mines - Hill",
-    "rudniki_02": "Mines - Village",
-    "rudniki_03": "Mines - Center",
     "savanna": "Oasis Palms",
     "skit": "Naval Frontier",
     "test": "World of Ducks",
     "tutorial": "Proving Grounds"
-}
+}

releases.md

@@ -2,8 +2,6 @@
 
 | Release | Date     | Notes       | Inaccurate? |
 |:-------:|:--------:|-------------|-----|
-| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |  |
-| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables  |  |
 | 7.7 | 2021-02-24 | Czechoslovakian medium tanks  |  |
 | 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera  |  |
 | 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |  |

tanks.json

@@ -1,7 +1,7 @@
 {
     "status": "ok",
     "meta": {
-        "count": 489
+        "count": 476
     },
     "data": {
         "1": {
@@ -414,7 +414,7 @@
         },
         "1649": {
             "is_premium": true,
-            "name": "Helsing",
+            "name": "Helsing H0",
             "nation": "other",
             "tank_id": 1649,
             "tier": 7,
@@ -894,7 +894,7 @@
         },
         "3937": {
             "is_premium": false,
-            "name": "Ho-Ri Type III",
+            "name": "Ho-Ri Type 3",
             "nation": "japan",
             "tank_id": 3937,
             "tier": 10,
@@ -2262,7 +2262,7 @@
         },
         "12321": {
             "is_premium": true,
-            "name": "T6E1 Grizzly",
+            "name": "T6E1",
             "nation": "usa",
             "tank_id": 12321,
             "tier": 4,
@@ -2614,7 +2614,7 @@
         },
         "17473": {
             "is_premium": true,
-            "name": "AMX Defender",
+            "name": "AMX 13 90 Defender",
             "nation": "france",
             "tank_id": 17473,
             "tier": 8,
@@ -2628,14 +2628,6 @@
             "tier": 8,
             "type": "AT-SPG"
         },
-        "17729": {
-            "is_premium": true,
-            "name": "Somua SM",
-            "nation": "france",
-            "tank_id": 17729,
-            "tier": 8,
-            "type": "heavyTank"
-        },
         "17745": {
             "is_premium": true,
             "name": "FV217 Badger",
@@ -2980,14 +2972,6 @@
             "tier": 8,
             "type": "mediumTank"
         },
-        "20817": {
-            "is_premium": true,
-            "name": "Explorer",
-            "nation": "uk",
-            "tank_id": 20817,
-            "tier": 6,
-            "type": "mediumTank"
-        },
         "20993": {
             "is_premium": true,
             "name": "T-2020",
@@ -3100,14 +3084,6 @@
             "tier": 10,
             "type": "heavyTank"
         },
-        "22305": {
-            "is_premium": true,
-            "name": "AE Phase I",
-            "nation": "usa",
-            "tank_id": 22305,
-            "tier": 9,
-            "type": "heavyTank"
-        },
         "22529": {
             "is_premium": true,
             "name": "LT-432",
@@ -3132,22 +3108,6 @@
             "tier": 6,
             "type": "heavyTank"
         },
-        "22817": {
-            "is_premium": true,
-            "name": "M-VI-Yoh",
-            "nation": "usa",
-            "tank_id": 22817,
-            "tier": 10,
-            "type": "heavyTank"
-        },
-        "23057": {
-            "is_premium": true,
-            "name": "Kunze Panzer",
-            "nation": "germany",
-            "tank_id": 23057,
-            "tier": 7,
-            "type": "lightTank"
-        },
         "51201": {
             "is_premium": true,
             "name": "KV-220 Beta-Test",
@@ -3638,7 +3598,7 @@
         },
         "63585": {
             "is_premium": true,
-            "name": "Tiger Kuromorimine SP",
+            "name": "Kuro Mori Mine",
             "nation": "japan",
             "tank_id": 63585,
             "tier": 6,
@@ -3646,7 +3606,7 @@
         },
         "63601": {
             "is_premium": true,
-            "name": "Dracula",
+            "name": "T6 Dracula",
             "nation": "other",
             "tank_id": 63601,
             "tier": 7,
@@ -3654,7 +3614,7 @@
         },
         "63841": {
             "is_premium": true,
-            "name": "Panzer IV Ankou Special",
+            "name": "Panzer IV Anko Special",
             "nation": "japan",
             "tank_id": 63841,
             "tier": 5,
@@ -3753,7 +3713,7 @@
             "name": "Pz.Kpfw. II",
             "nation": "germany",
             "tank_id": 64785,
-            "tier": 1,
+            "tier": 2,
             "type": "lightTank"
         },
         "64801": {
@@ -3851,70 +3811,6 @@
             "tank_id": 65377,
             "tier": 5,
             "type": "mediumTank"
-        },
-        "7041": {
-            "is_premium": true,
-            "name": "Turbo",
-            "nation": "european",
-            "tank_id": 7041,
-            "tier": 6,
-            "type": "mediumTank"
-        },
-        "17985": {
-            "is_premium": true,
-            "name": "Bretagne Panther",
-            "nation": "france",
-            "tank_id": 17985,
-            "tier": 6,
-            "type": "mediumTank"
-        },
-        "21537": {
-            "is_premium": true,
-            "name": "Sherman Easy 8",
-            "nation": "usa",
-            "tank_id": 21537,
-            "tier": 6,
-            "type": "mediumTank"
-        },
-        "22017": {
-            "is_premium": true,
-            "name": "T-34-85",
-            "nation": "ussr",
-            "tank_id": 22017,
-            "tier": 6,
-            "type": "mediumTank"
-        },
-        "22289": {
-            "is_premium": true,
-            "name": "Tiger I",
-            "nation": "germany",
-            "tank_id": 22289,
-            "tier": 6,
-            "type": "heavyTank"
-        },
-        "22785": {
-            "is_premium": true,
-            "name": "Triumphant",
-            "nation": "ussr",
-            "tank_id": 22785,
-            "tier": 6,
-            "type": "mediumTank"
-        },
-        "23313": {
-            "is_premium": true,
-            "name": "Kampfpanzer 50 t",
-            "nation": "germany",
-            "tank_id": 23313,
-            "tier": 10,
-            "type": "mediumTank"
-        },
-        "23841": {
-            "is_premium": true,
-            "name": "Super Hellcat",
-            "nation": "usa",
-            "tank_id": 23841,
-            "tier": 7,
-            "type": "AT-SPG"
         }
     },
     "userStr": {
@@ -3962,12 +3858,10 @@
         "A124_T54E2_Custom": "T54E2 Shark",
         "A124_T54E2_Custom_short": "T54E2 Shark",
         "A124_T54E2_short": "T54E2",
-        "A125_AEP_1": "AE Phase I",
-        "A125_AEP_1_short": "AE Phase I",
         "A68_T28D": "T28 Defender",
         "A68_T28D_short": "T28 Defender",
-        "A77_T6M": "T6E1 Grizzly",
+        "A77_T6M": "T6E1",
-        "A77_T6M_short": "T6E1 Grizzly",
+        "A77_T6M_short": "T6E1",
         "A94_T37": "T37",
         "A95_T95E6": "T95E6",
         "A95_T95E6_short": "T95E6",
@@ -4017,7 +3911,6 @@
         "AMX_dracula_Custom_short": "Dracula Two-faced",
         "AMX_dracula_short": "Dracula",
         "AOI-1490": "Continental AOI-1490",
-        "AOSI_1490-1": "Continental AOSI-1490-1",
         "AOS_895": "AOS-895",
         "APCNR": "Pzgr. 41 HK",
         "APCNR.erz": "Pzgr. 41 StK",
@@ -4205,7 +4098,6 @@
         "Continental_AOS-895-1_short": "Continental\\nAOS-895-1",
         "Continental_AOS-895-3": "AN/GRC-8",
         "Continental_AOS-895-3_short": "AN/GRC-8",
-        "Continental_AOSI-1195-5": "Continental AOSI-1195-5",
         "Continental_AOSI-895-5": "Continental AOSI-895-5",
         "Continental_AOSI-895-5_short": "Continental\\nAOSI-895-5",
         "Continental_AOSI_895_5_Prem": "Continental AOSI-895-5",
@@ -4344,10 +4236,6 @@
         "F82_AMX_M4_Mle1949_Ter_Custom": "AMX M4 54 Firebird",
         "F82_AMX_M4_Mle1949_Ter_Custom_short": "AMX M4 54 Firebird",
         "F82_AMX_M4_Mle1949_Ter_short": "AMX M4 mle. 54",
-        "F84_Somua_SM": "Somua SM",
-        "F84_Somua_SM_Custom": "Somua Grenadier",
-        "F84_Somua_SM_Custom_short": "Somua Grenadier",
-        "F84_Somua_SM_short": "Somua SM",
         "FCM50t_fwhite": "FCM 50 t Titi",
         "FCM50t_fwhite_short": "FCM 50 t Titi",
         "FCM50t_liliam": "FCM 50 t Zizou",
@@ -4421,8 +4309,6 @@
         "G136_Tiger_131_short": "Tiger 131",
         "G138_VK16801P": "VK 168.01 (P)",
         "G138_VK16801P_short": "VK 168.01 (P)",
-        "G147_mPanzer_64": "Kunze Panzer",
-        "G147_mPanzer_64_short": "Kunze Panzer",
         "G20_Marder_II": "Marder II",
         "G20_Marder_II_short": "Marder II",
         "GAZ-11": "GAZ-11",
@@ -4465,8 +4351,6 @@
         "GB19_Sherman_Firefly_short": "Sherman Firefly",
         "GB20_Crusader": "Crusader",
         "GB20_Crusader_short": "Crusader",
-        "GB21_A27BP": "Explorer",
-        "GB21_A27BP_short": "Explorer",
         "GB21_Cromwell": "Cromwell",
         "GB21_Cromwell_short": "Cromwell",
         "GB22_Comet": "Comet",
@@ -4817,8 +4701,6 @@
         "M-5-400F_Bis": "M-5-400F Bis",
         "M-50T": "M-50T",
         "M-50TD": "M-50TD",
-        "M-6-Y": "M-VI-Yoh",
-        "M-6-Y_short": "M-VI-Yoh",
         "M-80": "M-80",
         "M103": "M103",
         "M10_Wolverine": "M10 Wolverine",
@@ -4864,8 +4746,8 @@
         "M48A1_short": "M48 Patton",
         "M4A2E4": "M4A2E4 Sherman",
         "M4A2E4_short": "M4A2E4",
-        "M4A3E8TU": "Sherman Easy 8",
+        "M4A3E8TU": "M4A3E8 Sherman",
-        "M4A3E8TU_short": "Sherman Easy 8",
+        "M4A3E8TU_short": "M4A3E8",
         "M4A3E8_BP": "Rudolph",
         "M4A3E8_BP_short": "Rudolph",
         "M4A3E8_Sherman": "M4A3E8 Sherman",
@@ -4910,7 +4792,6 @@
         "MB517": "MB 517",
         "MB_837_Aa": "MB 837 Aa",
         "MB_837_Ea": "MB 837 Ea",
-        "MB_837_K": "MB 837 K",
         "MS-1": "MS-1",
         "MT-5-1": "MT-5-1",
         "MT25": "MT-25",
@@ -4998,7 +4879,6 @@
         "Maybach_HL_234_TRM_P45_1": "Maybach HL 234 TRM P45",
         "Maybach_HL_235": "Maybach HL 235",
         "Maybach_HL_295": "Maybach HL 295",
-        "Maybach_HL_295_003_Somua": "Maybach HL 295.003 \\\"Somua\\\"",
         "Maybach_HL_295_F": "Maybach HL 295 F",
         "Maybach_HL_295_FA": "Maybach HL 295 F",
         "Maybach_HL_45P": "Maybach HL 45 P",
@@ -5027,7 +4907,6 @@
         "Nuffield_liberty_MKII": "Nuffield Liberty Mk. II",
         "Nuffield_liberty_MKIV": "Nuffield Liberty Mk. III",
         "Nuffield_liberty_MKV": "Nuffield Liberty Mk. V",
-        "Nuffield_liberty_MKV_BP": "Nuffield Liberty Mk. VB",
         "OQF_17pdr_Gun_Mk_IV_SH_SH": "OQF 17-pdr Gun Mk. IV SH-SH",
         "OQF_17pdr_Gun_Mk_VII": "OQF 17-pdr Gun Mk. VII",
         "OQF_17pdr_Gun_Mk_VIIG": "OQF 17-pdr Gun Mk. VII G",
@@ -5321,8 +5200,6 @@
         "R109_T54S": "T-54 ltwt.",
         "R109_T54S_short": "T-54 ltwt.",
         "R110_Object_260": "Object 260",
-        "R110_Object_260_Custom": "Object 260 Pike",
-        "R110_Object_260_Custom_short": "Object 260 Pike",
         "R110_Object_260_short": "Obj. 260",
         "R111_ISU130": "ISU-130",
         "R111_ISU130_short": "ISU-130",
@@ -5524,8 +5401,8 @@
         "Skoda_AHK": "Škoda AHK",
         "Skoda_T-11": "Škoda T-11",
         "Skoda_T-15": "Škoda T-15",
-        "Skoda_V12_19_8_liter_450": "Škoda V12 mod. 1",
+        "Skoda_V12_19_8_liter_450": "Škoda V12 19.8 liter (450)",
-        "Skoda_V12_19_8_liter_500": "Škoda V12 mod. 2",
+        "Skoda_V12_19_8_liter_500": "Škoda V12 19.8 liter (500)",
         "Skoda_V16_diesel_600": "Škoda 16 cylinder diesel (600)",
         "Skoda_V16_diesel_660": "Škoda 16 cylinder diesel (660)",
         "Slpprj_m43": "Slpprj m/43",
@@ -5929,28 +5806,6 @@
         "chinaNY_short": "#other_vehicles:chinaNY_short",
         "skin_121B_Gold_dragon_Custom": "121B Invincible",
         "skin_121B_Gold_dragon_Custom_short": "121B Invincible",
-        "turret_IS-2Sh": "IS-2Sh",
+        "turret_IS-2Sh": "IS-2Sh"
-        "A130_Super_Hellcat": "Super Hellcat",
-        "A130_Super_Hellcat_short": "Super Hellcat",
-        "Cz13_T-27_Custom": "Škoda T 27 Rampant",
-        "Cz13_T-27_Custom_short": "Škoda T 27 Rampant",
-        "F113_PzV_FR": "Bretagne Panther",
-        "F113_PzV_FR_short": "Bretagne Panther",
-        "G144_Kpz_50t": "Kampfpanzer 50 t",
-        "G144_Kpz_50t_short": "Kampfpanzer 50 t",
-        "JagdPz_E100_Custom": "Jagdpanzer Mighty",
-        "JagdPz_E100_Custom_short": "Jg.Pz. Mighty",
-        "MB_837_F": "MB 837 F",
-        "Maybach_HL_230_TRM_P30_FR": "Maybach HL 230 TRM P30 fr.",
-        "Object_907_Custom": "Object 907 Ruthless",
-        "Object_907_Custom_short": "Object 907 Ruthless",
-        "S07_Strv74BP": "Turbo",
-        "S07_Strv74BP_Custom": "Turbo",
-        "S07_Strv74BP_Custom_short": "Turbo",
-        "S07_Strv74BP_short": "Turbo",
-        "T34_85BP": "Triumphant",
-        "T34_85BP_short": "Triumphant",
-        "V-2-34M_BP": "V-2-34M B",
-        "Wright_Continental_R_975_C4_858": "Wright Continental\\nR-975-C4 (858)"
     }
 }