WoTinspector.read_replay_id(): added back

Fix --filters and improve performance when filtering

.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+BlitzApp
+tmp/
+ssl/
+*.ini
+*.swp
+*.code-workspace
+*.sqlite3
+.vscode/
+*.sqlite3-journal
+.gitattributes
+export*
+*.csv
+!/stats/*.csv
+!*.py

README.md

@@ -1,5 +1,5 @@
 # blitz-tools
-Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or higher. I am using 3.8 myself so you may have to change the Python interpreter version in the scripts' first line. 
+Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or later. 
 
 ### Replays
 * [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader. 
@@ -13,46 +13,48 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
 * [blitzutils.py](blitzutils.py): A utils class for various Blitz related to functions. *Other scripts use this*, so you need to place it to the same folder with the other scripts
 
 ### Data
-* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py]](extractTankopedia.py]). Contains only limited information required by the scripts. 
-* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py]](extract_tankopedia.py]). 
+* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py](extractTankopedia.py). Contains only limited information required by the scripts. 
+* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py](extract_tankopedia.py). 
 
 ### Quick instructions
 
-1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
-2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
+1. Install [Python 3.8](https://www.python.org/downloads/) or later to your computer/laptop
+2. Install the required Python modules (see [requirements.txt](requirements.txt): motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
 ```
-# Remember to run the command with a Python interpreter >= 3.7
-python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress 
+python -m pip install -r requirements.txt
 ```
-3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. **The recommened way** is to install the scipts via `git clone`: [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for git. This makes it easy to receive the updates via `git pull`. 
-
-4. Upload your replays. 
+3. Clone the repository. [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for `git`. This makes it easy to receive the updates via `git pull`. 
 ```
-# Windows, run directly from Command Prompt or Powershell
-# Check the options
+git clone https://github.com/Jylpah/blitz-tools.git
+```
+4. Upload your replays with `upload_wotb_replays.py` 
+```
+## Check the options
+# Windows Command Prompt or Powershell
 python.exe .\upload_wotb_replays.py --help
 
-# Syntax
-python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
+# Linux or Windows Subsystem for Linux (WSL)
+./upload_wotb_replays.py --help
+
+# Upload replays (Windows)
+python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay  DIR_WITH_REPLAYS
 
 # example
 python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
-
-# Linux
-./upload_wotb_replays.py --help
 ```
-5. Analyze replays. 
+5. Analyze replays with `analyze_wotb_replays.py`
 ```
-# Windows, run directly from Command Prompt or Powershell
+## Check options
+# Windows Command Prompt or Powershell
 python.exe .\analyze_wotb_replays.py --help
 
-# Syntax
-python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
+# Syntax (Windows)
+python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json [DIR_WITH_REPLAYS ...]
 
-# Example
+# Example (Windows)
 python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
 
-# Linux
+# Linux or Windows Subsystem for Linux (WSL)
 ./analyze_wotb_replays.py --help
 ```
 

analyze_wotb_replays.py

@@ -29,6 +29,8 @@ STAT_TANK_BATTLE_MIN = 100
 BATTLE_TIME_BUCKET = 3600*24*14
 
 OPT_EXPORT_REPLAY_DIR = 'export_replays'
+OPT_EXPORT_CSV_FILE	 = 'export.csv'
+OPT_EXPORT_JSON_FILE = 'export.json'
 
 RE_SRC_IS_DB = re.compile(r'^DB:')
 
@@ -226,16 +228,41 @@ class BattleCategorizationList():
 		'top_tier'
 		]
 
+	## Player stats based categorizations
+	_categorizations_stats = [ group + '_' + measure for group in [ 'player', 'allies', 'enemies'] for measure in ['wins', 'battles', 'damage_dealt' ]]
+
+	
+	@classmethod
+	def get_categorizations_all(cls, cats: list = None) -> list:
+		try:
+			if cats == None:
+				return list(cls._categorizations.keys())
+			else:
+				return list(set(cls._categorizations.keys()) & set(cats)) 
+		except Exception as err:
+			bu.error(exception=err)
+		return None	
 
-	_categorizations_stats = [
-		'player_wins',
-		'player_battles',
-		'player_battles'
-	]
 
 	@classmethod
-	def get_categorizations_all(cls):
-		return cls._categorizations.keys()
+	def get_categorizations_stats(cls, cats=None, stats_cats=None):
+		try:
+			if cats == None:
+				return cls._categorizations_stats
+			else:
+				if stats_cats == None:
+					return cats
+				player_stat_filters = set(cls._categorizations_stats)
+				cats = set(cls.get_categorizations_all(cats))
+				if stats_cats:
+					cats = cats & player_stat_filters
+				else:
+					cats = cats - player_stat_filters
+				bu.debug(', '.join(list(cats)))
+				return list(cats)
+		except Exception as err:
+			bu.error(exception=err)
+		return None		
 
 
 	@classmethod
@@ -370,6 +397,10 @@ class BattleCategorizationList():
 		return BattleCategorizationListIterator(self)
 
 
+	def len(self):
+		return len(self.categorizations)
+
+
 	def get_categorization(self, cat: str):	
 		"""Return a BattleCategorization for a category"""
 		try:
@@ -378,6 +409,7 @@ class BattleCategorizationList():
 			bu.error(exception=err)
 		return None
 	
+
 	def record_result(self, result: dict):
 		try:
 			for cat in self.categorizations_list:
@@ -1388,7 +1420,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 ## main() -------------------------------------------------------------
 
 async def main(argv):
-	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT 
+	global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT, OPT_EXPORT_CSV_FILE, OPT_EXPORT_JSON_FILE 
 	# set the directory for the script
 	current_dir = os.getcwd()
 	os.chdir(os.path.dirname(sys.argv[0]))
@@ -1401,8 +1433,8 @@ async def main(argv):
 	OPT_HIST			= False
 	OPT_STAT_FUNC		= StatFunc.get_default()
 	OPT_WORKERS_N 		= 10
-	OPT_EXPORT_CSV_FILE	 = 'export.csv'
-	OPT_EXPORT_JSON_FILE = 'export.json'
+	# OPT_EXPORT_CSV_FILE	 = 'export.csv'
+	# OPT_EXPORT_JSON_FILE = 'export.json'
 
 	#WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					 	# China is not supported since WG API stats are not available there
@@ -1414,8 +1446,8 @@ async def main(argv):
 	## VERY unlikely you have a DB set up
 	DB_SERVER 	= 'localhost'
 	DB_PORT 	= 27017
-	DB_SSL		= False
-	DB_CERT_REQ = ssl.CERT_NONE
+	DB_TLS		= False
+	DB_CERT_REQ = False
 	DB_AUTH 	= 'admin'
 	DB_NAME 	= 'BlitzStats'
 	DB_USER		= 'mongouser'
@@ -1467,14 +1499,14 @@ async def main(argv):
 					OPT_DB		= configDB.getboolean('opt_DB', OPT_DB)
 					DB_SERVER 	= configDB.get('db_server', DB_SERVER)
 					DB_PORT 	= configDB.getint('db_port', DB_PORT)
-					DB_SSL		= configDB.getboolean('db_ssl', DB_SSL)
-					DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ)
+					DB_TLS		= configDB.getboolean('db_tls', DB_TLS)
+					DB_CERT_REQ = configDB.getboolean('db_tls_req', DB_CERT_REQ)
 					DB_AUTH 	= configDB.get('db_auth', DB_AUTH)
 					DB_NAME 	= configDB.get('db_name', DB_NAME)
 					DB_USER		= configDB.get('db_user', DB_USER)
 					DB_PASSWD 	= configDB.get('db_password', DB_PASSWD)
-					DB_CERT 	= configDB.get('db_ssl_cert_file', DB_CERT)
-					DB_CA 		= configDB.get('db_ssl_ca_file', DB_CA)
+					DB_CERT 	= configDB.get('db_tls_cert_file', DB_CERT)
+					DB_CA 		= configDB.get('db_tls_ca_file', DB_CA)
 			except (KeyError, configparser.NoSectionError)  as err:
 				bu.error(exception=err)
 
@@ -1520,7 +1552,7 @@ async def main(argv):
 		#### Connect to MongoDB. You are unlikely to have this set up... 
 		bu.debug('DB_SERVER: ' + DB_SERVER)
 		bu.debug('DB_PORT: ' + str(DB_PORT))
-		bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
+		bu.debug('DB_TLS: ' + "True" if DB_TLS else "False")
 		bu.debug('DB_AUTH: ' + DB_AUTH)
 		bu.debug('DB_NAME: ' + DB_NAME)
 		
@@ -1528,7 +1560,7 @@ async def main(argv):
 		db = None
 		if args.db:
 			try:
-				client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
+				client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, tls=DB_TLS, tlsAllowInvalidCertificates=DB_CERT_REQ, tlsCertificateKeyFile=DB_CERT, tlsCAFile=DB_CA)
 				db = client[DB_NAME]
 				bu.debug('Database connection initiated')
 			except Exception as err: 
@@ -1558,103 +1590,78 @@ async def main(argv):
 			args.account_id = await wg.get_account_id(args.account)
 			bu.debug('WG  account_id: ' + str(args.account_id))
 
-		try:
-			BattleCategory.set_fields(args.mode)
-			replayQ  = asyncio.Queue(maxsize=1000)			
-			reader_tasks = []
-			# Make replay Queue
+		# try:
+		BattleCategory.set_fields(args.mode)
+		replayQ  = asyncio.Queue(maxsize=1000)			
+		reader_tasks = []
+		# Make replay Queue
 
-			scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
-			bu.set_counter('Reading replays', 10)
-			# Start tasks to process the Queue
-			for i in range(OPT_WORKERS_N):
-				reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
-				bu.debug('ReplayReader ' + str(i) + ' started')
+		scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
+		bu.set_counter('Reading replays', 10)
+		# Start tasks to process the Queue
+		for i in range(OPT_WORKERS_N):
+			reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
+			bu.debug('ReplayReader ' + str(i) + ' started')
 
-			bu.debug('Waiting for the replay scanner to finish')
-			await asyncio.wait([scanner_task])
-			bu.finish_progress_bar()
-			# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
-			await replayQ.join()
-			await asyncio.sleep(0.1)
-			bu.debug('Replays read. Cancelling Readers and analyzing results')
-			for task in reader_tasks:
-				task.cancel()
-				await asyncio.sleep(0.1)	
-			results = []
-			players = set()
-			replays	= dict()
-			for res in await asyncio.gather(*reader_tasks):
-				results.extend(res[0])
-				players.update(res[1])
-				replays.update(res[2])
+		bu.debug('Waiting for the replay scanner to finish')
+		await asyncio.wait([scanner_task])
+		bu.finish_progress_bar()
+		# bu.debug('Scanner finished. Waiting for replay readers to finish the queue')
+		await replayQ.join()
+		await asyncio.sleep(0.1)
+		bu.debug('Replays read. Cancelling Readers and analyzing results')
+		for task in reader_tasks:
+			task.cancel()
+			await asyncio.sleep(0.1)	
+		results = []
+		players = set()
+		replays	= dict()
+		for res in await asyncio.gather(*reader_tasks):
+			results.extend(res[0])
+			players.update(res[1])
+			replays.update(res[2])
+		
+		if len(players) == 0:
+			raise Exception("No players found to fetch stats for. No replays found?")
+
+		if args.min != None:
+			results = filter_min_replays_by_player(results, args.min)
 			
-			if len(players) == 0:
-				raise Exception("No players found to fetch stats for. No replays found?")
+		# Filter based on non-stats filters
+		results = filter_results(results, args.filters, stats_filters=False)	
+		players = get_players(results)
+		
+		(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
+	
+		# Filter based on stats filters
+		results = filter_results(results, args.filters, stats_filters=True)
+		bu.debug(str(len(replays)) + ' replays to process')
+		replays = filter_replays(replays, results)				
+		
+		bu.debug('Number of player stats: ' + str(len(player_stats)))
+		teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
 
-			if args.min != None:
-				results = filter_min_replays_by_player(results, args.min)
-				players = get_players(results)
-				replays = filter_replays(replays, results)				
+		blt_cat_list = process_battle_results(teamresults, args)
+		bu.verbose_std('WR = ' + StatFunc.get_title())
+		blt_cat_list.print_results()
 
-			# Filter based on non-stats filters
-			results = filter_results(results, args.filters, stats_filters=False)	
+		histograms = None
+		if args.hist:
+			histograms = process_player_dist(results, player_stats, stat_id_map)
+			print_player_dist(histograms)			
+		
+		if args.json:
+			await export_json(args, blt_cat_list, histograms)
+		
+		if args.csv:
+			await export_csv(args, blt_cat_list, histograms)	
+		
+		if args.replays:
+			await export_replays(replays)
 
-      (player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
-      
-      # Filter based on stats filters
-			results = filter_results(results, args.filters, stats_filters=True)
-
-			bu.debug('Number of player stats: ' + str(len(player_stats)))
-			teamresults = calc_team_stats(results, player_stats, stat_id_map, args)
-
-			blt_cat_list = process_battle_results(teamresults, args)
-			bu.verbose_std('WR = ' + StatFunc.get_title())
-			blt_cat_list.print_results()
-
-			histograms = None
-			if args.hist:
-				histograms = process_player_dist(results, player_stats, stat_id_map)
-				print_player_dist(histograms)			
-			
-			if args.json:
-				res = blt_cat_list.get_results_json()
-				if histograms != None:
-					res_hist = dict()
-					for histogram in histograms.keys():
-						res_hist[histogram] = histograms[histogram].get_results()
-					res['histograms'] = res_hist
-				if args.outfile == '-':
-					export_file = OPT_EXPORT_JSON_FILE
-				else:
-					export_file = args.outfile
-				await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
-				bu.verbose_std('Results exported to: ' + export_file)
-			
-			if args.csv:
-				rows = blt_cat_list.get_results_list()
-				if args.hist:
-					rows.append('')
-					rows.append(['Player Histograms'])
-					for histogram in histograms.values():
-						rows.append('')
-						rows.extend(histogram.get_results_list())
-				if args.outfile == '-':
-					export_file = OPT_EXPORT_CSV_FILE
-				else:
-					export_file = args.outfile
-				with open(export_file, 'w') as csvfile:  
-					# creating a csv writer object  
-					csvwriter = csv.writer(csvfile)        					
-					csvwriter.writerows(rows)
-					bu.verbose_std('Results exported to: ' + export_file)			
-			
-			if args.replays:
-				await export_replays(replays)
-
-			bu.debug('Finished. Cleaning up..................')
-		except Exception as err:
-			bu.error(exception=err)
+		bu.debug('Finished. Cleaning up..................')
+		# except Exception as err:
+		# 	bu.error(exception=err)
 	except UserWarning as err:
 		bu.warning(str(err))
 	except asyncio.CancelledError:
@@ -1670,6 +1677,48 @@ async def main(argv):
 	return None
 
 
+async def export_csv(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
+	"""Export results into a CSV file"""
+	try:
+		rows = blt_cat_list.get_results_list()
+		if args.hist:
+			rows.append('')
+			rows.append(['Player Histograms'])
+			for histogram in histograms.values():
+				rows.append('')
+				rows.extend(histogram.get_results_list())
+		if args.outfile == '-':
+			export_file = OPT_EXPORT_CSV_FILE
+		else:
+			export_file = args.outfile
+		with open(export_file, 'w') as csvfile:  
+			# creating a csv writer object  
+			csvwriter = csv.writer(csvfile)        					
+			csvwriter.writerows(rows)
+			bu.verbose_std('Results exported to: ' + export_file)		
+	except Exception as err:
+		bu.error(exception=err)
+
+
+async def export_json(args : argparse.Namespace, blt_cat_list: BattleCategorizationList, histograms: dict = None):
+	"""Export results into a JSON file"""
+	try:
+		res = blt_cat_list.get_results_json()
+		if histograms != None:
+			res_hist = dict()
+			for histogram in histograms.keys():
+				res_hist[histogram] = histograms[histogram].get_results()
+			res['histograms'] = res_hist
+		if args.outfile == '-':
+			export_file = OPT_EXPORT_JSON_FILE
+		else:
+			export_file = args.outfile
+		await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
+		bu.verbose_std('Results exported to: ' + export_file)
+	except Exception as err:
+		bu.error(exception=err)
+
+
 async def help_extended(db : motor.motor_asyncio.AsyncIOMotorDatabase = None, parser: argparse.ArgumentParser = None):
 	"""Help for --filters"""
 	if parser != None:
@@ -1851,21 +1900,28 @@ def process_battle_results(results: dict, args : argparse.Namespace):
 	return blt_cat_list
 
 
-def filter_results(results: list, filter_json : str, stats_filters=False) -> bool:
+def filter_results(results: list, filters_json : str, stats_filters=None) -> bool:
 	"""Filter replays based on battle category filters"""
 	try:
-		filters = json.loads(filter_json)
+		if filters_json == None: 
+			return results
+		filters = json.loads(filters_json)
 		bu.debug(str(filters))
 		if type(filters) != dict:
 			bu.error('invalid --filter arguments: ' + str(filter))
 			sys.exit(1)
 		
-		filter_cats = BattleCategorizationList(filters.keys())
+		filter_cats = BattleCategorizationList.get_categorizations_stats(filters.keys(), stats_filters)
+		filter_cats = BattleCategorizationList(filter_cats)
+		if filter_cats.len() == 0:
+			bu.debug('Nothing to filter')
+			return results
+
 		bu.verbose('Filters applied:')
 		for category in filter_cats:
 			cat = category.category_key
 			categorization = filter_cats.get_categorization(cat)
-			bu.debug('Category: ' + cat + ' filters type: ' + str(type(filters)) + ' filters: ' + str(filters))
+			bu.debug('Category: ' + cat +  ' filters: ' + str(filters))
 			filters[cat] = categorization.get_filter_categories(filters[cat])
 			bu.verbose('  ' + cat + ' = ' + ','.join(filters[cat]))		
 		bu.verbose('')
@@ -1885,8 +1941,12 @@ def filter_results(results: list, filter_json : str, stats_filters=False) -> boo
 					res.append(result)
 			except Exception as err:
 				bu.error(exception=err)
+		if len(res) == 0:
+			raise UserWarning('No replays left after filtering')
 		bu.verbose_std('Replays after filtering: ' + str(len(res)))	
 		return res
+	except UserWarning:
+		raise
 	except Exception as err:
 		bu.error(exception=err)
 	return None
@@ -1961,9 +2021,9 @@ async def process_player_stats(players, N_workers: int, args : argparse.Namespac
 def remap_stat_id(stat_id_map: dict, stat_id_remap: dict):
 	bu.debug('Remapping stat_ids: stat_id_map: ' + str(len(stat_id_map)) + ' remap needed: ' + str(len(stat_id_remap)))
 	for stat_id in stat_id_map:
-	 	# bu.debug(stat_id)
-	 	if stat_id_map[stat_id] in stat_id_remap.keys():
-	 		stat_id_map[stat_id] = stat_id_remap[stat_id_map[stat_id]]
+		# bu.debug(stat_id)
+		if stat_id_map[stat_id] in stat_id_remap.keys():
+			stat_id_map[stat_id] = stat_id_remap[stat_id_map[stat_id]]
 	return stat_id_map
 
 
@@ -2439,7 +2499,7 @@ async def mk_readerQ_item(replay_json, filename : str = None, _id: str = None) -
 	try:
 		if wi.chk_JSON_replay(replay_json): 
 			if not '_id' in replay_json:
-				if _id  != None:			
+				if _id  == None:			
 					_id = wi.read_replay_id(replay_json)
 				replay_json['_id'] = _id
 		else:

blitzutils.py

@@ -36,132 +36,163 @@ _progress_obj = None
 UMASK = os.umask(0)
 os.umask(UMASK)
 
-# -----------------------------------------------------------
-# Class ThrottledClientSession(aiohttp.ClientSession)
-# -----------------------------------------------------------
+## -----------------------------------------------------------
+#### Class ThrottledClientSession(aiohttp.ClientSession)
+#
+#  Rate-limited async http client session
+#
+#  Inherits aiohttp.ClientSession 
+## -----------------------------------------------------------
 
+from typing import Optional, Union
+import aiohttp
+import asyncio
+import time
+import logging
+import re
 
 class ThrottledClientSession(aiohttp.ClientSession):
-    """Rate-throttled client session class inherited from aiohttp.ClientSession)"""
-    # MIN_SLEEP = 0.1
+    """Rate-throttled client session class inherited from aiohttp.ClientSession)""" 
 
-    def __init__(self, rate_limit: float = None, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        self.rate_limit = None
-        self._fillerTask = None
-        self._queue = None
-        self._start_time = None
-        self._count = 0
-        self._start_time = time.time()
+    def __init__(self, rate_limit: float = 0, filters: list[str] = list() , 
+                limit_filtered: bool = False, re_filter: bool = False, *args,**kwargs) -> None: 
+        assert isinstance(rate_limit, (int, float)),   "rate_limit has to be float"
+        assert isinstance(filters, list),       "filters has to be list"
+        assert isinstance(limit_filtered, bool),"limit_filtered has to be bool"
+        assert isinstance(re_filter, bool),     "re_filter has to be bool"
 
-        if rate_limit != None:
-            if rate_limit <= 0:
-                raise ValueError('rate_limit must be positive')
-            self.rate_limit = rate_limit
-            self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
-            self._fillerTask = asyncio.create_task(self._filler())
+        super().__init__(*args,**kwargs)
+        
+        self.rate_limit     : float
+        self._fillerTask    : Optional[asyncio.Task]    = None
+        self._queue         : Optional[asyncio.Queue]   = None
+        self._start_time    : float = time.time()
+        self._count         : int = 0
+        self._limit_filtered: bool = limit_filtered
+        self._re_filter     : bool = re_filter
+        self._filters       : list[Union[str, re.Pattern]] = list()
 
-    def _get_sleep(self) -> list:
-        if self.rate_limit == None:
-            return None
+        if re_filter:
+            for filter in filters:
+                self._filters.append(re.compile(filter))
         else:
+            for filter in filters:
+                self._filters.append(filter)
+        self.set_rate_limit(rate_limit)
+
+
+    def _get_sleep(self) -> float:        
+        if self.rate_limit > 0:
             return 1/self.rate_limit
+        return 0
+
 
     def get_rate(self) -> float:
         """Return rate of requests"""
         return self._count / (time.time() - self._start_time)
 
-    def get_stats(self):
+
+    def get_stats(self) -> dict[str, float]:
         """Get session statistics"""
-        res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
-                                     'count': self._count}
+        res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
         return res
+        
 
-    def get_stats_str(self):
+    def get_stats_str(self) -> str:
         """Print session statistics"""
-        return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
-                ' rate: ' + \
-                    "{0:.1f}".format(self.get_rate()) + \
-                                     ' requests: ' + str(self._count)
+        return f"rate limit: {str(self.rate_limit if self.rate_limit != None else '-')} \
+                rate:   {0:.1f}.format(self.get_rate()) requests: {str(self._count)}"
 
-    def reset_counters(self):
+
+    def reset_counters(self) -> dict[str, float]:
         """Reset rate counters and return current results"""
         res = self.get_stats()
         self._start_time = time.time()
         self._count = 0
         return res
 
-    def set_rate_limit(self, rate_limit: float = 1):
-        if rate_limit >= 0:
-            self.rate_limit = rate_limit
-            return self.rate_limit
-        return None
+
+    def set_rate_limit(self, rate_limit: float = 0) -> float:
+        assert rate_limit is not None, "rate_limit must not be None" 
+        assert isinstance(rate_limit, (int,float)) and rate_limit >= 0, "rate_limit has to be type of 'float' >= 0"
+        
+        self.rate_limit = rate_limit
+        if rate_limit > 0:
+            self._queue     = asyncio.Queue(int(rate_limit)+1) 
+            if self._fillerTask is not None: 
+                self._fillerTask.cancel()  
+            self._fillerTask = asyncio.create_task(self._filler())
+        return self.rate_limit
+        
 
     async def close(self) -> None:
         """Close rate-limiter's "bucket filler" task"""
-        # DEBUG
-        debug(self.get_stats_str())
+        # DEBUG 
+        logging.debug(self.get_stats_str())
         try:
-            if self._fillerTask != None:
+            if self._fillerTask is not None:
                 self._fillerTask.cancel()
-            await asyncio.wait_for(self._fillerTask, timeout=0.5)
-        except (asyncio.CancelledError, asyncio.TimeoutError):
-            pass
+                await asyncio.wait_for(self._fillerTask, timeout=0.5)
+        except asyncio.TimeoutError as err:
+            logging.error(str(err))
         await super().close()
 
-    async def _filler_simple(self):
+    
+    async def _filler(self) -> None:
         """Filler task to fill the leaky bucket algo"""
         try:
-            if self._queue == None:
-                return
-            sleep = self._get_sleep()
-            while True:
-                self._queue.put(None)
-                await asyncio.sleep(sleep)
-        except asyncio.CancelledError:
-            debug('Cancelled')
-        except Exception as err:
-            error(exception=err)
-
-    async def _filler(self):
-        """Filler task to fill the leaky bucket algo"""
-        try:
-            if self._queue == None:
-                return
-            sleep = self._get_sleep()
-            # debug('SLEEP: ' + str(sleep))
+            if self._queue is None:
+                return None            
+            logging.debug('SLEEP: ' + str(self._get_sleep()))
             updated_at = time.monotonic()
-            fraction = 0
-            extra_increment = 0
+            extra_increment : float = 0
             for i in range(0, self._queue.maxsize):
-                self._queue.put_nowait(i)
+                await self._queue.put(i)
             while True:
                 if not self._queue.full():
                     now = time.monotonic()
                     increment = self.rate_limit * (now - updated_at)
-                    fraction += increment % 1
-                    extra_increment = fraction // 1
-                    items_2_add = int(
-                        min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
-                    fraction = fraction % 1
-                    for i in range(0, items_2_add):
+                    items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment + extra_increment)))
+                    extra_increment = (increment + extra_increment) % 1
+                    for i in range(0,items_2_add):
                         self._queue.put_nowait(i)
                     updated_at = now
-                await asyncio.sleep(sleep)
+                await asyncio.sleep(self._get_sleep())
         except asyncio.CancelledError:
-            debug('Cancelled')
+            logging.debug('Cancelled')
         except Exception as err:
-            error(exception=err)
+            logging.error(str(err))
+        return None
 
-    async def _request(self, *args, **kwargs):
+
+    async def _request(self, *args,**kwargs) -> aiohttp.ClientResponse:
         """Throttled _request()"""
-        if self._queue != None:
+        if self._queue is not None and self.is_limited(*args):  
             await self._queue.get()
             self._queue.task_done()
         self._count += 1
-        return await super()._request(*args, **kwargs)
+        return await super()._request(*args,**kwargs)
 
 
+    def is_limited(self, *args: str) -> bool:
+        """Check wether the rate limit should be applied"""
+        try:
+            url: str = args[1]
+            for filter in self._filters:
+                if isinstance(filter, re.Pattern) and filter.match(url) is not None:
+                    return self._limit_filtered
+                elif isinstance(filter, str) and url.startswith(filter):
+                    return self._limit_filtered
+                    
+            return not self._limit_filtered
+        except Exception as err:
+            logging.error(str(err))
+        return True    
+
+# -----------------------------------------------------------
+# Class AsyncLogger()
+# -----------------------------------------------------------
+
 class AsyncLogger():
     """Async file logger"""
 
@@ -496,18 +527,17 @@ def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
     return None
 
 
-def rebase_file_args(current_dir, files):
-    """REbase file command line params after moving working dir to the script's dir""" 
-    if isinstance(files, list):    
-        if (files[0] == '-') or (files[0] == 'db:'):
-            return files
-        else:
-            return [ os.path.join(current_dir, fn) for fn in files ]
-    elif isinstance(files, str):
-        return os.path.join(current_dir, files)
+def rebase_file_args(current_dir : str, files: list[str]) -> list[str]:
+    """Rebase file command line params after moving working dir to the script's dir""" 
+    assert isinstance(files, list), "files has to be a list"
+
+    if (files[0] == '-') or (files[0] == 'db:'):
+        return [ files[0] ] 
+    else:
+        return [ os.path.join(current_dir, fn) for fn in files ]
 
 
-async def read_int_list(filename: str) -> list():
+async def read_int_list(filename: str) -> list[int]:
     """Read file to a list and return list of integers in the input file"""
     
     input_list = []
@@ -1596,6 +1626,8 @@ class WG:
 
     async def store_tank_stats(self, key: list , stats_data: list, update_time: int):
         """Save tank stats into cache"""
+        assert self.cache is not None, "cache must be initiated"
+        
         try:
             account_id  = key[0]
             tank_ids    = set(key[1])
@@ -1617,6 +1649,8 @@ class WG:
 
     async def store_player_stats(self, key: list , stats_data: list, update_time: int):
         """Save player stats into cache"""
+        assert self.cache is not None, "cache must be initiated"
+        
         try:
             account_id  = key[0]
             if stats_data != None:
@@ -1633,6 +1667,8 @@ class WG:
 
     async def store_player_achievements(self, key: list , stats_data: list, update_time: int):
         """Save player stats into cache"""
+        assert self.cache is not None, "cache must be initiated"
+        
         try:
             account_id  = key[0]
             if stats_data != None:
@@ -1648,6 +1684,8 @@ class WG:
 
 
     async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ):
+        assert self.cache is not None, "cache must be initiated"
+
         try:
             # test for cacheDB existence
             debug('Trying cached stats first')
@@ -1761,17 +1799,27 @@ class WoTinspector:
     URL_REPLAY_VIEW = URL_WI +'/en/view/'
     URL_REPLAY_UL   = 'https://api.wotinspector.com/replay/upload?'
     URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
-    
     URL_TANK_DB     = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
 
     REPLAY_N = 1
+    DEFAULT_RATE_LIMIT : float = 10/60  # 10 requests / min
+    DEFAULT_TOKEN : str = 'a5428af49bf7495986577d59b0e5bcff'
 
-    def __init__(self, rate_limit: int = 30):
-        self.session = ThrottledClientSession(rate_limit=rate_limit)
+    def __init__(self, rate_limit: float = DEFAULT_RATE_LIMIT, auth_token: Optional[str] = None):
+
+        headers : Optional[dict[str, str]] = None
+
+        if auth_token is None:
+            auth_token = self.DEFAULT_TOKEN        
+        headers = dict()
+        headers['Authorization'] = 'Token ' + auth_token
+        self.session = ThrottledClientSession(rate_limit=rate_limit, 
+                                                filters=[self.URL_REPLAY_LIST, self.URL_REPLAY_INFO], 
+                                                re_filter=False, limit_filtered=True, headers = headers)
         
 
-    async def close(self):
-        if self.session != None:
+    async def close(self) -> None:
+        if self.session is not None:
             debug('Closing aiohttp session')
             await self.session.close()
         
@@ -1835,7 +1883,7 @@ class WoTinspector:
             hash.update(data)
             replay_id = hash.hexdigest()
 
-            # Testing if the replay has already been posted
+            ##  Testing if the replay has already been posted
             json_resp = await self.get_replay_JSON(replay_id)
             if json_resp != None:
                 debug('Already uploaded: ' + title, id=N)
@@ -1850,7 +1898,7 @@ class WoTinspector:
             } 
 
             url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
-            # debug('URL: ' + url)
+            #debug('URL: ' + url)
             headers ={'Content-type':  'application/x-www-form-urlencoded'}
             payload = { 'file' : (filename, base64.b64encode(data)) }
         except Exception as err:
@@ -1875,8 +1923,6 @@ class WoTinspector:
             except Exception as err:
                 debug(exception=err, id=N)
             await asyncio.sleep(SLEEP)
-            _id  = self.read_replay_id(json_resp)
-            json_resp['_id']  =  _id
             
         debug(' Could not post replay: ' + title, id=N)
         return json_resp
@@ -1914,11 +1960,6 @@ class WoTinspector:
         return replay_links
     
 
-    @classmethod
-    def get_replay_id(cls, url):
-        return url.rsplit('/', 1)[-1]
-
-
     @classmethod
     def read_replay_id(cls, json_replay):
         """Read replay_id from replay JSON file""" 
@@ -1929,14 +1970,18 @@ class WoTinspector:
             error(exception=err)
         return None
 
+
     @classmethod
-    def chk_JSON_replay(cls, json_resp):
+    def get_replay_id(cls, url):
+        return url.rsplit('/', 1)[-1]
+
+
+    @classmethod
+    def chk_JSON_replay(cls, json_resp) -> bool:
         """"Check String for being a valid JSON file"""
         try:
-            # FIX The replay can have allies/enemies i.e. be useful for 
-            # team composition analysis, but does not have player contribution
             if ('status' in json_resp) and json_resp['status'] == 'ok' and \
-                (get_JSON_value(json_resp, key='data.summary.winner_team') != None) :
+                (get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
                 debug("JSON check OK")
                 return True 
         except KeyError as err:
@@ -1945,6 +1990,9 @@ class WoTinspector:
             debug("Replay JSON check failed: " + str(json_resp))
         return False      
 
+# -----------------------------------------------------------
+# Class BlitzStars 
+# -----------------------------------------------------------
 
 class BlitzStars:
 

extract_tankopedia.py

@@ -3,6 +3,7 @@
 
 # Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
 
+from os import path
 import sys, argparse, json, os, inspect, aiohttp, asyncio, aiofiles, re, logging, time, xmltodict, collections, configparser
 import blitzutils as bu
 from blitzutils import WG
@@ -11,9 +12,11 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG)
 
 FILE_CONFIG 	= 'blitzstats.ini'
 
-BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
-BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
-BLITZAPP_VEHICLE_FILE='/list.xml'
+# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
+# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
+BLITZAPP_STRINGS='Data/Strings/en.yaml'
+BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
+BLITZAPP_VEHICLE_FILE='list.xml'
 
 wg = None
 
@@ -104,8 +107,11 @@ async def main(argv):
 async def extract_tanks(blitz_app_base : str, nation: str):
 
     tanks = []
-    list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
-    if not os.path.isfile(list_xml_file): 
+    # WG has changed the location of Data directory - at least in steam client
+    if path.isdir(path.join(blitz_app_base, 'assets')):
+        blitz_app_base = path.join(blitz_app_base, 'assets')
+    list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
+    if not path.isfile(list_xml_file): 
         print('ERROR: cannot open ' + list_xml_file)
         return None
     bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
@@ -132,7 +138,7 @@ async def read_user_strs(blitz_app_base : str) -> dict:
     """Read user strings to convert map and tank names"""
     tank_strs = {}
     map_strs = {}
-    filename = blitz_app_base + BLITZAPP_STRINGS
+    filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
     bu.debug('Opening file: ' + filename + ' for reading UserStrings')
     try:
         async with aiofiles.open(filename, 'r', encoding="utf8") as f:

maps.json

@@ -18,6 +18,7 @@
     "fort": "Fort Despair",
     "grossberg": "Dynasty's Pearl",
     "himmelsdorf": "Himmelsdorf",
+    "holland": "Molendijk",
     "holmeisk": "Wasteland",
     "iceworld": "Everfrost",
     "idle": "Yukon",
@@ -56,4 +57,4 @@
     "skit": "Naval Frontier",
     "test": "World of Ducks",
     "tutorial": "Proving Grounds"
-}
+}

releases.md

@@ -1,88 +1,96 @@
 # WoT Blitz releases 
 
-| Release | Date     | Notes       | Inaccurate? |
-|:-------:|:--------:|-------------|-----|
-| 8.4 | 2021-10-20 | Polish heavy tanks |  |
-| 8.3 | 2021-09-16 | FV4005, FV4022, VK 91.01 (P) and WZ-121 legendary camos, -5 deg gun depression to WZ-120 |  |
-| 8.2 | 2021-08-18 | Ghost Factory map, XP/credits nerf for inactive players, new store UI |  |
-| 8.1 | 2021-07-15 | Armor highlighting updated |  |
-| 8.0 | 2021-06-08 | American experimental heavies (Yoh-line), PBR camos |  |
-| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |  |
-| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables  |  |
-| 7.7 | 2021-02-24 | Czechoslovakian medium tanks  |  |
-| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera  |  |
-| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |  |
-| 7.4 | 2020-11-12 | TD buffs, Clan missions |  |
-| 7.3 | 2020-10-08 | Swedish HT line |  |
-| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles |  |
-| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff  |  |
-| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes |  |
-| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes |  |
-| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff |  |
-| 6.8 | 2020-02-19 | ATGM nerf, Hellas map |  |
-| 6.7 | 2020-01-15 | ATGM US Lights |  | 
-| 6.6 | 2019-12-18 | T-62A + Leo buffs |  | 
-| 6.5 | 2019-11-21 | New UI |  | 
-| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... |  | 
-| 6.3 | 2019-09-18 | European line |  | 
-| 6.2 | 2019-08-14 | New MM |  | 
-| 6.1 | 2019-07-10 | Autoloader buff |  | 
-| 6.0 | 2019-06-05 | UK TD line #2 |   | 
-| 5.10 | 2019-04-24 | new missions |   | 
-| 5.9 | 2019-03-27 | New shell tracers, minor fixes |  | 
-| 5.8 | 2019-02-21 | Chinese TD line |  | 
-| 5.7 | 2019-01-16 | Avatars |  | 
-| 5.6 | 2018-12-05 | Normandy map |   | 
-| 5.5 | 2018-11-13 | Low-tier clean-up |  | 
-| 5.4 | 2018-10-10 | MM "rigging" adjusted |   | 
-| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added |   | 
-| 5.2 | 2018-08-22 | Faust map |  | 
-| 5.1 | 2018-07-31 | New MM: vehicle class matching |   | 
-| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts killed |   | 
-| 4.10 | 2018-05-24 | Alpenstadt map |   | 
-| 4.9 | 2018-04-25 | Chinese heavy tanks |  | 
-| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator |   | 
-| 4.7 | 2018-02-28 | New minimap |   | 
-| 4.6 | 2018-01-31 | Chinese nation |   | 
-| 4.5 | 2017-12-19 | Dynasty's Pearl map |  | 
-| 4.4 | 2017-11-09 | New British tanks |   | 
-| 4.3 | 2017-10-10 | Level 7 screw skills |   | 
-| 4.2 | 2017-09-12 | French LTs, Naval Frontier map |   | 
-| 4.1 | 2017-08-16 | New markers |   | 
-| 4.0 | 2017-07-13 | Rating battles |  Y | 
-| 3.10 | 2017-06-15 | Fixes | Y | 
-| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins | Y | 
-| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system | Y |
-| 3.7 | 2017-03-22 | Balance changes, Tournaments | Y |
-| 3.6 | 2017-02-16 | Vehicle Enrichment | Y |
-| 3.5 | 2017-01-13 | Boosters, Dead rail update | Y |
-| 3.4 | 2016-11-28 | Canyon map, USA T57 line |  |
-| 3.3 | 2016-11-01 | Updated Mines map |  |
-| 3.2 | 2016-10-04 | Copperfield update |  |
-| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map | Y |
-| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update | Y |
-| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs | |
-| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 |  |
-| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker | Y |
-| 2.8 | 2016-04-04 | New Russian TD line, new consumables system | Y |
-| 2.7 | 2016-03-07 | Mirage map | Y |
-| 2.6 | 2016-02-10 | German LTs, shooting through obstacles | Y |
-| 2.5 | 2016-01-11 | Camouflages, Clan supply | Y |
-| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version | Y |
-| 2.3 | 2015-11-12 | Canal map | Y |
-| 2.2 | 2015-10-05 | Desert Sands map update | Y |
-| 2.1 | 2015-09-07 | UK TDs, Clan Chat | Y |
-| 2.0 | 2015-08-05 | Clans, Oasis Palms updated | |
-| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo | Y |
-| 1.10 | 2015-06-11 | Port Bay map | Y |
-| 1.9 | 2015-05-13 | New German medium tanks (Leo) | Y |
-| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players  | Y |
-| 1.7 | 2015-02-16 | US Turreted TDs | Y |
-| 1.6 | 2015-01-19 | Castilla map, UK HT line | Y |
-| 1.5 | 2014-12-01 | UK Nation | Y |
-| 1.4 | 2014-10-29 | German HT line (Maus) | Y |
-| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps | Y |
-| 1.2 | 2014-08-20 | Oasis Palms map | Y |
-| 1.1.1 | 2014-07-23 | Fixes :-) |  |
-| 1.0 | 2014-06-26 | Worldwide launch |  | 
-| 0.9 | 2014-05-05 | Nordic soft launch |   |
+| Release | Date     | Notes       |
+|:-------:|:--------:|-------------|
+| 9.1 | 2022-07-14 | Releases updated via DLC, graphics updates for Object 140, Patton 46 |
+| 9.0 | 2022-06-08 | Japanies heavies, Castilla graphics update, new garage |
+| 8.10 | 2022-04-28 | New battle results screen, new "Operation" mechanics |
+| 8.9 | 2022-03-23 | "Free" camo resupply, VK 72.01 supplies |
+| 8.8 | 2022-02-16 | Molendijk map |
+| 8.7 | 2022-01-19 | Soviet Light Tanks |
+| 8.6 | 2021-12-15 | Joining Light and Medium Tank classes in MM |
+| 8.5 | 2021-11-17 | New national crew voices, economy update |
+| 8.4 | 2021-10-20 | Polish heavy tanks |
+| 8.3 | 2021-09-16 | FV4005, FV4022, VK 91.01 (P) and WZ-121 legendary camos, -5 deg gun depression to WZ-120 |
+| 8.2 | 2021-08-18 | Ghost Factory map, XP/credits nerf for inactive players, new store UI |
+| 8.1 | 2021-07-15 | Armor highlighting updated |
+| 8.0 | 2021-06-08 | American experimental heavies (Yoh-line), PBR camos |
+| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |
+| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables |
+| 7.7 | 2021-02-24 | Czechoslovakian medium tanks |
+| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera |
+| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |
+| 7.4 | 2020-11-12 | TD buffs, Clan missions |
+| 7.3 | 2020-10-08 | Swedish HT line |
+| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles |
+| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff  |
+| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes |
+| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes |
+| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff |
+| 6.8 | 2020-02-19 | ATGM nerf, Hellas map |
+| 6.7 | 2020-01-15 | ATGM US Lights |
+| 6.6 | 2019-12-18 | T-62A + Leo buffs |
+| 6.5 | 2019-11-21 | New UI |
+| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... |
+| 6.3 | 2019-09-18 | European line |
+| 6.2 | 2019-08-14 | New MM |
+| 6.1 | 2019-07-10 | Autoloader buff |
+| 6.0 | 2019-06-05 | UK TD line #2 | 
+| 5.10 | 2019-04-24 | new missions | 
+| 5.9 | 2019-03-27 | New shell tracers, minor fixes |
+| 5.8 | 2019-02-21 | Chinese TD line |
+| 5.7 | 2019-01-16 | Avatars |
+| 5.6 | 2018-12-05 | Normandy map |
+| 5.5 | 2018-11-13 | Low-tier clean-up :-( |
+| 5.4 | 2018-10-10 | MM "rigging" adjusted |
+| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added | 
+| 5.2 | 2018-08-22 | Faust map |
+| 5.1 | 2018-07-31 | New MM: vehicle class matching |
+| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts removed ! |
+| 4.10 | 2018-05-24 | Alpenstadt map | 
+| 4.9 | 2018-04-25 | Chinese heavy tanks |
+| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator |
+| 4.7 | 2018-02-28 | New minimap |
+| 4.6 | 2018-01-31 | Chinese nation |
+| 4.5 | 2017-12-19 | Dynasty's Pearl map |
+| 4.4 | 2017-11-09 | New British tanks | 
+| 4.3 | 2017-10-10 | Level 7 screw skills |
+| 4.2 | 2017-09-12 | French LTs, Naval Frontier map |
+| 4.1 | 2017-08-16 | New markers |
+| 4.0 | 2017-07-13 | Rating battles (date uncertain) | 
+| 3.10 | 2017-06-15 | Fixes (date uncertain) |
+| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins (date uncertain) |
+| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system (date uncertain) |
+| 3.7 | 2017-03-22 | Balance changes, Tournaments (date uncertain) |
+| 3.6 | 2017-02-16 | Vehicle Enrichment (date uncertain) |
+| 3.5 | 2017-01-13 | Boosters, Dead rail update (date uncertain) |
+| 3.4 | 2016-11-28 | Canyon map, USA T57 line |
+| 3.3 | 2016-11-01 | Updated Mines map |
+| 3.2 | 2016-10-04 | Copperfield update |
+| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map (date uncertain) |
+| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update (date uncertain) |
+| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs |
+| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 |
+| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker (date uncertain) |
+| 2.8 | 2016-04-04 | New Russian TD line, new consumables system (date uncertain) |
+| 2.7 | 2016-03-07 | Mirage map (date uncertain) |
+| 2.6 | 2016-02-10 | German LTs, shooting through obstacles (date uncertain) |
+| 2.5 | 2016-01-11 | Camouflages, Clan supply (date uncertain) |
+| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version (date uncertain) |
+| 2.3 | 2015-11-12 | Canal map (date uncertain) |
+| 2.2 | 2015-10-05 | Desert Sands map update (date uncertain) |
+| 2.1 | 2015-09-07 | UK TDs, Clan Chat (date uncertain) |
+| 2.0 | 2015-08-05 | Clans, Oasis Palms updated |
+| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo (date uncertain) |
+| 1.10 | 2015-06-11 | Port Bay map (date uncertain) |
+| 1.9 | 2015-05-13 | New German medium tanks (Leo) (date uncertain) |
+| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players  (date uncertain) |
+| 1.7 | 2015-02-16 | US Turreted TDs (date uncertain) |
+| 1.6 | 2015-01-19 | Castilla map, UK HT line (date uncertain) |
+| 1.5 | 2014-12-01 | UK Nation (date uncertain) |
+| 1.4 | 2014-10-29 | German HT line (Maus) (date uncertain) |
+| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps (date uncertain) |
+| 1.2 | 2014-08-20 | Oasis Palms map (date uncertain) |
+| 1.1.1 | 2014-07-23 | Fixes :-) |
+| 1.0 | 2014-06-26 | Worldwide launch | 
+| 0.9 | 2014-05-05 | Nordic soft launch |

upload_wotb_replays.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 ## PYTHON VERSION MUST BE 3.7 OR HIGHER
 
-import json, argparse, inspect, sys, os, io, base64, aiohttp, urllib, asyncio, aiofiles, aioconsole
-import logging, re, concurrent.futures, configparser, motor.motor_asyncio, ssl, zipfile
+import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
+import logging, re, configparser, zipfile
 import blitzutils as bu
 from blitzutils import WG
 from blitzutils import WoTinspector
@@ -21,17 +21,17 @@ SKIPPED_N 	= 0
 ERROR_N 	= 0
 WIurl='https://wotinspector.com/api/replay/upload?'
 WG_appID  = '81381d3f45fa4aa75b78a7198eb216ad'
-wg = None
-wi = None
+wg : WG | None = None
+wi : WoTinspector | None = None
 
-async def main(argv):
+async def main(argv) -> int:
 	global wg, wi
 	current_dir = os.getcwd()
 	# set the directory for the script
 	os.chdir(os.path.dirname(sys.argv[0]))
 	
 	# options defaults
-	OPT_WORKERS_N  	= 5
+	OPT_WORKERS_N  	= 2
 	WG_ID 			= None
 	WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					# China is not supported since WG API stats are not available there
@@ -43,13 +43,7 @@ async def main(argv):
 			bu.debug('Reading config file: ' + FILE_CONFIG)
 			config = configparser.ConfigParser()
 			config.read(FILE_CONFIG)
-			try:
-				if 'UPLOADER' in config.sections():
-					configUploader 	= config['UPLOADER']
-					OPT_WORKERS_N = configUploader.getint('opt_workers', OPT_WORKERS_N)
-			except (KeyError, configparser.NoSectionError) as err:
-				bu.error(exception=err)
-				
+							
 			try:
 				if 'WG' in config.sections():
 					configWG 		= config['WG']
@@ -82,18 +76,19 @@ async def main(argv):
 	bu.set_log_level(args.silent, args.verbose, args.debug)
 
 	wg = WG(WG_appID, args.tankopedia, args.mapfile)
-	wi = WoTinspector(rate_limit=WG_RATE_LIMIT)
+	wi = WoTinspector()
 
-	if args.account != None:
+	if args.account is not None:
 		args.accountID = await wg.get_account_id(args.account)
 		bu.debug('WG  account_id: ' + str(args.accountID))
 
-	if args.accountID == None: 
+	if args.accountID is None: 
 		args.accountID = 0
 
 	try:
-		queue  = asyncio.Queue()	
+		queue : asyncio.Queue = asyncio.Queue()	
 		# rebase file arguments due to moving the working directory to the script location
+		bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
 		args.files = bu.rebase_file_args(current_dir, args.files)
 
 		tasks = []
@@ -123,7 +118,7 @@ async def main(argv):
 		await wg.session.close()
 		await wi.close()
 
-	return None
+	return 0
 
 
 async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
@@ -137,13 +132,13 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
 			if not line: 
 				break
 			else:
-				if (p_replayfile.match(line) != None):
+				if (p_replayfile.match(line) is not None):
 					await queue.put(await mkQueueItem(line, title))
 	else:
 		for fn in files:
 			if fn.endswith('"'):
 				fn = fn[:-1]  
-			if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
+			if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
 				await queue.put(await mkQueueItem(fn, title))
 				bu.debug('File added to queue: ' + fn)
 			elif os.path.isdir(fn):
@@ -151,7 +146,7 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
 					for entry in dirEntry:
 						try:
 							bu.debug('Found: ' + entry.name)
-							if entry.is_file() and (p_replayfile.match(entry.name) != None): 
+							if entry.is_file() and (p_replayfile.match(entry.name) is not None): 
 								bu.debug(entry.name)
 								await queue.put(await mkQueueItem(entry.path, title))
 								bu.debug('File added to queue: ' + entry.path)
@@ -175,6 +170,9 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 	"""Async Worker to process the replay queue"""
 	global SKIPPED_N
 	global ERROR_N
+	global wi
+
+	assert wi is not None, "wi cannot be None"
 
 	while True:
 		item = await queue.get()
@@ -210,7 +208,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 				filename = os.path.basename(filename)
 				bu.debug(msg_str + 'File:  ' + filename)
 				json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
-				if json_resp != None:
+				if json_resp is not None:
 					if (await bu.save_JSON(replay_json_fn,json_resp)):
 						if wi.chk_JSON_replay(json_resp):
 							if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
@@ -235,7 +233,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 def getTitle_old(replayfile: str, title: str, i : int) -> str:
 	global wg
 
-	if title == None:
+	if title is None:
 		try:
 			filename = os.path.basename(replayfile)	
 			bu.debug(filename)
@@ -247,8 +245,8 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
 			p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
 			
 			m = p.match(filename)
-			if (m != None):
-				if wg.tanks != None:
+			if (m is not None):
+				if wg.tanks is not None:
 					tank = m.group(1)
 					if tank in wg.tanks['userStr']:
 						tank = wg.tanks['userStr'][tank]
@@ -273,7 +271,9 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
 
 def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
 	global wg
-	if title == None:
+	assert wg is not None, "wg must not be None"
+
+	if title is None:
 		try:
 			filename = os.path.basename(replayfile)	
 			bu.debug(filename)
@@ -289,10 +289,10 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
 						#player = metadata_json['playerName']						
 						tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
 						map_name = wg.get_map(metadata_json['mapName'])
-						bu.debug('Tank: ' + tank_name if tank_name != None else "NOT FOUND IN tanks.json" +  ' Map: ' + map_name)
+						bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" +  ' Map: ' + map_name)
 					except Exception as err:
 						bu.error(exception = err)
-			if (tank_name != None) and  (map_name != None):
+			if (tank_name is not None) and  (map_name is not None):
 				title = tank_name + ' @ ' + map_name
 			else:
 				title = re.sub('\\.wotbreplay$', '', filename)