From ba5197d4437854ca4c43c10566feb31d382f9b0b Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Wed, 8 Jul 2020 22:36:53 +0300
Subject: [PATCH 01/24] Update README.md

---
 README.md | 38 ++++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index af2fc83..5d23c5e 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
 
 ### Replays
 * [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader. 
-* [analyze_wotb_replays.py](analyze_wotb_replays.py): A tool to calculate statistics of a set of replays. Upload the replays first with [upload_wotb_replays.py](upload_wotb_replays.py) and then calculate the statistics. 
+* [analyze_wotb_replays.py](analyze_wotb_replays.py): A tool to calculate statistics of a set of replays. **Upload the replays first** with [upload_wotb_replays.py](upload_wotb_replays.py) and then calculate the statistics. 
 
 ### Tank data
 * [get_tankopedia.py](get_tankopedia.py): Get the lastest tankopedia data from wotinspector.com. The script fetches only a limited set of data. You may edit to your needs. 
@@ -19,16 +19,42 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
 ### Quick instructions
 
 1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
-1. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. The recommended way to do this is to use [GitHub Dekstop](https://desktop.github.com/) or do _git pull_
-1. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
-
+2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
 ```
 # Remember to run the command with a Python interpreter >= 3.7
 python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress 
 ```
+3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. A recommened way is to install [GitHub Desktop](https://desktop.github.com/) and clone the repository via _git_. This makes it easy to receive the updates. 
 
-4. Upload your replays. Run _python.exe .\upload_wotb_replays.py --help_ (Windows) or _./upload_wotb_replays.py --help_ (Linux)
-5. Analyze replays.  Run _python.exe .\analyze_wotb_replays.py --help_ (Windows) or _./analyze_wotb_replays.py --help_ (Linux)
+4. Upload your replays. 
+```
+# Windows
+# Check the options
+python.exe .\upload_wotb_replays.py --help
+
+# Syntax
+python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
+
+# example
+python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
+
+# Linux
+./upload_wotb_replays.py --help
+```
+5. Analyze replays. 
+```
+# Windows
+python.exe .\analyze_wotb_replays.py --help
+
+# Syntax
+python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
+
+# Example
+python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
+
+# Linux
+./analyze_wotb_replays.py --help
+```
 
 ### Blitz releases
 

From a4bd58965f36c5f7e47caf5d171169ce95bae1f7 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Wed, 8 Jul 2020 22:51:59 +0300
Subject: [PATCH 02/24] Clarify config options

---
 blitzstats.ini.default | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/blitzstats.ini.default b/blitzstats.ini.default
index 809d341..10cc583 100644
--- a/blitzstats.ini.default
+++ b/blitzstats.ini.default
@@ -7,7 +7,9 @@
 [WG]
 ## Find your WG account_id :  https://developers.wargaming.net/reference/all/wotb/account/list/
 #wg_id		    = 0
-## WG API limits requests / sec to 10 for client apps
+## WG API limits requests / sec to 10 for mobile apps. 
+## You can register your "own" server app and 
+## upgrade the rate limit to 20 requests / sec.
 ## Check https://developers.wargaming.net/documentation/guide/getting-started/
-#wg_app_id       = XXXXXXXXXXX
+# wg_app_id       = XXXXXXXXXXX
 

From 34a8d7055890c79e3ad5c89cd3ce8dce5c835416 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Wed, 8 Jul 2020 22:52:11 +0300
Subject: [PATCH 03/24] Update help text

---
 analyze_wotb_replays.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 9a40d68..f9608f0 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -521,7 +521,7 @@ async def main(argv):
 	DB_AUTH 	= 'admin'
 	DB_NAME 	= 'BlitzStats'
 	DB_USER		= 'mongouser'
-	DB_PASSWD 	= "PASSWORD"
+	DB_PASSWD 	= 'PASSWORD'
 	DB_CERT 	= None
 	DB_CA 		= None
 	
@@ -566,7 +566,7 @@ async def main(argv):
 			except configparser.NoSectionError as err:
 				bu.error(exception=err)
 
-		parser = ErrorCatchingArgumentParser(description='Analyze Blitz replay JSONs from WoTinspector.com')
+		parser = ErrorCatchingArgumentParser(description='Analyze Blitz replay JSON files from WoTinspector.com. Use \'upload_wotb_replays.py\' to upload the replay files first.')
 
 		parser.add_argument('--output', default='plain', choices=['plain', 'db'], help='Select output mode: plain text or database')
 		parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze')

From b24a2a8016e4610ab4aff672366a75937a79e31d Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Wed, 8 Jul 2020 22:53:11 +0300
Subject: [PATCH 04/24] Robust config file reading

---
 upload_wotb_replays.py | 40 ++++++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py
index 636da62..4d38edd 100644
--- a/upload_wotb_replays.py
+++ b/upload_wotb_replays.py
@@ -28,21 +28,37 @@ async def main(argv):
 	# set the directory for the script
 	os.chdir(os.path.dirname(sys.argv[0]))
 
-	## Read config
-	config = configparser.ConfigParser()
-	config.read(FILE_CONFIG)
+	# options defaults
+	OPT_WORKERS_N  = 5
+	WG_ID = None
+	WG_RATE_LIMIT = 10
 
-	configOptions 	= config['OPTIONS']
-	OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', 5)
+	try:
+		## Read config
+		if os.path.isfile(FILE_CONFIG):
+			config = configparser.ConfigParser()
+			config.read(FILE_CONFIG)
 
-	configWG 		= config['WG']
-	# WG account id of the uploader: 
-	# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
-	WG_ID			= configWG.getint('wg_id', None)
-	## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
-	WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', 10)
+			try:
+				configOptions 	= config['OPTIONS']
+				OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', OPT_WORKERS_N)
+			except (KeyError, configparser.NoSectionError) as err:
+				bu.debug("No section 'OPTIONS' found in configuration file: " + FILE_CONFIG)
+				
+			try:
+				configWG 		= config['WG']
+				# WG account id of the uploader: 
+				# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
+				WG_ID			= configWG.getint('wg_id', WG_ID)
+				## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
+				WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
+			except (KeyError, configparser.NoSectionError) as err:
+				bu.debug("No section 'WG' found in configuration file: " + FILE_CONFIG)
 
-	parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data')
+	except Exception as err:
+		bu.error(exception=err)
+
+	parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON')
 	parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id')
 	parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
 	parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering')

From aaa65a15549e37f5ca57eea15771413427116f92 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Thu, 9 Jul 2020 15:31:14 +0300
Subject: [PATCH 05/24] Update README.md

---
 README.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 5d23c5e..1290d2f 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
 
 ### Replays
 * [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader. 
-* [analyze_wotb_replays.py](analyze_wotb_replays.py): A tool to calculate statistics of a set of replays. **Upload the replays first** with [upload_wotb_replays.py](upload_wotb_replays.py) and then calculate the statistics. 
+* [analyze_wotb_replays.py](analyze_wotb_replays.py): A tool to calculate statistics of a set of replays. **Upload the replays first** with [upload_wotb_replays.py](upload_wotb_replays.py) and then calculate the statistics with analyze_wotb_replays.py based on the resulting **.json** files. 
 
 ### Tank data
 * [get_tankopedia.py](get_tankopedia.py): Get the lastest tankopedia data from wotinspector.com. The script fetches only a limited set of data. You may edit to your needs. 
@@ -28,7 +28,7 @@ python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4
 
 4. Upload your replays. 
 ```
-# Windows
+# Windows, run directly from Command Prompt or Powershell
 # Check the options
 python.exe .\upload_wotb_replays.py --help
 
@@ -43,7 +43,7 @@ python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbrep
 ```
 5. Analyze replays. 
 ```
-# Windows
+# Windows, run directly from Command Prompt or Powershell
 python.exe .\analyze_wotb_replays.py --help
 
 # Syntax
@@ -60,8 +60,9 @@ python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbre
 
 [WoT Blitz releases](releases.md): launch dates and key content.
 
-### Notes
+### Other
 
-Even though the code has an option to use MongoDB, you should not bother with that. The MongoDB support is solely for my own purposes and you do not need MongoDB to run the scripts. 
+* Please remember to run the scripts from command line, not from the Python interpreter. On Windows open Command Prompt or Powershell. 
+* Even though the code has an option to use MongoDB, you should not bother with that. The MongoDB support is solely for my own purposes and you do not need MongoDB to run the scripts. 
 
 You can reach me via email: [Jylpah@gmail.com](mailto:Jylpah@gmail.com) or [Discord](https://discordapp.com/): Jylpah#4662. Happy tanking and stats analysis!

From 99bfb3763660b46546778d99b9dc9a8941e46a3f Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Thu, 9 Jul 2020 15:34:00 +0300
Subject: [PATCH 06/24] Github desktop recommendation

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 1290d2f..c6161b9 100644
--- a/README.md
+++ b/README.md
@@ -63,6 +63,7 @@ python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbre
 ### Other
 
 * Please remember to run the scripts from command line, not from the Python interpreter. On Windows open Command Prompt or Powershell. 
+* [GitHub Desktop](https://desktop.github.com/) is the recommended way of downloading the scripts you can update the scripts easily via git/GitHub Deskop. To prevent GitHub Desktop from uploading your other files to github.com, you can "ignore" the files by right-clicking a file on GitHub Desktop or by manually editing the _.gitignore_ file.
 * Even though the code has an option to use MongoDB, you should not bother with that. The MongoDB support is solely for my own purposes and you do not need MongoDB to run the scripts. 
 
 You can reach me via email: [Jylpah@gmail.com](mailto:Jylpah@gmail.com) or [Discord](https://discordapp.com/): Jylpah#4662. Happy tanking and stats analysis!

From a95e0afdb0b172cc0f97ed884c1d3de7993b677a Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 11:38:32 +0300
Subject: [PATCH 07/24] Add: new utils funcs, update
 WoTinspector.chk_replay_JSON()

New util funcs:
- warning()
- get_JSON_keypath()
- get_JSON_value()
---
 blitzutils.py | 72 ++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 57 insertions(+), 15 deletions(-)

diff --git a/blitzutils.py b/blitzutils.py
index 939181d..48c2f77 100644
--- a/blitzutils.py
+++ b/blitzutils.py
@@ -210,6 +210,13 @@ def verbose_std(msg = "", id = None) -> bool:
         return True
     return False
 
+def warning(msg = "", id = None) -> bool:
+    """Print a warning message"""
+    if _log_level >= NORMAL:
+        _print_log_msg('', 'Warning: ' + msg, None, id)        
+        return True
+    return False
+
 def debug(msg = "", id = None, exception = None, force: bool = False) -> bool:
     """print a conditional debug message"""
     if (_log_level >= DEBUG) or force:
@@ -445,6 +452,40 @@ def bld_dict_hierarcy(d : dict, key : str, value) -> dict:
         error('Unexpected Exception', err)
     return None
 
+def  get_JSON_keypath(keypath: str, key: str):
+    if keypath == None:
+        return key
+    else:
+        return '.'.join([keypath, key])
+
+def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
+    if keys == None:
+        keys = key.split('.')   
+    
+    if len(keys) == 0:
+        return json
+
+    key = keys.pop(0)
+    if type(json) == dict:
+        if key in json:
+            return get_JSON_value(json[key], keys=keys, keypath=get_JSON_keypath(keypath, key))
+        else:
+            raise KeyError('Key: '+ get_JSON_keypath(keypath, key) + ' not found')
+    
+    if type(json) == list:
+        p = re.compile(r'^\[(\d+)\]$')
+        m = p.match(key)
+        if len(m.groups()) != 1:
+            raise KeyError('Invalid key given: ' + get_JSON_keypath(keypath, key))
+        ndx = m.group(1)
+        try:
+            return get_JSON_value(json[ndx], keys=keys, keypath=get_JSON_keypath(keypath, key))
+        except IndexError:
+            raise KeyError('JSON array index out of range: ' + get_JSON_keypath(keypath, key))
+    
+    raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
+
+
 ## -----------------------------------------------------------
 #### Class SlowBar 
 ## -----------------------------------------------------------
@@ -1462,41 +1503,39 @@ class WoTinspector:
             error(msg_str + 'Unexpected Exception', err)
             return None
 
+        json_resp  = None
         for retry in range(MAX_RETRIES):
             debug(msg_str + 'Posting: ' + title + ' Try #: ' + str(retry + 1) + '/' + str(MAX_RETRIES) )
             try:
                 async with self.session.post(url, headers=headers, data=payload) as resp:
                     debug(msg_str + 'HTTP response: '+ str(resp.status))
                     if resp.status == 200:								
-                        debug('HTTP POST 200 = Success. Reading response data')
+                        debug(msg_str + 'HTTP POST 200 = Success. Reading response data')
                         json_resp = await resp.json()
-                        if json_resp.get('status', None) == None:
-                            error(msg_str +' : ' + title + ' : Received invalid JSON')
-                        elif (json_resp['status'] == 'ok'): 
-                            debug('Response data read. Status OK')                            
+                        if self.chk_JSON_replay(json_resp):
+                            debug(msg_str + 'Response data read. Status OK')                            
                             return json_resp	
-                        elif (json_resp['status'] == 'error'):  
-                            error(msg_str + json_resp['error']['message'] + ' : ' + title)
-                        else:
-                            error(msg_str + ' Unspecified error: ' + title)											
+                        debug(msg_str +' : ' + title + ' : Receive invalid JSON')                        										
                     else:
                         debug(msg_str + 'Got HTTP/' + str(resp.status))
             except Exception as err:
-                error(exception=err)
+                debug(msg_str, exception=err)
             await asyncio.sleep(SLEEP)
             
-        error(msg_str + ' Could not post replay: ' + title)
-        return None
+        debug(msg_str + ' Could not post replay: ' + title)
+        return json_resp
 
 
     async def get_replay_listing(self, page: int = 0) -> aiohttp.ClientResponse:
         url = self.get_url_replay_listing(page)
         return await self.session.get(url)
 
+
     @classmethod
     def get_url_replay_listing(cls, page : int):
         return cls.URL_REPLAY_LIST + str(page) + '?vt=#filters'
 
+
     @classmethod
     def get_replay_links(cls, doc: str):
         """Get replay download links from WoTinspector.com replay listing page"""
@@ -1513,21 +1552,24 @@ class WoTinspector:
             error(exception=err)
         return replay_links
     
+
     @classmethod
     def get_replay_id(cls, url):
         return url.rsplit('/', 1)[-1]
 
+
     @classmethod
     def chk_JSON_replay(cls, json_resp):
         """"Check String for being a valid JSON file"""
         try:
-            if ('status' in json_resp) and json_resp['status'] == 'ok' and ('data' in json_resp) and json_resp['data'] != None:
+            if ('status' in json_resp) and json_resp['status'] == 'ok' and \
+                (get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
                 debug("JSON check OK")
                 return True 
         except KeyError as err:
-            error('Key not found', err)
+            debug('Replay JSON check failed', exception=err)
         except:
-            debug("JSON check failed: " + str(json_resp))
+            debug("Replay JSON check failed: " + str(json_resp))
         return False      
 
 class BlitzStars:

From 21ebe75b70fc842bd1805cdc9732d7616bbbcee8 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 11:39:19 +0300
Subject: [PATCH 08/24] more robust config file reading, handle invalid replay
 postings

---
 upload_wotb_replays.py | 61 ++++++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 23 deletions(-)

diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py
index 4d38edd..2d591d9 100644
--- a/upload_wotb_replays.py
+++ b/upload_wotb_replays.py
@@ -25,42 +25,48 @@ wi = None
 
 async def main(argv):
 	global wg, wi
+	current_dir = os.getcwd()
 	# set the directory for the script
 	os.chdir(os.path.dirname(sys.argv[0]))
-
+	
 	# options defaults
-	OPT_WORKERS_N  = 5
-	WG_ID = None
-	WG_RATE_LIMIT = 10
+	OPT_WORKERS_N  	= 5
+	WG_ID 			= None
+	WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
+	  					# China is not supported since WG API stats are not available there
+	WG_RATE_LIMIT 	= 10
 
 	try:
 		## Read config
 		if os.path.isfile(FILE_CONFIG):
+			bu.debug('Reading config file: ' + FILE_CONFIG)
 			config = configparser.ConfigParser()
 			config.read(FILE_CONFIG)
-
 			try:
-				configOptions 	= config['OPTIONS']
-				OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', OPT_WORKERS_N)
+				if 'OPTIONS' in config.sections():
+					configOptions 	= config['OPTIONS']
+					OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', OPT_WORKERS_N)
 			except (KeyError, configparser.NoSectionError) as err:
-				bu.debug("No section 'OPTIONS' found in configuration file: " + FILE_CONFIG)
+				bu.error(exception=err)
 				
 			try:
-				configWG 		= config['WG']
-				# WG account id of the uploader: 
-				# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
-				WG_ID			= configWG.getint('wg_id', WG_ID)
-				## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
-				WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
+				if 'WG' in config.sections():
+					configWG 		= config['WG']
+					# WG account id of the uploader: 
+					# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
+					WG_ID			= configWG.getint('wg_id', WG_ID)
+					WG_ACCOUNT		= configWG.get('wg_account', WG_ACCOUNT)
+					## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
+					WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
 			except (KeyError, configparser.NoSectionError) as err:
-				bu.debug("No section 'WG' found in configuration file: " + FILE_CONFIG)
-
+				bu.error(exception=err)
+		
 	except Exception as err:
 		bu.error(exception=err)
 
 	parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON')
 	parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id')
-	parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
+	parser.add_argument('-a', '--account', dest='account', type=str, default=WG_ACCOUNT, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
 	parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering')
 	parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com')
 	parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
@@ -86,10 +92,12 @@ async def main(argv):
 
 	try:
 		queue  = asyncio.Queue()	
-		
+		files = list()
+		for fn in args.files:
+			files.append(os.path.join(current_dir, fn))
 		tasks = []
 		# Make replay Queue
-		tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
+		tasks.append(asyncio.create_task(mkReplayQ(queue, files, args.title)))
 		# Start tasks to process the Queue
 		for i in range(OPT_WORKERS_N):
 			tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
@@ -104,7 +112,7 @@ async def main(argv):
 			task.cancel()
 		bu.debug('Waiting for workers to cancel')
 		await asyncio.gather(*tasks, return_exceptions=True)
-		bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
+		bu.verbose_std(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
 				
 	except KeyboardInterrupt:
 		print('Ctrl-c pressed ...')
@@ -203,11 +211,18 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 				json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
 				if json_resp != None:
 					if (await bu.save_JSON(replay_json_fn,json_resp)):
-						if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
-							bu.verbose_std(msg_str + title + ' posted')
+						if wi.chk_JSON_replay(json_resp):
+							if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
+								bu.verbose_std(msg_str + title + ' posted')
+						else:
+							bu.warning('Replay file is not valid: ' + filename)
+							ERROR_N += 1
 					else:
 						bu.error(msg_str + 'Error saving replay: ' + filename)
-						ERROR_N += 1					
+						ERROR_N += 1
+				else:
+					bu.error('Replay file is not valid: ' + filename)
+					ERROR_N += 1	
 		except Exception as err:
 			bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
 		bu.debug(msg_str + 'Marking task done')

From f934d4bb24eb5efb50afca1fabec71cec30e3be0 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 11:39:33 +0300
Subject: [PATCH 09/24] add wg_account_id option

---
 blitzstats.ini.default | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/blitzstats.ini.default b/blitzstats.ini.default
index 10cc583..8793231 100644
--- a/blitzstats.ini.default
+++ b/blitzstats.ini.default
@@ -5,8 +5,14 @@
 #opt_analyzer_stat_func   = player
 
 [WG]
+
 ## Find your WG account_id :  https://developers.wargaming.net/reference/all/wotb/account/list/
-#wg_id		    = 0
+# wg_id		    = 0
+
+## wg_account format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
+## China is not supported since WG API stats are not available there
+# wg_account    = your_nick@eu
+
 ## WG API limits requests / sec to 10 for mobile apps. 
 ## You can register your "own" server app and 
 ## upgrade the rate limit to 20 requests / sec.

From 49a520ddb5e77ec5c0c7c51cc9659757791417c7 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 11:40:10 +0300
Subject: [PATCH 10/24] More robust config file reading

---
 analyze_wotb_replays.py | 65 +++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 29 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index f9608f0..23c2754 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -508,8 +508,11 @@ async def main(argv):
 	OPT_STAT_FUNC	= 'player'
 	OPT_WORKERS_N 	= 10
 	
-	WG_ID			= None
-	#WG_APP_ID		= WG_APP_ID
+	WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
+	  					 	# China is not supported since WG API stats are not available there
+	WG_ID			= None  # WG account_id in integer format. 
+							# WG_ACCOUNT will be used to retrieve the account_id, but it can be set directly too
+	# WG_APP_ID		= WG_APP_ID
 	WG_RATE_LIMIT	= 10  ## WG standard. Do not edit unless you have your
 						  ## own server app ID, it will REDUCE the performance
 	
@@ -532,45 +535,49 @@ async def main(argv):
 			config.read(FILE_CONFIG)
 
 			try:
-				configOptions 	= config['OPTIONS']
-				# WG account id of the uploader: 
-				# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
-				OPT_DB			= configOptions.getboolean('opt_DB', OPT_DB)
-				OPT_EXTENDED 	= configOptions.getboolean('opt_analyzer_extended', OPT_EXTENDED)
-				OPT_HIST		= configOptions.getboolean('opt_analyzer_hist', OPT_HIST)
-				OPT_STAT_FUNC	= configOptions.get('opt_analyzer_stat_func', fallback=OPT_STAT_FUNC)
-				OPT_WORKERS_N 	= configOptions.getint('opt_analyzer_workers', OPT_WORKERS_N)
-			except configparser.NoSectionError as err:
+				if 'OPTIONS' in config.sections():
+					configOptions 	= config['OPTIONS']
+					# WG account id of the uploader: 
+					# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
+					OPT_DB			= configOptions.getboolean('opt_DB', OPT_DB)
+					OPT_EXTENDED 	= configOptions.getboolean('opt_analyzer_extended', OPT_EXTENDED)
+					OPT_HIST		= configOptions.getboolean('opt_analyzer_hist', OPT_HIST)
+					OPT_STAT_FUNC	= configOptions.get('opt_analyzer_stat_func', fallback=OPT_STAT_FUNC)
+					OPT_WORKERS_N 	= configOptions.getint('opt_analyzer_workers', OPT_WORKERS_N)
+			except (KeyError, configparser.NoSectionError) as err:
 				bu.error(exception=err)
 
 			try:
-				configWG 		= config['WG']
-				WG_ID			= configWG.getint('wg_id', WG_ID)
-				WG_APP_ID		= configWG.get('wg_app_id', WG_APP_ID)
-				WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
-			except configparser.NoSectionError as err:
+				if 'WG' in config.sections():
+					configWG 		= config['WG']
+					WG_ID			= configWG.getint('wg_id', WG_ID)
+					WG_ACCOUNT		= configWG.get('wg_account', WG_ACCOUNT)
+					WG_APP_ID		= configWG.get('wg_app_id', WG_APP_ID)
+					WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
+			except (KeyError, configparser.NoSectionError) as err:
 				bu.error(exception=err)
 
 			try:
-				configDB 	= config['DATABASE']
-				DB_SERVER 	= configDB.get('db_server', DB_SERVER)
-				DB_PORT 	= configDB.getint('db_port', DB_PORT)
-				DB_SSL		= configDB.getboolean('db_ssl', DB_SSL)
-				DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ)
-				DB_AUTH 	= configDB.get('db_auth', DB_AUTH)
-				DB_NAME 	= configDB.get('db_name', DB_NAME)
-				DB_USER		= configDB.get('db_user', DB_USER)
-				DB_PASSWD 	= configDB.get('db_password', DB_PASSWD)
-				DB_CERT 	= configDB.get('db_ssl_cert_file', DB_CERT)
-				DB_CA 		= configDB.get('db_ssl_ca_file', DB_CA)
-			except configparser.NoSectionError as err:
+				if 'DATABASE' in config.sections():
+					configDB 	= config['DATABASE']
+					DB_SERVER 	= configDB.get('db_server', DB_SERVER)
+					DB_PORT 	= configDB.getint('db_port', DB_PORT)
+					DB_SSL		= configDB.getboolean('db_ssl', DB_SSL)
+					DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ)
+					DB_AUTH 	= configDB.get('db_auth', DB_AUTH)
+					DB_NAME 	= configDB.get('db_name', DB_NAME)
+					DB_USER		= configDB.get('db_user', DB_USER)
+					DB_PASSWD 	= configDB.get('db_password', DB_PASSWD)
+					DB_CERT 	= configDB.get('db_ssl_cert_file', DB_CERT)
+					DB_CA 		= configDB.get('db_ssl_ca_file', DB_CA)
+			except (KeyError, configparser.NoSectionError)  as err:
 				bu.error(exception=err)
 
 		parser = ErrorCatchingArgumentParser(description='Analyze Blitz replay JSON files from WoTinspector.com. Use \'upload_wotb_replays.py\' to upload the replay files first.')
 
 		parser.add_argument('--output', default='plain', choices=['plain', 'db'], help='Select output mode: plain text or database')
 		parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze')
-		parser.add_argument('-a', '--account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER')
+		parser.add_argument('-a', '--account', type=str, default=WG_ACCOUNT, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER')
 		parser.add_argument('-x', '--extended', action='store_true', default=OPT_EXTENDED, help='Print Extended stats')
 		parser.add_argument('-X', '--extra_categories', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print Extended categories')
 		parser.add_argument('--hist', action='store_true', default=OPT_HIST, help='Print player histograms (WR/battles)')

From dd76d27b565454076aad0271a27e0055f32e015e Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:07:03 +0300
Subject: [PATCH 11/24] update args.files after rebasing the file args

---
 upload_wotb_replays.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py
index 2d591d9..8964a91 100644
--- a/upload_wotb_replays.py
+++ b/upload_wotb_replays.py
@@ -95,9 +95,11 @@ async def main(argv):
 		files = list()
 		for fn in args.files:
 			files.append(os.path.join(current_dir, fn))
+		args.files = files
+
 		tasks = []
 		# Make replay Queue
-		tasks.append(asyncio.create_task(mkReplayQ(queue, files, args.title)))
+		tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
 		# Start tasks to process the Queue
 		for i in range(OPT_WORKERS_N):
 			tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
@@ -194,7 +196,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 						bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
 					else:
 						os.remove(replay_json_fn)
-						bu.debug("Replay JSON not valid: Deleting " + replay_json_fn)
+						bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
 					SKIPPED_N += 1						
 					queue.task_done()						
 					continue
@@ -215,13 +217,13 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 							if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
 								bu.verbose_std(msg_str + title + ' posted')
 						else:
-							bu.warning('Replay file is not valid: ' + filename)
+							bu.warning(msg_str +'Replay file is not valid: ' + filename)
 							ERROR_N += 1
 					else:
 						bu.error(msg_str + 'Error saving replay: ' + filename)
 						ERROR_N += 1
 				else:
-					bu.error('Replay file is not valid: ' + filename)
+					bu.error(msg_str + 'Replay file is not valid: ' + filename)
 					ERROR_N += 1	
 		except Exception as err:
 			bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )

From ead9d1cad9b8c2029a9326326b9ec195bf2ee06d Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:15:54 +0300
Subject: [PATCH 12/24] Add rebase_file_args()

Since the scripts move their working dir to the script's location
---
 blitzutils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/blitzutils.py b/blitzutils.py
index 48c2f77..acc6bff 100644
--- a/blitzutils.py
+++ b/blitzutils.py
@@ -348,6 +348,10 @@ def NOW() -> int:
     return int(time.time())
 
 
+def rebase_file_args(current_dir, files):
+    """REbase file command line params after moving working dir to the script's dir""" 
+    return [ os.path.join(current_dir, fn) for fn in files ]
+
 async def read_int_list(filename: str) -> list():
     """Read file to a list and return list of integers in the input file"""
     

From 4e991b6900d55f323ba4941f1db5097ee56dab0a Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:16:15 +0300
Subject: [PATCH 13/24] Use bu.rebase_file_args() instead of inline code

---
 analyze_wotb_replays.py | 10 +++++++---
 upload_wotb_replays.py  |  6 ++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 23c2754..fce2701 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -499,6 +499,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 async def main(argv):
 	global wg, wi, WG_APP_ID
 	# set the directory for the script
+	current_dir = os.getcwd()
 	os.chdir(os.path.dirname(sys.argv[0]))
 
 	## Default options:
@@ -629,10 +630,12 @@ async def main(argv):
 				bu.error("Could no initiate DB connection: Disabling DB", err) 
 				args.db = False
 				pass
-
-		if not(args.db):
+		else:
 			bu.debug('No DB in use')
 
+		# rebase file arguments due to moving the working directory to the script location
+		args.files = bu.rebase_file_args(current_dir, args.files)
+
 		try:
 			replayQ  = asyncio.Queue(maxsize=1000)			
 			reader_tasks = []
@@ -1205,8 +1208,9 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 			replayID = item[1]
 
 			try:
+				msg_str = 'Replay[' + str(replayID) + ']:' 
 				if replay_json == None:
-					bu.verbose('Replay[' + str(replayID) + ']: is empty. Skipping.' )
+					bu.verbose(msg_str + 'Replay is empty. Skipping.' )
 					#SKIPPED_N += 1
 					queue.task_done()
 					continue
diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py
index 8964a91..15a1d2f 100644
--- a/upload_wotb_replays.py
+++ b/upload_wotb_replays.py
@@ -92,10 +92,8 @@ async def main(argv):
 
 	try:
 		queue  = asyncio.Queue()	
-		files = list()
-		for fn in args.files:
-			files.append(os.path.join(current_dir, fn))
-		args.files = files
+		# rebase file arguments due to moving the working directory to the script location
+		args.files = bu.rebase_file_args(current_dir, args.files)
 
 		tasks = []
 		# Make replay Queue

From 25b4f755334a1e06415e650409c5bfcf69b9dc26 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:20:02 +0300
Subject: [PATCH 14/24] SLEEP: 3s => 1.5s

---
 blitzutils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/blitzutils.py b/blitzutils.py
index acc6bff..298e4df 100644
--- a/blitzutils.py
+++ b/blitzutils.py
@@ -9,7 +9,7 @@ from progress.counter import Counter
 from decimal import Decimal
 
 MAX_RETRIES= 3
-SLEEP = 3
+SLEEP = 1.5
 
 LOG_LEVELS = { 'silent': 0, 'normal': 1, 'verbose': 2, 'debug': 3 }
 SILENT  = 0

From 001d73f3ae460a5b86aff5aca97abbc1ea891f43 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:29:21 +0300
Subject: [PATCH 15/24] Update warning messages

---
 analyze_wotb_replays.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index fce2701..ccc0279 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -1198,7 +1198,6 @@ async def mk_readerQ_item(replay_json) -> list:
 async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
 	"""Async Worker to process the replay queue"""
 	#global SKIPPED_N
-	# account_id = args.account_id
 	results = []
 	playerstanks = set()
 	try:
@@ -1208,9 +1207,9 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 			replayID = item[1]
 
 			try:
-				msg_str = 'Replay[' + str(replayID) + ']:' 
+				msg_str = 'Replay[' + str(replayID) + ']: ' 
 				if replay_json == None:
-					bu.verbose(msg_str + 'Replay is empty. Skipping.' )
+					bu.warning(msg_str + 'Invalid replay. Skipping.' )
 					#SKIPPED_N += 1
 					queue.task_done()
 					continue
@@ -1220,7 +1219,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 				result = await read_replay_JSON(replay_json, args)
 				bu.print_progress()
 				if result == None:
-					bu.debug('Replay[' + str(replayID) + ']: Invalid replay', id=readerID)
+					bu.warning(msg_str + 'Invalid replay', id=readerID)
 					queue.task_done()
 					continue
 				

From bbfc732aae0913a3ea84973d6b2491b7ac45077e Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:37:43 +0300
Subject: [PATCH 16/24] Add replay file name to error messages

---
 analyze_wotb_replays.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index ccc0279..4113f1d 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -1159,7 +1159,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 				
 				if (p_replayfile.match(line) != None):
 					replay_json = await bu.open_JSON(line, wi.chk_JSON_replay)			
-					await queue.put(await mk_readerQ_item(replay_json))
+					await queue.put(await mk_readerQ_item(replay_json, line))
 			except Exception as err:
 				bu.error(exception=err)
 	else:
@@ -1170,7 +1170,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 					fn = fn[:-1] 
 				if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
 					replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay)
-					await queue.put(await mk_readerQ_item(replay_json))
+					await queue.put(await mk_readerQ_item(replay_json, fn))
 					bu.debug('File added to queue: ' + fn)
 					Nreplays += 1
 				elif os.path.isdir(fn):
@@ -1179,7 +1179,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 							if entry.is_file() and (p_replayfile.match(entry.name) != None): 
 								bu.debug(entry.name)
 								replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay)
-								await queue.put(await mk_readerQ_item(replay_json))
+								await queue.put(await mk_readerQ_item(replay_json, fn))
 								bu.debug('File added to queue: ' + entry.path)
 								Nreplays += 1
 			except Exception as err:
@@ -1188,11 +1188,11 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 	return Nreplays
 
 
-async def mk_readerQ_item(replay_json) -> list:
+async def mk_readerQ_item(replay_json, filename : str = None) -> list:
 	"""Make an item to replay queue"""
 	global REPLAY_N
 	REPLAY_N +=1
-	return [replay_json, REPLAY_N]
+	return [replay_json, REPLAY_N, os.path.basename(filename) ]
 
 
 async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
@@ -1203,13 +1203,14 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 	try:
 		while True:
 			item = await queue.get()
-			replay_json = item[0]
-			replayID = item[1]
+			replay_json 	= item[0]
+			replayID 		= item[1]
+			replay_file 	= item[2]
 
 			try:
 				msg_str = 'Replay[' + str(replayID) + ']: ' 
 				if replay_json == None:
-					bu.warning(msg_str + 'Invalid replay. Skipping.' )
+					bu.warning(msg_str + 'Invalid replay. Skipping: '  + (replay_file if replay_file != None else '') )
 					#SKIPPED_N += 1
 					queue.task_done()
 					continue
@@ -1219,7 +1220,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 				result = await read_replay_JSON(replay_json, args)
 				bu.print_progress()
 				if result == None:
-					bu.warning(msg_str + 'Invalid replay', id=readerID)
+					bu.warning(msg_str + 'Invalid replay' + (replay_file if replay_file != None else '') )
 					queue.task_done()
 					continue
 				

From ca8b63e17a547aa8a29a95cbe35ad5ca05a62312 Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:39:17 +0300
Subject: [PATCH 17/24] Fix

---
 analyze_wotb_replays.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 4113f1d..4863c32 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -1179,7 +1179,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 							if entry.is_file() and (p_replayfile.match(entry.name) != None): 
 								bu.debug(entry.name)
 								replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay)
-								await queue.put(await mk_readerQ_item(replay_json, fn))
+								await queue.put(await mk_readerQ_item(replay_json, entry.name))
 								bu.debug('File added to queue: ' + entry.path)
 								Nreplays += 1
 			except Exception as err:

From def06ebbbd9d5d4019f5830d96e8b8e32f79b38d Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:45:27 +0300
Subject: [PATCH 18/24] more fine-grained histogram buckets

---
 analyze_wotb_replays.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 4863c32..5e78e79 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -74,9 +74,9 @@ replay_details_flds = [
 ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/  (all.KEY)
 ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ]
 histogram_fields = {
-	'wins'				: [ 'Win rate', [0, .35, .40, .45, .5, .55, .60, .65, 1], 100, '{:.0f}%' ],
-	'damage_dealt'		: [ 'Avg. Dmg.', [0, .25e3, .5e3, 1e3, 1.5e3, 2e3,2.5e3,3e3,100e3], 1, '{:.0f}' ],
-	'battles'			: [ 'Battles', 	[0, 1e3, 3e3, 5e3, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k']	# battles is a mandatory stat to include
+	'wins'				: [ 'Win rate', 	[0, .35, .40, .45, .5, .55, .60, .65, .70, 1], 100, '{:.0f}%' ],
+	'damage_dealt'		: [ 'Avg. Dmg.', 	[0, 250, 500, 750, 1000, 1350, 1500,1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
+	'battles'			: [ 'Battles', 		[0, 1000, 2500, 5000, 7500, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k']	# battles is a mandatory stat to include
 	}
 
 

From 2ec04f7e1868e63ab16b609d60caecfcdc98b7bf Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:48:06 +0300
Subject: [PATCH 19/24] update histogram WR / battle buckets

---
 analyze_wotb_replays.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 5e78e79..2132091 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -74,9 +74,9 @@ replay_details_flds = [
 ## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/  (all.KEY)
 ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ]
 histogram_fields = {
-	'wins'				: [ 'Win rate', 	[0, .35, .40, .45, .5, .55, .60, .65, .70, 1], 100, '{:.0f}%' ],
+	'wins'				: [ 'Win rate', 	[0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:.0f}%' ],
 	'damage_dealt'		: [ 'Avg. Dmg.', 	[0, 250, 500, 750, 1000, 1350, 1500,1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
-	'battles'			: [ 'Battles', 		[0, 1000, 2500, 5000, 7500, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k']	# battles is a mandatory stat to include
+	'battles'			: [ 'Battles', 		[0, 1000, 2500, 5000, 7000, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k']	# battles is a mandatory stat to include
 	}
 
 

From ca129f800d47906921ea5ab1487838b51591d61f Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:51:03 +0300
Subject: [PATCH 20/24] update avg dmg buckets

---
 analyze_wotb_replays.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 2132091..6f3a50e 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -75,7 +75,7 @@ replay_details_flds = [
 ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ]
 histogram_fields = {
 	'wins'				: [ 'Win rate', 	[0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:.0f}%' ],
-	'damage_dealt'		: [ 'Avg. Dmg.', 	[0, 250, 500, 750, 1000, 1350, 1500,1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
+	'damage_dealt'		: [ 'Avg. Dmg.', 	[0, 200, 300, 400, 500, 750, 1000, 1250, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
 	'battles'			: [ 'Battles', 		[0, 1000, 2500, 5000, 7000, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k']	# battles is a mandatory stat to include
 	}
 

From 6f847d7a05284146ee21af910dcbe76ae2bc7b2d Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 12:53:50 +0300
Subject: [PATCH 21/24] avg dmg bucket update

---
 analyze_wotb_replays.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 6f3a50e..615a672 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -75,7 +75,7 @@ replay_details_flds = [
 ## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ]
 histogram_fields = {
 	'wins'				: [ 'Win rate', 	[0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:.0f}%' ],
-	'damage_dealt'		: [ 'Avg. Dmg.', 	[0, 200, 300, 400, 500, 750, 1000, 1250, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
+	'damage_dealt'		: [ 'Avg. Dmg.', 	[0, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
 	'battles'			: [ 'Battles', 		[0, 1000, 2500, 5000, 7000, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k']	# battles is a mandatory stat to include
 	}
 

From 242352dd7c70ce8d0a6e594d1b2ab27253ac9f5b Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 14:32:12 +0300
Subject: [PATCH 22/24] Add DB _id to replay queue

---
 analyze_wotb_replays.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 615a672..14d13c4 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -27,6 +27,8 @@ REPLAY_I = 0
 STAT_TANK_BATTLE_MIN = 100
 BATTLE_TIME_BUCKET = 3600*24*14
 
+RE_SRC_IS_DB = re.compile(r'^DB:')
+
 ## For different player stat functions (global stats, tank tier stats, etc)
 # 1st function = for forming stat_id, 2nd for DB stats query, 3rd for WG API stats query
 STAT_FUNC	= {
@@ -1139,11 +1141,12 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 				cursor = dbc.find(filters, {'_id': 0 })
 			else:
 				# select all
-				cursor = dbc.find({}, {'_id': 0 })
+				cursor = dbc.find({}, {})
 			bu.debug('Reading replays...')	
 			async for replay_json in cursor:
-				#bu.debug(json.dumps(replay_json, indent=2))
-				await queue.put(await mk_readerQ_item(replay_json))
+				_id = replay_json['_id']
+				del(replay_json['_id'])
+				await queue.put(await mk_readerQ_item(replay_json, 'DB: _id = ' + _id))
 				Nreplays += 1
 			bu.debug('All the matching replays have been read from the DB')
 		except Exception as err:
@@ -1191,8 +1194,13 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 async def mk_readerQ_item(replay_json, filename : str = None) -> list:
 	"""Make an item to replay queue"""
 	global REPLAY_N
-	REPLAY_N +=1
-	return [replay_json, REPLAY_N, os.path.basename(filename) ]
+	REPLAY_N +=1	
+	if filename == None:
+		return [replay_json, REPLAY_N, '' ]
+	elif RE_SRC_IS_DB.match(filename) != None:
+		return [replay_json, REPLAY_N, filename ]
+	else:
+		return [replay_json, REPLAY_N, os.path.basename(filename) ]
 
 
 async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):

From 2adffa08e8201ea0ebe72e0d143537f28928a86e Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 15:00:00 +0300
Subject: [PATCH 23/24] Fix DB support

---
 analyze_wotb_replays.py | 7 ++++---
 blitzutils.py           | 5 ++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py
index 14d13c4..c0bcabf 100644
--- a/analyze_wotb_replays.py
+++ b/analyze_wotb_replays.py
@@ -1138,10 +1138,10 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
 				bu.debug(str(args.filters))
 				filters = json.loads(args.filters)
 				bu.debug(json.dumps(filters, indent=2))
-				cursor = dbc.find(filters, {'_id': 0 })
+				cursor = dbc.find(filters)
 			else:
 				# select all
-				cursor = dbc.find({}, {})
+				cursor = dbc.find({})
 			bu.debug('Reading replays...')	
 			async for replay_json in cursor:
 				_id = replay_json['_id']
@@ -1257,7 +1257,8 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
 	#db = args.db
 	result = {}
 	try:
-		if (replay_json['data']['summary']['mastery_badge'] == None) or (replay_json['data']['summary']['battle_result'] < 0):
+		# bu.debug(str(replay_json))
+		if not wi.chk_JSON_replay(replay_json):
 			bu.debug('Invalid replay')
 			return None
 		
diff --git a/blitzutils.py b/blitzutils.py
index 298e4df..0356170 100644
--- a/blitzutils.py
+++ b/blitzutils.py
@@ -350,7 +350,10 @@ def NOW() -> int:
 
 def rebase_file_args(current_dir, files):
     """REbase file command line params after moving working dir to the script's dir""" 
-    return [ os.path.join(current_dir, fn) for fn in files ]
+    if (files[0] == '-') or (files[0] == 'db:'):
+        return files
+    else:
+        return [ os.path.join(current_dir, fn) for fn in files ]
 
 async def read_int_list(filename: str) -> list():
     """Read file to a list and return list of integers in the input file"""

From d94318a744dea4d50312e41093027f33aa50455d Mon Sep 17 00:00:00 2001
From: Jylpah <jylpah@gmail.com>
Date: Fri, 10 Jul 2020 15:47:05 +0300
Subject: [PATCH 24/24] add: WoTInspector.get_url_replay_view()

---
 blitzutils.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/blitzutils.py b/blitzutils.py
index 0356170..728938a 100644
--- a/blitzutils.py
+++ b/blitzutils.py
@@ -1413,9 +1413,11 @@ class WoTinspector:
     URL_WI          = 'https://replays.wotinspector.com'
     URL_REPLAY_LIST = URL_WI + '/en/sort/ut/page/'
     URL_REPLAY_DL   = URL_WI + '/en/download/'  
+    URL_REPLAY_VIEW = URL_WI +'/en/view/'
     URL_REPLAY_UL   = 'https://api.wotinspector.com/replay/upload?'
     URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
-    URL_TANK_DB     ="https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
+    
+    URL_TANK_DB     = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
 
     REPLAY_N = 1
 
@@ -1543,6 +1545,11 @@ class WoTinspector:
         return cls.URL_REPLAY_LIST + str(page) + '?vt=#filters'
 
 
+    @classmethod
+    def get_url_replay_view(cls, replay_id):
+        return cls.URL_REPLAY_VIEW + replay_id
+
+
     @classmethod
     def get_replay_links(cls, doc: str):
         """Get replay download links from WoTinspector.com replay listing page"""
@@ -1579,6 +1586,7 @@ class WoTinspector:
             debug("Replay JSON check failed: " + str(json_resp))
         return False      
 
+
 class BlitzStars:
 
     URL_BLITZSTARS          = 'https://www.blitzstars.com'