Update 6.2 + bug fixes

blitzutils.py

@@ -5,7 +5,7 @@ import asyncio, aiofiles, aiohttp, aiosqlite, lxml
 from bs4 import BeautifulSoup
 
 MAX_RETRIES= 3
-SLEEP = 2
+SLEEP = 3
 DEBUG = False
 VERBOSE = False
 SILENT = False
@@ -60,10 +60,12 @@ def printWaiter(force = False):
     if not DEBUG and (not SILENT  or force):
         print('.', end='', flush=True)
 
+    
 def printNewline(force = False):
     if not DEBUG and (not SILENT  or force):
         print('', flush=True)
 
+
 def debug(msg = ""):
     """print a conditional debug message"""
     if DEBUG: 
@@ -73,6 +75,7 @@ def debug(msg = ""):
         print('DEBUG: ' + caller + '(): ' + msg)
     return None
 
+
 def error(msg = ""):
     """Print an error message"""
     curframe = inspect.currentframe()
@@ -80,9 +83,27 @@ def error(msg = ""):
     caller = calframe[1][3]
     print('ERROR: ' + caller + '(): ' + msg)
     return None
+
 def NOW() -> int:
     return int(time.time())
 
+
+async def readPlainList(filename: str) -> list():
+    """Read file to a list and return list of integers in the input file"""
+    
+    input_list = []
+    try:
+        async with aiofiles.open(filename) as fp:
+            async for line in fp:
+                try:
+                    input_list.append(int(line))
+                except (ValueError, TypeError) as err:
+                    pass
+    except Exception as err:
+        error('Unexpected error when reading file: ' + filename + ' : ' + str(type(err)) + ' : '+ str(err))
+    return input_list
+
+
 async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool:
     """Save JSON data into file"""
     try:
@@ -96,6 +117,7 @@ async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool:
         error(str(err))
     return False
 
+
 async def readJSON(filename: str, chkJSONfunc = None):
     try:
         async with aiofiles.open(filename) as fp:
@@ -231,6 +253,8 @@ class WG:
         "tutorial": "Proving Grounds"
     }
 
+    tanks = None
+
     nations = [ 'ussr', 'germany', 'usa', 'china', 'france', 'uk', 'japan', 'other']    
     nation_id = {
         'ussr'      : 0,
@@ -268,13 +292,13 @@ class WG:
     def __init__(self, WG_appID = None, tankopedia_fn =  None, maps_fn = None):
         
         self.WG_appID = WG_appID
-        self.tanks = None      
 
         if (tankopedia_fn != None):
             if os.path.exists(tankopedia_fn) and os.path.isfile(tankopedia_fn):
                 try:
                     with open(tankopedia_fn, 'rt', encoding='utf8') as f:
-                        self.tanks = json.loads(f.read())
+                        WG.tanks = json.loads(f.read())
+                        debug(str(WG.tanks["meta"]["count"]) + " tanks")
                 except Exception as err:
                     error('Could not read tankopedia: ' + tankopedia_fn + '\n' + str(err))  
             else:
@@ -356,6 +380,10 @@ class WG:
     def getMapUserStrs(cls) -> str:
         return cls.maps.keys()
 
+    @classmethod
+    def getTankUserStrs(cls) -> str:
+        return cls.tanks["userStr"].keys()
+
     @classmethod
     def chkJSONcontent(cls, json_obj, check = None) -> bool:
         try:
@@ -682,7 +710,6 @@ class WoTinspector:
                 'key'           : replay_id
             } 
 
-
             url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
             #debug('URL: ' + url)
             headers ={'Content-type':  'application/x-www-form-urlencoded'}
@@ -701,18 +728,21 @@ class WoTinspector:
                         json_resp = await resp.json()
                         if json_resp.get('status', None) == None:
                             error(msg_str +' : ' + title + ' : Received invalid JSON')
-                        elif (json_resp['status'] == 'error'):  
+                        elif (json_resp['status'] == 'ok'): 
-                            error(msg_str + json_resp['error']['message'] + ' : ' + title)
-                        else:
                             debug('Response data read')
                             verbose(msg_str + title + ' posted')
                             return json_resp	
+                        elif (json_resp['status'] == 'error'):  
+                            error(msg_str + json_resp['error']['message'] + ' : ' + title)
                         else:
-                        debug(msg_str + 'Got HTTP/' + str(resp.status) + ' Retrying... ' + str(retry))
+                            error(msg_str + ' Unspecified error: ' + title)											
-                        await asyncio.sleep(SLEEP)								
+                    else:
+                        debug(msg_str + 'Got HTTP/' + str(resp.status))
             except Exception as err:
                 error(str(err))
             await asyncio.sleep(SLEEP)
+            
+        error(msg_str + ' Could not post replay: ' + title)
         return None
 
     @classmethod

maps.json

@@ -7,6 +7,7 @@
     "desert_train": "Desert Sands",
     "erlenberg": "Middleburg",
     "faust": "Faust",
+    "forgecity": "New Bay",
     "fort": "Fort Despair",
     "grossberg": "Dynasty's Pearl",
     "himmelsdorf": "Himmelsdorf",

wotbreplay_uploader.py

@@ -18,6 +18,7 @@ SLEEP = 2
 MAX_RETRIES = 3
 REPLAY_N 	= 0
 SKIPPED_N 	= 0
+ERROR_N 	= 0
 WIurl='https://wotinspector.com/api/replay/upload?'
 WG_appID  = '81381d3f45fa4aa75b78a7198eb216ad'
 wg = None
@@ -33,7 +34,7 @@ async def main(argv):
 	parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
 	parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering')
 	parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com')
-	parser.add_argument('--tasks', dest='N_tasks', type=int, default=10, help='Number of worker threads')
+	parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads')
 	parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
 	parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
 	parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
@@ -77,7 +78,7 @@ async def main(argv):
 			task.cancel()
 		bu.debug('Waiting for workers to cancel')
 		await asyncio.gather(*tasks, return_exceptions=True)
-		bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped')
+		bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
 				
 	except KeyboardInterrupt:
 		print('Ctrl-c pressed ...')
@@ -136,6 +137,7 @@ async def mkQueueItem(filename : str, title : str) -> list:
 async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
 	"""Async Worker to process the replay queue"""
 	global SKIPPED_N
+	global ERROR_N
 
 	while True:
 		item = await queue.get()
@@ -158,6 +160,9 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 						SKIPPED_N += 1
 						queue.task_done()						
 						continue
+					else:
+						os.remove(replay_json_fn)
+						bu.debug("Replay JSON not valid: Deleting " + replay_json_fn)
 		except asyncio.CancelledError as err:
 			raise err
 		except Exception as err:
@@ -173,6 +178,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 						bu.debug(msg_str + 'Replay saved OK: ' + filename )
 					else:
 						bu.error(msg_str + 'Error saving replay: ' + filename)
+						ERROR_N += 1
 		except Exception as err:
 			bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
 		bu.debug(msg_str + 'Marking task done')
@@ -188,7 +194,12 @@ def getTitle(replayfile: str, title: str, i : int) -> str:
 			filename = os.path.basename(replayfile)	
 			bu.debug(filename)
 			map_usrStrs = wg.getMapUserStrs()
-			p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
+			tank_userStrs = wg.getTankUserStrs()
+			
+			#p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
+			# update 6.2 changed the file name format
+			p = re.compile('\\d{8}_\\d{4}_.+?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
+			
 			m = p.match(filename)
 			if (m != None):
 				if wg.tanks != None: