From 99a7e4530ea536fe67cc275c85edf728ccd25d16 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Fri, 16 Aug 2019 22:53:06 +0300 Subject: [PATCH] Update 6.2 + bug fixes --- blitzutils.py | 54 ++++++++++++++++++++++++++++++++---------- maps.json | 1 + wotbreplay_uploader.py | 21 ++++++++++++---- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index 816d553..679a3bd 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -5,7 +5,7 @@ import asyncio, aiofiles, aiohttp, aiosqlite, lxml from bs4 import BeautifulSoup MAX_RETRIES= 3 -SLEEP = 2 +SLEEP = 3 DEBUG = False VERBOSE = False SILENT = False @@ -59,11 +59,13 @@ def verbose_std(msg = ""): def printWaiter(force = False): if not DEBUG and (not SILENT or force): print('.', end='', flush=True) + def printNewline(force = False): if not DEBUG and (not SILENT or force): print('', flush=True) + def debug(msg = ""): """print a conditional debug message""" if DEBUG: @@ -73,6 +75,7 @@ def debug(msg = ""): print('DEBUG: ' + caller + '(): ' + msg) return None + def error(msg = ""): """Print an error message""" curframe = inspect.currentframe() @@ -80,9 +83,27 @@ def error(msg = ""): caller = calframe[1][3] print('ERROR: ' + caller + '(): ' + msg) return None + def NOW() -> int: return int(time.time()) + +async def readPlainList(filename: str) -> list(): + """Read file to a list and return list of integers in the input file""" + + input_list = [] + try: + async with aiofiles.open(filename) as fp: + async for line in fp: + try: + input_list.append(int(line)) + except (ValueError, TypeError) as err: + pass + except Exception as err: + error('Unexpected error when reading file: ' + filename + ' : ' + str(type(err)) + ' : '+ str(err)) + return input_list + + async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool: """Save JSON data into file""" try: @@ -96,6 +117,7 @@ async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool: error(str(err)) return False + async def readJSON(filename: str, chkJSONfunc = None): try: async with aiofiles.open(filename) as fp: @@ -231,6 +253,8 @@ class WG: "tutorial": "Proving Grounds" } + tanks = None + nations = [ 'ussr', 'germany', 'usa', 'china', 'france', 'uk', 'japan', 'other'] nation_id = { 'ussr' : 0, @@ -268,13 +292,13 @@ class WG: def __init__(self, WG_appID = None, tankopedia_fn = None, maps_fn = None): self.WG_appID = WG_appID - self.tanks = None if (tankopedia_fn != None): if os.path.exists(tankopedia_fn) and os.path.isfile(tankopedia_fn): try: with open(tankopedia_fn, 'rt', encoding='utf8') as f: - self.tanks = json.loads(f.read()) + WG.tanks = json.loads(f.read()) + debug(str(WG.tanks["meta"]["count"]) + " tanks") except Exception as err: error('Could not read tankopedia: ' + tankopedia_fn + '\n' + str(err)) else: @@ -356,6 +380,10 @@ class WG: def getMapUserStrs(cls) -> str: return cls.maps.keys() + @classmethod + def getTankUserStrs(cls) -> str: + return cls.tanks["userStr"].keys() + @classmethod def chkJSONcontent(cls, json_obj, check = None) -> bool: try: @@ -680,8 +708,7 @@ class WoTinspector: 'uploaded_by' : account_id, 'details' : 'full', 'key' : replay_id - } - + } url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote) #debug('URL: ' + url) @@ -701,19 +728,22 @@ class WoTinspector: json_resp = await resp.json() if json_resp.get('status', None) == None: error(msg_str +' : ' + title + ' : Received invalid JSON') + elif (json_resp['status'] == 'ok'): + debug('Response data read') + verbose(msg_str + title + ' posted') + return json_resp elif (json_resp['status'] == 'error'): error(msg_str + json_resp['error']['message'] + ' : ' + title) else: - debug('Response data read') - verbose(msg_str + title + ' posted') - return json_resp + error(msg_str + ' Unspecified error: ' + title) else: - debug(msg_str + 'Got HTTP/' + str(resp.status) + ' Retrying... ' + str(retry)) - await asyncio.sleep(SLEEP) + debug(msg_str + 'Got HTTP/' + str(resp.status)) except Exception as err: error(str(err)) - await asyncio.sleep(SLEEP) - return None + await asyncio.sleep(SLEEP) + + error(msg_str + ' Could not post replay: ' + title) + return None @classmethod def getUrlReplayListing(cls, page : int): diff --git a/maps.json b/maps.json index 75b6c76..5754cca 100644 --- a/maps.json +++ b/maps.json @@ -7,6 +7,7 @@ "desert_train": "Desert Sands", "erlenberg": "Middleburg", "faust": "Faust", + "forgecity": "New Bay", "fort": "Fort Despair", "grossberg": "Dynasty's Pearl", "himmelsdorf": "Himmelsdorf", diff --git a/wotbreplay_uploader.py b/wotbreplay_uploader.py index 4e1e9f2..3c0cf9d 100644 --- a/wotbreplay_uploader.py +++ b/wotbreplay_uploader.py @@ -16,8 +16,9 @@ DEBUG = False VERBOSE = False SLEEP = 2 MAX_RETRIES = 3 -REPLAY_N = 0 -SKIPPED_N = 0 +REPLAY_N = 0 +SKIPPED_N = 0 +ERROR_N = 0 WIurl='https://wotinspector.com/api/replay/upload?' WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' wg = None @@ -33,7 +34,7 @@ async def main(argv): parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering') parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com') - parser.add_argument('--tasks', dest='N_tasks', type=int, default=10, help='Number of worker threads') + parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') @@ -77,7 +78,7 @@ async def main(argv): task.cancel() bu.debug('Waiting for workers to cancel') await asyncio.gather(*tasks, return_exceptions=True) - bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped') + bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors') except KeyboardInterrupt: print('Ctrl-c pressed ...') @@ -136,6 +137,7 @@ async def mkQueueItem(filename : str, title : str) -> list: async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False): """Async Worker to process the replay queue""" global SKIPPED_N + global ERROR_N while True: item = await queue.get() @@ -158,6 +160,9 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri SKIPPED_N += 1 queue.task_done() continue + else: + os.remove(replay_json_fn) + bu.debug("Replay JSON not valid: Deleting " + replay_json_fn) except asyncio.CancelledError as err: raise err except Exception as err: @@ -173,6 +178,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri bu.debug(msg_str + 'Replay saved OK: ' + filename ) else: bu.error(msg_str + 'Error saving replay: ' + filename) + ERROR_N += 1 except Exception as err: bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.debug(msg_str + 'Marking task done') @@ -188,7 +194,12 @@ def getTitle(replayfile: str, title: str, i : int) -> str: filename = os.path.basename(replayfile) bu.debug(filename) map_usrStrs = wg.getMapUserStrs() - p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') + tank_userStrs = wg.getTankUserStrs() + + #p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') + # update 6.2 changed the file name format + p = re.compile('\\d{8}_\\d{4}_.+?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') + m = p.match(filename) if (m != None): if wg.tanks != None: