From 23e8d48bd9abaefb08fd700a5e70e5aebd14cd20 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sun, 23 Feb 2020 00:43:05 +0200 Subject: [PATCH 1/8] Rate-limiter added to keep server load modest --- blitzutils.py | 189 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 166 insertions(+), 23 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index b98c29d..2a2e410 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -27,6 +27,132 @@ _progress_obj = None UMASK= os.umask(0) os.umask(UMASK) +## ----------------------------------------------------------- +#### Class ThrottledClientSession(aiohttp.ClientSession) +## ----------------------------------------------------------- + + +class ThrottledClientSession(aiohttp.ClientSession): + """Rate-throttled client session class inherited from aiohttp.ClientSession)""" + MAX_STEPS = 10 + + def __init__(self, rate_limit: int =None, *args,**kwargs) -> None: + super().__init__(*args,**kwargs) + self.rate_limit = rate_limit + self._fillerTask = None + self._queue = None + + if self.rate_limit != None: + if self.rate_limit < 1: + raise ValueError('rate_limit must be positive integer') + (increment, sleep) = self._get_rate_increment() + self._queue = asyncio.Queue(rate_limit) + self._fillerTask = asyncio.create_task(self._filler(increment, sleep)) + self._start_time = None + self._count = 0 + + def _get_rate_increment(self) -> list: + if self.rate_limit <= self.MAX_STEPS: + return (1, 1/self.rate_limit) + else: + return (self.rate_limit // self.MAX_STEPS, 1/self.MAX_STEPS) + + async def close(self) -> None: + """Close rate-limiter's "bucket filler" task""" + # DEBUG + if self._start_time != None: + duration = time.time() - self._start_time + debug('Average WG API request rate: ' + '{:.1f}'.format(self._count / duration) + ' / sec') + self._fillerTask.cancel() + try: + await asyncio.wait_for(self._fillerTask, timeout= 3) + except asyncio.TimeoutError as err: + error(exception=err) + await super().close() + + + async def _filler(self, increment: int = 1, sleep: float = 1): + """Filler task to fill the leaky bucket algo""" + try: + if self._queue == None: + return + fraction = 0 + extra_increment = 0 + while True: + if not self._queue.full(): + if self.rate_limit > self.MAX_STEPS: + fraction += (self.rate_limit - increment / sleep) / self.MAX_STEPS + extra_increment = fraction // 1 + fraction -= extra_increment + items_2_add = int(min(self.rate_limit - self._queue.qsize(), increment + extra_increment)) + for i in range(0,items_2_add): + self._queue.put_nowait(i) + await asyncio.sleep(sleep) + except asyncio.CancelledError: + debug('Cancelled') + + + async def _allow(self) -> None: + if self._queue != None: + # debug + if self._start_time == None: + self._start_time = time.time() + await self._queue.get() + self._queue.task_done() + # DEBUG + self._count += 1 + return None + + + async def _request(self, *args,**kwargs): + """Throttled _request()""" + await self._allow() + return await super()._request(*args,**kwargs) + + +## ----------------------------------------------------------- +#### Class asyncThrottle +## ----------------------------------------------------------- + +# class asyncThrottle: +# def __init__(self, rate: int = 20): +# if rate < 1: +# error('Rate must be positive integer') +# self.rate = rate +# self.queue = asyncio.Queue(rate) +# self.fillerTask = asyncio.create_task(self.filler()) +# self.start_time = time.time() +# self.count = 0 + + +# async def close(self): +# if self.start_time != None: +# duration = time.time() - self.start_time +# debug('Average WG API request rate: ' + '{:.1f}'.format(self.count / duration) + ' / sec', force=True) +# self.fillerTask.cancel() +# try: +# await asyncio.wait_for(self.fillerTask, timeout= 3) +# except asyncio.TimeoutError as err: +# error(exception=err) + + +# async def filler(self): +# try: +# while True: +# if not self.queue.full(): +# items_2_add = self.rate - self.queue.qsize() +# for i in range(0,items_2_add): +# self.queue.put_nowait(i) +# self.count += 1 +# await asyncio.sleep(1) +# except asyncio.CancelledError: +# debug('Cancelled') + +# async def allow(self) -> None: +# await self.queue.get() +# self.queue.task_done() +# return None + def set_debug(debug: bool): global _log_level @@ -87,9 +213,9 @@ def verbose_std(msg = "", id = None): return None -def debug(msg = "", id = None, exception = None): +def debug(msg = "", id = None, exception = None, force: bool = False): """print a conditional debug message""" - if _log_level >= DEBUG: + if (_log_level >= DEBUG) or force: _print_log_msg('DEBUG', msg, exception, id) return None @@ -271,6 +397,9 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func = # debug("Received valid JSON: " + str(json_resp)) return json_resp # Sometimes WG API returns JSON error even a retry gives valid JSON + elif resp.status == 407: + json_resp_err = await resp.json() + error('WG API returned 407: ' + json_resp_err['error']['message']) if retry == max_tries: break debug('Retrying URL [' + str(retry) + '/' + str(max_tries) + ']: ' + url ) @@ -321,7 +450,6 @@ class SlowBar(IncrementalBar): return (self.eta - (self.eta // 3600)*3600) // 60 - ## ----------------------------------------------------------- #### Class StatsNotFound ## ----------------------------------------------------------- @@ -329,6 +457,9 @@ class SlowBar(IncrementalBar): class StatsNotFound(Exception): pass + + + ## ----------------------------------------------------------- #### Class WG ## ----------------------------------------------------------- @@ -456,25 +587,27 @@ class WG: } URL_WG_SERVER = { - 'eu' : 'https://api.wotblitz.eu/wotb/', - 'ru' : 'https://api.wotblitz.ru/wotb/', - 'na' : 'https://api.wotblitz.com/wotb/', - 'asia' : 'https://api.wotblitz.asia/wotb/' + 'eu' : 'https://api.wotblitz.eu/wotb/', + 'ru' : 'https://api.wotblitz.ru/wotb/', + 'na' : 'https://api.wotblitz.com/wotb/', + 'asia' : 'https://api.wotblitz.asia/wotb/', + 'china' : None } ACCOUNT_ID_SERVER= { 'ru' : range(0, int(5e8)), 'eu' : range(int(5e8), int(10e8)), 'na' : range(int(1e9),int(2e9)), - 'asia' : range(int(2e9),int(4e9)) + 'asia' : range(int(2e9),int(3e9)), + 'china' : range(int(3e9),int(4e9)) } - def __init__(self, WG_app_id = None, tankopedia_fn = None, maps_fn = None, stats_cache = False): + def __init__(self, WG_app_id : str = None, tankopedia_fn : str = None, maps_fn : str = None, stats_cache: bool = False, rate_limit: int = 10): self.WG_app_id = WG_app_id self.load_tanks(tankopedia_fn) WG.tanks = self.tanks - + if (maps_fn != None): if os.path.exists(maps_fn) and os.path.isfile(maps_fn): try: @@ -484,9 +617,10 @@ class WG: error('Could not read maps file: ' + maps_fn + '\n' + str(err)) else: verbose('Could not find maps file: ' + maps_fn) + self.rate_limiter = None if self.WG_app_id != None: - self.session = aiohttp.ClientSession() - debug('WG aiohttp session initiated') + self.session = ThrottledClientSession(rate_limit=rate_limit) + debug('WG aiohttp session initiated') else: self.session = None debug('WG aiohttp session NOT initiated') @@ -529,9 +663,7 @@ class WG: self.stat_saver_task.cancel() debug('statsCacheTask cancelled') await self.stat_saver_task - - - + # close cacheDB if self.cache != None: # prune old cache records @@ -540,7 +672,8 @@ class WG: await self.cache.close() if self.session != None: - await self.session.close() + await self.session.close() + return ## Class methods ---------------------------------------------------------- @@ -649,6 +782,7 @@ class WG: error("JSON format error", err) return False + @classmethod def chk_JSON_get_account_id(cls, json_resp: dict) -> bool: try: @@ -702,6 +836,7 @@ class WG: debug("JSON check failed") return False + @classmethod def chk_JSON_tank_stats(cls, json_resp: dict) -> bool: """"Check String for being a valid Tank list JSON file""" @@ -761,7 +896,7 @@ class WG: return self.get_url_player_tanks_stats(account_id, fields='tank_id') - def get_url_player_tanks_stats(self, account_id: int, tank_ids = [], fields = []) -> str: + def get_url_player_tanks_stats(self, account_id: int, tank_ids: list = [], fields: list = []) -> str: server = self.get_server(account_id) if server == None: return None @@ -823,6 +958,7 @@ class WG: if nick == None or server == None: raise ValueError('Invalid nickname given: ' + nickname) url = self.get_url_account_id(nick, server) + json_data = await get_url_JSON(self.session, url, self.chk_JSON_status) for res in json_data['data']: if res['nickname'].lower() == nick.lower(): @@ -1096,13 +1232,15 @@ class WoTinspector: REPLAY_N = 1 - def __init__(self): - self.session = aiohttp.ClientSession() + def __init__(self, rate_limit: int = 30): + self.session = ThrottledClientSession(rate_limit=rate_limit) + async def close(self): if self.session != None: debug('Closing aiohttp session') - await self.session.close() + await self.session.close() + async def get_tankopedia(self, filename = 'tanks.json'): """Retrieve Tankpedia from WoTinspector.com""" @@ -1143,7 +1281,7 @@ class WoTinspector: async def get_replay_JSON(self, replay_id: str): - json_resp = await get_url_JSON(self.session, self.URL_REPLAY_INFO + replay_id, None) + json_resp = await get_url_JSON(self.session, self.URL_REPLAY_INFO + replay_id, chk_JSON_func=None) try: if self.chk_JSON_replay(json_resp): return json_resp @@ -1212,6 +1350,11 @@ class WoTinspector: error(msg_str + ' Could not post replay: ' + title) return None + + async def get_replay_listing(self, page: int = 0) -> aiohttp.ClientResponse: + url = self.get_url_replay_listing(page) + return await self.session.get(url) + @classmethod def get_url_replay_listing(cls, page : int): return cls.URL_REPLAY_LIST + str(page) + '?vt=#filters' @@ -1258,8 +1401,8 @@ class BlitzStars: URL_ACTIVE_PLAYERS = URL_BLITZSTARS + '/api/playerstats/activeinlast30days' - def __init__(self): - self.session = aiohttp.ClientSession() + def __init__(self, rate_limit=30): + self.session = ThrottledClientSession(rate_limit=rate_limit) async def close(self): From 2c12782b506d688a3cde511c82304f94e62db571 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sun, 23 Feb 2020 00:49:27 +0200 Subject: [PATCH 2/8] update 6.8 --- maps.json | 1 + 1 file changed, 1 insertion(+) diff --git a/maps.json b/maps.json index 20ff83f..66d9fc4 100644 --- a/maps.json +++ b/maps.json @@ -26,6 +26,7 @@ "ordeal": "Trial by Fire", "pliego": "Castilla", "port": "Port Bay", + "rift": "Hellas", "rock": "Mayan Ruins", "rudniki": "Mines", "savanna": "Oasis Palms", From f4c7f13a0391fc0310633f97fdcb7cf9485361ce Mon Sep 17 00:00:00 2001 From: Jylpah Date: Sun, 23 Feb 2020 01:04:42 +0200 Subject: [PATCH 3/8] Update 6.8 --- tanks.json | 113 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 43 deletions(-) diff --git a/tanks.json b/tanks.json index 8cbb7eb..7d704bb 100644 --- a/tanks.json +++ b/tanks.json @@ -1,7 +1,7 @@ { "status": "ok", "meta": { - "count": 426 + "count": 429 }, "data": { "1": { @@ -37,12 +37,12 @@ "name": "Type 59" }, "81": { - "is_premium": false, - "name": "Vickers Medium Mk. I", - "nation": "uk", "tank_id": 81, + "nation": "uk", "tier": 1, - "type": "mediumTank" + "is_premium": false, + "type": "mediumTank", + "name": "Vickers Medium Mk. I" }, "113": { "tank_id": 113, @@ -117,12 +117,12 @@ "name": "Tiger I" }, "545": { - "is_premium": false, - "name": "T1 Cunningham", - "nation": "usa", "tank_id": 545, + "nation": "usa", "tier": 1, - "type": "lightTank" + "is_premium": false, + "type": "lightTank", + "name": "T1 Cunningham" }, "577": { "tank_id": 577, @@ -418,7 +418,7 @@ "tier": 7, "is_premium": true, "type": "AT-SPG", - "name": "Helsing H0" + "name": "Helsing" }, "1665": { "tank_id": 1665, @@ -757,12 +757,12 @@ "name": "Black Prince" }, "3329": { - "is_premium": false, - "name": "MS-1", - "nation": "ussr", "tank_id": 3329, + "nation": "ussr", "tier": 1, - "type": "lightTank" + "is_premium": false, + "type": "lightTank", + "name": "MS-1" }, "3345": { "tank_id": 3345, @@ -2624,7 +2624,7 @@ "tank_id": 20257, "nation": "usa", "tier": 10, - "is_premium": true, + "is_premium": false, "type": "lightTank", "name": "XM551 Sheridan" }, @@ -3186,7 +3186,7 @@ "tier": 7, "is_premium": true, "type": "mediumTank", - "name": "T6 Dracula" + "name": "Dracula" }, "63841": { "tank_id": 63841, @@ -3194,7 +3194,7 @@ "tier": 5, "is_premium": true, "type": "mediumTank", - "name": "Panzer IV Anko Special" + "name": "Panzer IV Ankou Special" }, "64001": { "tank_id": 64001, @@ -3285,12 +3285,12 @@ "name": "IS-6 Fearless" }, "64785": { - "is_premium": false, - "name": "Pz.Kpfw. II", - "nation": "germany", "tank_id": 64785, - "tier": 2, - "type": "lightTank" + "nation": "germany", + "tier": 1, + "is_premium": false, + "type": "lightTank", + "name": "Pz.Kpfw. II" }, "64801": { "tank_id": 64801, @@ -3317,20 +3317,20 @@ "name": "MS-1" }, "65041": { - "is_premium": false, - "name": "Leichttraktor", - "nation": "germany", "tank_id": 65041, + "nation": "germany", "tier": 1, - "type": "lightTank" + "is_premium": false, + "type": "lightTank", + "name": "Leichttraktor" }, "65057": { - "is_premium": false, - "name": "T1 Cunningham", - "nation": "usa", "tank_id": 65057, + "nation": "usa", "tier": 1, - "type": "lightTank" + "is_premium": false, + "type": "lightTank", + "name": "T1 Cunningham" }, "65105": { "tank_id": 65105, @@ -3341,20 +3341,20 @@ "name": "Vickers Medium Mk. I" }, "65281": { - "is_premium": false, - "name": "MS-1", - "nation": "ussr", "tank_id": 65281, + "nation": "ussr", "tier": 1, - "type": "lightTank" + "is_premium": false, + "type": "lightTank", + "name": "MS-1" }, "65297": { - "is_premium": false, - "name": "Leichttraktor", - "nation": "germany", "tank_id": 65297, + "nation": "germany", "tier": 1, - "type": "lightTank" + "is_premium": false, + "type": "lightTank", + "name": "Leichttraktor" }, "65313": { "tank_id": 65313, @@ -3373,12 +3373,12 @@ "name": "Type 62" }, "65361": { - "is_premium": false, - "name": "Vickers Medium Mk. I", - "nation": "uk", "tank_id": 65361, + "nation": "uk", "tier": 1, - "type": "mediumTank" + "is_premium": false, + "type": "mediumTank", + "name": "Vickers Medium Mk. I" }, "65377": { "tank_id": 65377, @@ -3411,6 +3411,30 @@ "is_premium": true, "type": "AT-SPG", "name": "SU-130PM" + }, + "18769": { + "tank_id": 18769, + "nation": "uk", + "tier": 8, + "is_premium": true, + "type": "heavyTank", + "name": "Caernarvon Action X" + }, + "19281": { + "tank_id": 19281, + "nation": "uk", + "tier": 10, + "is_premium": true, + "type": "heavyTank", + "name": "Super Conqueror" + }, + "19537": { + "tank_id": 19537, + "nation": "uk", + "tier": 10, + "is_premium": true, + "type": "lightTank", + "name": "Vickers Light 105" } }, "userStr": { @@ -3825,6 +3849,9 @@ "S04_Lago-I": "Lago", "A99_T92E1": "T92E1", "GB_Centurion_MkDF": "Defender Mk. 1", - "R159_SU_130PM": "SU-130PM" + "R159_SU_130PM": "SU-130PM", + "GB91_Super_Conqueror": "Super Conqueror", + "GB93_Caernarvon_AX": "Caernarvon Action X", + "GB_Vickers_Light_105": "Vickers Light 105" } } \ No newline at end of file From cd605348e9d6de4d5b4c841130e74093c33682df Mon Sep 17 00:00:00 2001 From: Jylpah Date: Mon, 24 Feb 2020 23:57:02 +0200 Subject: [PATCH 4/8] Deleted. Use upload_wotb_replays.py --- postwotreplay.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 postwotreplay.py diff --git a/postwotreplay.py b/postwotreplay.py deleted file mode 100644 index 0eb3058..0000000 --- a/postwotreplay.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python3 - -import json, requests, sys, base64 - -filename = str(sys.argv[1]) - -#Change this to your own WG id. Find it here: https://api.wotblitz.eu/wotb/account/list/?account_id%2Cnickname -# WG account id of the uploader: -uploader_account_ID = 0 - -# edit this or use arg.parse() to add proper suppod for commandline arguments -title="Some title" -url='https://wotinspector.com/api/replay/upload?title=' + title + '&details=full&uploaded_by=' + str(uploader_account_ID) - -# additional option: 'private=1' to keep the replay way from public listing at WoTinspector.com/replays - -with open(filename,'rb') as f: - payload = {'file' : (filename, base64.b64encode(f.read()))} - r = requests.post(url, data=payload) - print(r.json()) \ No newline at end of file From 5e4b7d858c501bcc5cc86016ef5a5f90f2341f28 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Mon, 24 Feb 2020 23:57:56 +0200 Subject: [PATCH 5/8] Rate throtling v2, removed WG.create(), fixed player_stats retrieval + cache --- blitzutils.py | 173 ++++++++++++++++++++------------------------------ 1 file changed, 68 insertions(+), 105 deletions(-) diff --git a/blitzutils.py b/blitzutils.py index 2a2e410..c8c086d 100644 --- a/blitzutils.py +++ b/blitzutils.py @@ -31,39 +31,36 @@ os.umask(UMASK) #### Class ThrottledClientSession(aiohttp.ClientSession) ## ----------------------------------------------------------- - class ThrottledClientSession(aiohttp.ClientSession): """Rate-throttled client session class inherited from aiohttp.ClientSession)""" - MAX_STEPS = 10 + MIN_SLEEP = 0.1 - def __init__(self, rate_limit: int =None, *args,**kwargs) -> None: + def __init__(self, rate_limit: float =None, *args,**kwargs) -> None: super().__init__(*args,**kwargs) self.rate_limit = rate_limit self._fillerTask = None self._queue = None - if self.rate_limit != None: - if self.rate_limit < 1: - raise ValueError('rate_limit must be positive integer') - (increment, sleep) = self._get_rate_increment() - self._queue = asyncio.Queue(rate_limit) - self._fillerTask = asyncio.create_task(self._filler(increment, sleep)) + if rate_limit != None: + if rate_limit <= 0: + raise ValueError('rate_limit must be positive') + #(increment, sleep) = self._get_rate_increment() + self._queue = asyncio.Queue(min(2, int(rate_limit)+1)) + self._fillerTask = asyncio.create_task(self._filler(rate_limit)) self._start_time = None self._count = 0 - def _get_rate_increment(self) -> list: - if self.rate_limit <= self.MAX_STEPS: - return (1, 1/self.rate_limit) - else: - return (self.rate_limit // self.MAX_STEPS, 1/self.MAX_STEPS) - + def _get_sleep(self) -> list: + return max(1/self.rate_limit, self.MIN_SLEEP) + async def close(self) -> None: """Close rate-limiter's "bucket filler" task""" # DEBUG if self._start_time != None: duration = time.time() - self._start_time debug('Average WG API request rate: ' + '{:.1f}'.format(self._count / duration) + ' / sec') - self._fillerTask.cancel() + if self._fillerTask != None: + self._fillerTask.cancel() try: await asyncio.wait_for(self._fillerTask, timeout= 3) except asyncio.TimeoutError as err: @@ -71,25 +68,35 @@ class ThrottledClientSession(aiohttp.ClientSession): await super().close() - async def _filler(self, increment: int = 1, sleep: float = 1): + async def _filler(self, rate_limit: float = 1): """Filler task to fill the leaky bucket algo""" try: if self._queue == None: return + self.rate_limit = rate_limit + sleep = self._get_sleep() + debug('SLEEP: ' + str(sleep)) + updated_at = time.monotonic() fraction = 0 extra_increment = 0 + for i in range(0,self._queue.maxsize): + self._queue.put_nowait(i) while True: if not self._queue.full(): - if self.rate_limit > self.MAX_STEPS: - fraction += (self.rate_limit - increment / sleep) / self.MAX_STEPS - extra_increment = fraction // 1 - fraction -= extra_increment - items_2_add = int(min(self.rate_limit - self._queue.qsize(), increment + extra_increment)) + now = time.monotonic() + increment = rate_limit * (now - updated_at) + fraction += increment % 1 + extra_increment = fraction // 1 + items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment)) + fraction = fraction % 1 for i in range(0,items_2_add): - self._queue.put_nowait(i) + self._queue.put_nowait(i) + updated_at = now await asyncio.sleep(sleep) except asyncio.CancelledError: debug('Cancelled') + except Exception as err: + error(exception=err) async def _allow(self) -> None: @@ -109,51 +116,6 @@ class ThrottledClientSession(aiohttp.ClientSession): await self._allow() return await super()._request(*args,**kwargs) - -## ----------------------------------------------------------- -#### Class asyncThrottle -## ----------------------------------------------------------- - -# class asyncThrottle: -# def __init__(self, rate: int = 20): -# if rate < 1: -# error('Rate must be positive integer') -# self.rate = rate -# self.queue = asyncio.Queue(rate) -# self.fillerTask = asyncio.create_task(self.filler()) -# self.start_time = time.time() -# self.count = 0 - - -# async def close(self): -# if self.start_time != None: -# duration = time.time() - self.start_time -# debug('Average WG API request rate: ' + '{:.1f}'.format(self.count / duration) + ' / sec', force=True) -# self.fillerTask.cancel() -# try: -# await asyncio.wait_for(self.fillerTask, timeout= 3) -# except asyncio.TimeoutError as err: -# error(exception=err) - - -# async def filler(self): -# try: -# while True: -# if not self.queue.full(): -# items_2_add = self.rate - self.queue.qsize() -# for i in range(0,items_2_add): -# self.queue.put_nowait(i) -# self.count += 1 -# await asyncio.sleep(1) -# except asyncio.CancelledError: -# debug('Cancelled') - -# async def allow(self) -> None: -# await self.queue.get() -# self.queue.task_done() -# return None - - def set_debug(debug: bool): global _log_level _log_level = DEBUG @@ -193,37 +155,39 @@ def get_log_level_str() -> str: error('Unknown log level: ' + str(_log_level)) -def verbose(msg = "", id = None): +def verbose(msg = "", id = None) -> bool: """Print a message""" if _log_level >= VERBOSE: if id == None: print(msg) else: print('[' + str(id) + ']: ' + msg) - return None + return True + return False -def verbose_std(msg = "", id = None): +def verbose_std(msg = "", id = None) -> bool: """Print a message""" if _log_level >= NORMAL: if id == None: print(msg) else: print('[' + str(id) + ']: ' + msg) - return None + return True + return False - -def debug(msg = "", id = None, exception = None, force: bool = False): +def debug(msg = "", id = None, exception = None, force: bool = False) -> bool: """print a conditional debug message""" if (_log_level >= DEBUG) or force: _print_log_msg('DEBUG', msg, exception, id) - return None + return True + return False -def error(msg = "", exception = None, id = None): +def error(msg = "", exception = None, id = None) -> bool: """Print an error message""" _print_log_msg('ERROR', msg, exception, id) - return None + return True def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None): @@ -625,34 +589,18 @@ class WG: self.session = None debug('WG aiohttp session NOT initiated') - # cache TBD + # cache self.cache = None self.statsQ = None self.stat_saver_task = None - - - @classmethod - async def create(cls, WG_app_id = None, tankopedia_fn = None, maps_fn = None, stats_cache = False): - """Separete Constuctor method to handle async calls required to initialize the object""" - self = WG(WG_app_id, tankopedia_fn , maps_fn ) - if stats_cache: try: - self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE) - ## Player Tank stats cache table - await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL) - - await self.cache.commit() - - async with self.cache.execute(WG.SQL_TANK_STATS_COUNT) as cursor: - debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' cached player tank stat records' ) - self.statsQ = asyncio.Queue() self.stat_saver_task = asyncio.create_task(self.stat_saver()) except Exception as err: error(exception=err) sys.exit(1) - return self + async def close(self): # close stats queue @@ -879,6 +827,7 @@ class WG: error('Invalid tier', err) return None + def get_url_clan_info(self, server: str, clan_id: int) -> str: try: if server == None: @@ -988,7 +937,7 @@ class WG: #debug('JSON Response received: ' + str(json_data)) stats = json_data['data'][str(account_id)] if cache: - await self.save_stats('tank_stats', [account_id, tank_ids], stats) + await self.put_2_statsQ('tank_stats', [account_id, tank_ids], stats) return stats except Exception as err: error(exception=err) @@ -1020,7 +969,7 @@ class WG: #debug('JSON Response received: ' + str(json_data)) stats = json_data['data'][str(account_id)] if cache: - await self.save_stats('player_stats', [account_id], stats) + await self.put_2_statsQ('player_stats', [account_id], stats) return stats except Exception as err: error(exception=err) @@ -1050,12 +999,13 @@ class WG: except KeyError as err: error('Key not found', err) return None - + + def get_tank_tier(self, tank_id: int): return self.get_tank_data(tank_id, 'tier') - async def save_stats(self, statsType: str, key: list, stats: list): + async def put_2_statsQ(self, statsType: str, key: list, stats: list): """Save stats to a async queue to be saved by the stat_saver -task""" if self.statsQ == None: return False @@ -1063,12 +1013,26 @@ class WG: await self.statsQ.put([ statsType, key, stats, NOW() ]) return True + async def stat_saver(self): """Async task for saving stats into cache in background""" if self.statsQ == None: error('No statsQ defined') return None + try: + self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE) + ## Create cache tables table + await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL) + await self.cache.execute(WG.SQL_PLAYER_STATS_CREATE_TBL) + await self.cache.commit() + + async with self.cache.execute(WG.SQL_TANK_STATS_COUNT) as cursor: + debug('Cache contains: ' + str((await cursor.fetchone())[0]) + ' cached player tank stat records' ) + except Exception as err: + error(exception=err) + sys.exit(1) + while True: try: stats = await self.statsQ.get() @@ -1193,7 +1157,7 @@ class WG: #debug('No cache DB') raise StatsNotFound('No cache DB in use') - async with self.cache.execute(WG.SQL_PLAYER_STATS_CACHED, (account_id, NOW() - WG.CACHE_GRACE_TIME) ) as cursor: + async with self.cache.execute(WG.SQL_PLAYER_STATS_CACHED, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor: row = await cursor.fetchone() #debug('account_id: ' + str(account_id) + ': 1') if row == None: @@ -1202,18 +1166,18 @@ class WG: raise StatsNotFound('No cached stats found') debug('Cached stats found') - if row[3] == None: + if row[2] == None: # None/null stats found in cache # i.e. stats have been requested, but not returned from WG API return None else: # Return proper stats - return json.loads(row[3]) + return json.loads(row[2]) except StatsNotFound as err: debug(exception=err) raise except Exception as err: - error('Error trying to look for cached stats', err) + error('Error trying to look for cached stats', exception=err) return None @@ -1334,8 +1298,7 @@ class WoTinspector: if json_resp.get('status', None) == None: error(msg_str +' : ' + title + ' : Received invalid JSON') elif (json_resp['status'] == 'ok'): - debug('Response data read') - verbose(msg_str + title + ' posted') + debug('Response data read. Status OK') return json_resp elif (json_resp['status'] == 'error'): error(msg_str + json_resp['error']['message'] + ' : ' + title) From 4fcfdd22967b00b6154444ec5146f78a7becaa27 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Mon, 24 Feb 2020 23:58:20 +0200 Subject: [PATCH 6/8] Fix, rate-limiter added, clean-up --- upload_wotb_replays.py | 89 ++++++++++-------------------------------- 1 file changed, 20 insertions(+), 69 deletions(-) diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py index 088cfa0..5fd5b81 100644 --- a/upload_wotb_replays.py +++ b/upload_wotb_replays.py @@ -11,9 +11,10 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG) FILE_CONFIG = 'blitzstats.ini' # DB_C_REPLAYS = 'Replays' -DEBUG = False +DEBUG = False VERBOSE = False -SLEEP = 2 +SLEEP = 2 +N_WORKERS = 5 MAX_RETRIES = 3 REPLAY_N = 0 SKIPPED_N = 0 @@ -31,29 +32,13 @@ async def main(argv): ## Read config config = configparser.ConfigParser() config.read(FILE_CONFIG) - # configDB = config['DATABASE'] - # DB_SERVER = configDB.get('db_server', 'localhost') - # DB_PORT = configDB.getint('db_port', 27017) - # DB_SSL = configDB.getboolean('db_ssl', False) - # DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) - # DB_AUTH = configDB.get('db_auth', 'admin') - # DB_NAME = configDB.get('db_name', 'BlitzStats') - # DB_USER = configDB.get('db_user', 'mongouser') - # DB_PASSWD = configDB.get('db_password', "PASSWORD") - # DB_CERT = configDB.get('db_ssl_cert_file', None) - # DB_CA = configDB.get('db_ssl_ca_file', None) - # bu.debug('DB_SERVER: ' + DB_SERVER) - # bu.debug('DB_PORT: ' + str(DB_PORT)) - # bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") - # bu.debug('DB_AUTH: ' + DB_AUTH) - # bu.debug('DB_NAME: ' + DB_NAME) - - configGeneral = config['GENERAL'] + configWG = config['WG'] # WG account id of the uploader: # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ - WG_ID = configGeneral.getint('wg_id', 0) -# USE_DB = configGeneral.getint('use_DB', False) + WG_ID = configWG.getint('wg_id', None) + ## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay + WG_RATE_LIMIT = configWG.getint('wg_rate_limit', 10) parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data') #parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database') @@ -61,7 +46,7 @@ async def main(argv): parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering') parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com') - parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads') + parser.add_argument('--tasks', dest='N_tasks', type=int, default=N_WORKERS, help='Number of worker threads') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') # parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') @@ -75,22 +60,7 @@ async def main(argv): bu.set_log_level(args.silent, args.verbose, args.debug) wg = WG(WG_appID, args.tankopedia, args.mapfile) - wi = WoTinspector() - - #### Connect to MongoDB (TBD) - # client = None - # db = None - # if args.db: - # try: - # client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) - # db = client[DB_NAME] - # bu.debug('Database connection initiated') - # except Exception as err: - # bu.error("Could no initiate DB connection: Disabling DB", err) - # args.db = False - # pass - # else: - # bu.debug('No DB in use') + wi = WoTinspector(rate_limit=WG_RATE_LIMIT) if args.account != None: args.accountID = await wg.get_account_id(args.account) @@ -170,13 +140,14 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str): bu.debug('Finished') return None + async def mkQueueItem(filename : str, title : str) -> list: """Make an item to replay queue""" global REPLAY_N REPLAY_N +=1 return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)] -#async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False): + async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False): """Async Worker to process the replay queue""" global SKIPPED_N @@ -199,14 +170,14 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri replay_json = json.loads(await fp.read()) #if replay_json['status'] == 'ok': if wi.chk_JSON_replay(replay_json): - bu.verbose(msg_str + title + ' has already been posted. Skipping.' ) - SKIPPED_N += 1 - #await saveReplay2DB(db, replay_json) - queue.task_done() - continue + bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' ) else: os.remove(replay_json_fn) bu.debug("Replay JSON not valid: Deleting " + replay_json_fn) + SKIPPED_N += 1 + queue.task_done() + continue + except asyncio.CancelledError as err: raise err except Exception as err: @@ -218,12 +189,12 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri bu.debug(msg_str + 'File: ' + filename) json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N) if json_resp != None: - if (await bu.save_JSON(replay_json_fn,json_resp)): - bu.debug(msg_str + 'Replay saved OK: ' + filename ) + if (await bu.save_JSON(replay_json_fn,json_resp)): + if not bu.debug(msg_str + 'Replay saved OK: ' + filename): + bu.verbose_std(msg_str + title + ' posted') else: bu.error(msg_str + 'Error saving replay: ' + filename) - ERROR_N += 1 - # await saveReplay2DB(db, json_resp) + ERROR_N += 1 except Exception as err: bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.debug(msg_str + 'Marking task done') @@ -232,26 +203,6 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri return None -# async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool: -# """Save retriever replay JSON into DB (mongo)""" -# try: -# if db == None: -# return False -# dbc = db[DB_C_REPLAYS] -# replay_link = replay['data']['download_url'] -# replay_id = wi.get_replay_id(replay_link) -# replay['_id'] = replay_id -# await dbc.insert_one(replay) -# bu.debug('Replay added to database') -# except pymongo.errors.DuplicateKeyError as err: -# bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) -# return False -# except Exception as err: -# bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) -# return False -# return True - - def getTitle(replayfile: str, title: str, i : int) -> str: global wg From 5bb9e1320d6fc932c8a9adbd558c2acc15800ed8 Mon Sep 17 00:00:00 2001 From: Jylpah Date: Mon, 24 Feb 2020 23:58:55 +0200 Subject: [PATCH 7/8] Options file format update, player_stats as stat_func fixed --- analyze_wotb_replays.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/analyze_wotb_replays.py b/analyze_wotb_replays.py index d931a0c..1b0d722 100644 --- a/analyze_wotb_replays.py +++ b/analyze_wotb_replays.py @@ -478,12 +478,19 @@ async def main(argv): ## Read config config = configparser.ConfigParser() config.read(FILE_CONFIG) - configGeneral = config['GENERAL'] + + configOptions = config['OPTIONS'] # WG account id of the uploader: # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ - WG_ID = configGeneral.getint('wg_id', None) - USE_DB = configGeneral.getboolean('use_DB', False) - + OPT_DB = configOptions.getboolean('opt_DB', False) + OPT_EXTENDED = configOptions.getboolean('opt_extended', False) + OPT_HIST = configOptions.getboolean('opt_hist', False) + OPT_STAT_FUNC = configOptions.get('opt_stat_func', fallback='tank_tier') + + configWG = config['WG'] + WG_ID = configWG.getint('wg_id', None) + WG_RATE_LIMIT = configWG.getint('wg_rate_limit', 10) + configDB = config['DATABASE'] DB_SERVER = configDB.get('db_server', 'localhost') DB_PORT = configDB.getint('db_port', 27017) @@ -502,15 +509,15 @@ async def main(argv): parser.add_argument('--output', default='plain', choices=['json', 'plain', 'db'], help='Select output mode: JSON, plain text or database') parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze') parser.add_argument('-a', '--account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER') - parser.add_argument('-x', '--extended', action='store_true', default=False, help='Print Extended stats') + parser.add_argument('-x', '--extended', action='store_true', default=OPT_EXTENDED, help='Print Extended stats') parser.add_argument('-X', '--extra_categories', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print Extended categories') - parser.add_argument('--hist', action='store_true', default=False, help='Print player histograms (WR/battles)') - parser.add_argument('--stat_func', default='tank_tier', choices=STAT_FUNC.keys(), help='Select how to calculate for ally/enemy performance: tank-tier stats, global player stats') + parser.add_argument('--hist', action='store_true', default=OPT_HIST, help='Print player histograms (WR/battles)') + parser.add_argument('--stat_func', default=OPT_STAT_FUNC, choices=STAT_FUNC.keys(), help='Select how to calculate for ally/enemy performance: tank-tier stats, global player stats') parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs') parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"') parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT') - parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') + parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it') parser.add_argument('--filters', type=str, default=None, help='Filters for DB based analyses. MongoDB find() filter JSON format.') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') @@ -521,8 +528,8 @@ async def main(argv): bu.set_log_level(args.silent, args.verbose, args.debug) bu.set_progress_step(250) # Set the frequency of the progress dots. - wg = await WG.create(WG_APP_ID, args.tankfile, args.mapfile, True) - wi = WoTinspector() + wg = WG(WG_APP_ID, args.tankfile, args.mapfile, stats_cache=True, rate_limit=WG_RATE_LIMIT) + wi = WoTinspector(rate_limit=10) if args.account != None: args.account_id = await wg.get_account_id(args.account) @@ -936,8 +943,8 @@ async def get_wg_player_stats(stat_id_str: str) -> dict: account_id = int(stat_id_str) # 'battles' must always be there - hist_stats = [ 'all.' + x for x in histogram_fields.keys() ] - hist_stats.append('tank_id') + hist_stats = [ 'statistics.all.' + x for x in histogram_fields.keys() ] + hist_stats.append('account_id') player_stats = await wg.get_player_stats(account_id, hist_stats) #bu.debug('account_id: ' + str(account_id) + ' ' + str(player_stats)) From b710ea55ca463c525f1dfc6116ff0c93b9beb73e Mon Sep 17 00:00:00 2001 From: Jylpah Date: Tue, 25 Feb 2020 00:02:33 +0200 Subject: [PATCH 8/8] Update blitzstats.ini.default --- blitzstats.ini.default | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/blitzstats.ini.default b/blitzstats.ini.default index e73bdcc..0aaf3dd 100644 --- a/blitzstats.ini.default +++ b/blitzstats.ini.default @@ -1,16 +1,14 @@ -[GENERAL] +[OPTIONS] +## You may set your preferred commandline param defaults here +#opt_extended = False +#opt_hist = False +#opt_stat_func = tank_tier + +[WG] ## Find your WG account_id : https://developers.wargaming.net/reference/all/wotb/account/list/ -#wg_id = 0 +#wg_id = 0 +## WG API limits requests / sec to 10 for client apps +## Check https://developers.wargaming.net/documentation/guide/getting-started/ +#wg_rate_limit = 10 + -# you are unlikely have MongoDB set up so just ignore these -#[DATABASE] -#db_server = localhost -#db_port = 27017 -#db_auth = admin -#db_name = BlitzStats -#db_user = USERNAME -#db_password = PASSWORD -#db_ssl = True -#db_cert_req = 2 -#db_ssl_cert_file= cert.pem -#db_ssl_ca_file = ca.pem