Class BlitzStars update

- PEB-8 naming
- tank_stats2WG() added
- get_player_stats() added
- get_tank_stats(9 added

analyze_wotb_replays.py

@@ -365,12 +365,12 @@ async def main(argv):
 
 	parser = argparse.ArgumentParser(description='Analyze Blitz replay JSONs from WoTinspector.com')
 	parser.add_argument('--output', default='plain', choices=['json', 'plain', 'db'], help='Select output mode: JSON, plain text or database')
-	parser.add_argument('-id', dest='accountID', type=int, default=None, help='WG account_id to analyze')
+	parser.add_argument('-id', dest='account_id', type=int, default=None, help='WG account_id to analyze')
-	parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER')
+	parser.add_argument('-a', '--account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER')
 	parser.add_argument('-x', '--extended', action='store_true', default=False, help='Print Extended stats')
 	parser.add_argument('--hist', action='store_true', default=False, help='Print player histograms (WR/battles)')
 	parser.add_argument('--stat_func', default='tank_tier', choices=STAT_FUNC.keys(), help='Select how to calculate for ally/enemy performance: tank-tier stats, global player stats')
-	parser.add_argument('-u', '--url', dest= 'url', action='store_true', default=False, help='Print replay URLs')
+	parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs')
 	parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"')
 	parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"')
 	parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
@@ -389,8 +389,8 @@ async def main(argv):
 
 
 	if args.account != None:
-		args.accountID = await wg.get_account_id(args.account)
+		args.account_id = await wg.get_account_id(args.account)
-		bu.debug('WG  account_id: ' + str(args.accountID))
+		bu.debug('WG  account_id: ' + str(args.account_id))
 
 	#### Connect to MongoDB (TBD)
 	bu.debug('DB_SERVER: ' + DB_SERVER)
@@ -905,7 +905,7 @@ async def mk_readerQ_item(filename : str) -> list:
 async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
 	"""Async Worker to process the replay queue"""
 	#global SKIPPED_N
-	# account_id = args.accountID
+	# account_id = args.account_id
 	results = []
 	playerstanks = set()
 	try:
@@ -950,7 +950,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
 async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict:
 	"""Parse replay JSON dict"""
 	global REPLAY_I
-	account_id = args.accountID
+	account_id = args.account_id
 	url = args.url
 	#db = args.db
 	result = {}

blitzutils.py

@@ -42,6 +42,7 @@ def set_log_level_normal():
     global _log_level
     _log_level = NORMAL
 
+
 def set_log_level(silent: bool,verbose: bool, debug: bool):
     global _log_level
     _log_level = NORMAL
@@ -81,42 +82,45 @@ def verbose_std(msg = "", id = None):
     return None
 
 
-def debug(msg = "", n = None):
+def debug(msg = "", id = None, exception = None):
     """print a conditional debug message"""
     if _log_level >= DEBUG:
-        curframe = inspect.currentframe()
+        _print_log_msg('DEBUG', msg, exception, id)
-        calframe = inspect.getouterframes(curframe, 2)
-        caller = calframe[1][3]
-        if n == None:
-            print('DEBUG: ' + caller + '(): ' + msg)
-        else:
-            print('DEBUG: ' + caller + '()' + '[' + str(n) + ']: ' + msg)
     return None
 
 
 def error(msg = "", exception = None, id = None):
     """Print an error message"""
+    _print_log_msg('ERROR', msg, exception, id)
+    return None
+
+
+def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None):
     curframe = inspect.currentframe()
-    calframe = inspect.getouterframes(curframe, 2)
+    calframe = inspect.getouterframes(curframe)
-    caller = calframe[1][3]
+    caller = calframe[2].function
+    
+    prefix = prefix + ': ' + caller + '(): '
+    if id != None:
+        prefix = prefix + '[' + str(id) + ']: '
+
     exception_msg = ''
     if (exception != None) and isinstance(exception, Exception):
         exception_msg = ' : Exception: ' + str(type(exception)) + ' : ' + str(exception)
 
-    if id == None:
+    print(prefix + msg + exception_msg)
-        print('ERROR: ' + caller + '(): ' + msg + exception_msg)
-    else:
-        print('ERROR: ' + caller + '()' + '[' + str(id) + ']: ' + msg + exception_msg)
     return None
 
 
-def print_progress(force = False):
+def print_progress(force = False) -> bool:
-    """Print progress dots"""
+    """Print progress dots. Returns True if the dot is being printed."""
     global _progress_N, _progress_i
     if (_log_level > SILENT) and ( force or (_log_level < DEBUG ) ):
         _progress_i = (_progress_i + 1) % _progress_N
         if _progress_i == 0:
             print('.', end='', flush=True)
+            return True
+    return False    
         
 def set_progress_step(n: int):
     """Set the frquency of the progress dots. The bigger 'n', the fewer dots"""
@@ -206,10 +210,10 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
                         if (chk_JSON_func == None) or chk_JSON_func(json_resp):
                             # debug("Received valid JSON: " + str(json_resp))
                             return json_resp
-                        else:
+                        # else:
-                            debug('Received JSON error. URL: ' + url)
+                        #     debug('Received JSON error. URL: ' + url)
-                            # Sometimes WG API returns JSON error even a retry gives valid JSON
+                        #     # Sometimes WG API returns JSON error even a retry gives valid JSON
-                            # return None                            
+                        #     # return None                            
                     if retry == max_tries:                        
                         if json_resp != None:
                             str_json = str(json_resp)
@@ -221,7 +225,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
 
         except aiohttp.ClientError as err:
             error("Could not retrieve URL: " + url)
-            error(exception=err)
+            error(str(err))
         except asyncio.CancelledError as err:
             error('Queue gets cancelled while still working.')        
         except Exception as err:
@@ -253,7 +257,7 @@ def bld_dict_hierarcy(d : dict, key : str, value) -> dict:
 #### Class StatsNotFound 
 ## -----------------------------------------------------------
 
-class CachedStatsNotFound(Exception):
+class StatsNotFound(Exception):
     pass
 
 ## -----------------------------------------------------------
@@ -751,8 +755,8 @@ class WG:
         return None
       
 
-    async def get_player_tanks_stats(self, account_id: int, tank_ids = [], fields = []) -> dict:
+    async def get_player_tank_stats(self, account_id: int, tank_ids = [], fields = [], cache=True) -> dict:
-        """Get player's stats (WR, # of battles) in a tank"""
+        """Get player's stats (WR, # of battles) in a tank or all tanks (empty tank_ids[])"""
         # debug('Started')        
         
         try:
@@ -760,9 +764,10 @@ class WG:
             stats = None
 
             # try cached stats first:
-            stats = await self.get_cached_tank_stats(account_id, tank_ids, fields)
+            if cache:
-            if stats != None:
+                stats = await self.get_cached_tank_stats(account_id, tank_ids, fields)
-                return stats
+                if stats != None:
+                    return stats
 
             # Cached stats not found, fetching new ones
             url = self.get_url_player_tanks_stats(account_id, tank_ids, fields)
@@ -770,27 +775,29 @@ class WG:
             if json_data != None:
                 #debug('JSON Response received: ' + str(json_data))
                 stats = json_data['data'][str(account_id)]
-                await self.save_stats('tank_stats', [account_id, tank_ids], stats)
+                if cache:
+                    await self.save_stats('tank_stats', [account_id, tank_ids], stats)
                 return stats
         except Exception as err:
             error(exception=err)
         return None
 
    
-    async def get_player_stats(self, account_id: int, fields = []) -> dict:
+    async def get_player_stats(self, account_id: int, fields = [], cache=True) -> dict:
         """Get player's global stats """
         try:
             #debug('account_id: ' + str(account_id) )
             stats = None
 
             # try cached stats first:
-            stats = await self.get_cached_player_stats(account_id,fields)
+            if cache:
-            # stats found unless CachedStatsNotFound exception is raised 
+                stats = await self.get_cached_player_stats(account_id,fields)
-            return stats
+                # stats found unless StatsNotFound exception is raised 
+                return stats
 
-        except CachedStatsNotFound as err:
+        except StatsNotFound as err:
             # No cached stats found, need to retrieve
-            debug(str(err))
+            debug(exception=err)
             pass
         
         try:
@@ -800,7 +807,8 @@ class WG:
             if json_data != None:
                 #debug('JSON Response received: ' + str(json_data))
                 stats = json_data['data'][str(account_id)]
-                await self.save_stats('player_stats', [account_id], stats)
+                if cache:
+                    await self.save_stats('player_stats', [account_id], stats)
                 return stats
         except Exception as err:
             error(exception=err)
@@ -971,7 +979,7 @@ class WG:
             debug('Trying cached stats first')
             if self.cache == None:
                 #debug('No cache DB')
-                raise CachedStatsNotFound('No cache DB in use')
+                raise StatsNotFound('No cache DB in use')
                       
             async with self.cache.execute(WG.SQL_PLAYER_STATS_CACHED, (account_id, NOW() - WG.CACHE_GRACE_TIME) ) as cursor:
                 row = await cursor.fetchone()
@@ -979,7 +987,7 @@ class WG:
                 if row == None:
                     # no cached stats found, marked with an empty array
                     #debug('No cached stats found')
-                    raise CachedStatsNotFound('No cached stats found')
+                    raise StatsNotFound('No cached stats found')
                 
                 debug('Cached stats found')    
                 if row[3] == None:
@@ -989,8 +997,8 @@ class WG:
                 else:
                     # Return proper stats 
                     return json.loads(row[3])
-        except CachedStatsNotFound as err:
+        except StatsNotFound as err:
-            debug(str(err))
+            debug(exception=err)
             raise
         except Exception as err:
             error('Error trying to look for cached stats', err)
@@ -1167,51 +1175,163 @@ class WoTinspector:
 
 class BlitzStars:
 
-    URL_BlitzStars      = 'https://www.blitzstars.com'
+    URL_BLITZSTARS          = 'https://www.blitzstars.com'
-    URL_playerStats     = URL_BlitzStars + '/api/playerstats'
+    URL_PLAYER_STATS        = URL_BLITZSTARS + '/api/playerstats'
-    URL_playerTankStats = URL_BlitzStars + '/api/tanks'
+    URL_PLAYER_TANK_STATS   = URL_BLITZSTARS + '/api/tanks'
-    URL_playersTankStats= URL_BlitzStars +  '/api/top/forjylpah?'
+    URL_TANK_AVERAGES       = URL_BLITZSTARS + '/tankaverages.json'
-    URL_tankAverages    = URL_BlitzStars + '/tankaverages.json'
+    URL_ACTIVE_PLAYERS      = URL_BLITZSTARS +  '/api/playerstats/activeinlast30days'
-    URL_activeplayers   = URL_BlitzStars +  '/api/playerstats/activeinlast30days'
 
 
     def __init__(self):
         self.session = aiohttp.ClientSession()
 
+
     async def close(self):
         if self.session != None:
             await self.session.close()        
 
+
     @classmethod
-    def getUrlTankAvgs(cls):
+    def get_url_tank_averages(cls):
-        return cls.URL_tankAverages
+        return cls.URL_TANK_AVERAGES
+
 
     @classmethod
     def get_url_player_stats(cls, account_id: int):
-        return cls.URL_playerStats + '/' + str(account_id)
+        return cls.URL_PLAYER_STATS + '/' + str(account_id)
+
 
     @classmethod
-    def getUrlPlayerTankStats(cls, account_id: int, tank_id: int):
+    def get_url_player_tank_stats(cls, account_id: int, tank_id: int):
-        return cls.URL_playerTankStats + '/' + str(account_id) + '/' + str(tank_id)
+        return cls.URL_PLAYER_TANK_STATS + '/' + str(account_id) + '/' + str(tank_id)
+
 
     @classmethod
-    def getUrlPlayersTankStats(cls, account_id: int):
+    def get_url_player_tanks_stats(cls, account_id: int):
-        return cls.URL_playersTankStats + '&accountId=' + str(account_id) 
+        return cls.URL_PLAYER_TANK_STATS + '/' + str(account_id) 
+
 
     @classmethod
-    def getUrlActivePlayers(cls):
+    def get_url_active_players(cls):
-        return cls.URL_activeplayers
+        return cls.URL_ACTIVE_PLAYERS
+
  
     @classmethod
-    def chkJSONtankStats(cls, json_resp):
+    def chk_JSON_tank_stats(cls, json_resp):
-        """"Check String for being a valid JSON file"""
+        """Check BlitzStars player tank stats"""
         try:
-            # print(str(json_resp['data']))
+            if (len(json_resp) > 0) and ('tank_id' in json_resp[0]):
-            if ('data' in json_resp):
+                debug('JSON check OK')
-                debug("JSON check OK")
                 return True
-        except KeyError as err:
+        except Exception as err:
-            error('Key not found', err)
+            error('JSON check FAILED: ' + str(json_resp) )
-        except:
+            error(exception=err)
-            debug("JSON check failed")
         return False
+
+    @classmethod
+    def chk_JSON_player_stats(cls, json_resp : list):
+        """Check BlitzStars player stats"""
+        try:
+            if (len(json_resp) > 0) and ('account_id' in json_resp[0]):
+                debug('JSON check OK')
+                return True
+        except Exception as err:
+            error('JSON check FAILED: ' + str(json_resp) )
+            error(exception=err)
+        return False
+
+
+    @classmethod
+    async def tank_stats2WG(cls, BS_tank_stats: list) -> list:
+        """Convert BlitzStars player tank stats to WG API format""" 
+        try:
+            if BS_tank_stats == None:
+                return None
+            WG_tank_stats = []
+            for stat in BS_tank_stats:
+                try:
+                    tmp = {}
+                    for field in ['all', 'last_battle_time', 'tank_id', 'battle_life_time', 'account_id']:
+                        tmp[field] = stat[field]
+                    WG_tank_stats.append(tmp)
+                except KeyError as err:
+                    error(exception=err)
+            if len(WG_tank_stats) > 0:
+                return WG_tank_stats
+        
+        except Exception as err:
+            error(exception=err)
+            pass
+        return None
+
+
+    async def get_player_stats(self, account_id: int, cache=True):
+        """Get player stats from BlitzStars"""
+        try:
+            #debug('account_id: ' + str(account_id) )
+            stats = None
+
+            if cache:
+                error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
+                sys.exit(1)
+                #stats = await self.get_cached_player_stats(account_id,fields)
+                # stats found unless StatsNotFound exception is raised 
+                return stats
+
+        except StatsNotFound as err:
+            # No cached stats found, need to retrieve
+            debug(exception=err)
+            pass
+        
+        try:
+            # Cached stats not found, fetching new ones
+            url = self.get_url_player_stats(account_id)
+            stats = await get_url_JSON(self.session, url, self.chk_JSON_player_stats)
+            if stats != None:
+                #debug('JSON Response received: ' + str(json_data))
+                if cache:
+                    error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
+                    sys.exit(1)
+                    # await self.save_stats('player_stats', [account_id], stats)
+                return stats
+        except Exception as err:
+            error(exception=err)
+        return None
+
+    async def get_player_tank_stats(self, account_id: int, tank_id = None, cache=True):
+        """Get player stats for all his tanks from BlitzStars"""
+        try:
+            #debug('account_id: ' + str(account_id) )
+            stats = None
+
+            if cache:
+                error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
+                sys.exit(1)
+                #stats = await self.get_cached_player_stats(account_id,fields)
+                # stats found unless StatsNotFound exception is raised 
+                return stats
+
+        except StatsNotFound as err:
+            # No cached stats found, need to retrieve
+            debug(exception=err)
+            pass
+        
+        try:
+            # Cached stats not found, fetching new ones
+            if tank_id == None:
+                # get stats for all the player's tanks
+                url = self.get_url_player_tanks_stats(account_id)
+            else:
+                url = self.get_url_player_tank_stats(account_id, tank_id)
+            stats = await get_url_JSON(self.session, url, self.chk_JSON_tank_stats)
+            if stats != None:
+                #debug('JSON Response received: ' + str(json_data))
+                if cache:
+                    error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
+                    sys.exit(1)
+                    # await self.save_stats('player_stats', [account_id], stats)
+                return stats
+        except Exception as err:
+            error(exception=err)
+        return None    
+