WoTinspector.read_replay_id(): added back

blitzutils.py

@@ -36,132 +36,163 @@ _progress_obj = None
 UMASK = os.umask(0)
 os.umask(UMASK)
 
-# -----------------------------------------------------------
+## -----------------------------------------------------------
-# Class ThrottledClientSession(aiohttp.ClientSession)
+#### Class ThrottledClientSession(aiohttp.ClientSession)
-# -----------------------------------------------------------
+#
+#  Rate-limited async http client session
+#
+#  Inherits aiohttp.ClientSession 
+## -----------------------------------------------------------
 
+from typing import Optional, Union
+import aiohttp
+import asyncio
+import time
+import logging
+import re
 
 class ThrottledClientSession(aiohttp.ClientSession):
     """Rate-throttled client session class inherited from aiohttp.ClientSession)""" 
-    # MIN_SLEEP = 0.1
 
-    def __init__(self, rate_limit: float = None, *args, **kwargs) -> None:
+    def __init__(self, rate_limit: float = 0, filters: list[str] = list() , 
-        super().__init__(*args, **kwargs)
+                limit_filtered: bool = False, re_filter: bool = False, *args,**kwargs) -> None: 
-        self.rate_limit = None
+        assert isinstance(rate_limit, (int, float)),   "rate_limit has to be float"
-        self._fillerTask = None
+        assert isinstance(filters, list),       "filters has to be list"
-        self._queue = None
+        assert isinstance(limit_filtered, bool),"limit_filtered has to be bool"
-        self._start_time = None
+        assert isinstance(re_filter, bool),     "re_filter has to be bool"
-        self._count = 0
-        self._start_time = time.time()
 
-        if rate_limit != None:
+        super().__init__(*args,**kwargs)
-            if rate_limit <= 0:
-                raise ValueError('rate_limit must be positive')
-            self.rate_limit = rate_limit
-            self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
-            self._fillerTask = asyncio.create_task(self._filler())
         
-    def _get_sleep(self) -> list:
+        self.rate_limit     : float
-        if self.rate_limit == None:
+        self._fillerTask    : Optional[asyncio.Task]    = None
-            return None
+        self._queue         : Optional[asyncio.Queue]   = None
+        self._start_time    : float = time.time()
+        self._count         : int = 0
+        self._limit_filtered: bool = limit_filtered
+        self._re_filter     : bool = re_filter
+        self._filters       : list[Union[str, re.Pattern]] = list()
+
+        if re_filter:
+            for filter in filters:
+                self._filters.append(re.compile(filter))
         else:
+            for filter in filters:
+                self._filters.append(filter)
+        self.set_rate_limit(rate_limit)
+
+
+    def _get_sleep(self) -> float:        
+        if self.rate_limit > 0:
             return 1/self.rate_limit
+        return 0
+
 
     def get_rate(self) -> float:
         """Return rate of requests"""
         return self._count / (time.time() - self._start_time)
 
-    def get_stats(self):
+
+    def get_stats(self) -> dict[str, float]:
         """Get session statistics"""
-        res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
+        res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
-                                     'count': self._count}
         return res
         
-    def get_stats_str(self):
-        """Print session statistics"""
-        return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
-                ' rate: ' + \
-                    "{0:.1f}".format(self.get_rate()) + \
-                                     ' requests: ' + str(self._count)
 
-    def reset_counters(self):
+    def get_stats_str(self) -> str:
+        """Print session statistics"""
+        return f"rate limit: {str(self.rate_limit if self.rate_limit != None else '-')} \
+                rate:   {0:.1f}.format(self.get_rate()) requests: {str(self._count)}"
+
+
+    def reset_counters(self) -> dict[str, float]:
         """Reset rate counters and return current results"""
         res = self.get_stats()
         self._start_time = time.time()
         self._count = 0
         return res
 
-    def set_rate_limit(self, rate_limit: float = 1):
+
-        if rate_limit >= 0:
+    def set_rate_limit(self, rate_limit: float = 0) -> float:
-            self.rate_limit = rate_limit
+        assert rate_limit is not None, "rate_limit must not be None" 
-            return self.rate_limit
+        assert isinstance(rate_limit, (int,float)) and rate_limit >= 0, "rate_limit has to be type of 'float' >= 0"
-        return None
+        
+        self.rate_limit = rate_limit
+        if rate_limit > 0:
+            self._queue     = asyncio.Queue(int(rate_limit)+1) 
+            if self._fillerTask is not None: 
+                self._fillerTask.cancel()  
+            self._fillerTask = asyncio.create_task(self._filler())
+        return self.rate_limit
+        
 
     async def close(self) -> None:
         """Close rate-limiter's "bucket filler" task"""
         # DEBUG 
-        debug(self.get_stats_str())
+        logging.debug(self.get_stats_str())
         try:
-            if self._fillerTask != None:
+            if self._fillerTask is not None:
                 self._fillerTask.cancel()
-            await asyncio.wait_for(self._fillerTask, timeout=0.5)
+                await asyncio.wait_for(self._fillerTask, timeout=0.5)
-        except (asyncio.CancelledError, asyncio.TimeoutError):
+        except asyncio.TimeoutError as err:
-            pass
+            logging.error(str(err))
         await super().close()
 
-    async def _filler_simple(self):
-        """Filler task to fill the leaky bucket algo"""
-        try:
-            if self._queue == None:
-                return
-            sleep = self._get_sleep()
-            while True:
-                self._queue.put(None)
-                await asyncio.sleep(sleep)
-        except asyncio.CancelledError:
-            debug('Cancelled')
-        except Exception as err:
-            error(exception=err)
     
-    async def _filler(self):
+    async def _filler(self) -> None:
         """Filler task to fill the leaky bucket algo"""
         try:
-            if self._queue == None:
+            if self._queue is None:
-                return
+                return None            
-            sleep = self._get_sleep()
+            logging.debug('SLEEP: ' + str(self._get_sleep()))
-            # debug('SLEEP: ' + str(sleep))
             updated_at = time.monotonic()
-            fraction = 0
+            extra_increment : float = 0
-            extra_increment = 0
             for i in range(0, self._queue.maxsize):
-                self._queue.put_nowait(i)
+                await self._queue.put(i)
             while True:
                 if not self._queue.full():
                     now = time.monotonic()
                     increment = self.rate_limit * (now - updated_at)
-                    fraction += increment % 1
+                    items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment + extra_increment)))
-                    extra_increment = fraction // 1
+                    extra_increment = (increment + extra_increment) % 1
-                    items_2_add = int(
+                    for i in range(0,items_2_add):
-                        min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
-                    fraction = fraction % 1
-                    for i in range(0, items_2_add):
                         self._queue.put_nowait(i)
                     updated_at = now
-                await asyncio.sleep(sleep)
+                await asyncio.sleep(self._get_sleep())
         except asyncio.CancelledError:
-            debug('Cancelled')
+            logging.debug('Cancelled')
         except Exception as err:
-            error(exception=err)
+            logging.error(str(err))
+        return None
 
-    async def _request(self, *args, **kwargs):
+
+    async def _request(self, *args,**kwargs) -> aiohttp.ClientResponse:
         """Throttled _request()"""
-        if self._queue != None:
+        if self._queue is not None and self.is_limited(*args):  
             await self._queue.get()
             self._queue.task_done()
         self._count += 1
-        return await super()._request(*args, **kwargs)
+        return await super()._request(*args,**kwargs)
 
 
+    def is_limited(self, *args: str) -> bool:
+        """Check wether the rate limit should be applied"""
+        try:
+            url: str = args[1]
+            for filter in self._filters:
+                if isinstance(filter, re.Pattern) and filter.match(url) is not None:
+                    return self._limit_filtered
+                elif isinstance(filter, str) and url.startswith(filter):
+                    return self._limit_filtered
+                    
+            return not self._limit_filtered
+        except Exception as err:
+            logging.error(str(err))
+        return True    
+
+# -----------------------------------------------------------
+# Class AsyncLogger()
+# -----------------------------------------------------------
+
 class AsyncLogger():
     """Async file logger"""
 
@@ -496,18 +527,17 @@ def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
     return None
 
 
-def rebase_file_args(current_dir, files):
+def rebase_file_args(current_dir : str, files: list[str]) -> list[str]:
-    """REbase file command line params after moving working dir to the script's dir""" 
+    """Rebase file command line params after moving working dir to the script's dir""" 
-    if isinstance(files, list):    
+    assert isinstance(files, list), "files has to be a list"
-        if (files[0] == '-') or (files[0] == 'db:'):
+
-            return files
+    if (files[0] == '-') or (files[0] == 'db:'):
-        else:
+        return [ files[0] ] 
-            return [ os.path.join(current_dir, fn) for fn in files ]
+    else:
-    elif isinstance(files, str):
+        return [ os.path.join(current_dir, fn) for fn in files ]
-        return os.path.join(current_dir, files)
 
 
-async def read_int_list(filename: str) -> list():
+async def read_int_list(filename: str) -> list[int]:
     """Read file to a list and return list of integers in the input file"""
     
     input_list = []
@@ -1596,6 +1626,8 @@ class WG:
 
     async def store_tank_stats(self, key: list , stats_data: list, update_time: int):
         """Save tank stats into cache"""
+        assert self.cache is not None, "cache must be initiated"
+        
         try:
             account_id  = key[0]
             tank_ids    = set(key[1])
@@ -1617,6 +1649,8 @@ class WG:
 
     async def store_player_stats(self, key: list , stats_data: list, update_time: int):
         """Save player stats into cache"""
+        assert self.cache is not None, "cache must be initiated"
+        
         try:
             account_id  = key[0]
             if stats_data != None:
@@ -1633,6 +1667,8 @@ class WG:
 
     async def store_player_achievements(self, key: list , stats_data: list, update_time: int):
         """Save player stats into cache"""
+        assert self.cache is not None, "cache must be initiated"
+        
         try:
             account_id  = key[0]
             if stats_data != None:
@@ -1648,6 +1684,8 @@ class WG:
 
 
     async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ):
+        assert self.cache is not None, "cache must be initiated"
+
         try:
             # test for cacheDB existence
             debug('Trying cached stats first')
@@ -1761,17 +1799,27 @@ class WoTinspector:
     URL_REPLAY_VIEW = URL_WI +'/en/view/'
     URL_REPLAY_UL   = 'https://api.wotinspector.com/replay/upload?'
     URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
-    
     URL_TANK_DB     = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
 
     REPLAY_N = 1
+    DEFAULT_RATE_LIMIT : float = 10/60  # 10 requests / min
+    DEFAULT_TOKEN : str = 'a5428af49bf7495986577d59b0e5bcff'
 
-    def __init__(self, rate_limit: int = 30):
+    def __init__(self, rate_limit: float = DEFAULT_RATE_LIMIT, auth_token: Optional[str] = None):
-        self.session = ThrottledClientSession(rate_limit=rate_limit)
+
+        headers : Optional[dict[str, str]] = None
+
+        if auth_token is None:
+            auth_token = self.DEFAULT_TOKEN        
+        headers = dict()
+        headers['Authorization'] = 'Token ' + auth_token
+        self.session = ThrottledClientSession(rate_limit=rate_limit, 
+                                                filters=[self.URL_REPLAY_LIST, self.URL_REPLAY_INFO], 
+                                                re_filter=False, limit_filtered=True, headers = headers)
         
 
-    async def close(self):
+    async def close(self) -> None:
-        if self.session != None:
+        if self.session is not None:
             debug('Closing aiohttp session')
             await self.session.close()
         
@@ -1835,7 +1883,7 @@ class WoTinspector:
             hash.update(data)
             replay_id = hash.hexdigest()
 
-            # Testing if the replay has already been posted
+            ##  Testing if the replay has already been posted
             json_resp = await self.get_replay_JSON(replay_id)
             if json_resp != None:
                 debug('Already uploaded: ' + title, id=N)
@@ -1850,7 +1898,7 @@ class WoTinspector:
             } 
 
             url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
-            # debug('URL: ' + url)
+            #debug('URL: ' + url)
             headers ={'Content-type':  'application/x-www-form-urlencoded'}
             payload = { 'file' : (filename, base64.b64encode(data)) }
         except Exception as err:
@@ -1875,8 +1923,6 @@ class WoTinspector:
             except Exception as err:
                 debug(exception=err, id=N)
             await asyncio.sleep(SLEEP)
-            _id  = self.read_replay_id(json_resp)
-            json_resp['_id']  =  _id
             
         debug(' Could not post replay: ' + title, id=N)
         return json_resp
@@ -1914,11 +1960,6 @@ class WoTinspector:
         return replay_links
     
 
-    @classmethod
-    def get_replay_id(cls, url):
-        return url.rsplit('/', 1)[-1]
-
-
     @classmethod
     def read_replay_id(cls, json_replay):
         """Read replay_id from replay JSON file""" 
@@ -1929,14 +1970,18 @@ class WoTinspector:
             error(exception=err)
         return None
 
+
     @classmethod
-    def chk_JSON_replay(cls, json_resp):
+    def get_replay_id(cls, url):
+        return url.rsplit('/', 1)[-1]
+
+
+    @classmethod
+    def chk_JSON_replay(cls, json_resp) -> bool:
         """"Check String for being a valid JSON file"""
         try:
-            # FIX The replay can have allies/enemies i.e. be useful for 
-            # team composition analysis, but does not have player contribution
             if ('status' in json_resp) and json_resp['status'] == 'ok' and \
-                (get_JSON_value(json_resp, key='data.summary.winner_team') != None) :
+                (get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
                 debug("JSON check OK")
                 return True 
         except KeyError as err:
@@ -1945,6 +1990,9 @@ class WoTinspector:
             debug("Replay JSON check failed: " + str(json_resp))
         return False      
 
+# -----------------------------------------------------------
+# Class BlitzStars 
+# -----------------------------------------------------------
 
 class BlitzStars:
 

tanks.json

@@ -1,7 +1,7 @@
 {
     "status": "ok",
     "meta": {
-        "count": 559
+        "count": 564
     },
     "data": {
         "1": {
@@ -2244,6 +2244,14 @@
             "tier": 7,
             "type": "AT-SPG"
         },
+        "10097": {
+            "is_premium": true,
+            "name": "Medjay",
+            "nation": "other",
+            "tank_id": 10097,
+            "tier": 5,
+            "type": "heavyTank"
+        },
         "10113": {
             "is_premium": true,
             "name": "Carro da Combattimento 45t",
@@ -2276,6 +2284,14 @@
             "tier": 4,
             "type": "AT-SPG"
         },
+        "10353": {
+            "is_premium": true,
+            "name": "Pharaoh",
+            "nation": "other",
+            "tank_id": 10353,
+            "tier": 7,
+            "type": "heavyTank"
+        },
         "10369": {
             "is_premium": true,
             "name": "Controcarro 3 Minotauro",
@@ -2492,6 +2508,14 @@
             "tier": 8,
             "type": "AT-SPG"
         },
+        "12161": {
+            "is_premium": true,
+            "name": "Strv K",
+            "nation": "european",
+            "tank_id": 12161,
+            "tier": 10,
+            "type": "heavyTank"
+        },
         "12305": {
             "is_premium": false,
             "name": "E 50 Ausf. M",
@@ -3148,6 +3172,14 @@
             "tier": 8,
             "type": "heavyTank"
         },
+        "19777": {
+            "is_premium": true,
+            "name": "AltProto AMX 30",
+            "nation": "france",
+            "tank_id": 19777,
+            "tier": 8,
+            "type": "mediumTank"
+        },
         "19793": {
             "is_premium": false,
             "name": "Vickers Cruiser",
@@ -3460,6 +3492,14 @@
             "tier": 9,
             "type": "heavyTank"
         },
+        "22353": {
+            "is_premium": true,
+            "name": "Churchill W",
+            "nation": "uk",
+            "tank_id": 22353,
+            "tier": 6,
+            "type": "heavyTank"
+        },
         "22529": {
             "is_premium": true,
             "name": "LT-432",
@@ -4557,6 +4597,8 @@
         "A95_T95E6_Custom_short": "T95E6 Piranha",
         "A95_T95E6_short": "T95E6",
         "A99_T92E1": "T92E1",
+        "A99_T92E1_Custom": "T92E1 Poison",
+        "A99_T92E1_Custom_short": "T92E1 Poison",
         "A99_T92E1_short": "T92E1",
         "AEC": "2x AEC",
         "AEC3": "AEC3",
@@ -4582,6 +4624,8 @@
         "AMX_50Fosh_155": "AMX 50 Foch (155)",
         "AMX_50Fosh_155_short": "Foch 155",
         "AMX_50_100": "AMX 50 100",
+        "AMX_50_100_Custom": "AMX 50 100 Basher",
+        "AMX_50_100_Custom_short": "AMX 50 100 Basher",
         "AMX_50_100_short": "AMX 50 100",
         "AMX_50_120": "AMX 50 120",
         "AMX_50_120_short": "AMX 50 120",
@@ -4626,6 +4670,7 @@
         "AT-1": "AT-1",
         "AT-1bis": "AT-1 Bis",
         "AVCR_1100_1": "AVCR 1100 1",
+        "AVS-1195A": "AVS-1195A",
         "AVS_1195": "AVS-1195",
         "AV_1195": "AV-1195",
         "AV_1790": "AV 1790",
@@ -4778,8 +4823,18 @@
         "Ch_WZ_112v2_Custom_short": "WZ-112-2 Celestial",
         "Chassis1_A140_ASTRON_REX_105": "Rex",
         "Chassis1_A140_ASTRON_REX_105_short": "Rex",
+        "Chassis1_F117_Alt_Proto_AMX_30": "AltProto AMX 30",
+        "Chassis1_F117_Alt_Proto_AMX_30_short": "A.P. AMX 30",
+        "Chassis1_GB09_Churchill_VII_BP": "Churchill W",
+        "Chassis1_GB09_Churchill_VII_BP_short": "Churchill W",
         "Chassis1_It10_P43_bis_BP": "Shadowhunter",
         "Chassis1_It10_P43_bis_BP_short": "Shadowhunter",
+        "Chassis1_Oth34_PanzerMummy": "Royal Tread",
+        "Chassis1_Oth34_PanzerMummy_short": "Royal Tread",
+        "Chassis1_Oth35_PanzerMejai": "Desert Wanderer",
+        "Chassis1_Oth35_PanzerMejai_short": "Desert Wanderer",
+        "Chassis1_S31_Strv_K": "Stridsvagn KRV",
+        "Chassis1_S31_Strv_K_short": "Stridsvagn KRV",
         "Chi_Ha": "Type 97 Chi-Ha",
         "Chi_Ha_short": "Chi-Ha",
         "Chi_He": "Type 1 Chi-He",
@@ -4936,12 +4991,16 @@
         "E-100": "E 100",
         "E-100_Custom": "E 100 Stalwart",
         "E-100_Custom_short": "E 100 Stalwart",
+        "E-100_Custom_short_v2": "E 100 Jötunn",
+        "E-100_Custom_v2": "E 100 Jötunn",
         "E-100_short": "E 100",
         "E-25": "E 25",
         "E-25_Custom": "E 25 \\\"#1\\\"",
         "E-50": "E 50",
         "E-50_short": "E 50",
         "E-75": "E 75",
+        "E-75_Custom": "E 75 Garm",
+        "E-75_Custom_short": "E 75 Garm",
         "E-75_short": "E 75",
         "E50_Ausf_M": "E 50 Ausf. M",
         "E50_Ausf_M_Custom": "E 50 M Roaring",
@@ -4961,6 +5020,8 @@
         "F116_Bat_Chat_Bourrasque_Custom": "Bourrasque Noir",
         "F116_Bat_Chat_Bourrasque_Custom_short": "Bourrasque Noir",
         "F116_Bat_Chat_Bourrasque_short": "B-C Bourrasque",
+        "F117_Alt_Proto_AMX_30": "AltProto AMX 30",
+        "F117_Alt_Proto_AMX_30_short": "A.P. AMX 30",
         "F19_Lorraine40t": "Lorraine 40 t",
         "F19_Lorraine40t_Com": "Lorraine 40 t Fearless",
         "F19_Lorraine40t_Com_short": "Lorraine Fearless",
@@ -5125,6 +5186,10 @@
         "GB08_Churchill_I": "Churchill I",
         "GB08_Churchill_I_short": "Churchill I",
         "GB09_Churchill_VII": "Churchill VII",
+        "GB09_Churchill_VII_BP": "Churchill W",
+        "GB09_Churchill_VII_BP_Custom": "Churchill W Defender",
+        "GB09_Churchill_VII_BP_Custom_short": "Churchill W Defender",
+        "GB09_Churchill_VII_BP_short": "Churchill W",
         "GB109_GSOR_1008": "GSOR 1008",
         "GB109_GSOR_1008_Custom": "GSOR 1008 Excalibur",
         "GB109_GSOR_1008_Custom_short": "GSOR 1008 Excalibur",
@@ -5383,6 +5448,8 @@
         "IS8": "IS-8",
         "ISU-152": "ISU-152",
         "ISU-152_01": "ISU-152",
+        "ISU-152_Custom_short_v2": "ISU-152 Tundra",
+        "ISU-152_Custom_v2": "ISU-152 Tundra",
         "ISU-152_mod_1945": "ISU-152M",
         "ISU152_Custom": "ISU-152 Hunter",
         "ISU152_Custom_short": "ISU-152 Hunter",
@@ -5766,6 +5833,7 @@
         "Maybach_HL_210_TRM_P45_short": "Maybach HL 210\\nTRM P45",
         "Maybach_HL_230": "Maybach HL 230",
         "Maybach_HL_230P30A": "Maybach HL 230 P30",
+        "Maybach_HL_230P30A_Desert": "Flaming Heart",
         "Maybach_HL_230P30_2500rpm": "Maybach HL 230 P30 2500 rpm",
         "Maybach_HL_230P45": "Maybach HL 230 P45F",
         "Maybach_HL_230P45C": "Maybach HL 230 P45C",
@@ -5843,6 +5911,8 @@
         "OQF_55_pdr_AT_Gun_short": "OQF 55-pdr AT Gun",
         "OQF_77mm_AT_Gun_MkII": "OQF 77 mm AT Gun Mk. II",
         "OQF_77mm_Gun_MkII": "OQF 77 mm Gun Mk. II",
+        "OQF_77mm_Gun_MkII_BP": "OQF 77 mm Gun Mk. II B",
+        "OQF_77mm_Gun_MkII_BP_short": "OQF 77 mm Gun Mk. II B",
         "Object263": "Object 263",
         "Object263_Custom": "Obj. 263 Graywolf",
         "Object263_Custom_short": "Obj. 263 Graywolf",
@@ -5939,6 +6009,12 @@
         "Oth33_Blizzard_BP_Custom": "Rover LunarX",
         "Oth33_Blizzard_BP_Custom_short": "Rover LunarX",
         "Oth33_Blizzard_BP_short": "Rover",
+        "Oth34_PanzerMummy": "Pharaoh",
+        "Oth34_PanzerMummy_Custom": "Pharaoh All-Seeing",
+        "Oth34_PanzerMummy_Custom_short": "Pharaoh All-Seeing",
+        "Oth34_PanzerMummy_short": "Pharaoh",
+        "Oth35_PanzerMejai": "Medjay",
+        "Oth35_PanzerMejai_short": "Medjay",
         "PLInz_RWA": "PLInz R.W.A.",
         "PLT_27": "PLT-27",
         "Paccar_120_325HT": "Gas Boiler",
@@ -6230,6 +6306,8 @@
         "R172_Object_752_Custom_short": "Obj. 752 Stargazer",
         "R172_Object_752_short": "Object 752",
         "R175_IS_2E": "IS-2 shielded",
+        "R175_IS_2E_Custom": "IS-2 shld Trapper",
+        "R175_IS_2E_Custom_short": "IS-2 shld Trapper",
         "R175_IS_2E_short": "IS-2 shielded",
         "R180_Object_274A": "Object 274a",
         "R180_Object_274A_Custom": "Object 274a Leshy",
@@ -6287,6 +6365,7 @@
         "Rolls_Royce_Meteor_MkIV_B12": "Rolls-Royce Meteor Mk. IVB/12",
         "Rolls_Royce_Meteor_P": "Rolls-Royce Meteor P",
         "Rolls_Royse_Meteor": "Rolls-Royce Meteor",
+        "Rolls_Royse_Meteor_Desert": "Ancient Spirit",
         "Rolls_Royse_Meteor_M120": "Rolls-Royce Meteor M120",
         "Rolls_Royse_Meteor_Mk12": "Rolls-Royce Meteor Mk. 12",
         "Rolls_Royse_Meteor_MkIV": "Rolls-Royce Meteor Mk. IVB",
@@ -6336,6 +6415,8 @@
         "S26_Lancen_C_Custom": "Lansen Moose",
         "S26_Lancen_C_Custom_short": "Lansen Moose",
         "S26_Lancen_C_short": "Lansen C",
+        "S31_Strv_K": "Strv K",
+        "S31_Strv_K_short": "Strv K",
         "S35_captured": "Pz.Kpfw. S35 739 (f)",
         "S35_captured_short": "Pz. S35",
         "SA35_4": "SA35 mle. 4",
@@ -6344,6 +6425,7 @@
         "SFA_F12": "SFA F12",
         "SN-230TR": "SN-230TR",
         "SN-230TRM": "SN-230TRM",
+        "SOFAM-12-GSds_B": "SOFAM 12 GSds B",
         "SOFAM_12GSds_A": "SOFAM 12 GSds",
         "SOFAM_12_58": "SOFAM 12 58",
         "SOFAM_8Gxb": "SOFAM 8 Gxb",
@@ -6369,6 +6451,8 @@
         "SU-14_1": "SU-14-2",
         "SU-152": "SU-152",
         "SU-152_01": "SU-152",
+        "SU-152_Custom": "SU-152 Taiga",
+        "SU-152_Custom_short": "SU-152 Тaiga",
         "SU-152_mod_1943": "SU-152",
         "SU-152_mod_1944": "SU-152M",
         "SU-18": "SU-18",
@@ -6815,6 +6899,7 @@
         "Van_Helsing_Custom_short": "Helsing Raven",
         "Van_Helsing_short": "Helsing",
         "Vauxhall_Bedford_Twin_Six": "Bedford Twin-Six",
+        "Vauxhall_Bedford_Twin_SixBP": "Bedford Twin-Six",
         "Vauxhall_Bedford_Twin_Six_B": "Bedford Twin-Six B",
         "Vauxhall_Bedford_Twin_Six_BP": "Bedford Twin-Six II",
         "Verbesserteturm": "L.K.A. 2",

upload_wotb_replays.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 ## PYTHON VERSION MUST BE 3.7 OR HIGHER
 
-import json, argparse, inspect, sys, os, io, base64, aiohttp, urllib, asyncio, aiofiles, aioconsole
+import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
-import logging, re, concurrent.futures, configparser, motor.motor_asyncio, ssl, zipfile
+import logging, re, configparser, zipfile
 import blitzutils as bu
 from blitzutils import WG
 from blitzutils import WoTinspector
@@ -21,17 +21,17 @@ SKIPPED_N 	= 0
 ERROR_N 	= 0
 WIurl='https://wotinspector.com/api/replay/upload?'
 WG_appID  = '81381d3f45fa4aa75b78a7198eb216ad'
-wg = None
+wg : WG | None = None
-wi = None
+wi : WoTinspector | None = None
 
-async def main(argv):
+async def main(argv) -> int:
 	global wg, wi
 	current_dir = os.getcwd()
 	# set the directory for the script
 	os.chdir(os.path.dirname(sys.argv[0]))
 	
 	# options defaults
-	OPT_WORKERS_N  	= 5
+	OPT_WORKERS_N  	= 2
 	WG_ID 			= None
 	WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					# China is not supported since WG API stats are not available there
@@ -43,12 +43,6 @@ async def main(argv):
 			bu.debug('Reading config file: ' + FILE_CONFIG)
 			config = configparser.ConfigParser()
 			config.read(FILE_CONFIG)
-			try:
-				if 'UPLOADER' in config.sections():
-					configUploader 	= config['UPLOADER']
-					OPT_WORKERS_N = configUploader.getint('opt_workers', OPT_WORKERS_N)
-			except (KeyError, configparser.NoSectionError) as err:
-				bu.error(exception=err)
 							
 			try:
 				if 'WG' in config.sections():
@@ -82,18 +76,19 @@ async def main(argv):
 	bu.set_log_level(args.silent, args.verbose, args.debug)
 
 	wg = WG(WG_appID, args.tankopedia, args.mapfile)
-	wi = WoTinspector(rate_limit=WG_RATE_LIMIT)
+	wi = WoTinspector()
 
-	if args.account != None:
+	if args.account is not None:
 		args.accountID = await wg.get_account_id(args.account)
 		bu.debug('WG  account_id: ' + str(args.accountID))
 
-	if args.accountID == None: 
+	if args.accountID is None: 
 		args.accountID = 0
 
 	try:
-		queue  = asyncio.Queue()	
+		queue : asyncio.Queue = asyncio.Queue()	
 		# rebase file arguments due to moving the working directory to the script location
+		bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
 		args.files = bu.rebase_file_args(current_dir, args.files)
 
 		tasks = []
@@ -123,7 +118,7 @@ async def main(argv):
 		await wg.session.close()
 		await wi.close()
 
-	return None
+	return 0
 
 
 async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
@@ -137,13 +132,13 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
 			if not line: 
 				break
 			else:
-				if (p_replayfile.match(line) != None):
+				if (p_replayfile.match(line) is not None):
 					await queue.put(await mkQueueItem(line, title))
 	else:
 		for fn in files:
 			if fn.endswith('"'):
 				fn = fn[:-1]  
-			if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
+			if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
 				await queue.put(await mkQueueItem(fn, title))
 				bu.debug('File added to queue: ' + fn)
 			elif os.path.isdir(fn):
@@ -151,7 +146,7 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
 					for entry in dirEntry:
 						try:
 							bu.debug('Found: ' + entry.name)
-							if entry.is_file() and (p_replayfile.match(entry.name) != None): 
+							if entry.is_file() and (p_replayfile.match(entry.name) is not None): 
 								bu.debug(entry.name)
 								await queue.put(await mkQueueItem(entry.path, title))
 								bu.debug('File added to queue: ' + entry.path)
@@ -175,6 +170,9 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 	"""Async Worker to process the replay queue"""
 	global SKIPPED_N
 	global ERROR_N
+	global wi
+
+	assert wi is not None, "wi cannot be None"
 
 	while True:
 		item = await queue.get()
@@ -210,7 +208,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 				filename = os.path.basename(filename)
 				bu.debug(msg_str + 'File:  ' + filename)
 				json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
-				if json_resp != None:
+				if json_resp is not None:
 					if (await bu.save_JSON(replay_json_fn,json_resp)):
 						if wi.chk_JSON_replay(json_resp):
 							if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
@@ -235,7 +233,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 def getTitle_old(replayfile: str, title: str, i : int) -> str:
 	global wg
 
-	if title == None:
+	if title is None:
 		try:
 			filename = os.path.basename(replayfile)	
 			bu.debug(filename)
@@ -247,8 +245,8 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
 			p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
 			
 			m = p.match(filename)
-			if (m != None):
+			if (m is not None):
-				if wg.tanks != None:
+				if wg.tanks is not None:
 					tank = m.group(1)
 					if tank in wg.tanks['userStr']:
 						tank = wg.tanks['userStr'][tank]
@@ -273,7 +271,9 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
 
 def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
 	global wg
-	if title == None:
+	assert wg is not None, "wg must not be None"
+
+	if title is None:
 		try:
 			filename = os.path.basename(replayfile)	
 			bu.debug(filename)
@@ -289,10 +289,10 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
 						#player = metadata_json['playerName']						
 						tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
 						map_name = wg.get_map(metadata_json['mapName'])
-						bu.debug('Tank: ' + tank_name if tank_name != None else "NOT FOUND IN tanks.json" +  ' Map: ' + map_name)
+						bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" +  ' Map: ' + map_name)
 					except Exception as err:
 						bu.error(exception = err)
-			if (tank_name != None) and  (map_name != None):
+			if (tank_name is not None) and  (map_name is not None):
 				title = tank_name + ' @ ' + map_name
 			else:
 				title = re.sub('\\.wotbreplay$', '', filename)