Author SHA1 Message Date
Jylpah bca474218e WoTinspector.read_replay_id(): added back 2022-10-29 22:21:14 +03:00
Jylpah e48759ce79 ThrottledClientSession(), WoTinspector(): updated 2022-10-29 22:14:33 +03:00
Jylpah 5c694fd721 Typing + remove unsed imports 2022-10-29 22:13:52 +03:00
Jylpah 1f41901042 update 9.2 2022-10-09 00:18:30 +03:00
Jylpah d275914b32 Update .gitignore 2022-08-18 09:44:39 +03:00
Jylpah 9f8c82f434 Stream client support 2022-08-18 09:42:55 +03:00
Jylpah 3a8f065f35 Update 9.2 2022-08-18 09:42:32 +03:00
6 changed files with 423 additions and 142 deletions

No files matched your search

Executable
+15
View File
@@ -0,0 +1,15 @@
__pycache__/
BlitzApp
tmp/
ssl/
*.ini
*.swp
*.code-workspace
*.sqlite3
.vscode/
*.sqlite3-journal
.gitattributes
export*
*.csv
!/stats/*.csv
!*.py
+152 -104
View File
@@ -36,132 +36,163 @@ _progress_obj = None
UMASK = os.umask(0)
os.umask(UMASK)
# -----------------------------------------------------------
# Class ThrottledClientSession(aiohttp.ClientSession)
# -----------------------------------------------------------
## -----------------------------------------------------------
#### Class ThrottledClientSession(aiohttp.ClientSession)
#
# Rate-limited async http client session
#
# Inherits aiohttp.ClientSession
## -----------------------------------------------------------
from typing import Optional, Union
import aiohttp
import asyncio
import time
import logging
import re
class ThrottledClientSession(aiohttp.ClientSession):
"""Rate-throttled client session class inherited from aiohttp.ClientSession)"""
# MIN_SLEEP = 0.1
"""Rate-throttled client session class inherited from aiohttp.ClientSession)"""
def __init__(self, rate_limit: float = None, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.rate_limit = None
self._fillerTask = None
self._queue = None
self._start_time = None
self._count = 0
self._start_time = time.time()
def __init__(self, rate_limit: float = 0, filters: list[str] = list() ,
limit_filtered: bool = False, re_filter: bool = False, *args,**kwargs) -> None:
assert isinstance(rate_limit, (int, float)), "rate_limit has to be float"
assert isinstance(filters, list), "filters has to be list"
assert isinstance(limit_filtered, bool),"limit_filtered has to be bool"
assert isinstance(re_filter, bool), "re_filter has to be bool"
if rate_limit != None:
if rate_limit <= 0:
raise ValueError('rate_limit must be positive')
self.rate_limit = rate_limit
self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
self._fillerTask = asyncio.create_task(self._filler())
super().__init__(*args,**kwargs)
self.rate_limit : float
self._fillerTask : Optional[asyncio.Task] = None
self._queue : Optional[asyncio.Queue] = None
self._start_time : float = time.time()
self._count : int = 0
self._limit_filtered: bool = limit_filtered
self._re_filter : bool = re_filter
self._filters : list[Union[str, re.Pattern]] = list()
def _get_sleep(self) -> list:
if self.rate_limit == None:
return None
if re_filter:
for filter in filters:
self._filters.append(re.compile(filter))
else:
for filter in filters:
self._filters.append(filter)
self.set_rate_limit(rate_limit)
def _get_sleep(self) -> float:
if self.rate_limit > 0:
return 1/self.rate_limit
return 0
def get_rate(self) -> float:
"""Return rate of requests"""
return self._count / (time.time() - self._start_time)
def get_stats(self):
def get_stats(self) -> dict[str, float]:
"""Get session statistics"""
res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
'count': self._count}
res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
return res
def get_stats_str(self):
def get_stats_str(self) -> str:
"""Print session statistics"""
return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
' rate: ' + \
"{0:.1f}".format(self.get_rate()) + \
' requests: ' + str(self._count)
return f"rate limit: {str(self.rate_limit if self.rate_limit != None else '-')} \
rate: {0:.1f}.format(self.get_rate()) requests: {str(self._count)}"
def reset_counters(self):
def reset_counters(self) -> dict[str, float]:
"""Reset rate counters and return current results"""
res = self.get_stats()
self._start_time = time.time()
self._count = 0
return res
def set_rate_limit(self, rate_limit: float = 1):
if rate_limit >= 0:
self.rate_limit = rate_limit
return self.rate_limit
return None
def set_rate_limit(self, rate_limit: float = 0) -> float:
assert rate_limit is not None, "rate_limit must not be None"
assert isinstance(rate_limit, (int,float)) and rate_limit >= 0, "rate_limit has to be type of 'float' >= 0"
self.rate_limit = rate_limit
if rate_limit > 0:
self._queue = asyncio.Queue(int(rate_limit)+1)
if self._fillerTask is not None:
self._fillerTask.cancel()
self._fillerTask = asyncio.create_task(self._filler())
return self.rate_limit
async def close(self) -> None:
"""Close rate-limiter's "bucket filler" task"""
# DEBUG
debug(self.get_stats_str())
# DEBUG
logging.debug(self.get_stats_str())
try:
if self._fillerTask != None:
if self._fillerTask is not None:
self._fillerTask.cancel()
await asyncio.wait_for(self._fillerTask, timeout=0.5)
except (asyncio.CancelledError, asyncio.TimeoutError):
pass
await asyncio.wait_for(self._fillerTask, timeout=0.5)
except asyncio.TimeoutError as err:
logging.error(str(err))
await super().close()
async def _filler_simple(self):
async def _filler(self) -> None:
"""Filler task to fill the leaky bucket algo"""
try:
if self._queue == None:
return
sleep = self._get_sleep()
while True:
self._queue.put(None)
await asyncio.sleep(sleep)
except asyncio.CancelledError:
debug('Cancelled')
except Exception as err:
error(exception=err)
async def _filler(self):
"""Filler task to fill the leaky bucket algo"""
try:
if self._queue == None:
return
sleep = self._get_sleep()
# debug('SLEEP: ' + str(sleep))
if self._queue is None:
return None
logging.debug('SLEEP: ' + str(self._get_sleep()))
updated_at = time.monotonic()
fraction = 0
extra_increment = 0
extra_increment : float = 0
for i in range(0, self._queue.maxsize):
self._queue.put_nowait(i)
await self._queue.put(i)
while True:
if not self._queue.full():
now = time.monotonic()
increment = self.rate_limit * (now - updated_at)
fraction += increment % 1
extra_increment = fraction // 1
items_2_add = int(
min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
fraction = fraction % 1
for i in range(0, items_2_add):
items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment + extra_increment)))
extra_increment = (increment + extra_increment) % 1
for i in range(0,items_2_add):
self._queue.put_nowait(i)
updated_at = now
await asyncio.sleep(sleep)
await asyncio.sleep(self._get_sleep())
except asyncio.CancelledError:
debug('Cancelled')
logging.debug('Cancelled')
except Exception as err:
error(exception=err)
logging.error(str(err))
return None
async def _request(self, *args, **kwargs):
async def _request(self, *args,**kwargs) -> aiohttp.ClientResponse:
"""Throttled _request()"""
if self._queue != None:
if self._queue is not None and self.is_limited(*args):
await self._queue.get()
self._queue.task_done()
self._count += 1
return await super()._request(*args, **kwargs)
return await super()._request(*args,**kwargs)
def is_limited(self, *args: str) -> bool:
"""Check wether the rate limit should be applied"""
try:
url: str = args[1]
for filter in self._filters:
if isinstance(filter, re.Pattern) and filter.match(url) is not None:
return self._limit_filtered
elif isinstance(filter, str) and url.startswith(filter):
return self._limit_filtered
return not self._limit_filtered
except Exception as err:
logging.error(str(err))
return True
# -----------------------------------------------------------
# Class AsyncLogger()
# -----------------------------------------------------------
class AsyncLogger():
"""Async file logger"""
@@ -496,18 +527,17 @@ def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
return None
def rebase_file_args(current_dir, files):
"""REbase file command line params after moving working dir to the script's dir"""
if isinstance(files, list):
if (files[0] == '-') or (files[0] == 'db:'):
return files
else:
return [ os.path.join(current_dir, fn) for fn in files ]
elif isinstance(files, str):
return os.path.join(current_dir, files)
def rebase_file_args(current_dir : str, files: list[str]) -> list[str]:
"""Rebase file command line params after moving working dir to the script's dir"""
assert isinstance(files, list), "files has to be a list"
if (files[0] == '-') or (files[0] == 'db:'):
return [ files[0] ]
else:
return [ os.path.join(current_dir, fn) for fn in files ]
async def read_int_list(filename: str) -> list():
async def read_int_list(filename: str) -> list[int]:
"""Read file to a list and return list of integers in the input file"""
input_list = []
@@ -1596,6 +1626,8 @@ class WG:
async def store_tank_stats(self, key: list , stats_data: list, update_time: int):
"""Save tank stats into cache"""
assert self.cache is not None, "cache must be initiated"
try:
account_id = key[0]
tank_ids = set(key[1])
@@ -1617,6 +1649,8 @@ class WG:
async def store_player_stats(self, key: list , stats_data: list, update_time: int):
"""Save player stats into cache"""
assert self.cache is not None, "cache must be initiated"
try:
account_id = key[0]
if stats_data != None:
@@ -1633,6 +1667,8 @@ class WG:
async def store_player_achievements(self, key: list , stats_data: list, update_time: int):
"""Save player stats into cache"""
assert self.cache is not None, "cache must be initiated"
try:
account_id = key[0]
if stats_data != None:
@@ -1648,6 +1684,8 @@ class WG:
async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ):
assert self.cache is not None, "cache must be initiated"
try:
# test for cacheDB existence
debug('Trying cached stats first')
@@ -1761,17 +1799,27 @@ class WoTinspector:
URL_REPLAY_VIEW = URL_WI +'/en/view/'
URL_REPLAY_UL = 'https://api.wotinspector.com/replay/upload?'
URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
URL_TANK_DB = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
REPLAY_N = 1
DEFAULT_RATE_LIMIT : float = 10/60 # 10 requests / min
DEFAULT_TOKEN : str = 'a5428af49bf7495986577d59b0e5bcff'
def __init__(self, rate_limit: int = 30):
self.session = ThrottledClientSession(rate_limit=rate_limit)
def __init__(self, rate_limit: float = DEFAULT_RATE_LIMIT, auth_token: Optional[str] = None):
headers : Optional[dict[str, str]] = None
if auth_token is None:
auth_token = self.DEFAULT_TOKEN
headers = dict()
headers['Authorization'] = 'Token ' + auth_token
self.session = ThrottledClientSession(rate_limit=rate_limit,
filters=[self.URL_REPLAY_LIST, self.URL_REPLAY_INFO],
re_filter=False, limit_filtered=True, headers = headers)
async def close(self):
if self.session != None:
async def close(self) -> None:
if self.session is not None:
debug('Closing aiohttp session')
await self.session.close()
@@ -1835,7 +1883,7 @@ class WoTinspector:
hash.update(data)
replay_id = hash.hexdigest()
# Testing if the replay has already been posted
## Testing if the replay has already been posted
json_resp = await self.get_replay_JSON(replay_id)
if json_resp != None:
debug('Already uploaded: ' + title, id=N)
@@ -1850,7 +1898,7 @@ class WoTinspector:
}
url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
# debug('URL: ' + url)
#debug('URL: ' + url)
headers ={'Content-type': 'application/x-www-form-urlencoded'}
payload = { 'file' : (filename, base64.b64encode(data)) }
except Exception as err:
@@ -1875,8 +1923,6 @@ class WoTinspector:
except Exception as err:
debug(exception=err, id=N)
await asyncio.sleep(SLEEP)
_id = self.read_replay_id(json_resp)
json_resp['_id'] = _id
debug(' Could not post replay: ' + title, id=N)
return json_resp
@@ -1914,11 +1960,6 @@ class WoTinspector:
return replay_links
@classmethod
def get_replay_id(cls, url):
return url.rsplit('/', 1)[-1]
@classmethod
def read_replay_id(cls, json_replay):
"""Read replay_id from replay JSON file"""
@@ -1929,14 +1970,18 @@ class WoTinspector:
error(exception=err)
return None
@classmethod
def chk_JSON_replay(cls, json_resp):
def get_replay_id(cls, url):
return url.rsplit('/', 1)[-1]
@classmethod
def chk_JSON_replay(cls, json_resp) -> bool:
""""Check String for being a valid JSON file"""
try:
# FIX The replay can have allies/enemies i.e. be useful for
# team composition analysis, but does not have player contribution
if ('status' in json_resp) and json_resp['status'] == 'ok' and \
(get_JSON_value(json_resp, key='data.summary.winner_team') != None) :
(get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
debug("JSON check OK")
return True
except KeyError as err:
@@ -1945,6 +1990,9 @@ class WoTinspector:
debug("Replay JSON check failed: " + str(json_resp))
return False
# -----------------------------------------------------------
# Class BlitzStars
# -----------------------------------------------------------
class BlitzStars:
+12 -6
View File
@@ -3,6 +3,7 @@
# Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
from os import path
import sys, argparse, json, os, inspect, aiohttp, asyncio, aiofiles, re, logging, time, xmltodict, collections, configparser
import blitzutils as bu
from blitzutils import WG
@@ -11,9 +12,11 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini'
BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
BLITZAPP_VEHICLE_FILE='/list.xml'
# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
BLITZAPP_STRINGS='Data/Strings/en.yaml'
BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
BLITZAPP_VEHICLE_FILE='list.xml'
wg = None
@@ -104,8 +107,11 @@ async def main(argv):
async def extract_tanks(blitz_app_base : str, nation: str):
tanks = []
list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
if not os.path.isfile(list_xml_file):
# WG has changed the location of Data directory - at least in steam client
if path.isdir(path.join(blitz_app_base, 'assets')):
blitz_app_base = path.join(blitz_app_base, 'assets')
list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
if not path.isfile(list_xml_file):
print('ERROR: cannot open ' + list_xml_file)
return None
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
@@ -132,7 +138,7 @@ async def read_user_strs(blitz_app_base : str) -> dict:
"""Read user strings to convert map and tank names"""
tank_strs = {}
map_strs = {}
filename = blitz_app_base + BLITZAPP_STRINGS
filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
try:
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
+1 -1
View File
@@ -57,4 +57,4 @@
"skit": "Naval Frontier",
"test": "World of Ducks",
"tutorial": "Proving Grounds"
}
}
+215 -3
View File
@@ -1,7 +1,7 @@
{
"status": "ok",
"meta": {
"count": 550
"count": 564
},
"data": {
"1": {
@@ -2188,6 +2188,14 @@
"tier": 6,
"type": "AT-SPG"
},
"9841": {
"is_premium": true,
"name": "Rover",
"nation": "other",
"tank_id": 9841,
"tier": 6,
"type": "lightTank"
},
"9857": {
"is_premium": true,
"name": "Škoda T 45",
@@ -2236,6 +2244,14 @@
"tier": 7,
"type": "AT-SPG"
},
"10097": {
"is_premium": true,
"name": "Medjay",
"nation": "other",
"tank_id": 10097,
"tier": 5,
"type": "heavyTank"
},
"10113": {
"is_premium": true,
"name": "Carro da Combattimento 45t",
@@ -2268,6 +2284,22 @@
"tier": 4,
"type": "AT-SPG"
},
"10353": {
"is_premium": true,
"name": "Pharaoh",
"nation": "other",
"tank_id": 10353,
"tier": 7,
"type": "heavyTank"
},
"10369": {
"is_premium": true,
"name": "Controcarro 3 Minotauro",
"nation": "european",
"tank_id": 10369,
"tier": 10,
"type": "AT-SPG"
},
"10497": {
"is_premium": false,
"name": "KV-2",
@@ -2436,6 +2468,14 @@
"tier": 7,
"type": "mediumTank"
},
"11905": {
"is_premium": true,
"name": "Shadowhunter",
"nation": "european",
"tank_id": 11905,
"tier": 6,
"type": "mediumTank"
},
"12033": {
"is_premium": false,
"name": "SU-122-54",
@@ -2468,6 +2508,14 @@
"tier": 8,
"type": "AT-SPG"
},
"12161": {
"is_premium": true,
"name": "Strv K",
"nation": "european",
"tank_id": 12161,
"tier": 10,
"type": "heavyTank"
},
"12305": {
"is_premium": false,
"name": "E 50 Ausf. M",
@@ -3124,6 +3172,14 @@
"tier": 8,
"type": "heavyTank"
},
"19777": {
"is_premium": true,
"name": "AltProto AMX 30",
"nation": "france",
"tank_id": 19777,
"tier": 8,
"type": "mediumTank"
},
"19793": {
"is_premium": false,
"name": "Vickers Cruiser",
@@ -3404,6 +3460,14 @@
"tier": 6,
"type": "heavyTank"
},
"22097": {
"is_premium": true,
"name": "Churchill VIII",
"nation": "uk",
"tank_id": 22097,
"tier": 6,
"type": "heavyTank"
},
"22273": {
"is_premium": true,
"name": "Object 260",
@@ -3428,6 +3492,14 @@
"tier": 9,
"type": "heavyTank"
},
"22353": {
"is_premium": true,
"name": "Churchill W",
"nation": "uk",
"tank_id": 22353,
"tier": 6,
"type": "heavyTank"
},
"22529": {
"is_premium": true,
"name": "LT-432",
@@ -3676,6 +3748,14 @@
"tier": 8,
"type": "mediumTank"
},
"25361": {
"is_premium": true,
"name": "Waffenträger Camelot",
"nation": "germany",
"tank_id": 25361,
"tier": 9,
"type": "AT-SPG"
},
"25377": {
"is_premium": true,
"name": "T77",
@@ -3684,6 +3764,22 @@
"tier": 8,
"type": "heavyTank"
},
"25601": {
"is_premium": true,
"name": "IS-2 shielded",
"nation": "ussr",
"tank_id": 25601,
"tier": 7,
"type": "heavyTank"
},
"25633": {
"is_premium": true,
"name": "M-IV-Y",
"nation": "usa",
"tank_id": 25633,
"tier": 8,
"type": "heavyTank"
},
"25889": {
"is_premium": true,
"name": "Ranger",
@@ -3700,6 +3796,22 @@
"tier": 5,
"type": "mediumTank"
},
"26401": {
"is_premium": true,
"name": "Enforcer",
"nation": "usa",
"tank_id": 26401,
"tier": 6,
"type": "AT-SPG"
},
"26657": {
"is_premium": true,
"name": "ASTRON Rex 105 mm",
"nation": "usa",
"tank_id": 26657,
"tier": 8,
"type": "mediumTank"
},
"51201": {
"is_premium": true,
"name": "KV-220 Beta-Test",
@@ -4470,6 +4582,12 @@
"A132_T77_Custom": "T77 Cannonball",
"A132_T77_Custom_short": "T77 Cannonball",
"A132_T77_short": "T77",
"A140_ASTRON_REX_105": "ASTRON Rex 105 mm",
"A140_ASTRON_REX_105_short": "ASTRON Rex",
"A141_M-4-Y": "M-IV-Y",
"A141_M-4-Y_Custom": "M-IV-Y Stalker",
"A141_M-4-Y_Custom_short": "M-IV-Y Stalker",
"A141_M-4-Y_short": "M-IV-Y",
"A68_T28D": "T28 Defender",
"A68_T28D_short": "T28 Defender",
"A77_T6M": "T6E1 Grizzly",
@@ -4479,6 +4597,8 @@
"A95_T95E6_Custom_short": "T95E6 Piranha",
"A95_T95E6_short": "T95E6",
"A99_T92E1": "T92E1",
"A99_T92E1_Custom": "T92E1 Poison",
"A99_T92E1_Custom_short": "T92E1 Poison",
"A99_T92E1_short": "T92E1",
"AEC": "2x AEC",
"AEC3": "AEC3",
@@ -4504,6 +4624,8 @@
"AMX_50Fosh_155": "AMX 50 Foch (155)",
"AMX_50Fosh_155_short": "Foch 155",
"AMX_50_100": "AMX 50 100",
"AMX_50_100_Custom": "AMX 50 100 Basher",
"AMX_50_100_Custom_short": "AMX 50 100 Basher",
"AMX_50_100_short": "AMX 50 100",
"AMX_50_120": "AMX 50 120",
"AMX_50_120_short": "AMX 50 120",
@@ -4548,6 +4670,7 @@
"AT-1": "AT-1",
"AT-1bis": "AT-1 Bis",
"AVCR_1100_1": "AVCR 1100 1",
"AVS-1195A": "AVS-1195A",
"AVS_1195": "AVS-1195",
"AV_1195": "AV-1195",
"AV_1790": "AV 1790",
@@ -4602,6 +4725,8 @@
"Bison_I_short": "Bison",
"Buick_series_60": "Buick Series 60",
"CH38_WZ113G-FT": "WZ-113G FT",
"CH38_WZ113G-FT_Custom_short_v2": "WZ-113G FT Cute Baby",
"CH38_WZ113G-FT_Custom_v2": "WZ-113G FT Cute Baby",
"CH38_WZ113G-FT_short": "WZ-113G FT",
"CH38_WZ113G_FT_Custom": "WZ-113G FT Lucky",
"CH38_WZ113G_FT_Custom_short": "WZ-113G FT Lucky",
@@ -4696,6 +4821,20 @@
"Ch_WZ-112v2_short": "WZ-112-2",
"Ch_WZ_112v2_Custom": "WZ-112-2 Celestial",
"Ch_WZ_112v2_Custom_short": "WZ-112-2 Celestial",
"Chassis1_A140_ASTRON_REX_105": "Rex",
"Chassis1_A140_ASTRON_REX_105_short": "Rex",
"Chassis1_F117_Alt_Proto_AMX_30": "AltProto AMX 30",
"Chassis1_F117_Alt_Proto_AMX_30_short": "A.P. AMX 30",
"Chassis1_GB09_Churchill_VII_BP": "Churchill W",
"Chassis1_GB09_Churchill_VII_BP_short": "Churchill W",
"Chassis1_It10_P43_bis_BP": "Shadowhunter",
"Chassis1_It10_P43_bis_BP_short": "Shadowhunter",
"Chassis1_Oth34_PanzerMummy": "Royal Tread",
"Chassis1_Oth34_PanzerMummy_short": "Royal Tread",
"Chassis1_Oth35_PanzerMejai": "Desert Wanderer",
"Chassis1_Oth35_PanzerMejai_short": "Desert Wanderer",
"Chassis1_S31_Strv_K": "Stridsvagn KRV",
"Chassis1_S31_Strv_K_short": "Stridsvagn KRV",
"Chi_Ha": "Type 97 Chi-Ha",
"Chi_Ha_short": "Chi-Ha",
"Chi_He": "Type 1 Chi-He",
@@ -4725,17 +4864,20 @@
"Continental_AOI-628-1": "Continental AOI-628-1",
"Continental_AOI-628-1_short": "Continental\\nAOI-628-1",
"Continental_AOI-628-2": "Continental AOI-628-2",
"Continental_AOI-628-2A": "Continental AOI-628-2A",
"Continental_AOI-628-2_short": "Continental\\nAOI-628-2",
"Continental_AOI_1490_1": "Continental AOI-1490-1",
"Continental_AOS-895-1": "Continental AOS-895-1",
"Continental_AOS-895-1_short": "Continental\\nAOS-895-1",
"Continental_AOS-895-3": "AN/GRC-8",
"Continental_AOS-895-3V2": "Continental AOS-895-3-2",
"Continental_AOS-895-3_short": "AN/GRC-8",
"Continental_AOSI-1195-1": "Continental AOSI 1195-1",
"Continental_AOSI-1195-1A": "Continental AOSI-1195-1A",
"Continental_AOSI-1195-5": "Continental AOSI-1195-5",
"Continental_AOSI-895-5": "Continental AOSI-895-5",
"Continental_AOSI-895-5_short": "Continental\\nAOSI-895-5",
"Continental_AOSI_1195_5A": "Continental AOSI-1195-5 Y IV",
"Continental_AOSI_895_5_Prem": "Continental AOSI-895-5",
"Continental_AV-1790-1": "Continental AV-1790-1",
"Continental_AV-1790-1A": "Continental AV-1790-1",
@@ -4811,6 +4953,8 @@
"Cz14_Skoda_T-56_Custom_short": "Škoda T 56 Golem",
"Cz14_Skoda_T-56_short": "Škoda T 56",
"Cz15_Skoda_T-45": "Škoda T 45",
"Cz15_Skoda_T-45_Custom": "Škoda T 45 Skvělý",
"Cz15_Skoda_T-45_Custom_short": "Škoda T 45 Skvělý",
"Cz15_Skoda_T-45_short": "Škoda T 45",
"D-744": "D-744",
"D1": "D1",
@@ -4847,12 +4991,16 @@
"E-100": "E 100",
"E-100_Custom": "E 100 Stalwart",
"E-100_Custom_short": "E 100 Stalwart",
"E-100_Custom_short_v2": "E 100 Jötunn",
"E-100_Custom_v2": "E 100 Jötunn",
"E-100_short": "E 100",
"E-25": "E 25",
"E-25_Custom": "E 25 \\\"#1\\\"",
"E-50": "E 50",
"E-50_short": "E 50",
"E-75": "E 75",
"E-75_Custom": "E 75 Garm",
"E-75_Custom_short": "E 75 Garm",
"E-75_short": "E 75",
"E50_Ausf_M": "E 50 Ausf. M",
"E50_Ausf_M_Custom": "E 50 M Roaring",
@@ -4872,6 +5020,8 @@
"F116_Bat_Chat_Bourrasque_Custom": "Bourrasque Noir",
"F116_Bat_Chat_Bourrasque_Custom_short": "Bourrasque Noir",
"F116_Bat_Chat_Bourrasque_short": "B-C Bourrasque",
"F117_Alt_Proto_AMX_30": "AltProto AMX 30",
"F117_Alt_Proto_AMX_30_short": "A.P. AMX 30",
"F19_Lorraine40t": "Lorraine 40 t",
"F19_Lorraine40t_Com": "Lorraine 40 t Fearless",
"F19_Lorraine40t_Com_short": "Lorraine Fearless",
@@ -5036,11 +5186,19 @@
"GB08_Churchill_I": "Churchill I",
"GB08_Churchill_I_short": "Churchill I",
"GB09_Churchill_VII": "Churchill VII",
"GB09_Churchill_VII_BP": "Churchill W",
"GB09_Churchill_VII_BP_Custom": "Churchill W Defender",
"GB09_Churchill_VII_BP_Custom_short": "Churchill W Defender",
"GB09_Churchill_VII_BP_short": "Churchill W",
"GB109_GSOR_1008": "GSOR 1008",
"GB109_GSOR_1008_Custom": "GSOR 1008 Excalibur",
"GB109_GSOR_1008_Custom_short": "GSOR 1008 Excalibur",
"GB109_GSOR_1008_short": "GSOR 1008",
"GB10_Black_Prince": "Black Prince",
"GB10_Black_Prince_short": "Black Prince",
"GB112_Caliban": "Caliban",
"GB112_Caliban_Custom": "Caliban Hugo",
"GB112_Caliban_Custom_short": "Caliban Hugo",
"GB112_Caliban_short": "Caliban",
"GB11_Caernarvon": "Caernarvon",
"GB11_Caernarvon_short": "Caernarvon",
@@ -5163,6 +5321,8 @@
"GB_Centurion_MkDF": "Defender Mk. 1",
"GB_Centurion_MkDF_short": "Defender Mk. 1",
"GB_Churchill_MkVI": "Churchill Mk. VI",
"GB_Churchill_MkVIII": "Churchill VIII",
"GB_Churchill_MkVIII_short": "Churchill VIII",
"GB_Churchill_MkVI_short": "Churchill Mk. VI",
"GB_FV301": "FV301",
"GB_FV301_short": "FV301",
@@ -5182,6 +5342,7 @@
"GMC_6004": "GMC 6004T",
"GMC_6046": "GMC 6046",
"GMC_6046A": "GMC 6046A",
"GMC_6046_BP": "GMC 6046B",
"GMC_Allison_V1710-E32": "GMC Allison V1710-E32",
"GMC_Allison_V1710-E32_short": "GMC Allison\\nV1710-E32",
"GMS-6046B": "GMS 6046B",
@@ -5287,6 +5448,8 @@
"IS8": "IS-8",
"ISU-152": "ISU-152",
"ISU-152_01": "ISU-152",
"ISU-152_Custom_short_v2": "ISU-152 Tundra",
"ISU-152_Custom_v2": "ISU-152 Tundra",
"ISU-152_mod_1945": "ISU-152M",
"ISU152_Custom": "ISU-152 Hunter",
"ISU152_Custom_short": "ISU-152 Hunter",
@@ -5307,6 +5470,10 @@
"It10_P43_BD": "P.43/06 anniversario",
"It10_P43_BD_short": "P.43/06 ann.",
"It10_P43_bis": "P.43 bis",
"It10_P43_bis_BP": "Shadowhunter",
"It10_P43_bis_BP_Custom": "Shadowhunter Enchanted",
"It10_P43_bis_BP_Custom_short": "Shadowhunter Enchanted",
"It10_P43_bis_BP_short": "Shadowhunter",
"It10_P43_bis_short": "P.43 bis",
"It12_Prototipo_Standard_B": "Prototipo Standard B",
"It12_Prototipo_Standard_B_short": "Standard B",
@@ -5319,7 +5486,11 @@
"It14_P44_Pantera_Custom_short": "P.44 Pantera Furious",
"It14_P44_Pantera_short": "P.44 Pantera",
"It20_Car_Comb_45t": "Carro da Combattimento 45t",
"It20_Car_Comb_45t_Custom": "Carro 45t Glitch",
"It20_Car_Comb_45t_Custom_short": "Carro 45t Glitch",
"It20_Car_Comb_45t_short": "Carro 45t",
"It21_Controcarro3_Minotauro": "Controcarro 3 Minotauro",
"It21_Controcarro3_Minotauro_short": "Minotauro",
"J12_Shinobi": "Shinobi",
"J12_Shinobi_short": "Shinobi",
"J20_Ho_Ri_type3": "Ho-Ri Type III",
@@ -5333,8 +5504,8 @@
"J23_Chi_To_SPG": "Chi-To SPG",
"J23_Chi_To_SPG_short": "Chi-To SPG",
"J24_Type_57": "Type 57",
"J24_Type_57_Custom": "Type 57 Mechacamo",
"J24_Type_57_Custom_short": "Type 57 Mechacamo",
"J24_Type_57_Custom": "Type 57 Mecha",
"J24_Type_57_Custom_short": "Type 57 Mecha",
"J24_Type_57_short": "Type 57",
"J25_Type71": "Type 71",
"J25_Type71_Custom": "Type 71 Sankai",
@@ -5487,6 +5658,8 @@
"M18E2BP": "BLTZ9000",
"M18E2BP_short": "BLTZ9000",
"M18_Hellcat": "M18 Hellcat",
"M18_Hellcat_Custom": "Hellcat Arracourt",
"M18_Hellcat_Custom_short": "Hellcat Arracourt",
"M18_Hellcat_short": "Hellcat",
"M22_Locust": "M22 Locust",
"M22_Locust_short": "Locust",
@@ -5501,6 +5674,10 @@
"M2_lt_short": "M2 Light",
"M2_med": "M2 Medium Tank",
"M2_med_short": "M2 Medium",
"M36_BP": "Enforcer",
"M36_BP_Custom": "Vandal",
"M36_BP_Custom_short": "Vandal",
"M36_BP_short": "Enforcer",
"M36_Slagger": "M36 Jackson",
"M36_Slagger_short": "Jackson",
"M36_Slagger_skin_volar": "M36 Valor",
@@ -5588,6 +5765,7 @@
"MTLS-1G14": "MTLS-1G14",
"MTLS_D1": "MTLS D1",
"MTLS_T1": "MTLS T1",
"MTU_MB-838_Ca_M-500": "MTU MB 838 CaM 500B",
"MT_50_T": "MT-50T",
"M_40": "M-40",
"MarderII": "Marder II",
@@ -5605,6 +5783,7 @@
"Mausketten": "Maus",
"Mausturm": "Maus",
"Maybach_HL100": "Maybach HL 100",
"Maybach_HL100V2": "Maybach HL 100 B",
"Maybach_HL45P": "Maybach HL 45 P",
"Maybach_HL57TR": "Maybach HL 57 TR",
"Maybach_HL62TR": "Maybach HL 62 TR",
@@ -5654,6 +5833,7 @@
"Maybach_HL_210_TRM_P45_short": "Maybach HL 210\\nTRM P45",
"Maybach_HL_230": "Maybach HL 230",
"Maybach_HL_230P30A": "Maybach HL 230 P30",
"Maybach_HL_230P30A_Desert": "Flaming Heart",
"Maybach_HL_230P30_2500rpm": "Maybach HL 230 P30 2500 rpm",
"Maybach_HL_230P45": "Maybach HL 230 P45F",
"Maybach_HL_230P45C": "Maybach HL 230 P45C",
@@ -5731,6 +5911,8 @@
"OQF_55_pdr_AT_Gun_short": "OQF 55-pdr AT Gun",
"OQF_77mm_AT_Gun_MkII": "OQF 77 mm AT Gun Mk. II",
"OQF_77mm_Gun_MkII": "OQF 77 mm Gun Mk. II",
"OQF_77mm_Gun_MkII_BP": "OQF 77 mm Gun Mk. II B",
"OQF_77mm_Gun_MkII_BP_short": "OQF 77 mm Gun Mk. II B",
"Object263": "Object 263",
"Object263_Custom": "Obj. 263 Graywolf",
"Object263_Custom_short": "Obj. 263 Graywolf",
@@ -5823,6 +6005,16 @@
"Oth31_T150BP_Custom": "Titan-150 Trailblazer",
"Oth31_T150BP_Custom_short": "Titan-150 Trailblazer",
"Oth31_T150BP_short": "Titan-150",
"Oth33_Blizzard_BP": "Rover",
"Oth33_Blizzard_BP_Custom": "Rover LunarX",
"Oth33_Blizzard_BP_Custom_short": "Rover LunarX",
"Oth33_Blizzard_BP_short": "Rover",
"Oth34_PanzerMummy": "Pharaoh",
"Oth34_PanzerMummy_Custom": "Pharaoh All-Seeing",
"Oth34_PanzerMummy_Custom_short": "Pharaoh All-Seeing",
"Oth34_PanzerMummy_short": "Pharaoh",
"Oth35_PanzerMejai": "Medjay",
"Oth35_PanzerMejai_short": "Medjay",
"PLInz_RWA": "PLInz R.W.A.",
"PLT_27": "PLT-27",
"Paccar_120_325HT": "Gas Boiler",
@@ -5999,6 +6191,10 @@
"PzVI_AusfBP_short": "Icebreaker",
"PzVI_AusfTU": "Tiger I",
"PzVI_AusfTU_short": "Tiger I",
"PzVI_Custom_short_v2": "Tiger I Fox",
"PzVI_Custom_short_v3": "Tiger I Baron",
"PzVI_Custom_v2": "Tiger I Fox",
"PzVI_Custom_v3": "Tiger I Baron",
"PzVI_GuP": "Tiger Kuromorimine SP",
"PzVI_GuP_short": "Tiger Kuromorimine SP",
"PzVI_Porsche_Turm": "Pz.Kpfw. Tiger Ausf. E",
@@ -6109,6 +6305,10 @@
"R172_Object_752_Custom": "Object 752 Stargazer",
"R172_Object_752_Custom_short": "Obj. 752 Stargazer",
"R172_Object_752_short": "Object 752",
"R175_IS_2E": "IS-2 shielded",
"R175_IS_2E_Custom": "IS-2 shld Trapper",
"R175_IS_2E_Custom_short": "IS-2 shld Trapper",
"R175_IS_2E_short": "IS-2 shielded",
"R180_Object_274A": "Object 274a",
"R180_Object_274A_Custom": "Object 274a Leshy",
"R180_Object_274A_Custom_short": "Object 274a Leshy",
@@ -6165,6 +6365,7 @@
"Rolls_Royce_Meteor_MkIV_B12": "Rolls-Royce Meteor Mk. IVB/12",
"Rolls_Royce_Meteor_P": "Rolls-Royce Meteor P",
"Rolls_Royse_Meteor": "Rolls-Royce Meteor",
"Rolls_Royse_Meteor_Desert": "Ancient Spirit",
"Rolls_Royse_Meteor_M120": "Rolls-Royce Meteor M120",
"Rolls_Royse_Meteor_Mk12": "Rolls-Royce Meteor Mk. 12",
"Rolls_Royse_Meteor_MkIV": "Rolls-Royce Meteor Mk. IVB",
@@ -6214,6 +6415,8 @@
"S26_Lancen_C_Custom": "Lansen Moose",
"S26_Lancen_C_Custom_short": "Lansen Moose",
"S26_Lancen_C_short": "Lansen C",
"S31_Strv_K": "Strv K",
"S31_Strv_K_short": "Strv K",
"S35_captured": "Pz.Kpfw. S35 739 (f)",
"S35_captured_short": "Pz. S35",
"SA35_4": "SA35 mle. 4",
@@ -6222,6 +6425,7 @@
"SFA_F12": "SFA F12",
"SN-230TR": "SN-230TR",
"SN-230TRM": "SN-230TRM",
"SOFAM-12-GSds_B": "SOFAM 12 GSds B",
"SOFAM_12GSds_A": "SOFAM 12 GSds",
"SOFAM_12_58": "SOFAM 12 58",
"SOFAM_8Gxb": "SOFAM 8 Gxb",
@@ -6247,6 +6451,8 @@
"SU-14_1": "SU-14-2",
"SU-152": "SU-152",
"SU-152_01": "SU-152",
"SU-152_Custom": "SU-152 Taiga",
"SU-152_Custom_short": "SU-152 Тaiga",
"SU-152_mod_1943": "SU-152",
"SU-152_mod_1944": "SU-152M",
"SU-18": "SU-18",
@@ -6569,6 +6775,7 @@
"V-2-54IS_GuP": "V-2-54 GuP",
"V-2IS": "V-2IS",
"V-2IS_B": "V-2IS",
"V-2IS_B_2": "V-2IS-2",
"V-2IS_R": "V-2-10",
"V-2IS_R1": "V-2-10",
"V-2K": "V-2K",
@@ -6680,6 +6887,7 @@
"V_2_34": "V-2-34",
"V_2_34M": "V-2-34M",
"V_2_34_modificato": "V-2-34 modificato",
"V_2_34_modificato_BP": "V-2-34 modificato B",
"V_2_54SCG": "V-2-54SCG",
"V_46_5": "V-46-5",
"V_54": "V-54",
@@ -6691,7 +6899,9 @@
"Van_Helsing_Custom_short": "Helsing Raven",
"Van_Helsing_short": "Helsing",
"Vauxhall_Bedford_Twin_Six": "Bedford Twin-Six",
"Vauxhall_Bedford_Twin_SixBP": "Bedford Twin-Six",
"Vauxhall_Bedford_Twin_Six_B": "Bedford Twin-Six B",
"Vauxhall_Bedford_Twin_Six_BP": "Bedford Twin-Six II",
"Verbesserteturm": "L.K.A. 2",
"Vickers-Armstrong": "Armstrong Siddeley V8",
"Vickers-Armstrong_MkII": "Armstrong Siddeley V8 SC",
@@ -6707,6 +6917,8 @@
"W_12_5_B": "W12-5B",
"W_12_5_P": "W12-5P",
"Waffentrager_IV": "Waffenträger auf Pz. IV",
"Waffentrager_IV_CL": "Waffenträger Camelot",
"Waffentrager_IV_CL_short": "WT Camelot",
"Waffentrager_IV_short": "WT auf Pz. IV",
"Waterspit_1": "Waterspit",
"Wespe": "Wespe",
+28 -28
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
## PYTHON VERSION MUST BE 3.7 OR HIGHER
import json, argparse, inspect, sys, os, io, base64, aiohttp, urllib, asyncio, aiofiles, aioconsole
import logging, re, concurrent.futures, configparser, motor.motor_asyncio, ssl, zipfile
import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
import logging, re, configparser, zipfile
import blitzutils as bu
from blitzutils import WG
from blitzutils import WoTinspector
@@ -21,17 +21,17 @@ SKIPPED_N = 0
ERROR_N = 0
WIurl='https://wotinspector.com/api/replay/upload?'
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
wg = None
wi = None
wg : WG | None = None
wi : WoTinspector | None = None
async def main(argv):
async def main(argv) -> int:
global wg, wi
current_dir = os.getcwd()
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
# options defaults
OPT_WORKERS_N = 5
OPT_WORKERS_N = 2
WG_ID = None
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
# China is not supported since WG API stats are not available there
@@ -43,13 +43,7 @@ async def main(argv):
bu.debug('Reading config file: ' + FILE_CONFIG)
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
try:
if 'UPLOADER' in config.sections():
configUploader = config['UPLOADER']
OPT_WORKERS_N = configUploader.getint('opt_workers', OPT_WORKERS_N)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
try:
if 'WG' in config.sections():
configWG = config['WG']
@@ -82,18 +76,19 @@ async def main(argv):
bu.set_log_level(args.silent, args.verbose, args.debug)
wg = WG(WG_appID, args.tankopedia, args.mapfile)
wi = WoTinspector(rate_limit=WG_RATE_LIMIT)
wi = WoTinspector()
if args.account != None:
if args.account is not None:
args.accountID = await wg.get_account_id(args.account)
bu.debug('WG account_id: ' + str(args.accountID))
if args.accountID == None:
if args.accountID is None:
args.accountID = 0
try:
queue = asyncio.Queue()
queue : asyncio.Queue = asyncio.Queue()
# rebase file arguments due to moving the working directory to the script location
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
args.files = bu.rebase_file_args(current_dir, args.files)
tasks = []
@@ -123,7 +118,7 @@ async def main(argv):
await wg.session.close()
await wi.close()
return None
return 0
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
@@ -137,13 +132,13 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
if not line:
break
else:
if (p_replayfile.match(line) != None):
if (p_replayfile.match(line) is not None):
await queue.put(await mkQueueItem(line, title))
else:
for fn in files:
if fn.endswith('"'):
fn = fn[:-1]
if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
await queue.put(await mkQueueItem(fn, title))
bu.debug('File added to queue: ' + fn)
elif os.path.isdir(fn):
@@ -151,7 +146,7 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
for entry in dirEntry:
try:
bu.debug('Found: ' + entry.name)
if entry.is_file() and (p_replayfile.match(entry.name) != None):
if entry.is_file() and (p_replayfile.match(entry.name) is not None):
bu.debug(entry.name)
await queue.put(await mkQueueItem(entry.path, title))
bu.debug('File added to queue: ' + entry.path)
@@ -175,6 +170,9 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
"""Async Worker to process the replay queue"""
global SKIPPED_N
global ERROR_N
global wi
assert wi is not None, "wi cannot be None"
while True:
item = await queue.get()
@@ -210,7 +208,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
filename = os.path.basename(filename)
bu.debug(msg_str + 'File: ' + filename)
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
if json_resp != None:
if json_resp is not None:
if (await bu.save_JSON(replay_json_fn,json_resp)):
if wi.chk_JSON_replay(json_resp):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
@@ -235,7 +233,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
def getTitle_old(replayfile: str, title: str, i : int) -> str:
global wg
if title == None:
if title is None:
try:
filename = os.path.basename(replayfile)
bu.debug(filename)
@@ -247,8 +245,8 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
m = p.match(filename)
if (m != None):
if wg.tanks != None:
if (m is not None):
if wg.tanks is not None:
tank = m.group(1)
if tank in wg.tanks['userStr']:
tank = wg.tanks['userStr'][tank]
@@ -273,7 +271,9 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
global wg
if title == None:
assert wg is not None, "wg must not be None"
if title is None:
try:
filename = os.path.basename(replayfile)
bu.debug(filename)
@@ -289,10 +289,10 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
#player = metadata_json['playerName']
tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
map_name = wg.get_map(metadata_json['mapName'])
bu.debug('Tank: ' + tank_name if tank_name != None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name)
bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name)
except Exception as err:
bu.error(exception = err)
if (tank_name != None) and (map_name != None):
if (tank_name is not None) and (map_name is not None):
title = tank_name + ' @ ' + map_name
else:
title = re.sub('\\.wotbreplay$', '', filename)