GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-21 02:07:06 +02:00
Code Issues Projects Releases Wiki Activity

Update 6.2 + bug fixes

Browse Source
This commit is contained in:
Jylpah committed 2019-08-16 22:53:06 +03:00
1 parent e911ca634a
commit 99a7e4530e
3 files changed
+59 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

blitzutils.py
+42 -12
View File
@@ -5,7 +5,7 @@ import asyncio, aiofiles, aiohttp, aiosqlite, lxml
from bs4 import BeautifulSoup
MAX_RETRIES= 3
SLEEP = 2
SLEEP = 3
DEBUG = False
VERBOSE = False
SILENT = False
@@ -59,11 +59,13 @@ def verbose_std(msg = ""):
def printWaiter(force = False):
if not DEBUG and (not SILENT or force):
print('.', end='', flush=True)
def printNewline(force = False):
if not DEBUG and (not SILENT or force):
print('', flush=True)
def debug(msg = ""):
"""print a conditional debug message"""
if DEBUG:
@@ -73,6 +75,7 @@ def debug(msg = ""):
print('DEBUG: ' + caller + '(): ' + msg)
return None
def error(msg = ""):
"""Print an error message"""
curframe = inspect.currentframe()
@@ -80,9 +83,27 @@ def error(msg = ""):
caller = calframe[1][3]
print('ERROR: ' + caller + '(): ' + msg)
return None
def NOW() -> int:
return int(time.time())
async def readPlainList(filename: str) -> list():
"""Read file to a list and return list of integers in the input file"""
input_list = []
try:
async with aiofiles.open(filename) as fp:
async for line in fp:
try:
input_list.append(int(line))
except (ValueError, TypeError) as err:
pass
except Exception as err:
error('Unexpected error when reading file: ' + filename + ' : ' + str(type(err)) + ' : '+ str(err))
return input_list
async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool:
"""Save JSON data into file"""
try:
@@ -96,6 +117,7 @@ async def saveJSON(filename: str, json_data: dict, sort_keys = False) -> bool:
error(str(err))
return False
async def readJSON(filename: str, chkJSONfunc = None):
try:
async with aiofiles.open(filename) as fp:
@@ -231,6 +253,8 @@ class WG:
"tutorial": "Proving Grounds"
}
tanks = None
nations = [ 'ussr', 'germany', 'usa', 'china', 'france', 'uk', 'japan', 'other']
nation_id = {
'ussr' : 0,
@@ -268,13 +292,13 @@ class WG:
def __init__(self, WG_appID = None, tankopedia_fn = None, maps_fn = None):
self.WG_appID = WG_appID
self.tanks = None
if (tankopedia_fn != None):
if os.path.exists(tankopedia_fn) and os.path.isfile(tankopedia_fn):
try:
with open(tankopedia_fn, 'rt', encoding='utf8') as f:
self.tanks = json.loads(f.read())
WG.tanks = json.loads(f.read())
debug(str(WG.tanks["meta"]["count"]) + " tanks")
except Exception as err:
error('Could not read tankopedia: ' + tankopedia_fn + '\n' + str(err))
else:
@@ -356,6 +380,10 @@ class WG:
def getMapUserStrs(cls) -> str:
return cls.maps.keys()
@classmethod
def getTankUserStrs(cls) -> str:
return cls.tanks["userStr"].keys()
@classmethod
def chkJSONcontent(cls, json_obj, check = None) -> bool:
try:
@@ -680,8 +708,7 @@ class WoTinspector:
'uploaded_by' : account_id,
'details' : 'full',
'key' : replay_id
}
}
url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
#debug('URL: ' + url)
@@ -701,19 +728,22 @@ class WoTinspector:
json_resp = await resp.json()
if json_resp.get('status', None) == None:
error(msg_str +' : ' + title + ' : Received invalid JSON')
elif (json_resp['status'] == 'ok'):
debug('Response data read')
verbose(msg_str + title + ' posted')
return json_resp
elif (json_resp['status'] == 'error'):
error(msg_str + json_resp['error']['message'] + ' : ' + title)
else:
debug('Response data read')
verbose(msg_str + title + ' posted')
return json_resp
error(msg_str + ' Unspecified error: ' + title)
else:
debug(msg_str + 'Got HTTP/' + str(resp.status) + ' Retrying... ' + str(retry))
await asyncio.sleep(SLEEP)
debug(msg_str + 'Got HTTP/' + str(resp.status))
except Exception as err:
error(str(err))
await asyncio.sleep(SLEEP)
return None
await asyncio.sleep(SLEEP)
error(msg_str + ' Could not post replay: ' + title)
return None
@classmethod
def getUrlReplayListing(cls, page : int):
maps.json
+1
View File
@@ -7,6 +7,7 @@
"desert_train": "Desert Sands",
"erlenberg": "Middleburg",
"faust": "Faust",
"forgecity": "New Bay",
"fort": "Fort Despair",
"grossberg": "Dynasty's Pearl",
"himmelsdorf": "Himmelsdorf",
wotbreplay_uploader.py
+16 -5
View File
@@ -16,8 +16,9 @@ DEBUG = False
VERBOSE = False
SLEEP = 2
MAX_RETRIES = 3
REPLAY_N = 0
SKIPPED_N = 0
REPLAY_N = 0
SKIPPED_N = 0
ERROR_N = 0
WIurl='https://wotinspector.com/api/replay/upload?'
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
wg = None
@@ -33,7 +34,7 @@ async def main(argv):
parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering')
parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com')
parser.add_argument('--tasks', dest='N_tasks', type=int, default=10, help='Number of worker threads')
parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads')
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
@@ -77,7 +78,7 @@ async def main(argv):
task.cancel()
bu.debug('Waiting for workers to cancel')
await asyncio.gather(*tasks, return_exceptions=True)
bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped')
bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
except KeyboardInterrupt:
print('Ctrl-c pressed ...')
@@ -136,6 +137,7 @@ async def mkQueueItem(filename : str, title : str) -> list:
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
"""Async Worker to process the replay queue"""
global SKIPPED_N
global ERROR_N
while True:
item = await queue.get()
@@ -158,6 +160,9 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
SKIPPED_N += 1
queue.task_done()
continue
else:
os.remove(replay_json_fn)
bu.debug("Replay JSON not valid: Deleting " + replay_json_fn)
except asyncio.CancelledError as err:
raise err
except Exception as err:
@@ -173,6 +178,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
bu.debug(msg_str + 'Replay saved OK: ' + filename )
else:
bu.error(msg_str + 'Error saving replay: ' + filename)
ERROR_N += 1
except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
bu.debug(msg_str + 'Marking task done')
@@ -188,7 +194,12 @@ def getTitle(replayfile: str, title: str, i : int) -> str:
filename = os.path.basename(replayfile)
bu.debug(filename)
map_usrStrs = wg.getMapUserStrs()
p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
tank_userStrs = wg.getTankUserStrs()
#p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
# update 6.2 changed the file name format
p = re.compile('\\d{8}_\\d{4}_.+?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
m = p.match(filename)
if (m != None):
if wg.tanks != None: