rename legacy blitzutils

This commit is contained in:
jylpah committed 2023-09-10 19:58:39 +03:00
1 parent 82fa381720
commit c27d7ce4fc
3 files changed
+2598 -2277

No files matched your search

-2165
View File
File diff suppressed because it is too large. Load diff
Executable
+2384
View File
File diff suppressed because it is too large. Load diff
+214 -112
View File
@@ -3,13 +3,13 @@
import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
import logging, re, configparser, zipfile import logging, re, configparser, zipfile
import blitzutils as bu import blitzutils1 as bu
from blitzutils import WG from blitzutils1 import WG
from blitzutils import WoTinspector from blitzutils1 import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG) logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini' FILE_CONFIG = "blitzstats.ini"
# DB_C_REPLAYS = 'Replays' # DB_C_REPLAYS = 'Replays'
DEBUG = False DEBUG = False
@@ -19,10 +19,11 @@ MAX_RETRIES = 3
REPLAY_N = 0 REPLAY_N = 0
SKIPPED_N = 0 SKIPPED_N = 0
ERROR_N = 0 ERROR_N = 0
WIurl='https://wotinspector.com/api/replay/upload?' WIurl = "https://wotinspector.com/api/replay/upload?"
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' WG_appID = "81381d3f45fa4aa75b78a7198eb216ad"
wg : WG | None = None wg: WG | None = None
wi : WoTinspector | None = None wi: WoTinspector | None = None
async def main(argv) -> int: async def main(argv) -> int:
global wg, wi global wg, wi
@@ -40,36 +41,82 @@ async def main(argv) -> int:
try: try:
## Read config ## Read config
if os.path.isfile(FILE_CONFIG): if os.path.isfile(FILE_CONFIG):
bu.debug('Reading config file: ' + FILE_CONFIG) bu.debug("Reading config file: " + FILE_CONFIG)
config = configparser.ConfigParser() config = configparser.ConfigParser()
config.read(FILE_CONFIG) config.read(FILE_CONFIG)
try: try:
if 'WG' in config.sections(): if "WG" in config.sections():
configWG = config['WG'] configWG = config["WG"]
# WG account id of the uploader: # WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configWG.getint('wg_id', WG_ID) WG_ID = configWG.getint("wg_id", WG_ID)
WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT) WG_ACCOUNT = configWG.get("wg_account", WG_ACCOUNT)
## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay ## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT) WG_RATE_LIMIT = configWG.getint("wg_rate_limit", WG_RATE_LIMIT)
except (KeyError, configparser.NoSectionError) as err: except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err) bu.error(exception=err)
except Exception as err: except Exception as err:
bu.error(exception=err) bu.error(exception=err)
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON') parser = argparse.ArgumentParser(
parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id') description="Post replays(s) to WoTinspector.com and retrieve replay data as JSON"
parser.add_argument('-a', '--account', dest='account', type=str, default=WG_ACCOUNT, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') )
parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use "NN" for continous numbering. Default is automatic naming') parser.add_argument(
parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com') "-id", dest="accountID", type=int, default=WG_ID, help="WG account_id"
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') )
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') parser.add_argument(
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') "-a",
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') "--account",
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') dest="account",
parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"') type=str,
default=WG_ACCOUNT,
help="Uploader's WG account name. Format: ACCOUNT_NAME@SERVER",
)
parser.add_argument(
"-t",
"--title",
type=str,
default=None,
help='Title for replays. Use "NN" for continous numbering. Default is automatic naming',
)
parser.add_argument(
"-p",
"--private",
dest="private",
action="store_true",
default=False,
help="Set replays private on WoTinspector.com",
)
parser.add_argument(
"--tankopedia",
type=str,
default="tanks.json",
help='JSON file to read Tankopedia from. Default: "tanks.json"',
)
parser.add_argument(
"--mapfile",
type=str,
default="maps.json",
help='JSON file to read Blitz map names from. Default: "maps.json"',
)
parser.add_argument(
"-d", "--debug", action="store_true", default=False, help="Debug mode"
)
parser.add_argument(
"-v", "--verbose", action="store_true", default=False, help="Verbose mode"
)
parser.add_argument(
"-s", "--silent", action="store_true", default=False, help="Silent mode"
)
parser.add_argument(
"files",
metavar="FILE1 [FILE2 ...]",
type=str,
nargs="+",
help="Files to read. Use '-' for STDIN\"",
)
args = parser.parse_args() args = parser.parse_args()
bu.set_verbose(args.verbose) bu.set_verbose(args.verbose)
@@ -80,13 +127,13 @@ async def main(argv) -> int:
if args.account is not None: if args.account is not None:
args.accountID = await wg.get_account_id(args.account) args.accountID = await wg.get_account_id(args.account)
bu.debug('WG account_id: ' + str(args.accountID)) bu.debug("WG account_id: " + str(args.accountID))
if args.accountID is None: if args.accountID is None:
args.accountID = 0 args.accountID = 0
try: try:
queue : asyncio.Queue = asyncio.Queue() queue: asyncio.Queue = asyncio.Queue()
# rebase file arguments due to moving the working directory to the script location # rebase file arguments due to moving the working directory to the script location
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute") bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
args.files = bu.rebase_file_args(current_dir, args.files) args.files = bu.rebase_file_args(current_dir, args.files)
@@ -96,22 +143,35 @@ async def main(argv) -> int:
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
# Start tasks to process the Queue # Start tasks to process the Queue
for i in range(OPT_WORKERS_N): for i in range(OPT_WORKERS_N):
tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private))) tasks.append(
bu.debug('Task ' + str(i) + ' started') asyncio.create_task(
replayWorker(queue, i, args.accountID, args.private)
)
)
bu.debug("Task " + str(i) + " started")
bu.debug('Waiting for the replay scanner to finish') bu.debug("Waiting for the replay scanner to finish")
await asyncio.wait([tasks[0]]) await asyncio.wait([tasks[0]])
bu.debug('Scanner finished. Waiting for workers to finish queue') bu.debug("Scanner finished. Waiting for workers to finish queue")
await queue.join() await queue.join()
bu.debug('Cancelling workers') bu.debug("Cancelling workers")
for task in tasks: for task in tasks:
task.cancel() task.cancel()
bu.debug('Waiting for workers to cancel') bu.debug("Waiting for workers to cancel")
await asyncio.gather(*tasks, return_exceptions=True) await asyncio.gather(*tasks, return_exceptions=True)
bu.verbose_std(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors') bu.verbose_std(
str(REPLAY_N)
+ " replays: "
+ str(REPLAY_N - SKIPPED_N - ERROR_N)
+ " uploaded, "
+ str(SKIPPED_N)
+ " skipped, "
+ str(ERROR_N)
+ " errors"
)
except KeyboardInterrupt: except KeyboardInterrupt:
print('Ctrl-c pressed ...') print("Ctrl-c pressed ...")
sys.exit(1) sys.exit(1)
finally: finally:
## Need to close the aiohttp.session since Python destructors do not support async methods... ## Need to close the aiohttp.session since Python destructors do not support async methods...
@@ -121,18 +181,18 @@ async def main(argv) -> int:
return 0 return 0
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str): async def mkReplayQ(queue: asyncio.Queue, files: list, title: str):
"""Create queue of replays to post""" """Create queue of replays to post"""
p_replayfile = re.compile('.*\\.wotbreplay$') p_replayfile = re.compile(".*\\.wotbreplay$")
if files[0] == '-': if files[0] == "-":
bu.debug('reading replay file list from STDIN') bu.debug("reading replay file list from STDIN")
stdin, _ = await aioconsole.get_standard_streams() stdin, _ = await aioconsole.get_standard_streams()
while True: while True:
line = (await stdin.readline()).decode('utf-8').rstrip() line = (await stdin.readline()).decode("utf-8").rstrip()
if not line: if not line:
break break
else: else:
if (p_replayfile.match(line) is not None): if p_replayfile.match(line) is not None:
await queue.put(await mkQueueItem(line, title)) await queue.put(await mkQueueItem(line, title))
else: else:
for fn in files: for fn in files:
@@ -140,33 +200,37 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
fn = fn[:-1] fn = fn[:-1]
if os.path.isfile(fn) and (p_replayfile.match(fn) is not None): if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
await queue.put(await mkQueueItem(fn, title)) await queue.put(await mkQueueItem(fn, title))
bu.debug('File added to queue: ' + fn) bu.debug("File added to queue: " + fn)
elif os.path.isdir(fn): elif os.path.isdir(fn):
with os.scandir(fn) as dirEntry: with os.scandir(fn) as dirEntry:
for entry in dirEntry: for entry in dirEntry:
try: try:
bu.debug('Found: ' + entry.name) bu.debug("Found: " + entry.name)
if entry.is_file() and (p_replayfile.match(entry.name) is not None): if entry.is_file() and (
p_replayfile.match(entry.name) is not None
):
bu.debug(entry.name) bu.debug(entry.name)
await queue.put(await mkQueueItem(entry.path, title)) await queue.put(await mkQueueItem(entry.path, title))
bu.debug('File added to queue: ' + entry.path) bu.debug("File added to queue: " + entry.path)
except Exception as err: except Exception as err:
bu.error(str(err)) bu.error(str(err))
else: else:
bu.error('File not found: ' + fn) bu.error("File not found: " + fn)
bu.debug('Finished') bu.debug("Finished")
return None return None
async def mkQueueItem(filename : str, title : str) -> list: async def mkQueueItem(filename: str, title: str) -> list:
"""Make an item to replay queue""" """Make an item to replay queue"""
global REPLAY_N global REPLAY_N
REPLAY_N +=1 REPLAY_N += 1
return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)] return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)]
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False): async def replayWorker(
queue: asyncio.Queue, workerID: int, account_id: int, priv=False
):
"""Async Worker to process the replay queue""" """Async Worker to process the replay queue"""
global SKIPPED_N global SKIPPED_N
global ERROR_N global ERROR_N
@@ -180,20 +244,27 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
N = item[1] N = item[1]
title = item[2] title = item[2]
replay_json_fn = filename + '.json' replay_json_fn = filename + ".json"
msg_str = 'Replay[' + str(N) + ']: ' msg_str = "Replay[" + str(N) + "]: "
#bu.debug(msg_str + replay_json_fn) # bu.debug(msg_str + replay_json_fn)
try: try:
if os.path.isfile(replay_json_fn): if os.path.isfile(replay_json_fn):
async with aiofiles.open(replay_json_fn) as fp: async with aiofiles.open(replay_json_fn) as fp:
replay_json = json.loads(await fp.read()) replay_json = json.loads(await fp.read())
#if replay_json['status'] == 'ok': # if replay_json['status'] == 'ok':
if wi.chk_JSON_replay(replay_json): if wi.chk_JSON_replay(replay_json):
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' ) bu.verbose_std(
msg_str + title + " has already been posted. Skipping."
)
else: else:
os.remove(replay_json_fn) os.remove(replay_json_fn)
bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID) bu.debug(
msg_str
+ "Replay JSON not valid/complete: Deleting "
+ replay_json_fn,
id=workerID,
)
SKIPPED_N += 1 SKIPPED_N += 1
queue.task_done() queue.task_done()
continue continue
@@ -201,36 +272,44 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
except asyncio.CancelledError as err: except asyncio.CancelledError as err:
raise err raise err
except Exception as err: except Exception as err:
bu.error(msg_str + 'Unexpected error: ' + str(type(err)) + ' : '+ str(err)) bu.error(msg_str + "Unexpected error: " + str(type(err)) + " : " + str(err))
try: try:
#bu.debug('Opening file [' + str(N) +']: ' + filename) # bu.debug('Opening file [' + str(N) +']: ' + filename)
async with aiofiles.open(filename,'rb') as fp: async with aiofiles.open(filename, "rb") as fp:
filename = os.path.basename(filename) filename = os.path.basename(filename)
bu.debug(msg_str + 'File: ' + filename) bu.debug(msg_str + "File: " + filename)
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N) json_resp = await wi.post_replay(
await fp.read(), filename, account_id, title, priv, N
)
if json_resp is not None: if json_resp is not None:
if (await bu.save_JSON(replay_json_fn,json_resp)): if await bu.save_JSON(replay_json_fn, json_resp):
if wi.chk_JSON_replay(json_resp): if wi.chk_JSON_replay(json_resp):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename): if not bu.debug(msg_str + "Replay saved OK: " + filename):
bu.verbose_std(msg_str + title + ' posted') bu.verbose_std(msg_str + title + " posted")
else: else:
bu.warning(msg_str +'Replay file is not valid/complete: ' + filename) bu.warning(
msg_str
+ "Replay file is not valid/complete: "
+ filename
)
ERROR_N += 1 ERROR_N += 1
else: else:
bu.error(msg_str + 'Error saving replay: ' + filename) bu.error(msg_str + "Error saving replay: " + filename)
ERROR_N += 1 ERROR_N += 1
else: else:
bu.error(msg_str + 'Replay file is not valid/complete: ' + filename) bu.error(msg_str + "Replay file is not valid/complete: " + filename)
ERROR_N += 1 ERROR_N += 1
except Exception as err: except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.error(
bu.debug(msg_str + 'Marking task done') msg_str + "Unexpected Exception: " + str(type(err)) + " : " + str(err)
)
bu.debug(msg_str + "Marking task done")
queue.task_done() queue.task_done()
return None return None
def getTitle_old(replayfile: str, title: str, i : int) -> str: def getTitle_old(replayfile: str, title: str, i: int) -> str:
global wg global wg
if title is None: if title is None:
@@ -240,36 +319,50 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
map_usrStrs = wg.get_map_user_strs() map_usrStrs = wg.get_map_user_strs()
tank_userStrs = wg.get_tank_user_strs() tank_userStrs = wg.get_tank_user_strs()
#p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') # p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
# update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah # update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') p = re.compile(
"\\d{8}_\\d{4}_.*?("
+ "|".join(tank_userStrs)
+ ")_("
+ "|".join(map_usrStrs)
+ ")(?:-\\d)?\\.wotbreplay$"
)
m = p.match(filename) m = p.match(filename)
if (m is not None): if m is not None:
if wg.tanks is not None: if wg.tanks is not None:
tank = m.group(1) tank = m.group(1)
if tank in wg.tanks['userStr']: if tank in wg.tanks["userStr"]:
tank = wg.tanks['userStr'][tank] tank = wg.tanks["userStr"][tank]
else: else:
bu.error('Tank code: "' + tank + '" not found from Tankopedia (tanks.json)') bu.error(
'Tank code: "'
+ tank
+ '" not found from Tankopedia (tanks.json)'
)
else: else:
tank = m.group(1) tank = m.group(1)
blitz_map = m.group(2) blitz_map = m.group(2)
if blitz_map in wg.maps: if blitz_map in wg.maps:
blitz_map = wg.maps[blitz_map] blitz_map = wg.maps[blitz_map]
else: else:
bu.error('Mapcode: "' + blitz_map + '" not found from map database (maps.json)') bu.error(
title = tank + ' @ ' + blitz_map 'Mapcode: "'
+ blitz_map
+ '" not found from map database (maps.json)'
)
title = tank + " @ " + blitz_map
else: else:
title = re.sub('\\.wotbreplay$', '', filename) title = re.sub("\\.wotbreplay$", "", filename)
except Exception as err: except Exception as err:
bu.error(err) bu.error(err)
else: else:
title.replace('NN', str(i)) title.replace("NN", str(i))
return title return title
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str: def getTitle(replayfile: str, title: str = None, i: int = 0) -> str:
global wg global wg
assert wg is not None, "wg must not be None" assert wg is not None, "wg must not be None"
@@ -280,54 +373,61 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
tank = None tank = None
map_name = None map_name = None
with zipfile.ZipFile(replayfile, 'r') as archive: with zipfile.ZipFile(replayfile, "r") as archive:
# bu.debug('Replay file: ' + replayfile + ' opened') # bu.debug('Replay file: ' + replayfile + ' opened')
with io.TextIOWrapper(archive.open('meta.json')) as meta: with io.TextIOWrapper(archive.open("meta.json")) as meta:
# bu.debug('Replay file\'s metadata: opened') # bu.debug('Replay file\'s metadata: opened')
try: try:
metadata_json = json.load(meta) metadata_json = json.load(meta)
#player = metadata_json['playerName'] # player = metadata_json['playerName']
tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor']) tank_name = wg.get_tank_name(
map_name = wg.get_map(metadata_json['mapName']) metadata_json["vehicleCompDescriptor"]
bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name) )
map_name = wg.get_map(metadata_json["mapName"])
bu.debug(
"Tank: " + tank_name
if tank_name is not None
else "NOT FOUND IN tanks.json" + " Map: " + map_name
)
except Exception as err: except Exception as err:
bu.error(exception = err) bu.error(exception=err)
if (tank_name is not None) and (map_name is not None): if (tank_name is not None) and (map_name is not None):
title = tank_name + ' @ ' + map_name title = tank_name + " @ " + map_name
else: else:
title = re.sub('\\.wotbreplay$', '', filename) title = re.sub("\\.wotbreplay$", "", filename)
except Exception as err: except Exception as err:
bu.error(err) bu.error(err)
else: else:
title.replace('NN', str(i)) title.replace("NN", str(i))
bu.debug('Returning: ' + title) bu.debug("Returning: " + title)
return title return title
### UNIT TESTS - pytest ### UNIT TESTS - pytest
# class TestClass: # class TestClass:
# def test_getTitle(self): # def test_getTitle(self):
# #player = "jylpah" # #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' # WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json') # wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D" # tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle" # map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay' # replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name # res = tank_name + ' @ ' + map_name
# assert getTitle(replay) == res # assert getTitle(replay) == res
# @pytest.mark.asyncio # @pytest.mark.asyncio
# async def test_getTitle(self): # async def test_getTitle(self):
# #player = "jylpah" # #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' # WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json') # wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D" # tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle" # map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay' # replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name # res = tank_name + ' @ ' + map_name
# assert getTitle(replay) == res # assert getTitle(replay) == res
### main() ------------------------------------------- ### main() -------------------------------------------
if __name__ == "__main__": if __name__ == "__main__":
@@ -338,6 +438,8 @@ if __name__ == "__main__":
except ImportError: except ImportError:
pass pass
else: else:
if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): if not isinstance(
asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy
):
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
asyncio.run(main(sys.argv[1:]), debug=False) asyncio.run(main(sys.argv[1:]), debug=False)