GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-21 02:07:06 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #37 from Jylpah/dev-fix-tankopedia

Browse Source 
Use new tankopedia format
This commit is contained in:
Jylpah authored and GitHub committed 2023-09-10 20:08:29 +03:00
2 parents b5efc4bc9d b001bb44be
commit 76bda7e4e7
8 files changed
+4068 -10875
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

analyze_wotb_replays.py
+1468 -880
View File
File diff suppressed because it is too large. Load diff
blitzutils.py
-2165
View File
File diff suppressed because it is too large. Load diff
blitzutils1.py
Executable
+2384
View File
File diff suppressed because it is too large. Load diff
extract_tankopedia.py
-248
View File
@@ -1,248 +0,0 @@
#!/usr/bin/env python3
## PYTHON VERSION MUST BE 3.7 OR HIGHER
# Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
from os import path
import aiofiles # type: ignore
import xmltodict # type: ignore
from asyncio import set_event_loop_policy, run, create_task, get_event_loop_policy
import sys, argparse, json, os, inspect, asyncio, re, logging, time, configparser
from collections import OrderedDict
import blitzutils as bu
from blitzutils import WG
logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini'
# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
BLITZAPP_STRINGS='Data/Strings/en.yaml'
BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
BLITZAPP_VEHICLE_FILE='list.xml'
wg : WG | None = None
## main() -------------------------------------------------------------
async def main() -> None:
global wg
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
## Read config
BLITZAPP_FOLDER = '.'
try:
if os.path.isfile(FILE_CONFIG):
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
configOptions = config['EXTRACT_TANKOPEDIA']
BLITZAPP_FOLDER = configOptions.get('blitz_app_dir', BLITZAPP_FOLDER)
except:
pass
parser = argparse.ArgumentParser(description='Extract Tankopedia data from Blitz game files')
parser.add_argument('blitz_app_base', type=str, nargs='?', metavar="BLITZAPP_FOLDER", default=BLITZAPP_FOLDER, help='Base dir of the Blitz App files')
parser.add_argument('tanks', type=str, default='tanks.json', nargs='?', metavar="TANKS_FILE", help='File to write Tankopedia')
parser.add_argument('maps', type=str, default='maps.json', nargs='?', metavar='MAPS_FILE', help='File to write map names')
arggroup = parser.add_mutually_exclusive_group()
arggroup.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
arggroup.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
arggroup.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
args = parser.parse_args()
bu.set_log_level(args.silent, args.verbose, args.debug)
wg = WG()
tasks = []
for nation in wg.NATION:
tasks.append(asyncio.create_task(extract_tanks(args.blitz_app_base, nation)))
tanklist = []
for tanklist_tmp in await asyncio.gather(*tasks):
tanklist.extend(tanklist_tmp)
tank_strs, map_strs = await read_user_strs(args.blitz_app_base)
json_data = None
userStrs = {}
tanks = {}
if os.path.exists(args.tanks):
try:
async with aiofiles.open(args.tanks) as infile:
json_data = json.loads(await infile.read())
userStrs = json_data['userStr']
tanks = json_data['data']
except Exception as err:
bu.error('Unexpected error when reading file: ' + args.tanks + ' : ' + str(err))
async with aiofiles.open(args.tanks, 'w', encoding="utf8") as outfile:
new_tanks, new_userStrs = await convert_tank_names(tanklist, tank_strs)
# merge old and new tankopedia
tanks.update(new_tanks)
userStrs.update(new_userStrs)
tankopedia : OrderedDict[str, str| int | dict ]= OrderedDict()
tankopedia['status'] = 'ok'
tankopedia['meta'] = { "count": len(tanks) }
tankopedia['data'] = bu.sort_dict(tanks, number=True)
tankopedia['userStr'] = bu.sort_dict(userStrs)
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')
await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
if args.maps is not None:
maps = {}
if os.path.exists(args.maps):
try:
async with aiofiles.open(args.maps) as infile:
maps = json.loads(await infile.read())
except Exception as err:
bu.error('Unexpected error when reading file: ' + args.maps + ' : ' + str(err))
# merge old and new map data
maps.update(map_strs)
async with aiofiles.open(args.maps, 'w', encoding="utf8") as outfile:
bu.verbose_std('New maps file \'' + args.maps + '\' contains ' + str(len(maps)) + ' maps')
await outfile.write(json.dumps(maps, ensure_ascii=False, indent=4, sort_keys=True))
return None
async def extract_tanks(blitz_app_base : str, nation: str):
tanks = []
# WG has changed the location of Data directory - at least in steam client
if path.isdir(path.join(blitz_app_base, 'assets')):
blitz_app_base = path.join(blitz_app_base, 'assets')
list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
if not path.isfile(list_xml_file):
print('ERROR: cannot open ' + list_xml_file)
return None
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
async with aiofiles.open(list_xml_file, 'r', encoding="utf8") as f:
try:
tankList = xmltodict.parse(await f.read())
for data in tankList['root'].keys():
tank_xml = tankList['root'][data]
tank : dict[str, bool | int | str]= dict()
tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
tank['nation'] = nation
tank['tank_id'] = get_tank_id(nation, int(tank_xml['id']))
tank['tier'] = int(tank_xml['level'])
tank['type'] = get_tank_type(tank_xml['tags'])
tank['userStr'] = tank_xml['userString']
#bu.debug('Reading tank string: ' + tank['userStr'], force=True)
tanks.append(tank)
except Exception as err:
bu.error(err)
sys.exit(2)
return tanks
async def read_user_strs(blitz_app_base : str) -> tuple[dict, dict]:
"""Read user strings to convert map and tank names"""
tank_strs = {}
map_strs = {}
filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
try:
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
p_tank = re.compile('^"(#\\w+?_vehicles:.+?)": "(.+)"$')
p_map = re.compile('^"#maps:(.+?):.+?: "(.+?)"$')
async for l in f:
m = p_tank.match(l)
if m is not None:
tank_strs[m.group(1)] = m.group(2)
m = p_map.match(l)
if m is not None and m.group(2) != 'Macragge':
map_strs[m.group(1)] = m.group(2)
except Exception as err:
bu.error(err)
sys.exit(1)
return tank_strs, map_strs
async def convert_tank_names(tanklist : list, tank_strs: dict) -> tuple[dict, dict]:
"""Convert tank names for Tankopedia"""
tankopedia = {}
userStrs = {}
bu.debug(f'tank_strs:')
for key, value in tank_strs.items():
bu.debug(f'{key}: {value}')
bu.debug('---------')
try:
for tank in tanklist:
try:
bu.debug(f'tank: {tank}')
if tank['userStr'] in tank_strs:
tank['name'] = tank_strs[tank['userStr']]
else:
tank['name'] = tank['userStr'].split(':')[1]
tank.pop('userStr', None)
tank_tmp = OrderedDict()
for key in sorted(tank.keys()):
tank_tmp[key] = tank[key]
tankopedia[str(tank['tank_id'])] = tank_tmp
except:
bu.error(f'Could not process tank: {tank}')
for tank_str in tank_strs:
skip = False
key = tank_str.split(':')[1]
bu.debug('Tank string: ' + key + ' = ' + tank_strs[tank_str])
re_strs = [r'^Chassis_', r'^Turret_', r'^_', r'_short$' ]
for re_str in re_strs:
p = re.compile(re_str)
if p.match(key):
skip = True
break
if skip:
continue
userStrs[key] = tank_strs[tank_str]
# sorting
tankopedia_sorted = OrderedDict()
for tank_id in sorted(tankopedia.keys(), key=int):
tankopedia_sorted[str(tank_id)] = tankopedia[str(tank_id)]
userStrs_sorted = OrderedDict()
for userStr in sorted(userStrs.keys()):
userStrs_sorted[userStr] = userStrs[userStr]
# bu.debug('Tank strings: ' + str(len(userStrs_sorted)))
except Exception as err:
bu.error(err)
sys.exit(1)
return tankopedia_sorted, userStrs_sorted
def get_tank_id(nation: str, tank_id : int) -> int:
assert wg is not None, "wg has to be set"
return (tank_id << 8) + (wg.NATION_ID[nation] << 4) + 1
def get_tank_type(tagstr : str):
assert wg is not None, "wg has to be set"
tags = tagstr.split(' ')
for t_type in wg.TANK_TYPE:
if tags[0] == t_type:
return t_type
return None
### main()
if __name__ == "__main__":
# To avoid 'Event loop is closed' RuntimeError due to compatibility issue with aiohttp
if sys.platform.startswith("win") and sys.version_info >= (3, 8):
try:
from asyncio import WindowsSelectorEventLoopPolicy
except ImportError:
pass
else:
if not isinstance(get_event_loop_policy(), WindowsSelectorEventLoopPolicy):
set_event_loop_policy(WindowsSelectorEventLoopPolicy())
run(main())
get_tankopedia.py
-91
View File
@@ -1,91 +0,0 @@
#!/usr/bin/env python3
# Get Tankpedia from WoTinspector.com
import sys
import argparse
import json
import os
import inspect
import aiohttp
import asyncio
import re
import logging
from pathlib import Path
import blitzutils as bu
from blitzutils import WG
from blitzutils import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG)
VERBOSE = False
DEBUG = False
tank_type = ['lightTank', 'mediumTank', 'heavyTank', 'AT-SPG']
# @asyncio.coroutine
async def get_tankopedia(filename: str):
url = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
async with aiohttp.ClientSession() as client:
async with client.get(url) as r:
if r.status == 200:
WI_tank_db = await r.text()
WI_tank_db = WI_tank_db.split("\n")
else:
print('Error: Could not get valid HTTPS response. HTTP: ' + str(r.status) )
sys.exit(1)
tanks = {}
n = 0
p = re.compile('\\s*(\\d+):\\s{"en":"([^"]+)",.*?"tier":(\\d+), "type":(\\d), "premium":(\\d).*')
for line in WI_tank_db[1:-1]:
m = p.match(line)
tank = {}
tank['tank_id'] = int(m.group(1))
tank['name'] = m.group(2)
tank['tier'] = int(m.group(3))
tank['type'] = tank_type[int(m.group(4))]
tank['is_premium'] = (int(m.group(5)) == 1)
tanks[str(m.group(1))] = tank
n += 1
tankopedia = {}
tankopedia['status'] = "ok"
tankopedia['meta'] = {"count" : n}
tankopedia['data'] = tanks
print("Tankopedia has " + str(n) + " tanks in: " + filename)
with open(filename,'w') as outfile:
outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=True))
return None
async def main(argv):
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
parser = argparse.ArgumentParser(description='Retrieve Tankopedia from WoTinspector.com')
parser.add_argument('--file', dest="outfile", help='Write Tankopedia to file')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
args = parser.parse_args()
bu.set_log_level(args.silent, args.verbose, args.debug)
wi = WoTinspector()
await wi.get_tankopedia(args.outfile)
await wi.close()
# main()
if __name__ == "__main__":
# To avoid 'Event loop is closed' RuntimeError due to compatibility issue with aiohttp
if sys.platform.startswith("win") and sys.version_info >= (3, 8):
try:
from asyncio import WindowsSelectorEventLoopPolicy
except ImportError:
pass
else:
if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy):
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
asyncio.run(main(sys.argv[1:]), debug=False)
requirements.txt
+1
View File
@@ -8,3 +8,4 @@ lxml
xmltodict
progress
pytest-asyncio
blitz-utils @ git+https://github.com/Jylpah/blitz-utils
tanks.json
+1 -7379
View File
File diff suppressed because it is too large. Load diff
upload_wotb_replays.py
+214 -112
View File
@@ -3,13 +3,13 @@
import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
import logging, re, configparser, zipfile
import blitzutils as bu
from blitzutils import WG
from blitzutils import WoTinspector
import blitzutils1 as bu
from blitzutils1 import WG
from blitzutils1 import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini'
FILE_CONFIG = "blitzstats.ini"
# DB_C_REPLAYS = 'Replays'
DEBUG = False
@@ -19,10 +19,11 @@ MAX_RETRIES = 3
REPLAY_N = 0
SKIPPED_N = 0
ERROR_N = 0
WIurl='https://wotinspector.com/api/replay/upload?'
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
wg : WG | None = None
wi : WoTinspector | None = None
WIurl = "https://wotinspector.com/api/replay/upload?"
WG_appID = "81381d3f45fa4aa75b78a7198eb216ad"
wg: WG | None = None
wi: WoTinspector | None = None
async def main(argv) -> int:
global wg, wi
@@ -40,36 +41,82 @@ async def main(argv) -> int:
try:
## Read config
if os.path.isfile(FILE_CONFIG):
bu.debug('Reading config file: ' + FILE_CONFIG)
bu.debug("Reading config file: " + FILE_CONFIG)
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
try:
if 'WG' in config.sections():
configWG = config['WG']
if "WG" in config.sections():
configWG = config["WG"]
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configWG.getint('wg_id', WG_ID)
WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT)
WG_ID = configWG.getint("wg_id", WG_ID)
WG_ACCOUNT = configWG.get("wg_account", WG_ACCOUNT)
## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
WG_RATE_LIMIT = configWG.getint("wg_rate_limit", WG_RATE_LIMIT)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
except Exception as err:
bu.error(exception=err)
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON')
parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id')
parser.add_argument('-a', '--account', dest='account', type=str, default=WG_ACCOUNT, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use "NN" for continous numbering. Default is automatic naming')
parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com')
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"')
parser = argparse.ArgumentParser(
description="Post replays(s) to WoTinspector.com and retrieve replay data as JSON"
)
parser.add_argument(
"-id", dest="accountID", type=int, default=WG_ID, help="WG account_id"
)
parser.add_argument(
"-a",
"--account",
dest="account",
type=str,
default=WG_ACCOUNT,
help="Uploader's WG account name. Format: ACCOUNT_NAME@SERVER",
)
parser.add_argument(
"-t",
"--title",
type=str,
default=None,
help='Title for replays. Use "NN" for continous numbering. Default is automatic naming',
)
parser.add_argument(
"-p",
"--private",
dest="private",
action="store_true",
default=False,
help="Set replays private on WoTinspector.com",
)
parser.add_argument(
"--tankopedia",
type=str,
default="tanks.json",
help='JSON file to read Tankopedia from. Default: "tanks.json"',
)
parser.add_argument(
"--mapfile",
type=str,
default="maps.json",
help='JSON file to read Blitz map names from. Default: "maps.json"',
)
parser.add_argument(
"-d", "--debug", action="store_true", default=False, help="Debug mode"
)
parser.add_argument(
"-v", "--verbose", action="store_true", default=False, help="Verbose mode"
)
parser.add_argument(
"-s", "--silent", action="store_true", default=False, help="Silent mode"
)
parser.add_argument(
"files",
metavar="FILE1 [FILE2 ...]",
type=str,
nargs="+",
help="Files to read. Use '-' for STDIN\"",
)
args = parser.parse_args()
bu.set_verbose(args.verbose)
@@ -80,13 +127,13 @@ async def main(argv) -> int:
if args.account is not None:
args.accountID = await wg.get_account_id(args.account)
bu.debug('WG account_id: ' + str(args.accountID))
bu.debug("WG account_id: " + str(args.accountID))
if args.accountID is None:
args.accountID = 0
try:
queue : asyncio.Queue = asyncio.Queue()
queue: asyncio.Queue = asyncio.Queue()
# rebase file arguments due to moving the working directory to the script location
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
args.files = bu.rebase_file_args(current_dir, args.files)
@@ -96,22 +143,35 @@ async def main(argv) -> int:
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
# Start tasks to process the Queue
for i in range(OPT_WORKERS_N):
tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
bu.debug('Task ' + str(i) + ' started')
tasks.append(
asyncio.create_task(
replayWorker(queue, i, args.accountID, args.private)
)
)
bu.debug("Task " + str(i) + " started")
bu.debug('Waiting for the replay scanner to finish')
bu.debug("Waiting for the replay scanner to finish")
await asyncio.wait([tasks[0]])
bu.debug('Scanner finished. Waiting for workers to finish queue')
bu.debug("Scanner finished. Waiting for workers to finish queue")
await queue.join()
bu.debug('Cancelling workers')
bu.debug("Cancelling workers")
for task in tasks:
task.cancel()
bu.debug('Waiting for workers to cancel')
bu.debug("Waiting for workers to cancel")
await asyncio.gather(*tasks, return_exceptions=True)
bu.verbose_std(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
bu.verbose_std(
str(REPLAY_N)
+ " replays: "
+ str(REPLAY_N - SKIPPED_N - ERROR_N)
+ " uploaded, "
+ str(SKIPPED_N)
+ " skipped, "
+ str(ERROR_N)
+ " errors"
)
except KeyboardInterrupt:
print('Ctrl-c pressed ...')
print("Ctrl-c pressed ...")
sys.exit(1)
finally:
## Need to close the aiohttp.session since Python destructors do not support async methods...
@@ -121,18 +181,18 @@ async def main(argv) -> int:
return 0
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
async def mkReplayQ(queue: asyncio.Queue, files: list, title: str):
"""Create queue of replays to post"""
p_replayfile = re.compile('.*\\.wotbreplay$')
if files[0] == '-':
bu.debug('reading replay file list from STDIN')
p_replayfile = re.compile(".*\\.wotbreplay$")
if files[0] == "-":
bu.debug("reading replay file list from STDIN")
stdin, _ = await aioconsole.get_standard_streams()
while True:
line = (await stdin.readline()).decode('utf-8').rstrip()
line = (await stdin.readline()).decode("utf-8").rstrip()
if not line:
break
else:
if (p_replayfile.match(line) is not None):
if p_replayfile.match(line) is not None:
await queue.put(await mkQueueItem(line, title))
else:
for fn in files:
@@ -140,33 +200,37 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
fn = fn[:-1]
if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
await queue.put(await mkQueueItem(fn, title))
bu.debug('File added to queue: ' + fn)
bu.debug("File added to queue: " + fn)
elif os.path.isdir(fn):
with os.scandir(fn) as dirEntry:
for entry in dirEntry:
try:
bu.debug('Found: ' + entry.name)
if entry.is_file() and (p_replayfile.match(entry.name) is not None):
bu.debug("Found: " + entry.name)
if entry.is_file() and (
p_replayfile.match(entry.name) is not None
):
bu.debug(entry.name)
await queue.put(await mkQueueItem(entry.path, title))
bu.debug('File added to queue: ' + entry.path)
bu.debug("File added to queue: " + entry.path)
except Exception as err:
bu.error(str(err))
else:
bu.error('File not found: ' + fn)
bu.error("File not found: " + fn)
bu.debug('Finished')
bu.debug("Finished")
return None
async def mkQueueItem(filename : str, title : str) -> list:
async def mkQueueItem(filename: str, title: str) -> list:
"""Make an item to replay queue"""
global REPLAY_N
REPLAY_N +=1
REPLAY_N += 1
return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)]
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
async def replayWorker(
queue: asyncio.Queue, workerID: int, account_id: int, priv=False
):
"""Async Worker to process the replay queue"""
global SKIPPED_N
global ERROR_N
@@ -180,20 +244,27 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
N = item[1]
title = item[2]
replay_json_fn = filename + '.json'
msg_str = 'Replay[' + str(N) + ']: '
replay_json_fn = filename + ".json"
msg_str = "Replay[" + str(N) + "]: "
#bu.debug(msg_str + replay_json_fn)
# bu.debug(msg_str + replay_json_fn)
try:
if os.path.isfile(replay_json_fn):
async with aiofiles.open(replay_json_fn) as fp:
replay_json = json.loads(await fp.read())
#if replay_json['status'] == 'ok':
# if replay_json['status'] == 'ok':
if wi.chk_JSON_replay(replay_json):
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
bu.verbose_std(
msg_str + title + " has already been posted. Skipping."
)
else:
os.remove(replay_json_fn)
bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID)
bu.debug(
msg_str
+ "Replay JSON not valid/complete: Deleting "
+ replay_json_fn,
id=workerID,
)
SKIPPED_N += 1
queue.task_done()
continue
@@ -201,36 +272,44 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
except asyncio.CancelledError as err:
raise err
except Exception as err:
bu.error(msg_str + 'Unexpected error: ' + str(type(err)) + ' : '+ str(err))
bu.error(msg_str + "Unexpected error: " + str(type(err)) + " : " + str(err))
try:
#bu.debug('Opening file [' + str(N) +']: ' + filename)
async with aiofiles.open(filename,'rb') as fp:
# bu.debug('Opening file [' + str(N) +']: ' + filename)
async with aiofiles.open(filename, "rb") as fp:
filename = os.path.basename(filename)
bu.debug(msg_str + 'File: ' + filename)
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
bu.debug(msg_str + "File: " + filename)
json_resp = await wi.post_replay(
await fp.read(), filename, account_id, title, priv, N
)
if json_resp is not None:
if (await bu.save_JSON(replay_json_fn,json_resp)):
if await bu.save_JSON(replay_json_fn, json_resp):
if wi.chk_JSON_replay(json_resp):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
bu.verbose_std(msg_str + title + ' posted')
if not bu.debug(msg_str + "Replay saved OK: " + filename):
bu.verbose_std(msg_str + title + " posted")
else:
bu.warning(msg_str +'Replay file is not valid/complete: ' + filename)
bu.warning(
msg_str
+ "Replay file is not valid/complete: "
+ filename
)
ERROR_N += 1
else:
bu.error(msg_str + 'Error saving replay: ' + filename)
bu.error(msg_str + "Error saving replay: " + filename)
ERROR_N += 1
else:
bu.error(msg_str + 'Replay file is not valid/complete: ' + filename)
bu.error(msg_str + "Replay file is not valid/complete: " + filename)
ERROR_N += 1
except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
bu.debug(msg_str + 'Marking task done')
bu.error(
msg_str + "Unexpected Exception: " + str(type(err)) + " : " + str(err)
)
bu.debug(msg_str + "Marking task done")
queue.task_done()
return None
def getTitle_old(replayfile: str, title: str, i : int) -> str:
def getTitle_old(replayfile: str, title: str, i: int) -> str:
global wg
if title is None:
@@ -240,36 +319,50 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
map_usrStrs = wg.get_map_user_strs()
tank_userStrs = wg.get_tank_user_strs()
#p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
# p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
# update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
p = re.compile(
"\\d{8}_\\d{4}_.*?("
+ "|".join(tank_userStrs)
+ ")_("
+ "|".join(map_usrStrs)
+ ")(?:-\\d)?\\.wotbreplay$"
)
m = p.match(filename)
if (m is not None):
if m is not None:
if wg.tanks is not None:
tank = m.group(1)
if tank in wg.tanks['userStr']:
tank = wg.tanks['userStr'][tank]
if tank in wg.tanks["userStr"]:
tank = wg.tanks["userStr"][tank]
else:
bu.error('Tank code: "' + tank + '" not found from Tankopedia (tanks.json)')
bu.error(
'Tank code: "'
+ tank
+ '" not found from Tankopedia (tanks.json)'
)
else:
tank = m.group(1)
blitz_map = m.group(2)
if blitz_map in wg.maps:
blitz_map = wg.maps[blitz_map]
else:
bu.error('Mapcode: "' + blitz_map + '" not found from map database (maps.json)')
title = tank + ' @ ' + blitz_map
bu.error(
'Mapcode: "'
+ blitz_map
+ '" not found from map database (maps.json)'
)
title = tank + " @ " + blitz_map
else:
title = re.sub('\\.wotbreplay$', '', filename)
title = re.sub("\\.wotbreplay$", "", filename)
except Exception as err:
bu.error(err)
else:
title.replace('NN', str(i))
title.replace("NN", str(i))
return title
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
def getTitle(replayfile: str, title: str = None, i: int = 0) -> str:
global wg
assert wg is not None, "wg must not be None"
@@ -280,54 +373,61 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
tank = None
map_name = None
with zipfile.ZipFile(replayfile, 'r') as archive:
with zipfile.ZipFile(replayfile, "r") as archive:
# bu.debug('Replay file: ' + replayfile + ' opened')
with io.TextIOWrapper(archive.open('meta.json')) as meta:
with io.TextIOWrapper(archive.open("meta.json")) as meta:
# bu.debug('Replay file\'s metadata: opened')
try:
metadata_json = json.load(meta)
#player = metadata_json['playerName']
tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
map_name = wg.get_map(metadata_json['mapName'])
bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name)
# player = metadata_json['playerName']
tank_name = wg.get_tank_name(
metadata_json["vehicleCompDescriptor"]
)
map_name = wg.get_map(metadata_json["mapName"])
bu.debug(
"Tank: " + tank_name
if tank_name is not None
else "NOT FOUND IN tanks.json" + " Map: " + map_name
)
except Exception as err:
bu.error(exception = err)
bu.error(exception=err)
if (tank_name is not None) and (map_name is not None):
title = tank_name + ' @ ' + map_name
title = tank_name + " @ " + map_name
else:
title = re.sub('\\.wotbreplay$', '', filename)
title = re.sub("\\.wotbreplay$", "", filename)
except Exception as err:
bu.error(err)
else:
title.replace('NN', str(i))
bu.debug('Returning: ' + title)
title.replace("NN", str(i))
bu.debug("Returning: " + title)
return title
### UNIT TESTS - pytest
# class TestClass:
# def test_getTitle(self):
# #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name
# def test_getTitle(self):
# #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name
# assert getTitle(replay) == res
# assert getTitle(replay) == res
# @pytest.mark.asyncio
# async def test_getTitle(self):
# #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name
# @pytest.mark.asyncio
# async def test_getTitle(self):
# #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name
# assert getTitle(replay) == res
# assert getTitle(replay) == res
### main() -------------------------------------------
if __name__ == "__main__":
@@ -338,6 +438,8 @@ if __name__ == "__main__":
except ImportError:
pass
else:
if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy):
if not isinstance(
asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy
):
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
asyncio.run(main(sys.argv[1:]), debug=False)