GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-23 19:19:53 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #37 from Jylpah/dev-fix-tankopedia

Browse Source 
Use new tankopedia format
This commit is contained in:
Jylpah authored and GitHub committed 2023-09-10 20:08:29 +03:00
2 parents b5efc4bc9d b001bb44be
commit 76bda7e4e7
8 files changed
+5878 -12685
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

analyze_wotb_replays.py
+3098 -2510
View File
File diff suppressed because it is too large. Load diff
blitzutils.py
-2165
View File
File diff suppressed because it is too large. Load diff
blitzutils1.py
Executable
+2384
View File
File diff suppressed because it is too large. Load diff
extract_tankopedia.py
-248
View File
@@ -1,248 +0,0 @@
#!/usr/bin/env python3
## PYTHON VERSION MUST BE 3.7 OR HIGHER
# Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
from os import path
import aiofiles # type: ignore
import xmltodict # type: ignore
from asyncio import set_event_loop_policy, run, create_task, get_event_loop_policy
import sys, argparse, json, os, inspect, asyncio, re, logging, time, configparser
from collections import OrderedDict
import blitzutils as bu
from blitzutils import WG
logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini'
# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
BLITZAPP_STRINGS='Data/Strings/en.yaml'
BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
BLITZAPP_VEHICLE_FILE='list.xml'
wg : WG | None = None
## main() -------------------------------------------------------------
async def main() -> None:
global wg
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
## Read config
BLITZAPP_FOLDER = '.'
try:
if os.path.isfile(FILE_CONFIG):
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
configOptions = config['EXTRACT_TANKOPEDIA']
BLITZAPP_FOLDER = configOptions.get('blitz_app_dir', BLITZAPP_FOLDER)
except:
pass
parser = argparse.ArgumentParser(description='Extract Tankopedia data from Blitz game files')
parser.add_argument('blitz_app_base', type=str, nargs='?', metavar="BLITZAPP_FOLDER", default=BLITZAPP_FOLDER, help='Base dir of the Blitz App files')
parser.add_argument('tanks', type=str, default='tanks.json', nargs='?', metavar="TANKS_FILE", help='File to write Tankopedia')
parser.add_argument('maps', type=str, default='maps.json', nargs='?', metavar='MAPS_FILE', help='File to write map names')
arggroup = parser.add_mutually_exclusive_group()
arggroup.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
arggroup.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
arggroup.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
args = parser.parse_args()
bu.set_log_level(args.silent, args.verbose, args.debug)
wg = WG()
tasks = []
for nation in wg.NATION:
tasks.append(asyncio.create_task(extract_tanks(args.blitz_app_base, nation)))
tanklist = []
for tanklist_tmp in await asyncio.gather(*tasks):
tanklist.extend(tanklist_tmp)
tank_strs, map_strs = await read_user_strs(args.blitz_app_base)
json_data = None
userStrs = {}
tanks = {}
if os.path.exists(args.tanks):
try:
async with aiofiles.open(args.tanks) as infile:
json_data = json.loads(await infile.read())
userStrs = json_data['userStr']
tanks = json_data['data']
except Exception as err:
bu.error('Unexpected error when reading file: ' + args.tanks + ' : ' + str(err))
async with aiofiles.open(args.tanks, 'w', encoding="utf8") as outfile:
new_tanks, new_userStrs = await convert_tank_names(tanklist, tank_strs)
# merge old and new tankopedia
tanks.update(new_tanks)
userStrs.update(new_userStrs)
tankopedia : OrderedDict[str, str| int | dict ]= OrderedDict()
tankopedia['status'] = 'ok'
tankopedia['meta'] = { "count": len(tanks) }
tankopedia['data'] = bu.sort_dict(tanks, number=True)
tankopedia['userStr'] = bu.sort_dict(userStrs)
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')
await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
if args.maps is not None:
maps = {}
if os.path.exists(args.maps):
try:
async with aiofiles.open(args.maps) as infile:
maps = json.loads(await infile.read())
except Exception as err:
bu.error('Unexpected error when reading file: ' + args.maps + ' : ' + str(err))
# merge old and new map data
maps.update(map_strs)
async with aiofiles.open(args.maps, 'w', encoding="utf8") as outfile:
bu.verbose_std('New maps file \'' + args.maps + '\' contains ' + str(len(maps)) + ' maps')
await outfile.write(json.dumps(maps, ensure_ascii=False, indent=4, sort_keys=True))
return None
async def extract_tanks(blitz_app_base : str, nation: str):
tanks = []
# WG has changed the location of Data directory - at least in steam client
if path.isdir(path.join(blitz_app_base, 'assets')):
blitz_app_base = path.join(blitz_app_base, 'assets')
list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
if not path.isfile(list_xml_file):
print('ERROR: cannot open ' + list_xml_file)
return None
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
async with aiofiles.open(list_xml_file, 'r', encoding="utf8") as f:
try:
tankList = xmltodict.parse(await f.read())
for data in tankList['root'].keys():
tank_xml = tankList['root'][data]
tank : dict[str, bool | int | str]= dict()
tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
tank['nation'] = nation
tank['tank_id'] = get_tank_id(nation, int(tank_xml['id']))
tank['tier'] = int(tank_xml['level'])
tank['type'] = get_tank_type(tank_xml['tags'])
tank['userStr'] = tank_xml['userString']
#bu.debug('Reading tank string: ' + tank['userStr'], force=True)
tanks.append(tank)
except Exception as err:
bu.error(err)
sys.exit(2)
return tanks
async def read_user_strs(blitz_app_base : str) -> tuple[dict, dict]:
"""Read user strings to convert map and tank names"""
tank_strs = {}
map_strs = {}
filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
try:
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
p_tank = re.compile('^"(#\\w+?_vehicles:.+?)": "(.+)"$')
p_map = re.compile('^"#maps:(.+?):.+?: "(.+?)"$')
async for l in f:
m = p_tank.match(l)
if m is not None:
tank_strs[m.group(1)] = m.group(2)
m = p_map.match(l)
if m is not None and m.group(2) != 'Macragge':
map_strs[m.group(1)] = m.group(2)
except Exception as err:
bu.error(err)
sys.exit(1)
return tank_strs, map_strs
async def convert_tank_names(tanklist : list, tank_strs: dict) -> tuple[dict, dict]:
"""Convert tank names for Tankopedia"""
tankopedia = {}
userStrs = {}
bu.debug(f'tank_strs:')
for key, value in tank_strs.items():
bu.debug(f'{key}: {value}')
bu.debug('---------')
try:
for tank in tanklist:
try:
bu.debug(f'tank: {tank}')
if tank['userStr'] in tank_strs:
tank['name'] = tank_strs[tank['userStr']]
else:
tank['name'] = tank['userStr'].split(':')[1]
tank.pop('userStr', None)
tank_tmp = OrderedDict()
for key in sorted(tank.keys()):
tank_tmp[key] = tank[key]
tankopedia[str(tank['tank_id'])] = tank_tmp
except:
bu.error(f'Could not process tank: {tank}')
for tank_str in tank_strs:
skip = False
key = tank_str.split(':')[1]
bu.debug('Tank string: ' + key + ' = ' + tank_strs[tank_str])
re_strs = [r'^Chassis_', r'^Turret_', r'^_', r'_short$' ]
for re_str in re_strs:
p = re.compile(re_str)
if p.match(key):
skip = True
break
if skip:
continue
userStrs[key] = tank_strs[tank_str]
# sorting
tankopedia_sorted = OrderedDict()
for tank_id in sorted(tankopedia.keys(), key=int):
tankopedia_sorted[str(tank_id)] = tankopedia[str(tank_id)]
userStrs_sorted = OrderedDict()
for userStr in sorted(userStrs.keys()):
userStrs_sorted[userStr] = userStrs[userStr]
# bu.debug('Tank strings: ' + str(len(userStrs_sorted)))
except Exception as err:
bu.error(err)
sys.exit(1)
return tankopedia_sorted, userStrs_sorted
def get_tank_id(nation: str, tank_id : int) -> int:
assert wg is not None, "wg has to be set"
return (tank_id << 8) + (wg.NATION_ID[nation] << 4) + 1
def get_tank_type(tagstr : str):
assert wg is not None, "wg has to be set"
tags = tagstr.split(' ')
for t_type in wg.TANK_TYPE:
if tags[0] == t_type:
return t_type
return None
### main()
if __name__ == "__main__":
# To avoid 'Event loop is closed' RuntimeError due to compatibility issue with aiohttp
if sys.platform.startswith("win") and sys.version_info >= (3, 8):
try:
from asyncio import WindowsSelectorEventLoopPolicy
except ImportError:
pass
else:
if not isinstance(get_event_loop_policy(), WindowsSelectorEventLoopPolicy):
set_event_loop_policy(WindowsSelectorEventLoopPolicy())
run(main())
get_tankopedia.py
-91
View File
@@ -1,91 +0,0 @@
#!/usr/bin/env python3
# Get Tankpedia from WoTinspector.com
import sys
import argparse
import json
import os
import inspect
import aiohttp
import asyncio
import re
import logging
from pathlib import Path
import blitzutils as bu
from blitzutils import WG
from blitzutils import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG)
VERBOSE = False
DEBUG = False
tank_type = ['lightTank', 'mediumTank', 'heavyTank', 'AT-SPG']
# @asyncio.coroutine
async def get_tankopedia(filename: str):
url = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
async with aiohttp.ClientSession() as client:
async with client.get(url) as r:
if r.status == 200:
WI_tank_db = await r.text()
WI_tank_db = WI_tank_db.split("\n")
else:
print('Error: Could not get valid HTTPS response. HTTP: ' + str(r.status) )
sys.exit(1)
tanks = {}
n = 0
p = re.compile('\\s*(\\d+):\\s{"en":"([^"]+)",.*?"tier":(\\d+), "type":(\\d), "premium":(\\d).*')
for line in WI_tank_db[1:-1]:
m = p.match(line)
tank = {}
tank['tank_id'] = int(m.group(1))
tank['name'] = m.group(2)
tank['tier'] = int(m.group(3))
tank['type'] = tank_type[int(m.group(4))]
tank['is_premium'] = (int(m.group(5)) == 1)
tanks[str(m.group(1))] = tank
n += 1
tankopedia = {}
tankopedia['status'] = "ok"
tankopedia['meta'] = {"count" : n}
tankopedia['data'] = tanks
print("Tankopedia has " + str(n) + " tanks in: " + filename)
with open(filename,'w') as outfile:
outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=True))
return None
async def main(argv):
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
parser = argparse.ArgumentParser(description='Retrieve Tankopedia from WoTinspector.com')
parser.add_argument('--file', dest="outfile", help='Write Tankopedia to file')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
args = parser.parse_args()
bu.set_log_level(args.silent, args.verbose, args.debug)
wi = WoTinspector()
await wi.get_tankopedia(args.outfile)
await wi.close()
# main()
if __name__ == "__main__":
# To avoid 'Event loop is closed' RuntimeError due to compatibility issue with aiohttp
if sys.platform.startswith("win") and sys.version_info >= (3, 8):
try:
from asyncio import WindowsSelectorEventLoopPolicy
except ImportError:
pass
else:
if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy):
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
asyncio.run(main(sys.argv[1:]), debug=False)
requirements.txt
+2 -1
View File
@@ -7,4 +7,5 @@ beautifulsoup4
lxml lxml
xmltodict xmltodict
progress progress
pytest-asyncio pytest-asyncio
blitz-utils @ git+https://github.com/Jylpah/blitz-utils
tanks.json
+1 -7379
View File
File diff suppressed because it is too large. Load diff
upload_wotb_replays.py
+393 -291
View File
@@ -3,341 +3,443 @@
import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
import logging, re, configparser, zipfile import logging, re, configparser, zipfile
import blitzutils as bu import blitzutils1 as bu
from blitzutils import WG from blitzutils1 import WG
from blitzutils import WoTinspector from blitzutils1 import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG) logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini' FILE_CONFIG = "blitzstats.ini"
# DB_C_REPLAYS = 'Replays' # DB_C_REPLAYS = 'Replays'
DEBUG = False DEBUG = False
VERBOSE = False VERBOSE = False
SLEEP = 2 SLEEP = 2
MAX_RETRIES = 3 MAX_RETRIES = 3
REPLAY_N = 0 REPLAY_N = 0
SKIPPED_N = 0 SKIPPED_N = 0
ERROR_N = 0 ERROR_N = 0
WIurl='https://wotinspector.com/api/replay/upload?' WIurl = "https://wotinspector.com/api/replay/upload?"
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' WG_appID = "81381d3f45fa4aa75b78a7198eb216ad"
wg : WG | None = None wg: WG | None = None
wi : WoTinspector | None = None wi: WoTinspector | None = None
async def main(argv) -> int: async def main(argv) -> int:
global wg, wi global wg, wi
current_dir = os.getcwd() current_dir = os.getcwd()
# set the directory for the script # set the directory for the script
os.chdir(os.path.dirname(sys.argv[0])) os.chdir(os.path.dirname(sys.argv[0]))
# options defaults
OPT_WORKERS_N = 2
WG_ID = None
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
# China is not supported since WG API stats are not available there
WG_RATE_LIMIT = 10
try: # options defaults
## Read config OPT_WORKERS_N = 2
if os.path.isfile(FILE_CONFIG): WG_ID = None
bu.debug('Reading config file: ' + FILE_CONFIG) WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
config = configparser.ConfigParser() # China is not supported since WG API stats are not available there
config.read(FILE_CONFIG) WG_RATE_LIMIT = 10
try:
if 'WG' in config.sections():
configWG = config['WG']
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configWG.getint('wg_id', WG_ID)
WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT)
## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
except Exception as err:
bu.error(exception=err)
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON') try:
parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id') ## Read config
parser.add_argument('-a', '--account', dest='account', type=str, default=WG_ACCOUNT, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER') if os.path.isfile(FILE_CONFIG):
parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use "NN" for continous numbering. Default is automatic naming') bu.debug("Reading config file: " + FILE_CONFIG)
parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com') config = configparser.ConfigParser()
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') config.read(FILE_CONFIG)
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"')
args = parser.parse_args()
bu.set_verbose(args.verbose) try:
bu.set_log_level(args.silent, args.verbose, args.debug) if "WG" in config.sections():
configWG = config["WG"]
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configWG.getint("wg_id", WG_ID)
WG_ACCOUNT = configWG.get("wg_account", WG_ACCOUNT)
## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
WG_RATE_LIMIT = configWG.getint("wg_rate_limit", WG_RATE_LIMIT)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
wg = WG(WG_appID, args.tankopedia, args.mapfile) except Exception as err:
wi = WoTinspector() bu.error(exception=err)
if args.account is not None: parser = argparse.ArgumentParser(
args.accountID = await wg.get_account_id(args.account) description="Post replays(s) to WoTinspector.com and retrieve replay data as JSON"
bu.debug('WG account_id: ' + str(args.accountID)) )
parser.add_argument(
"-id", dest="accountID", type=int, default=WG_ID, help="WG account_id"
)
parser.add_argument(
"-a",
"--account",
dest="account",
type=str,
default=WG_ACCOUNT,
help="Uploader's WG account name. Format: ACCOUNT_NAME@SERVER",
)
parser.add_argument(
"-t",
"--title",
type=str,
default=None,
help='Title for replays. Use "NN" for continous numbering. Default is automatic naming',
)
parser.add_argument(
"-p",
"--private",
dest="private",
action="store_true",
default=False,
help="Set replays private on WoTinspector.com",
)
parser.add_argument(
"--tankopedia",
type=str,
default="tanks.json",
help='JSON file to read Tankopedia from. Default: "tanks.json"',
)
parser.add_argument(
"--mapfile",
type=str,
default="maps.json",
help='JSON file to read Blitz map names from. Default: "maps.json"',
)
parser.add_argument(
"-d", "--debug", action="store_true", default=False, help="Debug mode"
)
parser.add_argument(
"-v", "--verbose", action="store_true", default=False, help="Verbose mode"
)
parser.add_argument(
"-s", "--silent", action="store_true", default=False, help="Silent mode"
)
parser.add_argument(
"files",
metavar="FILE1 [FILE2 ...]",
type=str,
nargs="+",
help="Files to read. Use '-' for STDIN\"",
)
args = parser.parse_args()
if args.accountID is None: bu.set_verbose(args.verbose)
args.accountID = 0 bu.set_log_level(args.silent, args.verbose, args.debug)
try: wg = WG(WG_appID, args.tankopedia, args.mapfile)
queue : asyncio.Queue = asyncio.Queue() wi = WoTinspector()
# rebase file arguments due to moving the working directory to the script location
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
args.files = bu.rebase_file_args(current_dir, args.files)
tasks = [] if args.account is not None:
# Make replay Queue args.accountID = await wg.get_account_id(args.account)
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) bu.debug("WG account_id: " + str(args.accountID))
# Start tasks to process the Queue
for i in range(OPT_WORKERS_N):
tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
bu.debug('Task ' + str(i) + ' started')
bu.debug('Waiting for the replay scanner to finish')
await asyncio.wait([tasks[0]])
bu.debug('Scanner finished. Waiting for workers to finish queue')
await queue.join()
bu.debug('Cancelling workers')
for task in tasks:
task.cancel()
bu.debug('Waiting for workers to cancel')
await asyncio.gather(*tasks, return_exceptions=True)
bu.verbose_std(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
except KeyboardInterrupt:
print('Ctrl-c pressed ...')
sys.exit(1)
finally:
## Need to close the aiohttp.session since Python destructors do not support async methods...
await wg.session.close()
await wi.close()
return 0 if args.accountID is None:
args.accountID = 0
try:
queue: asyncio.Queue = asyncio.Queue()
# rebase file arguments due to moving the working directory to the script location
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
args.files = bu.rebase_file_args(current_dir, args.files)
tasks = []
# Make replay Queue
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
# Start tasks to process the Queue
for i in range(OPT_WORKERS_N):
tasks.append(
asyncio.create_task(
replayWorker(queue, i, args.accountID, args.private)
)
)
bu.debug("Task " + str(i) + " started")
bu.debug("Waiting for the replay scanner to finish")
await asyncio.wait([tasks[0]])
bu.debug("Scanner finished. Waiting for workers to finish queue")
await queue.join()
bu.debug("Cancelling workers")
for task in tasks:
task.cancel()
bu.debug("Waiting for workers to cancel")
await asyncio.gather(*tasks, return_exceptions=True)
bu.verbose_std(
str(REPLAY_N)
+ " replays: "
+ str(REPLAY_N - SKIPPED_N - ERROR_N)
+ " uploaded, "
+ str(SKIPPED_N)
+ " skipped, "
+ str(ERROR_N)
+ " errors"
)
except KeyboardInterrupt:
print("Ctrl-c pressed ...")
sys.exit(1)
finally:
## Need to close the aiohttp.session since Python destructors do not support async methods...
await wg.session.close()
await wi.close()
return 0
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str): async def mkReplayQ(queue: asyncio.Queue, files: list, title: str):
"""Create queue of replays to post""" """Create queue of replays to post"""
p_replayfile = re.compile('.*\\.wotbreplay$') p_replayfile = re.compile(".*\\.wotbreplay$")
if files[0] == '-': if files[0] == "-":
bu.debug('reading replay file list from STDIN') bu.debug("reading replay file list from STDIN")
stdin, _ = await aioconsole.get_standard_streams() stdin, _ = await aioconsole.get_standard_streams()
while True: while True:
line = (await stdin.readline()).decode('utf-8').rstrip() line = (await stdin.readline()).decode("utf-8").rstrip()
if not line: if not line:
break break
else: else:
if (p_replayfile.match(line) is not None): if p_replayfile.match(line) is not None:
await queue.put(await mkQueueItem(line, title)) await queue.put(await mkQueueItem(line, title))
else: else:
for fn in files: for fn in files:
if fn.endswith('"'): if fn.endswith('"'):
fn = fn[:-1] fn = fn[:-1]
if os.path.isfile(fn) and (p_replayfile.match(fn) is not None): if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
await queue.put(await mkQueueItem(fn, title)) await queue.put(await mkQueueItem(fn, title))
bu.debug('File added to queue: ' + fn) bu.debug("File added to queue: " + fn)
elif os.path.isdir(fn): elif os.path.isdir(fn):
with os.scandir(fn) as dirEntry: with os.scandir(fn) as dirEntry:
for entry in dirEntry: for entry in dirEntry:
try: try:
bu.debug('Found: ' + entry.name) bu.debug("Found: " + entry.name)
if entry.is_file() and (p_replayfile.match(entry.name) is not None): if entry.is_file() and (
bu.debug(entry.name) p_replayfile.match(entry.name) is not None
await queue.put(await mkQueueItem(entry.path, title)) ):
bu.debug('File added to queue: ' + entry.path) bu.debug(entry.name)
except Exception as err: await queue.put(await mkQueueItem(entry.path, title))
bu.error(str(err)) bu.debug("File added to queue: " + entry.path)
else: except Exception as err:
bu.error('File not found: ' + fn) bu.error(str(err))
else:
bu.debug('Finished') bu.error("File not found: " + fn)
return None
bu.debug("Finished")
return None
async def mkQueueItem(filename : str, title : str) -> list: async def mkQueueItem(filename: str, title: str) -> list:
"""Make an item to replay queue""" """Make an item to replay queue"""
global REPLAY_N global REPLAY_N
REPLAY_N +=1 REPLAY_N += 1
return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)] return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)]
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False): async def replayWorker(
"""Async Worker to process the replay queue""" queue: asyncio.Queue, workerID: int, account_id: int, priv=False
global SKIPPED_N ):
global ERROR_N """Async Worker to process the replay queue"""
global wi global SKIPPED_N
global ERROR_N
global wi
assert wi is not None, "wi cannot be None" assert wi is not None, "wi cannot be None"
while True: while True:
item = await queue.get() item = await queue.get()
filename = item[0] filename = item[0]
N = item[1] N = item[1]
title = item[2] title = item[2]
replay_json_fn = filename + '.json' replay_json_fn = filename + ".json"
msg_str = 'Replay[' + str(N) + ']: ' msg_str = "Replay[" + str(N) + "]: "
#bu.debug(msg_str + replay_json_fn) # bu.debug(msg_str + replay_json_fn)
try: try:
if os.path.isfile(replay_json_fn): if os.path.isfile(replay_json_fn):
async with aiofiles.open(replay_json_fn) as fp: async with aiofiles.open(replay_json_fn) as fp:
replay_json = json.loads(await fp.read()) replay_json = json.loads(await fp.read())
#if replay_json['status'] == 'ok': # if replay_json['status'] == 'ok':
if wi.chk_JSON_replay(replay_json): if wi.chk_JSON_replay(replay_json):
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' ) bu.verbose_std(
else: msg_str + title + " has already been posted. Skipping."
os.remove(replay_json_fn) )
bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID) else:
SKIPPED_N += 1 os.remove(replay_json_fn)
queue.task_done() bu.debug(
continue msg_str
+ "Replay JSON not valid/complete: Deleting "
+ replay_json_fn,
id=workerID,
)
SKIPPED_N += 1
queue.task_done()
continue
except asyncio.CancelledError as err: except asyncio.CancelledError as err:
raise err raise err
except Exception as err: except Exception as err:
bu.error(msg_str + 'Unexpected error: ' + str(type(err)) + ' : '+ str(err)) bu.error(msg_str + "Unexpected error: " + str(type(err)) + " : " + str(err))
try: try:
#bu.debug('Opening file [' + str(N) +']: ' + filename) # bu.debug('Opening file [' + str(N) +']: ' + filename)
async with aiofiles.open(filename,'rb') as fp: async with aiofiles.open(filename, "rb") as fp:
filename = os.path.basename(filename) filename = os.path.basename(filename)
bu.debug(msg_str + 'File: ' + filename) bu.debug(msg_str + "File: " + filename)
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N) json_resp = await wi.post_replay(
if json_resp is not None: await fp.read(), filename, account_id, title, priv, N
if (await bu.save_JSON(replay_json_fn,json_resp)): )
if wi.chk_JSON_replay(json_resp): if json_resp is not None:
if not bu.debug(msg_str + 'Replay saved OK: ' + filename): if await bu.save_JSON(replay_json_fn, json_resp):
bu.verbose_std(msg_str + title + ' posted') if wi.chk_JSON_replay(json_resp):
else: if not bu.debug(msg_str + "Replay saved OK: " + filename):
bu.warning(msg_str +'Replay file is not valid/complete: ' + filename) bu.verbose_std(msg_str + title + " posted")
ERROR_N += 1 else:
else: bu.warning(
bu.error(msg_str + 'Error saving replay: ' + filename) msg_str
ERROR_N += 1 + "Replay file is not valid/complete: "
else: + filename
bu.error(msg_str + 'Replay file is not valid/complete: ' + filename) )
ERROR_N += 1 ERROR_N += 1
except Exception as err: else:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.error(msg_str + "Error saving replay: " + filename)
bu.debug(msg_str + 'Marking task done') ERROR_N += 1
queue.task_done() else:
bu.error(msg_str + "Replay file is not valid/complete: " + filename)
ERROR_N += 1
except Exception as err:
bu.error(
msg_str + "Unexpected Exception: " + str(type(err)) + " : " + str(err)
)
bu.debug(msg_str + "Marking task done")
queue.task_done()
return None return None
def getTitle_old(replayfile: str, title: str, i : int) -> str: def getTitle_old(replayfile: str, title: str, i: int) -> str:
global wg global wg
if title is None: if title is None:
try: try:
filename = os.path.basename(replayfile) filename = os.path.basename(replayfile)
bu.debug(filename) bu.debug(filename)
map_usrStrs = wg.get_map_user_strs() map_usrStrs = wg.get_map_user_strs()
tank_userStrs = wg.get_tank_user_strs() tank_userStrs = wg.get_tank_user_strs()
#p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') # p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
# update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah # update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$') p = re.compile(
"\\d{8}_\\d{4}_.*?("
m = p.match(filename) + "|".join(tank_userStrs)
if (m is not None): + ")_("
if wg.tanks is not None: + "|".join(map_usrStrs)
tank = m.group(1) + ")(?:-\\d)?\\.wotbreplay$"
if tank in wg.tanks['userStr']: )
tank = wg.tanks['userStr'][tank]
else: m = p.match(filename)
bu.error('Tank code: "' + tank + '" not found from Tankopedia (tanks.json)') if m is not None:
else: if wg.tanks is not None:
tank = m.group(1) tank = m.group(1)
blitz_map = m.group(2) if tank in wg.tanks["userStr"]:
if blitz_map in wg.maps: tank = wg.tanks["userStr"][tank]
blitz_map = wg.maps[blitz_map] else:
else: bu.error(
bu.error('Mapcode: "' + blitz_map + '" not found from map database (maps.json)') 'Tank code: "'
title = tank + ' @ ' + blitz_map + tank
else: + '" not found from Tankopedia (tanks.json)'
title = re.sub('\\.wotbreplay$', '', filename) )
except Exception as err: else:
bu.error(err) tank = m.group(1)
else: blitz_map = m.group(2)
title.replace('NN', str(i)) if blitz_map in wg.maps:
return title blitz_map = wg.maps[blitz_map]
else:
bu.error(
'Mapcode: "'
+ blitz_map
+ '" not found from map database (maps.json)'
)
title = tank + " @ " + blitz_map
else:
title = re.sub("\\.wotbreplay$", "", filename)
except Exception as err:
bu.error(err)
else:
title.replace("NN", str(i))
return title
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str: def getTitle(replayfile: str, title: str = None, i: int = 0) -> str:
global wg global wg
assert wg is not None, "wg must not be None" assert wg is not None, "wg must not be None"
if title is None: if title is None:
try: try:
filename = os.path.basename(replayfile) filename = os.path.basename(replayfile)
bu.debug(filename) bu.debug(filename)
tank = None tank = None
map_name = None map_name = None
with zipfile.ZipFile(replayfile, "r") as archive:
# bu.debug('Replay file: ' + replayfile + ' opened')
with io.TextIOWrapper(archive.open("meta.json")) as meta:
# bu.debug('Replay file\'s metadata: opened')
try:
metadata_json = json.load(meta)
# player = metadata_json['playerName']
tank_name = wg.get_tank_name(
metadata_json["vehicleCompDescriptor"]
)
map_name = wg.get_map(metadata_json["mapName"])
bu.debug(
"Tank: " + tank_name
if tank_name is not None
else "NOT FOUND IN tanks.json" + " Map: " + map_name
)
except Exception as err:
bu.error(exception=err)
if (tank_name is not None) and (map_name is not None):
title = tank_name + " @ " + map_name
else:
title = re.sub("\\.wotbreplay$", "", filename)
except Exception as err:
bu.error(err)
else:
title.replace("NN", str(i))
bu.debug("Returning: " + title)
return title
with zipfile.ZipFile(replayfile, 'r') as archive:
# bu.debug('Replay file: ' + replayfile + ' opened')
with io.TextIOWrapper(archive.open('meta.json')) as meta:
# bu.debug('Replay file\'s metadata: opened')
try:
metadata_json = json.load(meta)
#player = metadata_json['playerName']
tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
map_name = wg.get_map(metadata_json['mapName'])
bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name)
except Exception as err:
bu.error(exception = err)
if (tank_name is not None) and (map_name is not None):
title = tank_name + ' @ ' + map_name
else:
title = re.sub('\\.wotbreplay$', '', filename)
except Exception as err:
bu.error(err)
else:
title.replace('NN', str(i))
bu.debug('Returning: ' + title)
return title
### UNIT TESTS - pytest ### UNIT TESTS - pytest
# class TestClass: # class TestClass:
# def test_getTitle(self): # def test_getTitle(self):
# #player = "jylpah" # #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' # WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json') # wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D" # tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle" # map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay' # replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name # res = tank_name + ' @ ' + map_name
# assert getTitle(replay) == res # assert getTitle(replay) == res
# @pytest.mark.asyncio # @pytest.mark.asyncio
# async def test_getTitle(self): # async def test_getTitle(self):
# #player = "jylpah" # #player = "jylpah"
# WG_appID = '81381d3f45fa4aa75b78a7198eb216ad' # WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
# wg = WG(WG_appID, 'tanks.json', 'maps.json') # wg = WG(WG_appID, 'tanks.json', 'maps.json')
# tank_name = "T28 Defender" # "A68_T28D" # tank_name = "T28 Defender" # "A68_T28D"
# map_name = "Yukon" # "idle" # map_name = "Yukon" # "idle"
# replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay' # replay = './test/' + '20201030_2358__jylpah_A_1157485188135816346.wotbreplay'
# res = tank_name + ' @ ' + map_name # res = tank_name + ' @ ' + map_name
# assert getTitle(replay) == res # assert getTitle(replay) == res
### main() ------------------------------------------- ### main() -------------------------------------------
if __name__ == "__main__": if __name__ == "__main__":
# To avoid 'Event loop is closed' RuntimeError due to compatibility issue with aiohttp # To avoid 'Event loop is closed' RuntimeError due to compatibility issue with aiohttp
if sys.platform.startswith("win") and sys.version_info >= (3, 8): if sys.platform.startswith("win") and sys.version_info >= (3, 8):
try: try:
from asyncio import WindowsSelectorEventLoopPolicy from asyncio import WindowsSelectorEventLoopPolicy
except ImportError: except ImportError:
pass pass
else: else:
if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): if not isinstance(
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy
asyncio.run(main(sys.argv[1:]), debug=False) ):
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
asyncio.run(main(sys.argv[1:]), debug=False)