GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-21 10:17:05 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #23 from Jylpah/master

Browse Source 
Fixes: config file reading, DB, warning messages
This commit is contained in:
Jylpah authored and GitHub committed 2020-07-10 15:49:10 +03:00
2 parents 420410eecf db7d9144b1
commit 3748922933
5 files changed
+245 -101
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+36 -8
View File
@@ -3,7 +3,7 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
### Replays
* [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader.
* [analyze_wotb_replays.py](analyze_wotb_replays.py): A tool to calculate statistics of a set of replays. Upload the replays first with [upload_wotb_replays.py](upload_wotb_replays.py) and then calculate the statistics.
* [analyze_wotb_replays.py](analyze_wotb_replays.py): A tool to calculate statistics of a set of replays. **Upload the replays first** with [upload_wotb_replays.py](upload_wotb_replays.py) and then calculate the statistics with analyze_wotb_replays.py based on the resulting **.json** files.
### Tank data
* [get_tankopedia.py](get_tankopedia.py): Get the lastest tankopedia data from wotinspector.com. The script fetches only a limited set of data. You may edit to your needs.
@@ -19,23 +19,51 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
### Quick instructions
1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
1. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. The recommended way to do this is to use [GitHub Dekstop](https://desktop.github.com/) or do _git pull_
1. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
```
# Remember to run the command with a Python interpreter >= 3.7
python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress
```
3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. A recommened way is to install [GitHub Desktop](https://desktop.github.com/) and clone the repository via _git_. This makes it easy to receive the updates.
4. Upload your replays. Run _python.exe .\upload_wotb_replays.py --help_ (Windows) or _./upload_wotb_replays.py --help_ (Linux)
5. Analyze replays. Run _python.exe .\analyze_wotb_replays.py --help_ (Windows) or _./analyze_wotb_replays.py --help_ (Linux)
4. Upload your replays.
```
# Windows, run directly from Command Prompt or Powershell
# Check the options
python.exe .\upload_wotb_replays.py --help
# Syntax
python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
# example
python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
# Linux
./upload_wotb_replays.py --help
```
5. Analyze replays.
```
# Windows, run directly from Command Prompt or Powershell
python.exe .\analyze_wotb_replays.py --help
# Syntax
python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
# Example
python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
# Linux
./analyze_wotb_replays.py --help
```
### Blitz releases
[WoT Blitz releases](releases.md): launch dates and key content.
### Notes
### Other
Even though the code has an option to use MongoDB, you should not bother with that. The MongoDB support is solely for my own purposes and you do not need MongoDB to run the scripts.
* Please remember to run the scripts from command line, not from the Python interpreter. On Windows open Command Prompt or Powershell.
* [GitHub Desktop](https://desktop.github.com/) is the recommended way of downloading the scripts you can update the scripts easily via git/GitHub Deskop. To prevent GitHub Desktop from uploading your other files to github.com, you can "ignore" the files by right-clicking a file on GitHub Desktop or by manually editing the _.gitignore_ file.
* Even though the code has an option to use MongoDB, you should not bother with that. The MongoDB support is solely for my own purposes and you do not need MongoDB to run the scripts.
You can reach me via email: [[email protected]](mailto:[email protected]) or [Discord](https://discordapp.com/): Jylpah#4662. Happy tanking and stats analysis!
analyze_wotb_replays.py
+72 -52
View File
@@ -27,6 +27,8 @@ REPLAY_I = 0
STAT_TANK_BATTLE_MIN = 100
BATTLE_TIME_BUCKET = 3600*24*14
RE_SRC_IS_DB = re.compile(r'^DB:')
## For different player stat functions (global stats, tank tier stats, etc)
# 1st function = for forming stat_id, 2nd for DB stats query, 3rd for WG API stats query
STAT_FUNC = {
@@ -74,9 +76,9 @@ replay_details_flds = [
## Syntax: key == stat field in https://api.wotblitz.eu/wotb/tanks/stats/ (all.KEY)
## Value array [ 'Stat Title', [ 0, data buckets ....], scaling_factor_for_bucket_values, 'print_format' ]
histogram_fields = {
'wins' : [ 'Win rate', [0, .35, .40, .45, .5, .55, .60, .65, 1], 100, '{:.0f}%' ],
'damage_dealt' : [ 'Avg. Dmg.', [0, .25e3, .5e3, 1e3, 1.5e3, 2e3,2.5e3,3e3,100e3], 1, '{:.0f}' ],
'battles' : [ 'Battles', [0, 1e3, 3e3, 5e3, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k'] # battles is a mandatory stat to include
'wins' : [ 'Win rate', [0, .35, .40, .45, .48, .5, .52, .55, .60, .65, .70, 1], 100, '{:.0f}%' ],
'damage_dealt' : [ 'Avg. Dmg.', [0, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 100e3], 1, '{:.0f}' ],
'battles' : [ 'Battles', [0, 1000, 2500, 5000, 7000, 10e3, 15e3, 25e3, 50e3, 5e7], .001, '{:.0f}k'] # battles is a mandatory stat to include
}
@@ -499,6 +501,7 @@ class ErrorCatchingArgumentParser(argparse.ArgumentParser):
async def main(argv):
global wg, wi, WG_APP_ID
# set the directory for the script
current_dir = os.getcwd()
os.chdir(os.path.dirname(sys.argv[0]))
## Default options:
@@ -508,8 +511,11 @@ async def main(argv):
OPT_STAT_FUNC = 'player'
OPT_WORKERS_N = 10
WG_ID = None
#WG_APP_ID = WG_APP_ID
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
# China is not supported since WG API stats are not available there
WG_ID = None # WG account_id in integer format.
# WG_ACCOUNT will be used to retrieve the account_id, but it can be set directly too
# WG_APP_ID = WG_APP_ID
WG_RATE_LIMIT = 10 ## WG standard. Do not edit unless you have your
## own server app ID, it will REDUCE the performance
@@ -521,7 +527,7 @@ async def main(argv):
DB_AUTH = 'admin'
DB_NAME = 'BlitzStats'
DB_USER = 'mongouser'
DB_PASSWD = "PASSWORD"
DB_PASSWD = 'PASSWORD'
DB_CERT = None
DB_CA = None
@@ -532,45 +538,49 @@ async def main(argv):
config.read(FILE_CONFIG)
try:
configOptions = config['OPTIONS']
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
OPT_DB = configOptions.getboolean('opt_DB', OPT_DB)
OPT_EXTENDED = configOptions.getboolean('opt_analyzer_extended', OPT_EXTENDED)
OPT_HIST = configOptions.getboolean('opt_analyzer_hist', OPT_HIST)
OPT_STAT_FUNC = configOptions.get('opt_analyzer_stat_func', fallback=OPT_STAT_FUNC)
OPT_WORKERS_N = configOptions.getint('opt_analyzer_workers', OPT_WORKERS_N)
except configparser.NoSectionError as err:
if 'OPTIONS' in config.sections():
configOptions = config['OPTIONS']
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
OPT_DB = configOptions.getboolean('opt_DB', OPT_DB)
OPT_EXTENDED = configOptions.getboolean('opt_analyzer_extended', OPT_EXTENDED)
OPT_HIST = configOptions.getboolean('opt_analyzer_hist', OPT_HIST)
OPT_STAT_FUNC = configOptions.get('opt_analyzer_stat_func', fallback=OPT_STAT_FUNC)
OPT_WORKERS_N = configOptions.getint('opt_analyzer_workers', OPT_WORKERS_N)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
try:
configWG = config['WG']
WG_ID = configWG.getint('wg_id', WG_ID)
WG_APP_ID = configWG.get('wg_app_id', WG_APP_ID)
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
except configparser.NoSectionError as err:
if 'WG' in config.sections():
configWG = config['WG']
WG_ID = configWG.getint('wg_id', WG_ID)
WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT)
WG_APP_ID = configWG.get('wg_app_id', WG_APP_ID)
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
try:
configDB = config['DATABASE']
DB_SERVER = configDB.get('db_server', DB_SERVER)
DB_PORT = configDB.getint('db_port', DB_PORT)
DB_SSL = configDB.getboolean('db_ssl', DB_SSL)
DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ)
DB_AUTH = configDB.get('db_auth', DB_AUTH)
DB_NAME = configDB.get('db_name', DB_NAME)
DB_USER = configDB.get('db_user', DB_USER)
DB_PASSWD = configDB.get('db_password', DB_PASSWD)
DB_CERT = configDB.get('db_ssl_cert_file', DB_CERT)
DB_CA = configDB.get('db_ssl_ca_file', DB_CA)
except configparser.NoSectionError as err:
if 'DATABASE' in config.sections():
configDB = config['DATABASE']
DB_SERVER = configDB.get('db_server', DB_SERVER)
DB_PORT = configDB.getint('db_port', DB_PORT)
DB_SSL = configDB.getboolean('db_ssl', DB_SSL)
DB_CERT_REQ = configDB.getint('db_ssl_req', DB_CERT_REQ)
DB_AUTH = configDB.get('db_auth', DB_AUTH)
DB_NAME = configDB.get('db_name', DB_NAME)
DB_USER = configDB.get('db_user', DB_USER)
DB_PASSWD = configDB.get('db_password', DB_PASSWD)
DB_CERT = configDB.get('db_ssl_cert_file', DB_CERT)
DB_CA = configDB.get('db_ssl_ca_file', DB_CA)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
parser = ErrorCatchingArgumentParser(description='Analyze Blitz replay JSONs from WoTinspector.com')
parser = ErrorCatchingArgumentParser(description='Analyze Blitz replay JSON files from WoTinspector.com. Use \'upload_wotb_replays.py\' to upload the replay files first.')
parser.add_argument('--output', default='plain', choices=['plain', 'db'], help='Select output mode: plain text or database')
parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze')
parser.add_argument('-a', '--account', type=str, default=None, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER')
parser.add_argument('-a', '--account', type=str, default=WG_ACCOUNT, help='WG account nameto analyze. Format: ACCOUNT_NAME@SERVER')
parser.add_argument('-x', '--extended', action='store_true', default=OPT_EXTENDED, help='Print Extended stats')
parser.add_argument('-X', '--extra_categories', choices=BattleRecordCategory.get_extra_categories(), default=None, nargs='*', help='Print Extended categories')
parser.add_argument('--hist', action='store_true', default=OPT_HIST, help='Print player histograms (WR/battles)')
@@ -622,10 +632,12 @@ async def main(argv):
bu.error("Could no initiate DB connection: Disabling DB", err)
args.db = False
pass
if not(args.db):
else:
bu.debug('No DB in use')
# rebase file arguments due to moving the working directory to the script location
args.files = bu.rebase_file_args(current_dir, args.files)
try:
replayQ = asyncio.Queue(maxsize=1000)
reader_tasks = []
@@ -1126,14 +1138,15 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
bu.debug(str(args.filters))
filters = json.loads(args.filters)
bu.debug(json.dumps(filters, indent=2))
cursor = dbc.find(filters, {'_id': 0 })
cursor = dbc.find(filters)
else:
# select all
cursor = dbc.find({}, {'_id': 0 })
cursor = dbc.find({})
bu.debug('Reading replays...')
async for replay_json in cursor:
#bu.debug(json.dumps(replay_json, indent=2))
await queue.put(await mk_readerQ_item(replay_json))
_id = replay_json['_id']
del(replay_json['_id'])
await queue.put(await mk_readerQ_item(replay_json, 'DB: _id = ' + _id))
Nreplays += 1
bu.debug('All the matching replays have been read from the DB')
except Exception as err:
@@ -1149,7 +1162,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
if (p_replayfile.match(line) != None):
replay_json = await bu.open_JSON(line, wi.chk_JSON_replay)
await queue.put(await mk_readerQ_item(replay_json))
await queue.put(await mk_readerQ_item(replay_json, line))
except Exception as err:
bu.error(exception=err)
else:
@@ -1160,7 +1173,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
fn = fn[:-1]
if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
replay_json = await bu.open_JSON(fn, wi.chk_JSON_replay)
await queue.put(await mk_readerQ_item(replay_json))
await queue.put(await mk_readerQ_item(replay_json, fn))
bu.debug('File added to queue: ' + fn)
Nreplays += 1
elif os.path.isdir(fn):
@@ -1169,7 +1182,7 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
if entry.is_file() and (p_replayfile.match(entry.name) != None):
bu.debug(entry.name)
replay_json = await bu.open_JSON(entry.path, wi.chk_JSON_replay)
await queue.put(await mk_readerQ_item(replay_json))
await queue.put(await mk_readerQ_item(replay_json, entry.name))
bu.debug('File added to queue: ' + entry.path)
Nreplays += 1
except Exception as err:
@@ -1178,28 +1191,34 @@ async def mk_replayQ(queue : asyncio.Queue, args : argparse.Namespace, db : moto
return Nreplays
async def mk_readerQ_item(replay_json) -> list:
async def mk_readerQ_item(replay_json, filename : str = None) -> list:
"""Make an item to replay queue"""
global REPLAY_N
REPLAY_N +=1
return [replay_json, REPLAY_N]
REPLAY_N +=1
if filename == None:
return [replay_json, REPLAY_N, '' ]
elif RE_SRC_IS_DB.match(filename) != None:
return [replay_json, REPLAY_N, filename ]
else:
return [replay_json, REPLAY_N, os.path.basename(filename) ]
async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
"""Async Worker to process the replay queue"""
#global SKIPPED_N
# account_id = args.account_id
results = []
playerstanks = set()
try:
while True:
item = await queue.get()
replay_json = item[0]
replayID = item[1]
replay_json = item[0]
replayID = item[1]
replay_file = item[2]
try:
msg_str = 'Replay[' + str(replayID) + ']: '
if replay_json == None:
bu.verbose('Replay[' + str(replayID) + ']: is empty. Skipping.' )
bu.warning(msg_str + 'Invalid replay. Skipping: ' + (replay_file if replay_file != None else '') )
#SKIPPED_N += 1
queue.task_done()
continue
@@ -1209,7 +1228,7 @@ async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Nam
result = await read_replay_JSON(replay_json, args)
bu.print_progress()
if result == None:
bu.debug('Replay[' + str(replayID) + ']: Invalid replay', id=readerID)
bu.warning(msg_str + 'Invalid replay' + (replay_file if replay_file != None else '') )
queue.task_done()
continue
@@ -1238,7 +1257,8 @@ async def read_replay_JSON(replay_json: dict, args : argparse.Namespace) -> dict
#db = args.db
result = {}
try:
if (replay_json['data']['summary']['mastery_badge'] == None) or (replay_json['data']['summary']['battle_result'] < 0):
# bu.debug(str(replay_json))
if not wi.chk_JSON_replay(replay_json):
bu.debug('Invalid replay')
return None
blitzstats.ini.default
+11 -3
View File
@@ -5,9 +5,17 @@
#opt_analyzer_stat_func = player
[WG]
## Find your WG account_id : https://developers.wargaming.net/reference/all/wotb/account/list/
#wg_id = 0
## WG API limits requests / sec to 10 for client apps
# wg_id = 0
## wg_account format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
## China is not supported since WG API stats are not available there
# wg_account = your_nick@eu
## WG API limits requests / sec to 10 for mobile apps.
## You can register your "own" server app and
## upgrade the rate limit to 20 requests / sec.
## Check https://developers.wargaming.net/documentation/guide/getting-started/
#wg_app_id = XXXXXXXXXXX
# wg_app_id = XXXXXXXXXXX
blitzutils.py
+74 -17
View File
@@ -9,7 +9,7 @@ from progress.counter import Counter
from decimal import Decimal
MAX_RETRIES= 3
SLEEP = 3
SLEEP = 1.5
LOG_LEVELS = { 'silent': 0, 'normal': 1, 'verbose': 2, 'debug': 3 }
SILENT = 0
@@ -210,6 +210,13 @@ def verbose_std(msg = "", id = None) -> bool:
return True
return False
def warning(msg = "", id = None) -> bool:
"""Print a warning message"""
if _log_level >= NORMAL:
_print_log_msg('', 'Warning: ' + msg, None, id)
return True
return False
def debug(msg = "", id = None, exception = None, force: bool = False) -> bool:
"""print a conditional debug message"""
if (_log_level >= DEBUG) or force:
@@ -341,6 +348,13 @@ def NOW() -> int:
return int(time.time())
def rebase_file_args(current_dir, files):
"""REbase file command line params after moving working dir to the script's dir"""
if (files[0] == '-') or (files[0] == 'db:'):
return files
else:
return [ os.path.join(current_dir, fn) for fn in files ]
async def read_int_list(filename: str) -> list():
"""Read file to a list and return list of integers in the input file"""
@@ -445,6 +459,40 @@ def bld_dict_hierarcy(d : dict, key : str, value) -> dict:
error('Unexpected Exception', err)
return None
def get_JSON_keypath(keypath: str, key: str):
if keypath == None:
return key
else:
return '.'.join([keypath, key])
def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
if keys == None:
keys = key.split('.')
if len(keys) == 0:
return json
key = keys.pop(0)
if type(json) == dict:
if key in json:
return get_JSON_value(json[key], keys=keys, keypath=get_JSON_keypath(keypath, key))
else:
raise KeyError('Key: '+ get_JSON_keypath(keypath, key) + ' not found')
if type(json) == list:
p = re.compile(r'^\[(\d+)\]$')
m = p.match(key)
if len(m.groups()) != 1:
raise KeyError('Invalid key given: ' + get_JSON_keypath(keypath, key))
ndx = m.group(1)
try:
return get_JSON_value(json[ndx], keys=keys, keypath=get_JSON_keypath(keypath, key))
except IndexError:
raise KeyError('JSON array index out of range: ' + get_JSON_keypath(keypath, key))
raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
## -----------------------------------------------------------
#### Class SlowBar
## -----------------------------------------------------------
@@ -1365,9 +1413,11 @@ class WoTinspector:
URL_WI = 'https://replays.wotinspector.com'
URL_REPLAY_LIST = URL_WI + '/en/sort/ut/page/'
URL_REPLAY_DL = URL_WI + '/en/download/'
URL_REPLAY_VIEW = URL_WI +'/en/view/'
URL_REPLAY_UL = 'https://api.wotinspector.com/replay/upload?'
URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
URL_TANK_DB ="https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
URL_TANK_DB = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
REPLAY_N = 1
@@ -1462,41 +1512,44 @@ class WoTinspector:
error(msg_str + 'Unexpected Exception', err)
return None
json_resp = None
for retry in range(MAX_RETRIES):
debug(msg_str + 'Posting: ' + title + ' Try #: ' + str(retry + 1) + '/' + str(MAX_RETRIES) )
try:
async with self.session.post(url, headers=headers, data=payload) as resp:
debug(msg_str + 'HTTP response: '+ str(resp.status))
if resp.status == 200:
debug('HTTP POST 200 = Success. Reading response data')
debug(msg_str + 'HTTP POST 200 = Success. Reading response data')
json_resp = await resp.json()
if json_resp.get('status', None) == None:
error(msg_str +' : ' + title + ' : Received invalid JSON')
elif (json_resp['status'] == 'ok'):
debug('Response data read. Status OK')
if self.chk_JSON_replay(json_resp):
debug(msg_str + 'Response data read. Status OK')
return json_resp
elif (json_resp['status'] == 'error'):
error(msg_str + json_resp['error']['message'] + ' : ' + title)
else:
error(msg_str + ' Unspecified error: ' + title)
debug(msg_str +' : ' + title + ' : Receive invalid JSON')
else:
debug(msg_str + 'Got HTTP/' + str(resp.status))
except Exception as err:
error(exception=err)
debug(msg_str, exception=err)
await asyncio.sleep(SLEEP)
error(msg_str + ' Could not post replay: ' + title)
return None
debug(msg_str + ' Could not post replay: ' + title)
return json_resp
async def get_replay_listing(self, page: int = 0) -> aiohttp.ClientResponse:
url = self.get_url_replay_listing(page)
return await self.session.get(url)
@classmethod
def get_url_replay_listing(cls, page : int):
return cls.URL_REPLAY_LIST + str(page) + '?vt=#filters'
@classmethod
def get_url_replay_view(cls, replay_id):
return cls.URL_REPLAY_VIEW + replay_id
@classmethod
def get_replay_links(cls, doc: str):
"""Get replay download links from WoTinspector.com replay listing page"""
@@ -1513,23 +1566,27 @@ class WoTinspector:
error(exception=err)
return replay_links
@classmethod
def get_replay_id(cls, url):
return url.rsplit('/', 1)[-1]
@classmethod
def chk_JSON_replay(cls, json_resp):
""""Check String for being a valid JSON file"""
try:
if ('status' in json_resp) and json_resp['status'] == 'ok' and ('data' in json_resp) and json_resp['data'] != None:
if ('status' in json_resp) and json_resp['status'] == 'ok' and \
(get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
debug("JSON check OK")
return True
except KeyError as err:
error('Key not found', err)
debug('Replay JSON check failed', exception=err)
except:
debug("JSON check failed: " + str(json_resp))
debug("Replay JSON check failed: " + str(json_resp))
return False
class BlitzStars:
URL_BLITZSTARS = 'https://www.blitzstars.com'
upload_wotb_replays.py
+52 -21
View File
@@ -25,26 +25,48 @@ wi = None
async def main(argv):
global wg, wi
current_dir = os.getcwd()
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
# options defaults
OPT_WORKERS_N = 5
WG_ID = None
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
# China is not supported since WG API stats are not available there
WG_RATE_LIMIT = 10
## Read config
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
try:
## Read config
if os.path.isfile(FILE_CONFIG):
bu.debug('Reading config file: ' + FILE_CONFIG)
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
try:
if 'OPTIONS' in config.sections():
configOptions = config['OPTIONS']
OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', OPT_WORKERS_N)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
try:
if 'WG' in config.sections():
configWG = config['WG']
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configWG.getint('wg_id', WG_ID)
WG_ACCOUNT = configWG.get('wg_account', WG_ACCOUNT)
## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', WG_RATE_LIMIT)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
except Exception as err:
bu.error(exception=err)
configOptions = config['OPTIONS']
OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', 5)
configWG = config['WG']
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configWG.getint('wg_id', None)
## WG API Rules limit 10 request / sec. Higher rate of requests will return errors ==> extra delay
WG_RATE_LIMIT = configWG.getint('wg_rate_limit', 10)
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data')
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve replay data as JSON')
parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id')
parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
parser.add_argument('-a', '--account', dest='account', type=str, default=WG_ACCOUNT, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering')
parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com')
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
@@ -70,7 +92,9 @@ async def main(argv):
try:
queue = asyncio.Queue()
# rebase file arguments due to moving the working directory to the script location
args.files = bu.rebase_file_args(current_dir, args.files)
tasks = []
# Make replay Queue
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
@@ -88,7 +112,7 @@ async def main(argv):
task.cancel()
bu.debug('Waiting for workers to cancel')
await asyncio.gather(*tasks, return_exceptions=True)
bu.verbose(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
bu.verbose_std(str(REPLAY_N) + ' replays: ' + str(REPLAY_N - SKIPPED_N - ERROR_N) + ' uploaded, ' + str(SKIPPED_N) + ' skipped, ' + str(ERROR_N) + ' errors')
except KeyboardInterrupt:
print('Ctrl-c pressed ...')
@@ -170,7 +194,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
else:
os.remove(replay_json_fn)
bu.debug("Replay JSON not valid: Deleting " + replay_json_fn)
bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
SKIPPED_N += 1
queue.task_done()
continue
@@ -187,11 +211,18 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
if json_resp != None:
if (await bu.save_JSON(replay_json_fn,json_resp)):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
bu.verbose_std(msg_str + title + ' posted')
if wi.chk_JSON_replay(json_resp):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
bu.verbose_std(msg_str + title + ' posted')
else:
bu.warning(msg_str +'Replay file is not valid: ' + filename)
ERROR_N += 1
else:
bu.error(msg_str + 'Error saving replay: ' + filename)
ERROR_N += 1
ERROR_N += 1
else:
bu.error(msg_str + 'Replay file is not valid: ' + filename)
ERROR_N += 1
except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
bu.debug(msg_str + 'Marking task done')