GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-23 19:19:53 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #23 from Jylpah/stable-csv

Browse Source 
CSV export
This commit is contained in:
Jylpah authored and GitHub committed 2021-02-05 21:42:35 +02:00
2 parents be0e43ad1c d5cd5f5b12
commit 9f91ef85b6
2 files changed
+302 -145
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

analyze_wotb_replays.py
+185 -41
View File
@@ -4,7 +4,7 @@
# Script Analyze WoT Blitz replays # Script Analyze WoT Blitz replays
import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole
import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections, csv
from datetime import datetime, timedelta, date from datetime import datetime, timedelta, date
import blitzutils as bu import blitzutils as bu
from blitzutils import WG from blitzutils import WG
@@ -89,7 +89,6 @@ class StatFunc:
return cls.STAT_FUNC[cls._stat_func][3] return cls.STAT_FUNC[cls._stat_func][3]
replay_summary_flds = [ replay_summary_flds = [
'battle_result', 'battle_result',
'battle_type', 'battle_type',
@@ -434,6 +433,22 @@ class BattleCategorizationList():
return None return None
def get_results_list(self, urls: bool = False):
try:
res = list()
for cat in self.categorizations_list:
res.append('')
res.extend(self.categorizations[cat].get_results_list())
if urls:
res.append('')
for url in self.urls:
res.append(url)
return res
except Exception as err:
bu.error(exception=err)
return None
class BattleCategorization(): class BattleCategorization():
RESULT_CAT_HEADER_LEN = 15 RESULT_CAT_HEADER_LEN = 15
@@ -479,7 +494,7 @@ class BattleCategorization():
print(FORMAT.format(self.category_key, self.title), end=' ') print(FORMAT.format(self.category_key, self.title), end=' ')
def record_result(self, result: dict): def record_result(self, result: dict) -> bool:
try: try:
cat = self.get_category(result) cat = self.get_category(result)
self.categories[cat].record_result(result) self.categories[cat].record_result(result)
@@ -501,12 +516,24 @@ class BattleCategorization():
def print_headers(self): def print_headers(self):
if len(self.title) > self.RESULT_CAT_HEADER_LEN: try:
cat_name = self.title[:self.RESULT_CAT_HEADER_LEN] if len(self.title) > self.RESULT_CAT_HEADER_LEN:
else: cat_name = self.title[:self.RESULT_CAT_HEADER_LEN]
cat_name = self.title else:
header = self.RESULT_CAT_HEADER_FRMT.format(cat_name) cat_name = self.title
BattleCategory.print_headers(header) header = self.RESULT_CAT_HEADER_FRMT.format(cat_name)
BattleCategory.print_headers(header)
return None
except Exception as err:
bu.error(exception = err)
def get_headers(self) -> list:
"""Get BattleCategory headers as list"""
try:
return BattleCategory.get_headers(self.title)
except Exception as err:
bu.error(exception = err)
return None return None
@@ -528,7 +555,7 @@ class BattleCategorization():
return None return None
def get_results_json(self): def get_results_json(self) -> dict:
"""Get results in JSON format""" """Get results in JSON format"""
try: try:
results = dict() results = dict()
@@ -542,6 +569,19 @@ class BattleCategorization():
bu.error('Key not found', err) bu.error('Key not found', err)
return None return None
def get_results_list(self) -> list:
"""Get results in list format (for CSV printing)"""
try:
results = list()
results.append(self.get_headers())
for cat in self.get_categories():
results.append( [cat] + self.categories[cat].get_results_list() )
return results
except KeyError as err:
bu.error('Key not found', err)
return None
class BattleTotals(BattleCategorization): class BattleTotals(BattleCategorization):
TOTAL = 'Total' TOTAL = 'Total'
@@ -1008,6 +1048,20 @@ class BattleCategory():
return None return None
@classmethod
def get_headers(cls, header: str = None) -> list:
try:
headers = [ header ]
for field in cls.result_fields:
headers.append(cls.get_field_name(field))
return headers
except KeyError as err:
bu.error('Key not found', exception=err)
except Exception as err:
bu.error(exception=err)
return None
def __init__(self): def __init__(self):
try: try:
@@ -1089,6 +1143,22 @@ class BattleCategory():
except Exception as err: except Exception as err:
bu.error(exception=err) bu.error(exception=err)
return None return None
def get_results_list(self) -> list:
"""Return results as a list"""
if not self.results_ready:
bu.error('Stats have not been calculated yet. call calc_results() before get_results()')
try:
results = list()
for field in self.result_fields:
results.append(self._result_fields[field][3].format(self.results[field]))
return results
except KeyError as err:
bu.error('Key not found', err)
except Exception as err:
bu.error(exception=err)
return None
def print_results(self): def print_results(self):
@@ -1121,6 +1191,7 @@ class PlayerHistogram():
self.allies = [0] * self.ncat self.allies = [0] * self.ncat
self.enemies = [0] * self.ncat self.enemies = [0] * self.ncat
self.total = [0] * self.ncat self.total = [0] * self.ncat
self.results = None
def record_ally(self, stat: float): def record_ally(self, stat: float):
for cat in range(0, self.ncat): for cat in range(0, self.ncat):
@@ -1150,7 +1221,7 @@ class PlayerHistogram():
return self.cat_format.format(float(val * self.cat_factor)) return self.cat_format.format(float(val * self.cat_factor))
def print(self, ret_json: bool = False): def calc_results(self):
N_enemies = 0 N_enemies = 0
N_allies = 0 N_allies = 0
res = dict() res = dict()
@@ -1159,23 +1230,60 @@ class PlayerHistogram():
N_enemies += self.enemies[cat] N_enemies += self.enemies[cat]
N_allies += self.allies[cat] N_allies += self.allies[cat]
N_total = N_allies + N_enemies N_total = N_allies + N_enemies
print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL")) for cat in range(0, self.ncat):
for cat in range(0, self.ncat): stat = dict()
print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(self.get_category_name(cat), self.allies[cat], self.allies[cat]/N_allies*100, self.enemies[cat], self.enemies[cat]/N_enemies*100, self.allies[cat] + self.enemies[cat], (self.allies[cat] + self.enemies[cat])/N_total*100 )) stat['allies'] = self.allies[cat]
if ret_json: stat['allies%'] = stat['allies'] / N_allies
stat = dict() stat['enemies'] = self.enemies[cat]
stat['allies'] = self.allies[cat] stat['enemies%'] = stat['enemies'] / N_enemies
stat['allies%'] = stat['allies'] / N_allies stat['total'] = self.allies[cat] + self.enemies[cat]
stat['enemies'] = self.enemies[cat] stat['total%'] = stat['total'] / N_total
stat['enemies%'] = stat['enemies'] / N_enemies
stat['total'] = self.allies[cat] + self.enemies[cat]
stat['total%'] = stat['total'] / N_total
res[self.get_category_name(cat)] = stat
res[self.get_category_name(cat)] = stat
self.results = res
return res return res
def get_results(self) -> dict:
return self.results
def get_results_list(self) -> list:
try:
if (self.results != None):
res = list()
res.append([self.name, "Allies", "Allies %", "Enemies", "Enemies %" ,"TOTAL", "TOTAL %"])
for cat in self.results:
stat = self.results[cat]
res.append([ cat, stat['allies'], stat['allies%']*100,
stat['enemies'], stat['enemies%']*100,
stat['total'], stat['total%']*100 ])
return res
else:
bu.error('Results have not been calculated yet.')
return None
except Exception as err:
bu.error(exception=err)
return None
def print(self):
try:
if (self.results != None):
print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL"))
for cat in self.results:
stat = self.results[cat]
print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(cat, stat['allies'], stat['allies%']*100, stat['enemies'], stat['enemies%']*100, stat['total'], stat['total%']*100 ))
else:
bu.error('Results have not been calculated yet.')
except Exception as err:
bu.error(exception=err)
return None
class ErrorCatchingArgumentParser(argparse.ArgumentParser): class ErrorCatchingArgumentParser(argparse.ArgumentParser):
def exit(self, status=0, message=None): def exit(self, status=0, message=None):
if status: if status:
@@ -1192,7 +1300,7 @@ OPT_MODE_HELP = 'help'
OPT_MODES = BattleCategory.get_modes() + [OPT_MODE_HELP] OPT_MODES = BattleCategory.get_modes() + [OPT_MODE_HELP]
async def main(argv): async def main(argv):
global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT global wg, wi, WG_APP_ID, OPT_MODE_DEFAULT
# set the directory for the script # set the directory for the script
current_dir = os.getcwd() current_dir = os.getcwd()
os.chdir(os.path.dirname(sys.argv[0])) os.chdir(os.path.dirname(sys.argv[0]))
@@ -1202,7 +1310,8 @@ async def main(argv):
OPT_HIST = False OPT_HIST = False
OPT_STAT_FUNC = StatFunc.get_default() OPT_STAT_FUNC = StatFunc.get_default()
OPT_WORKERS_N = 10 OPT_WORKERS_N = 10
OPT_EXPORT_CSV_FILE = 'export.csv'
OPT_EXPORT_JSON_FILE = 'export.json'
#WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. #WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
# China is not supported since WG API stats are not available there # China is not supported since WG API stats are not available there
@@ -1236,6 +1345,8 @@ async def main(argv):
OPT_HIST = configAnalyzer.getboolean('histograms', OPT_HIST) OPT_HIST = configAnalyzer.getboolean('histograms', OPT_HIST)
OPT_STAT_FUNC = configAnalyzer.get('stat_func', fallback=OPT_STAT_FUNC) OPT_STAT_FUNC = configAnalyzer.get('stat_func', fallback=OPT_STAT_FUNC)
OPT_WORKERS_N = configAnalyzer.getint('workers', OPT_WORKERS_N) OPT_WORKERS_N = configAnalyzer.getint('workers', OPT_WORKERS_N)
OPT_EXPORT_CSV_FILE = configAnalyzer.get('csv_file', OPT_EXPORT_CSV_FILE)
OPT_EXPORT_JSON_FILE = configAnalyzer.get('json_file', OPT_EXPORT_JSON_FILE)
res_categorizations = configAnalyzer.get('categorizations', None) res_categorizations = configAnalyzer.get('categorizations', None)
if res_categorizations != None: if res_categorizations != None:
res_categorizations.replace(' ','') res_categorizations.replace(' ','')
@@ -1289,7 +1400,8 @@ async def main(argv):
parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs') parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs')
parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"') parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"')
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"')
parser.add_argument('-j', '--json', action='store_true', default=False, help='Export data in JSON') parser.add_argument('--json', action='store_true', default=False, help='Export data in JSON format')
parser.add_argument('--csv', action='store_true', default=False, help='Export data in CSV format')
parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT') parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it') parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,20] }, { "player_wins" : 5 }]') parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,20] }, { "player_wins" : 5 }]')
@@ -1395,16 +1507,44 @@ async def main(argv):
blt_cat_list = process_battle_results(teamresults, args) blt_cat_list = process_battle_results(teamresults, args)
bu.verbose_std('WR = ' + StatFunc.get_title()) bu.verbose_std('WR = ' + StatFunc.get_title())
blt_cat_list.print_results() blt_cat_list.print_results()
hist = None
if args.hist: histograms = None
hist = process_player_dist(results, player_stats, stat_id_map, args.json) if args.hist:
histograms = process_player_dist(results, player_stats, stat_id_map)
print_player_dist(histograms)
if args.json: if args.json:
res = blt_cat_list.get_results_json() res = blt_cat_list.get_results_json()
res['histograms'] = hist if histograms != None:
res_hist = dict()
for histogram in histograms.keys():
res_hist[histogram] = histograms[histogram].get_results()
res['histograms'] = res_hist
if args.outfile == '-': if args.outfile == '-':
args.outfile = 'export.json' export_file = OPT_EXPORT_JSON_FILE
await bu.save_JSON(args.outfile, res, pretty=False) ## Excel does not understand pretty JSON else:
bu.verbose_std('Data exported to ' + args.outfile) export_file = args.outfile
await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
bu.verbose_std('Results exported to: ' + export_file)
if args.csv:
rows = blt_cat_list.get_results_list()
if args.hist:
rows.append('')
rows.append(['Player Histograms'])
for histogram in histograms.values():
rows.append('')
rows.extend(histogram.get_results_list())
if args.outfile == '-':
export_file = OPT_EXPORT_CSV_FILE
else:
export_file = args.outfile
with open(export_file, 'w') as csvfile:
# creating a csv writer object
csvwriter = csv.writer(csvfile)
csvwriter.writerows(rows)
bu.verbose_std('Results exported to: ' + export_file)
bu.debug('Finished. Cleaning up..................') bu.debug('Finished. Cleaning up..................')
except Exception as err: except Exception as err:
bu.error(exception=err) bu.error(exception=err)
@@ -1510,7 +1650,7 @@ def filter_min_replays_by_player(results: list, min_replays: int) -> list:
return None return None
def process_player_dist(results: list, player_stats: dict, stat_id_map: dict, res_json: bool = False) -> dict: def process_player_dist(results: list, player_stats: dict, stat_id_map: dict) -> dict:
"""Process player distribution""" """Process player distribution"""
hist_stats = dict() hist_stats = dict()
try: try:
@@ -1527,20 +1667,24 @@ def process_player_dist(results: list, player_stats: dict, stat_id_map: dict, re
else: else:
for stat_field in hist_stats: for stat_field in hist_stats:
hist_stats[stat_field].record_enemy(player_stats[player_remapped][stat_field]) hist_stats[stat_field].record_enemy(player_stats[player_remapped][stat_field])
# Print results
print('\nPlayer Histograms______', end='', flush=True)
res = dict()
for stat_field in hist_stats: for stat_field in hist_stats:
res[stat_field] = hist_stats[stat_field].print(res_json) hist_stats[stat_field].calc_results()
return res return hist_stats
except Exception as err: except Exception as err:
bu.error(exception=err) bu.error(exception=err)
return None return None
def print_player_dist(histograms: dict):
# Print results
print('\nPlayer Histograms______', end='', flush=True)
for hist in histograms.values():
print('')
hist.print()
def process_battle_results(results: dict, args : argparse.Namespace): def process_battle_results(results: dict, args : argparse.Namespace):
"""Process replay battle results""" """Process replay battle results"""
try: try:
blitzutils.py
+117 -104
View File
@@ -10,37 +10,38 @@ from progress.counter import Counter
from decimal import Decimal from decimal import Decimal
from datetime import datetime from datetime import datetime
MAX_RETRIES= 3 MAX_RETRIES = 3
SLEEP = 1.5 SLEEP = 1.5
LOG_LEVELS = { 'silent': 0, 'normal': 1, 'verbose': 2, 'debug': 3 } LOG_LEVELS = {'silent': 0, 'normal': 1, 'verbose': 2, 'debug': 3}
SILENT = 0 SILENT = 0
NORMAL = 1 NORMAL = 1
VERBOSE = 2 VERBOSE = 2
DEBUG = 3 DEBUG = 3
_log_level = NORMAL _log_level = NORMAL
LOG = False LOG = False
LOGGER = None LOGGER = None
## Progress display # Progress display
_progress_N = 100 _progress_N = 100
_progress_i = 0 _progress_i = 0
_progress_id = None _progress_id = None
_progress_obj = None _progress_obj = None
UMASK= os.umask(0) UMASK = os.umask(0)
os.umask(UMASK) os.umask(UMASK)
## ----------------------------------------------------------- # -----------------------------------------------------------
#### Class ThrottledClientSession(aiohttp.ClientSession) # Class ThrottledClientSession(aiohttp.ClientSession)
## ----------------------------------------------------------- # -----------------------------------------------------------
class ThrottledClientSession(aiohttp.ClientSession): class ThrottledClientSession(aiohttp.ClientSession):
"""Rate-throttled client session class inherited from aiohttp.ClientSession)""" """Rate-throttled client session class inherited from aiohttp.ClientSession)"""
#MIN_SLEEP = 0.1 # MIN_SLEEP = 0.1
def __init__(self, rate_limit: float = None, *args,**kwargs) -> None: def __init__(self, rate_limit: float = None, *args, **kwargs) -> None:
super().__init__(*args,**kwargs) super().__init__(*args, **kwargs)
self.rate_limit = None self.rate_limit = None
self._fillerTask = None self._fillerTask = None
self._queue = None self._queue = None
@@ -54,8 +55,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
self.rate_limit = rate_limit self.rate_limit = rate_limit
self._queue = asyncio.Queue(min(2, int(rate_limit)+1)) self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
self._fillerTask = asyncio.create_task(self._filler()) self._fillerTask = asyncio.create_task(self._filler())
def _get_sleep(self) -> list: def _get_sleep(self) -> list:
if self.rate_limit == None: if self.rate_limit == None:
@@ -63,23 +62,22 @@ class ThrottledClientSession(aiohttp.ClientSession):
else: else:
return 1/self.rate_limit return 1/self.rate_limit
def get_rate(self) -> float: def get_rate(self) -> float:
"""Return rate of requests""" """Return rate of requests"""
return self._count / (time.time() - self._start_time) return self._count / (time.time() - self._start_time)
def get_stats(self): def get_stats(self):
"""Get session statistics""" """Get session statistics"""
res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count } res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
'count': self._count}
return res return res
def get_stats_str(self): def get_stats_str(self):
"""Print session statistics""" """Print session statistics"""
return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \ return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
' rate: ' + "{0:.1f}".format(self.get_rate()) + ' requests: ' + str(self._count) ' rate: ' + \
"{0:.1f}".format(self.get_rate()) + \
' requests: ' + str(self._count)
def reset_counters(self): def reset_counters(self):
"""Reset rate counters and return current results""" """Reset rate counters and return current results"""
@@ -88,32 +86,29 @@ class ThrottledClientSession(aiohttp.ClientSession):
self._count = 0 self._count = 0
return res return res
def set_rate_limit(self, rate_limit: float = 1): def set_rate_limit(self, rate_limit: float = 1):
if rate_limit >= 0: if rate_limit >= 0:
self.rate_limit = rate_limit self.rate_limit = rate_limit
return self.rate_limit return self.rate_limit
return None return None
async def close(self) -> None: async def close(self) -> None:
"""Close rate-limiter's "bucket filler" task""" """Close rate-limiter's "bucket filler" task"""
# DEBUG # DEBUG
debug(self.get_stats_str()) debug(self.get_stats_str())
try: try:
if self._fillerTask != None: if self._fillerTask != None:
self._fillerTask.cancel() self._fillerTask.cancel()
await asyncio.wait_for(self._fillerTask, timeout= 0.5) await asyncio.wait_for(self._fillerTask, timeout=0.5)
except (asyncio.CancelledError, asyncio.TimeoutError): except (asyncio.CancelledError, asyncio.TimeoutError):
pass pass
await super().close() await super().close()
async def _filler_simple(self): async def _filler_simple(self):
"""Filler task to fill the leaky bucket algo""" """Filler task to fill the leaky bucket algo"""
try: try:
if self._queue == None: if self._queue == None:
return return
sleep = self._get_sleep() sleep = self._get_sleep()
while True: while True:
self._queue.put(None) self._queue.put(None)
@@ -123,18 +118,17 @@ class ThrottledClientSession(aiohttp.ClientSession):
except Exception as err: except Exception as err:
error(exception=err) error(exception=err)
async def _filler(self): async def _filler(self):
"""Filler task to fill the leaky bucket algo""" """Filler task to fill the leaky bucket algo"""
try: try:
if self._queue == None: if self._queue == None:
return return
sleep = self._get_sleep() sleep = self._get_sleep()
# debug('SLEEP: ' + str(sleep)) # debug('SLEEP: ' + str(sleep))
updated_at = time.monotonic() updated_at = time.monotonic()
fraction = 0 fraction = 0
extra_increment = 0 extra_increment = 0
for i in range(0,self._queue.maxsize): for i in range(0, self._queue.maxsize):
self._queue.put_nowait(i) self._queue.put_nowait(i)
while True: while True:
if not self._queue.full(): if not self._queue.full():
@@ -142,9 +136,10 @@ class ThrottledClientSession(aiohttp.ClientSession):
increment = self.rate_limit * (now - updated_at) increment = self.rate_limit * (now - updated_at)
fraction += increment % 1 fraction += increment % 1
extra_increment = fraction // 1 extra_increment = fraction // 1
items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment)) items_2_add = int(
min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
fraction = fraction % 1 fraction = fraction % 1
for i in range(0,items_2_add): for i in range(0, items_2_add):
self._queue.put_nowait(i) self._queue.put_nowait(i)
updated_at = now updated_at = now
await asyncio.sleep(sleep) await asyncio.sleep(sleep)
@@ -153,24 +148,23 @@ class ThrottledClientSession(aiohttp.ClientSession):
except Exception as err: except Exception as err:
error(exception=err) error(exception=err)
async def _request(self, *args, **kwargs):
async def _request(self, *args,**kwargs):
"""Throttled _request()""" """Throttled _request()"""
if self._queue != None: if self._queue != None:
await self._queue.get() await self._queue.get()
self._queue.task_done() self._queue.task_done()
self._count += 1 self._count += 1
return await super()._request(*args,**kwargs) return await super()._request(*args, **kwargs)
class AsyncLogger(): class AsyncLogger():
"""Async file logger""" """Async file logger"""
def __init__(self) -> None:
def __init__(self) -> None:
self._queue = asyncio.Queue() self._queue = asyncio.Queue()
self._task = None self._task = None
self._file = None self._file = None
async def open(self, logfn: str = None): async def open(self, logfn: str = None):
"""Set logging to file""" """Set logging to file"""
if logfn == None: if logfn == None:
@@ -184,12 +178,11 @@ class AsyncLogger():
self._file = None self._file = None
return False return False
async def logger(self): async def logger(self):
"""Async file logger""" """Async file logger"""
if self._file == None: if self._file == None:
error('No log file defined') error('No log file defined')
return False return False
while True: while True:
try: try:
msg = await self._queue.get() msg = await self._queue.get()
@@ -199,25 +192,24 @@ class AsyncLogger():
return None return None
except Exception as err: except Exception as err:
error(exception=err) error(exception=err)
def log(self, msg: str = ''): def log(self, msg: str = ''):
self._queue.put_nowait(msg) self._queue.put_nowait(msg)
async def close(self): async def close(self):
try: try:
## empty queue & close # empty queue & close
await self._queue.join() await self._queue.join()
self._task.cancel() self._task.cancel()
self._file.close() self._file.close()
except Exception as err: except Exception as err:
error('Error closing log file', err) error('Error closing log file', err)
return None return None
## ----------------------------------------------------------- # -----------------------------------------------------------
#### Utils # Utils
## ----------------------------------------------------------- # -----------------------------------------------------------
def set_debug(debug: bool): def set_debug(debug: bool):
global _log_level global _log_level
@@ -255,7 +247,7 @@ def is_silent() -> bool:
return _log_level == SILENT return _log_level == SILENT
def set_log_level(silent: bool,verbose: bool, debug: bool): def set_log_level(silent: bool, verbose: bool, debug: bool):
global _log_level global _log_level
_log_level = NORMAL _log_level = NORMAL
if silent: _log_level = SILENT if silent: _log_level = SILENT
@@ -274,7 +266,7 @@ def get_log_level_str() -> str:
error('Unknown log level: ' + str(_log_level)) error('Unknown log level: ' + str(_log_level))
async def set_file_logging(logfn = None, add_timestamp = False): async def set_file_logging(logfn=None, add_timestamp=False):
"""Set logging to file""" """Set logging to file"""
global LOG, LOGGER global LOG, LOGGER
LOG = True LOG = True
@@ -297,47 +289,63 @@ async def set_file_logging(logfn = None, add_timestamp = False):
async def close_file_logging(): async def close_file_logging():
global LOG, LOGGER global LOG, LOGGER
LOG=False LOG = False
await LOGGER.close() await LOGGER.close()
LOGGER = None LOGGER = None
def _randomword(length): def _randomword(length):
letters = string.ascii_lowercase letters = string.ascii_lowercase
return ''.join(random.choice(letters) for i in range(length)) return ''.join(random.choice(letters) for i in range(length))
def verbose(msg = "", id = None) -> bool: def verbose(msg="", id=None) -> bool:
"""Print a message""" """Print a message"""
return _print_log_msg('', msg, exception=None, id=id, print_msg= (_log_level >= VERBOSE) ) return _print_log_msg('', msg, exception=None, id=id, print_msg=(_log_level >= VERBOSE))
def verbose_std(msg = "", id = None) -> bool: def verbose_std(msg="", id=None) -> bool:
"""Print a message""" """Print a message"""
return _print_log_msg('', msg, exception=None, id=id, print_msg= (_log_level >= NORMAL) ) return _print_log_msg('', msg, exception=None, id=id, print_msg=(_log_level >= NORMAL))
def warning(msg = "", id = None, force: bool = False) -> bool: def warning(msg="", id=None, force: bool = False) -> bool:
"""Print a warning message""" """Print a warning message"""
return _print_log_msg('Warning', msg, None, id, print_msg= (force or (_log_level >= NORMAL)) ) return _print_log_msg('Warning', msg, None, id, print_msg=(force or (_log_level >= NORMAL)))
def debug(msg = "", id = None, exception = None, force: bool = False) -> bool: def debug(msg="", id=None, exception=None, force: bool = False) -> bool:
"""print a conditional debug message""" """print a conditional debug message"""
if (_log_level >= DEBUG) or force: if (_log_level >= DEBUG) or force:
return _print_log_msg('DEBUG', msg, exception, id) return _print_log_msg('DEBUG', msg, exception, id)
return False return False
def error(msg = "", exception = None, id = None) -> bool: def error(msg="", exception=None, id=None) -> bool:
"""Print an error message""" """Print an error message"""
return _print_log_msg('ERROR', msg, exception, id) return _print_log_msg('ERROR', msg, exception, id)
def log(msg = "", id = None, exception = None) -> bool: def log(msg="", id=None, exception=None) -> bool:
"""print a conditional debug message""" """print a conditional debug message"""
return _print_log_msg('LOG', msg=msg, exception=exception, id=id, print_msg=(_log_level >= DEBUG)) return _print_log_msg('LOG', msg=msg, exception=exception, id=id, print_msg=(_log_level >= DEBUG))
## Copy with pride: https://stackoverflow.com/questions/2203424/python-how-to-retrieve-class-information-from-a-frame-object
def get_class_from_frame(fr):
args, _, _, value_dict = inspect.getargvalues(fr)
# we check the first parameter for the frame function is
# named 'self'
if len(args) and args[0] == 'self':
# in that case, 'self' will be referenced in value_dict
instance = value_dict.get('self', None)
if instance:
# return its class
return getattr(instance, '__class__', None)
# return None otherwise
return None
def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_msg : bool = True): def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_msg : bool = True):
# Use empty prefix to determine standard verbose messages # Use empty prefix to determine standard verbose messages
if not (print_msg or LOG): if not (print_msg or LOG):
@@ -346,8 +354,13 @@ def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_
if prefix != '': if prefix != '':
curframe = inspect.currentframe() curframe = inspect.currentframe()
calframe = inspect.getouterframes(curframe) calframe = inspect.getouterframes(curframe)
class_name = get_class_from_frame(curframe)
caller = calframe[2].function caller = calframe[2].function
prefix = prefix + ': ' + caller + '(): ' if class_name != None:
prefix = prefix + ': ' + class_name + '.' + caller
else:
prefix = prefix + ': ' + caller
prefix = prefix + '(): '
if id != None: if id != None:
prefix = prefix + '[' + str(id) + ']: ' prefix = prefix + '[' + str(id) + ']: '
@@ -535,7 +548,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
if url == None: if url == None:
return None return None
## To avoid excessive use of servers # To avoid excessive use of servers
for retry in range(1,max_tries+1): for retry in range(1,max_tries+1):
try: try:
async with session.get(url) as resp: async with session.get(url) as resp:
@@ -613,9 +626,9 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0])) raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
## ----------------------------------------------------------- # -----------------------------------------------------------
#### Class SlowBar # Class SlowBar
## ----------------------------------------------------------- # -----------------------------------------------------------
class SlowBar(IncrementalBar): class SlowBar(IncrementalBar):
suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).0f h %(remaining_mins).0f mins' suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).0f h %(remaining_mins).0f mins'
@@ -628,9 +641,9 @@ class SlowBar(IncrementalBar):
return (self.eta - (self.eta // 3600)*3600) // 60 return (self.eta - (self.eta // 3600)*3600) // 60
## ----------------------------------------------------------- # -----------------------------------------------------------
#### Class StatsNotFound # Class StatsNotFound
## ----------------------------------------------------------- # -----------------------------------------------------------
class StatsNotFound(Exception): class StatsNotFound(Exception):
pass pass
@@ -638,15 +651,15 @@ class StatsNotFound(Exception):
## ----------------------------------------------------------- # -----------------------------------------------------------
#### Class WG # Class WG
## ----------------------------------------------------------- # -----------------------------------------------------------
class WG: class WG:
URL_WG_CLAN_INFO = 'clans/info/?application_id=' URL_WG_CLAN_INFO = 'clans/info/?application_id='
#URL_WG_PLAYER_TANK_LIST = 'tanks/stats/?fields=tank_id%2Clast_battle_time&application_id=' # URL_WG_PLAYER_TANK_LIST = 'tanks/stats/?fields=tank_id%2Clast_battle_time&application_id='
#URL_WG_PLAYER_TANK_LIST = 'tanks/stats/?fields=account_id%2Ctank_id%2Clast_battle_time%2Cbattle_life_time%2Call&application_id=' # URL_WG_PLAYER_TANK_LIST = 'tanks/stats/?fields=account_id%2Ctank_id%2Clast_battle_time%2Cbattle_life_time%2Call&application_id='
URL_WG_PLAYER_TANK_STATS = 'tanks/stats/?application_id=' URL_WG_PLAYER_TANK_STATS = 'tanks/stats/?application_id='
URL_WG_ACCOUNT_ID = 'account/list/?fields=account_id%2Cnickname&application_id=' URL_WG_ACCOUNT_ID = 'account/list/?fields=account_id%2Cnickname&application_id='
URL_WG_PLAYER_STATS = 'account/info/?application_id=' URL_WG_PLAYER_STATS = 'account/info/?application_id='
@@ -719,7 +732,7 @@ class WG:
SQL_PRUNE_CACHE = """DELETE from {} WHERE update_time < {}""" SQL_PRUNE_CACHE = """DELETE from {} WHERE update_time < {}"""
## Default data. Please use the latest maps.json # Default data. Please use the latest maps.json
maps = { maps = {
"Random": "Random map", "Random": "Random map",
@@ -868,7 +881,7 @@ class WG:
error(exception=err) error(exception=err)
## Class methods ---------------------------------------------------------- # Class methods ----------------------------------------------------------
@classmethod @classmethod
def get_server(cls, account_id: int) -> str: def get_server(cls, account_id: int) -> str:
@@ -1053,7 +1066,7 @@ class WG:
return False return False
## Methods -------------------------------------------------- # Methods --------------------------------------------------
def load_tanks(self, tankopedia_fn: str): def load_tanks(self, tankopedia_fn: str):
"""Load tanks from tankopedia JSON""" """Load tanks from tankopedia JSON"""
if tankopedia_fn == None: if tankopedia_fn == None:
@@ -1245,7 +1258,7 @@ class WG:
url = self.get_url_player_tanks_stats(account_id, tank_ids, fields) url = self.get_url_player_tanks_stats(account_id, tank_ids, fields)
json_data = await self.get_url_JSON(url, self.chk_JSON_status) json_data = await self.get_url_JSON(url, self.chk_JSON_status)
if json_data != None: if json_data != None:
#debug('JSON Response received: ' + str(json_data)) # debug('JSON Response received: ' + str(json_data))
stats = json_data['data'][str(account_id)] stats = json_data['data'][str(account_id)]
if cache: if cache:
await self.put_2_statsQ('tank_stats', [account_id, tank_ids], stats) await self.put_2_statsQ('tank_stats', [account_id, tank_ids], stats)
@@ -1258,7 +1271,7 @@ class WG:
async def get_player_stats(self, account_id: int, fields = [], cache=True, cache_only = False) -> dict: async def get_player_stats(self, account_id: int, fields = [], cache=True, cache_only = False) -> dict:
"""Get player's global stats """ """Get player's global stats """
try: try:
#debug('account_id: ' + str(account_id) ) # debug('account_id: ' + str(account_id) )
stats = None stats = None
# try cached stats first: # try cached stats first:
@@ -1278,7 +1291,7 @@ class WG:
url = self.get_url_player_stats(account_id, fields) url = self.get_url_player_stats(account_id, fields)
json_data = await self.get_url_JSON(url, self.chk_JSON_status) json_data = await self.get_url_JSON(url, self.chk_JSON_status)
if json_data != None: if json_data != None:
#debug('JSON Response received: ' + str(json_data)) # debug('JSON Response received: ' + str(json_data))
stats = json_data['data'][str(account_id)] stats = json_data['data'][str(account_id)]
if cache: if cache:
await self.put_2_statsQ('player_stats', [account_id], stats) await self.put_2_statsQ('player_stats', [account_id], stats)
@@ -1316,7 +1329,7 @@ class WG:
url = self.get_url_player_achievements(list(account_ids), fields) url = self.get_url_player_achievements(list(account_ids), fields)
json_data = await self.get_url_JSON(url, self.chk_JSON_status) json_data = await self.get_url_JSON(url, self.chk_JSON_status)
if (json_data != None) and ('data' in json_data): if (json_data != None) and ('data' in json_data):
#debug('JSON Response received: ' + str(json_data)) # debug('JSON Response received: ' + str(json_data))
for account_id in json_data['data'].keys(): for account_id in json_data['data'].keys():
stats[account_id] = json_data['data'][account_id] stats[account_id] = json_data['data'][account_id]
if cache: if cache:
@@ -1444,7 +1457,7 @@ class WG:
return None return None
try: try:
self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE) self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE)
## Create cache tables table # Create cache tables table
await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL) await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL)
await self.cache.execute(WG.SQL_PLAYER_STATS_CREATE_TBL) await self.cache.execute(WG.SQL_PLAYER_STATS_CREATE_TBL)
await self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_CREATE_TBL) await self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_CREATE_TBL)
@@ -1569,13 +1582,13 @@ class WG:
async with self.cache.execute(sql_query, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor: async with self.cache.execute(sql_query, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor:
tank_ids = set(tank_ids) tank_ids = set(tank_ids)
async for row in cursor: async for row in cursor:
#debug('account_id: ' + str(account_id) + ': 1') # debug('account_id: ' + str(account_id) + ': 1')
if row[3] == None: if row[3] == None:
# None/null stats found in cache # None/null stats found in cache
# i.e. stats have been requested, but not returned from WG API # i.e. stats have been requested, but not returned from WG API
tank_ids.remove(row[1]) tank_ids.remove(row[1])
continue continue
#debug('account_id: ' + str(account_id) + ': 2') # debug('account_id: ' + str(account_id) + ': 2')
stats.append(json.loads(row[3])) stats.append(json.loads(row[3]))
# debug('account_id: ' + str(account_id) + ': 3') # debug('account_id: ' + str(account_id) + ': 3')
tank_ids.remove(row[1]) tank_ids.remove(row[1])
@@ -1596,15 +1609,15 @@ class WG:
# test for cacheDB existence # test for cacheDB existence
debug('Trying cached stats first') debug('Trying cached stats first')
if self.cache == None: if self.cache == None:
#debug('No cache DB') # debug('No cache DB')
raise StatsNotFound('No cache DB in use') raise StatsNotFound('No cache DB in use')
async with self.cache.execute(WG.SQL_PLAYER_STATS_CACHED, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor: async with self.cache.execute(WG.SQL_PLAYER_STATS_CACHED, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor:
row = await cursor.fetchone() row = await cursor.fetchone()
#debug('account_id: ' + str(account_id) + ': 1') # debug('account_id: ' + str(account_id) + ': 1')
if row == None: if row == None:
# no cached stats found, marked with an empty array # no cached stats found, marked with an empty array
#debug('No cached stats found') # debug('No cached stats found')
raise StatsNotFound('No cached stats found') raise StatsNotFound('No cached stats found')
debug('Cached stats found') debug('Cached stats found')
@@ -1628,15 +1641,15 @@ class WG:
# test for cacheDB existence # test for cacheDB existence
debug('Trying cached stats first') debug('Trying cached stats first')
if self.cache == None: if self.cache == None:
#debug('No cache DB') # debug('No cache DB')
raise StatsNotFound('No cache DB in use') raise StatsNotFound('No cache DB in use')
async with self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_CACHED, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor: async with self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_CACHED, [account_id, NOW() - WG.CACHE_GRACE_TIME] ) as cursor:
row = await cursor.fetchone() row = await cursor.fetchone()
#debug('account_id: ' + str(account_id) + ': 1') # debug('account_id: ' + str(account_id) + ': 1')
if row == None: if row == None:
# no cached stats found, marked with an empty array # no cached stats found, marked with an empty array
#debug('No cached stats found') # debug('No cached stats found')
raise StatsNotFound('No cached stats found') raise StatsNotFound('No cached stats found')
debug('Cached stats found') debug('Cached stats found')
@@ -1655,9 +1668,9 @@ class WG:
return None return None
## ----------------------------------------------------------- # -----------------------------------------------------------
#### Class WoTinspector # Class WoTinspector
## ----------------------------------------------------------- # -----------------------------------------------------------
class WoTinspector: class WoTinspector:
URL_WI = 'https://replays.wotinspector.com' URL_WI = 'https://replays.wotinspector.com'
@@ -1740,7 +1753,7 @@ class WoTinspector:
hash.update(data) hash.update(data)
replay_id = hash.hexdigest() replay_id = hash.hexdigest()
## Testing if the replay has already been posted # Testing if the replay has already been posted
json_resp = await self.get_replay_JSON(replay_id) json_resp = await self.get_replay_JSON(replay_id)
if json_resp != None: if json_resp != None:
debug('Already uploaded: ' + title, id=N) debug('Already uploaded: ' + title, id=N)
@@ -1755,7 +1768,7 @@ class WoTinspector:
} }
url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote) url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
#debug('URL: ' + url) # debug('URL: ' + url)
headers ={'Content-type': 'application/x-www-form-urlencoded'} headers ={'Content-type': 'application/x-www-form-urlencoded'}
payload = { 'file' : (filename, base64.b64encode(data)) } payload = { 'file' : (filename, base64.b64encode(data)) }
except Exception as err: except Exception as err:
@@ -1944,13 +1957,13 @@ class BlitzStars:
async def get_player_stats(self, account_id: int, cache=True): async def get_player_stats(self, account_id: int, cache=True):
"""Get player stats from BlitzStars""" """Get player stats from BlitzStars"""
try: try:
#debug('account_id: ' + str(account_id) ) # debug('account_id: ' + str(account_id) )
stats = None stats = None
if cache: if cache:
error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()') error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
sys.exit(1) sys.exit(1)
#stats = await self.get_cached_player_stats(account_id,fields) # stats = await self.get_cached_player_stats(account_id,fields)
# stats found unless StatsNotFound exception is raised # stats found unless StatsNotFound exception is raised
return stats return stats
@@ -1964,7 +1977,7 @@ class BlitzStars:
url = self.get_url_player_stats(account_id) url = self.get_url_player_stats(account_id)
stats = await get_url_JSON(self.session, url, self.chk_JSON_player_stats) stats = await get_url_JSON(self.session, url, self.chk_JSON_player_stats)
if stats != None: if stats != None:
#debug('JSON Response received: ' + str(json_data)) # debug('JSON Response received: ' + str(json_data))
if cache: if cache:
error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()') error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
sys.exit(1) sys.exit(1)
@@ -1977,13 +1990,13 @@ class BlitzStars:
async def get_player_tank_stats(self, account_id: int, tank_id = None, cache=True): async def get_player_tank_stats(self, account_id: int, tank_id = None, cache=True):
"""Get player stats for all his tanks from BlitzStars""" """Get player stats for all his tanks from BlitzStars"""
try: try:
#debug('account_id: ' + str(account_id) ) # debug('account_id: ' + str(account_id) )
stats = None stats = None
if cache: if cache:
error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()') error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
sys.exit(1) sys.exit(1)
#stats = await self.get_cached_player_stats(account_id,fields) # stats = await self.get_cached_player_stats(account_id,fields)
# stats found unless StatsNotFound exception is raised # stats found unless StatsNotFound exception is raised
return stats return stats
@@ -2001,7 +2014,7 @@ class BlitzStars:
url = self.get_url_player_tank_stats(account_id, tank_id) url = self.get_url_player_tank_stats(account_id, tank_id)
stats = await get_url_JSON(self.session, url, self.chk_JSON_tank_stats) stats = await get_url_JSON(self.session, url, self.chk_JSON_tank_stats)
if stats != None: if stats != None:
#debug('JSON Response received: ' + str(json_data)) # debug('JSON Response received: ' + str(json_data))
if cache: if cache:
error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()') error('CACHE NOT IMPLEMENTED YET FOR BlitzStars()')
sys.exit(1) sys.exit(1)