GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-21 10:17:05 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4 from Jylpah/dev

Browse Source 
New categorization options, fancy progress bar, database support
This commit is contained in:
Jylpah authored and GitHub committed 2020-02-15 21:10:30 +02:00
2 parents b4123c479d e51ab20f1e
commit bb560a0766
3 files changed
+613 -375
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

analyze_wotb_replays.py
+533 -309
View File
File diff suppressed because it is too large. Load diff
blitzutils.py
+24 -12
View File
@@ -257,8 +257,10 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
if session == None:
error('Session must be initialized first')
sys.exit(1)
## To avoid excessive use of servers
if url == None:
return None
## To avoid excessive use of servers
for retry in range(1,max_tries+1):
try:
async with session.get(url) as resp:
@@ -272,7 +274,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
if retry == max_tries:
break
debug('Retrying URL [' + str(retry) + '/' + str(max_tries) + ']: ' + url )
await asyncio.sleep(SLEEP)
await asyncio.sleep(SLEEP)
except aiohttp.ClientError as err:
debug("Could not retrieve URL: " + url)
@@ -309,10 +311,15 @@ def bld_dict_hierarcy(d : dict, key : str, value) -> dict:
## -----------------------------------------------------------
class SlowBar(IncrementalBar):
suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).1f hours'
suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).0f h %(remaining_mins).0f mins'
@property
def remaining_hours(self):
return Decimal(self.eta / 3600).quantize(Decimal('1.0'))
return self.eta // 3600
@property
def remaining_mins(self):
return (self.eta - (self.eta // 3600)*3600) // 60
## -----------------------------------------------------------
@@ -541,8 +548,11 @@ class WG:
@classmethod
def get_server(cls, account_id: int) -> str:
"""Get Realm/server of an account based on account ID"""
if account_id > 1e9:
if account_id > 2e9:
if account_id >= 1e9:
if account_id >= 3e9:
debug('Chinese account/server: not stats available')
return None
if account_id >= 2e9:
return 'asia'
return 'na'
else:
@@ -736,7 +746,9 @@ class WG:
def get_url_clan_info(self, server: str, clan_id: int) -> str:
try:
return self.URL_WG_SERVER[server] + self.URL_WG_CLAN_INFO + self.WG_app_id + '&clan_id=' + str(clan_id)
if server == None:
return None
return self.URL_WG_SERVER[server] + self.URL_WG_CLAN_INFO + self.WG_app_id + '&clan_id=' + str(clan_id)
except Exception as err:
if (server == None) or (server.lower() not in WG.ACCOUNT_ID_SERVER.keys()):
error('No server name or invalid server name given: ' + server if (server != None) else '')
@@ -751,7 +763,8 @@ class WG:
def get_url_player_tanks_stats(self, account_id: int, tank_ids = [], fields = []) -> str:
server = self.get_server(account_id)
if server == None:
return None
if (tank_ids != None) and (len(tank_ids) > 0):
tank_id_str= '&tank_id=' + '%2C'.join([ str(x) for x in tank_ids])
else:
@@ -770,6 +783,8 @@ class WG:
def get_url_player_stats(self, account_id, fields) -> str:
try:
server = self.get_server(account_id)
if server == None:
return None
if (fields != None) and (len(fields) > 0):
field_str = '&fields=' + '%2C'.join(fields)
else:
@@ -821,10 +836,7 @@ class WG:
async def get_player_tank_stats(self, account_id: int, tank_ids = [], fields = [], cache=True) -> dict:
"""Get player's stats (WR, # of battles) in a tank or all tanks (empty tank_ids[])"""
# debug('Started')
try:
#debug('account_id: ' + str(account_id) + ' TankID: ' + ','.join([ str(id) for id in tank_ids]))
stats = None
# try cached stats first:
upload_wotb_replays.py
+56 -54
View File
@@ -9,7 +9,7 @@ from blitzutils import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini'
DB_C_REPLAYS = 'Replays'
# DB_C_REPLAYS = 'Replays'
DEBUG = False
VERBOSE = False
@@ -31,29 +31,29 @@ async def main(argv):
## Read config
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
configDB = config['DATABASE']
DB_SERVER = configDB.get('db_server', 'localhost')
DB_PORT = configDB.getint('db_port', 27017)
DB_SSL = configDB.getboolean('db_ssl', False)
DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE)
DB_AUTH = configDB.get('db_auth', 'admin')
DB_NAME = configDB.get('db_name', 'BlitzStats')
DB_USER = configDB.get('db_user', 'mongouser')
DB_PASSWD = configDB.get('db_password', "PASSWORD")
DB_CERT = configDB.get('db_ssl_cert_file', None)
DB_CA = configDB.get('db_ssl_ca_file', None)
# configDB = config['DATABASE']
# DB_SERVER = configDB.get('db_server', 'localhost')
# DB_PORT = configDB.getint('db_port', 27017)
# DB_SSL = configDB.getboolean('db_ssl', False)
# DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE)
# DB_AUTH = configDB.get('db_auth', 'admin')
# DB_NAME = configDB.get('db_name', 'BlitzStats')
# DB_USER = configDB.get('db_user', 'mongouser')
# DB_PASSWD = configDB.get('db_password', "PASSWORD")
# DB_CERT = configDB.get('db_ssl_cert_file', None)
# DB_CA = configDB.get('db_ssl_ca_file', None)
bu.debug('DB_SERVER: ' + DB_SERVER)
bu.debug('DB_PORT: ' + str(DB_PORT))
bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
bu.debug('DB_AUTH: ' + DB_AUTH)
bu.debug('DB_NAME: ' + DB_NAME)
# bu.debug('DB_SERVER: ' + DB_SERVER)
# bu.debug('DB_PORT: ' + str(DB_PORT))
# bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
# bu.debug('DB_AUTH: ' + DB_AUTH)
# bu.debug('DB_NAME: ' + DB_NAME)
configGeneral = config['GENERAL']
# WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configGeneral.getint('wg_id', 0)
USE_DB = configGeneral.getint('use_DB', False)
# USE_DB = configGeneral.getint('use_DB', False)
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data')
#parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database')
@@ -64,7 +64,7 @@ async def main(argv):
parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads')
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it')
# parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
@@ -78,19 +78,19 @@ async def main(argv):
wi = WoTinspector()
#### Connect to MongoDB (TBD)
client = None
db = None
if args.db:
try:
client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
db = client[DB_NAME]
bu.debug('Database connection initiated')
except Exception as err:
bu.error("Could no initiate DB connection: Disabling DB", err)
args.db = False
pass
else:
bu.debug('No DB in use')
# client = None
# db = None
# if args.db:
# try:
# client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
# db = client[DB_NAME]
# bu.debug('Database connection initiated')
# except Exception as err:
# bu.error("Could no initiate DB connection: Disabling DB", err)
# args.db = False
# pass
# else:
# bu.debug('No DB in use')
if args.account != None:
args.accountID = await wg.get_account_id(args.account)
@@ -107,7 +107,8 @@ async def main(argv):
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
# Start tasks to process the Queue
for i in range(args.N_tasks):
tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private)))
#tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private)))
tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
bu.debug('Task ' + str(i) + ' started')
bu.debug('Waiting for the replay scanner to finish')
@@ -175,7 +176,8 @@ async def mkQueueItem(filename : str, title : str) -> list:
REPLAY_N +=1
return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)]
async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False):
#async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False):
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
"""Async Worker to process the replay queue"""
global SKIPPED_N
global ERROR_N
@@ -199,7 +201,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
if wi.chk_JSON_replay(replay_json):
bu.verbose(msg_str + title + ' has already been posted. Skipping.' )
SKIPPED_N += 1
await saveReplay2DB(db, replay_json)
#await saveReplay2DB(db, replay_json)
queue.task_done()
continue
else:
@@ -221,7 +223,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
else:
bu.error(msg_str + 'Error saving replay: ' + filename)
ERROR_N += 1
await saveReplay2DB(db, json_resp)
# await saveReplay2DB(db, json_resp)
except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
bu.debug(msg_str + 'Marking task done')
@@ -230,24 +232,24 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
return None
async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool:
"""Save retriever replay JSON into DB (mongo)"""
try:
if db == None:
return False
dbc = db[DB_C_REPLAYS]
replay_link = replay['data']['download_url']
replay_id = wi.get_replay_id(replay_link)
replay['_id'] = replay_id
await dbc.insert_one(replay)
bu.debug('Replay added to database')
except pymongo.errors.DuplicateKeyError as err:
bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id)
return False
except Exception as err:
bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err))
return False
return True
# async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool:
# """Save retriever replay JSON into DB (mongo)"""
# try:
# if db == None:
# return False
# dbc = db[DB_C_REPLAYS]
# replay_link = replay['data']['download_url']
# replay_id = wi.get_replay_id(replay_link)
# replay['_id'] = replay_id
# await dbc.insert_one(replay)
# bu.debug('Replay added to database')
# except pymongo.errors.DuplicateKeyError as err:
# bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id)
# return False
# except Exception as err:
# bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err))
# return False
# return True
def getTitle(replayfile: str, title: str, i : int) -> str: