GonnaHetzMe/blitz-tools
0
0 Fork 0
mirror of https://github.com/Jylpah/blitz-tools.git synced 2026-10-24 11:39:52 +02:00
Code Issues Projects Releases Wiki Activity

Removing database support. It breaks the replay fetching logic.

Browse Source
This commit is contained in:
Jylpah committed 2020-02-11 21:22:18 +02:00
1 parent 8cce091fa8
commit 4259f967ef
1 file changed
+56 -54
Download Patch File Download Diff File Expand all files Collapse all files
upload_wotb_replays.py
+56 -54
View File
@@ -9,7 +9,7 @@ from blitzutils import WoTinspector
logging.getLogger("asyncio").setLevel(logging.DEBUG) logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini' FILE_CONFIG = 'blitzstats.ini'
DB_C_REPLAYS = 'Replays' # DB_C_REPLAYS = 'Replays'
DEBUG = False DEBUG = False
VERBOSE = False VERBOSE = False
@@ -31,29 +31,29 @@ async def main(argv):
## Read config ## Read config
config = configparser.ConfigParser() config = configparser.ConfigParser()
config.read(FILE_CONFIG) config.read(FILE_CONFIG)
configDB = config['DATABASE'] # configDB = config['DATABASE']
DB_SERVER = configDB.get('db_server', 'localhost') # DB_SERVER = configDB.get('db_server', 'localhost')
DB_PORT = configDB.getint('db_port', 27017) # DB_PORT = configDB.getint('db_port', 27017)
DB_SSL = configDB.getboolean('db_ssl', False) # DB_SSL = configDB.getboolean('db_ssl', False)
DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) # DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE)
DB_AUTH = configDB.get('db_auth', 'admin') # DB_AUTH = configDB.get('db_auth', 'admin')
DB_NAME = configDB.get('db_name', 'BlitzStats') # DB_NAME = configDB.get('db_name', 'BlitzStats')
DB_USER = configDB.get('db_user', 'mongouser') # DB_USER = configDB.get('db_user', 'mongouser')
DB_PASSWD = configDB.get('db_password', "PASSWORD") # DB_PASSWD = configDB.get('db_password', "PASSWORD")
DB_CERT = configDB.get('db_ssl_cert_file', None) # DB_CERT = configDB.get('db_ssl_cert_file', None)
DB_CA = configDB.get('db_ssl_ca_file', None) # DB_CA = configDB.get('db_ssl_ca_file', None)
bu.debug('DB_SERVER: ' + DB_SERVER) # bu.debug('DB_SERVER: ' + DB_SERVER)
bu.debug('DB_PORT: ' + str(DB_PORT)) # bu.debug('DB_PORT: ' + str(DB_PORT))
bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") # bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
bu.debug('DB_AUTH: ' + DB_AUTH) # bu.debug('DB_AUTH: ' + DB_AUTH)
bu.debug('DB_NAME: ' + DB_NAME) # bu.debug('DB_NAME: ' + DB_NAME)
configGeneral = config['GENERAL'] configGeneral = config['GENERAL']
# WG account id of the uploader: # WG account id of the uploader:
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
WG_ID = configGeneral.getint('wg_id', 0) WG_ID = configGeneral.getint('wg_id', 0)
USE_DB = configGeneral.getint('use_DB', False) # USE_DB = configGeneral.getint('use_DB', False)
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data') parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data')
#parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database') #parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database')
@@ -64,7 +64,7 @@ async def main(argv):
parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads') parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads')
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') # parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
@@ -78,19 +78,19 @@ async def main(argv):
wi = WoTinspector() wi = WoTinspector()
#### Connect to MongoDB (TBD) #### Connect to MongoDB (TBD)
client = None # client = None
db = None # db = None
if args.db: # if args.db:
try: # try:
client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) # client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
db = client[DB_NAME] # db = client[DB_NAME]
bu.debug('Database connection initiated') # bu.debug('Database connection initiated')
except Exception as err: # except Exception as err:
bu.error("Could no initiate DB connection: Disabling DB", err) # bu.error("Could no initiate DB connection: Disabling DB", err)
args.db = False # args.db = False
pass # pass
else: # else:
bu.debug('No DB in use') # bu.debug('No DB in use')
if args.account != None: if args.account != None:
args.accountID = await wg.get_account_id(args.account) args.accountID = await wg.get_account_id(args.account)
@@ -107,7 +107,8 @@ async def main(argv):
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
# Start tasks to process the Queue # Start tasks to process the Queue
for i in range(args.N_tasks): for i in range(args.N_tasks):
tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private))) #tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private)))
tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
bu.debug('Task ' + str(i) + ' started') bu.debug('Task ' + str(i) + ' started')
bu.debug('Waiting for the replay scanner to finish') bu.debug('Waiting for the replay scanner to finish')
@@ -175,7 +176,8 @@ async def mkQueueItem(filename : str, title : str) -> list:
REPLAY_N +=1 REPLAY_N +=1
return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)] return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)]
async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False): #async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False):
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
"""Async Worker to process the replay queue""" """Async Worker to process the replay queue"""
global SKIPPED_N global SKIPPED_N
global ERROR_N global ERROR_N
@@ -199,7 +201,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
if wi.chk_JSON_replay(replay_json): if wi.chk_JSON_replay(replay_json):
bu.verbose(msg_str + title + ' has already been posted. Skipping.' ) bu.verbose(msg_str + title + ' has already been posted. Skipping.' )
SKIPPED_N += 1 SKIPPED_N += 1
await saveReplay2DB(db, replay_json) #await saveReplay2DB(db, replay_json)
queue.task_done() queue.task_done()
continue continue
else: else:
@@ -221,7 +223,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
else: else:
bu.error(msg_str + 'Error saving replay: ' + filename) bu.error(msg_str + 'Error saving replay: ' + filename)
ERROR_N += 1 ERROR_N += 1
await saveReplay2DB(db, json_resp) # await saveReplay2DB(db, json_resp)
except Exception as err: except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
bu.debug(msg_str + 'Marking task done') bu.debug(msg_str + 'Marking task done')
@@ -230,24 +232,24 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
return None return None
async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool: # async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool:
"""Save retriever replay JSON into DB (mongo)""" # """Save retriever replay JSON into DB (mongo)"""
try: # try:
if db == None: # if db == None:
return False # return False
dbc = db[DB_C_REPLAYS] # dbc = db[DB_C_REPLAYS]
replay_link = replay['data']['download_url'] # replay_link = replay['data']['download_url']
replay_id = wi.get_replay_id(replay_link) # replay_id = wi.get_replay_id(replay_link)
replay['_id'] = replay_id # replay['_id'] = replay_id
await dbc.insert_one(replay) # await dbc.insert_one(replay)
bu.debug('Replay added to database') # bu.debug('Replay added to database')
except pymongo.errors.DuplicateKeyError as err: # except pymongo.errors.DuplicateKeyError as err:
bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) # bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id)
return False # return False
except Exception as err: # except Exception as err:
bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) # bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err))
return False # return False
return True # return True
def getTitle(replayfile: str, title: str, i : int) -> str: def getTitle(replayfile: str, title: str, i : int) -> str: