mirror of
https://github.com/Jylpah/blitz-tools.git
synced 2026-10-21 02:07:06 +02:00
Removing database support. It breaks the replay fetching logic.
This commit is contained in:
Jylpah committed
1 parent
8cce091fa8
commit
4259f967ef
1 file changed
+56
-54
+56
-54
@@ -9,7 +9,7 @@ from blitzutils import WoTinspector
|
||||
logging.getLogger("asyncio").setLevel(logging.DEBUG)
|
||||
|
||||
FILE_CONFIG = 'blitzstats.ini'
|
||||
DB_C_REPLAYS = 'Replays'
|
||||
# DB_C_REPLAYS = 'Replays'
|
||||
|
||||
DEBUG = False
|
||||
VERBOSE = False
|
||||
@@ -31,29 +31,29 @@ async def main(argv):
|
||||
## Read config
|
||||
config = configparser.ConfigParser()
|
||||
config.read(FILE_CONFIG)
|
||||
configDB = config['DATABASE']
|
||||
DB_SERVER = configDB.get('db_server', 'localhost')
|
||||
DB_PORT = configDB.getint('db_port', 27017)
|
||||
DB_SSL = configDB.getboolean('db_ssl', False)
|
||||
DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE)
|
||||
DB_AUTH = configDB.get('db_auth', 'admin')
|
||||
DB_NAME = configDB.get('db_name', 'BlitzStats')
|
||||
DB_USER = configDB.get('db_user', 'mongouser')
|
||||
DB_PASSWD = configDB.get('db_password', "PASSWORD")
|
||||
DB_CERT = configDB.get('db_ssl_cert_file', None)
|
||||
DB_CA = configDB.get('db_ssl_ca_file', None)
|
||||
# configDB = config['DATABASE']
|
||||
# DB_SERVER = configDB.get('db_server', 'localhost')
|
||||
# DB_PORT = configDB.getint('db_port', 27017)
|
||||
# DB_SSL = configDB.getboolean('db_ssl', False)
|
||||
# DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE)
|
||||
# DB_AUTH = configDB.get('db_auth', 'admin')
|
||||
# DB_NAME = configDB.get('db_name', 'BlitzStats')
|
||||
# DB_USER = configDB.get('db_user', 'mongouser')
|
||||
# DB_PASSWD = configDB.get('db_password', "PASSWORD")
|
||||
# DB_CERT = configDB.get('db_ssl_cert_file', None)
|
||||
# DB_CA = configDB.get('db_ssl_ca_file', None)
|
||||
|
||||
bu.debug('DB_SERVER: ' + DB_SERVER)
|
||||
bu.debug('DB_PORT: ' + str(DB_PORT))
|
||||
bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
|
||||
bu.debug('DB_AUTH: ' + DB_AUTH)
|
||||
bu.debug('DB_NAME: ' + DB_NAME)
|
||||
# bu.debug('DB_SERVER: ' + DB_SERVER)
|
||||
# bu.debug('DB_PORT: ' + str(DB_PORT))
|
||||
# bu.debug('DB_SSL: ' + "True" if DB_SSL else "False")
|
||||
# bu.debug('DB_AUTH: ' + DB_AUTH)
|
||||
# bu.debug('DB_NAME: ' + DB_NAME)
|
||||
|
||||
configGeneral = config['GENERAL']
|
||||
# WG account id of the uploader:
|
||||
# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
|
||||
WG_ID = configGeneral.getint('wg_id', 0)
|
||||
USE_DB = configGeneral.getint('use_DB', False)
|
||||
# USE_DB = configGeneral.getint('use_DB', False)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data')
|
||||
#parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database')
|
||||
@@ -64,7 +64,7 @@ async def main(argv):
|
||||
parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads')
|
||||
parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
|
||||
parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
|
||||
parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it')
|
||||
# parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it')
|
||||
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
|
||||
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
|
||||
parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
|
||||
@@ -78,19 +78,19 @@ async def main(argv):
|
||||
wi = WoTinspector()
|
||||
|
||||
#### Connect to MongoDB (TBD)
|
||||
client = None
|
||||
db = None
|
||||
if args.db:
|
||||
try:
|
||||
client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
|
||||
db = client[DB_NAME]
|
||||
bu.debug('Database connection initiated')
|
||||
except Exception as err:
|
||||
bu.error("Could no initiate DB connection: Disabling DB", err)
|
||||
args.db = False
|
||||
pass
|
||||
else:
|
||||
bu.debug('No DB in use')
|
||||
# client = None
|
||||
# db = None
|
||||
# if args.db:
|
||||
# try:
|
||||
# client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
|
||||
# db = client[DB_NAME]
|
||||
# bu.debug('Database connection initiated')
|
||||
# except Exception as err:
|
||||
# bu.error("Could no initiate DB connection: Disabling DB", err)
|
||||
# args.db = False
|
||||
# pass
|
||||
# else:
|
||||
# bu.debug('No DB in use')
|
||||
|
||||
if args.account != None:
|
||||
args.accountID = await wg.get_account_id(args.account)
|
||||
@@ -107,7 +107,8 @@ async def main(argv):
|
||||
tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
|
||||
# Start tasks to process the Queue
|
||||
for i in range(args.N_tasks):
|
||||
tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private)))
|
||||
#tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private)))
|
||||
tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
|
||||
bu.debug('Task ' + str(i) + ' started')
|
||||
|
||||
bu.debug('Waiting for the replay scanner to finish')
|
||||
@@ -175,7 +176,8 @@ async def mkQueueItem(filename : str, title : str) -> list:
|
||||
REPLAY_N +=1
|
||||
return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)]
|
||||
|
||||
async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False):
|
||||
#async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False):
|
||||
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
|
||||
"""Async Worker to process the replay queue"""
|
||||
global SKIPPED_N
|
||||
global ERROR_N
|
||||
@@ -199,7 +201,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
|
||||
if wi.chk_JSON_replay(replay_json):
|
||||
bu.verbose(msg_str + title + ' has already been posted. Skipping.' )
|
||||
SKIPPED_N += 1
|
||||
await saveReplay2DB(db, replay_json)
|
||||
#await saveReplay2DB(db, replay_json)
|
||||
queue.task_done()
|
||||
continue
|
||||
else:
|
||||
@@ -221,7 +223,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
|
||||
else:
|
||||
bu.error(msg_str + 'Error saving replay: ' + filename)
|
||||
ERROR_N += 1
|
||||
await saveReplay2DB(db, json_resp)
|
||||
# await saveReplay2DB(db, json_resp)
|
||||
except Exception as err:
|
||||
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
|
||||
bu.debug(msg_str + 'Marking task done')
|
||||
@@ -230,24 +232,24 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
|
||||
return None
|
||||
|
||||
|
||||
async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool:
|
||||
"""Save retriever replay JSON into DB (mongo)"""
|
||||
try:
|
||||
if db == None:
|
||||
return False
|
||||
dbc = db[DB_C_REPLAYS]
|
||||
replay_link = replay['data']['download_url']
|
||||
replay_id = wi.get_replay_id(replay_link)
|
||||
replay['_id'] = replay_id
|
||||
await dbc.insert_one(replay)
|
||||
bu.debug('Replay added to database')
|
||||
except pymongo.errors.DuplicateKeyError as err:
|
||||
bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id)
|
||||
return False
|
||||
except Exception as err:
|
||||
bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err))
|
||||
return False
|
||||
return True
|
||||
# async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool:
|
||||
# """Save retriever replay JSON into DB (mongo)"""
|
||||
# try:
|
||||
# if db == None:
|
||||
# return False
|
||||
# dbc = db[DB_C_REPLAYS]
|
||||
# replay_link = replay['data']['download_url']
|
||||
# replay_id = wi.get_replay_id(replay_link)
|
||||
# replay['_id'] = replay_id
|
||||
# await dbc.insert_one(replay)
|
||||
# bu.debug('Replay added to database')
|
||||
# except pymongo.errors.DuplicateKeyError as err:
|
||||
# bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id)
|
||||
# return False
|
||||
# except Exception as err:
|
||||
# bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err))
|
||||
# return False
|
||||
# return True
|
||||
|
||||
|
||||
def getTitle(replayfile: str, title: str, i : int) -> str:
|
||||
|
||||
Reference in new issue
Block a user