diff --git a/upload_wotb_replays.py b/upload_wotb_replays.py index ec41ee2..088cfa0 100644 --- a/upload_wotb_replays.py +++ b/upload_wotb_replays.py @@ -9,7 +9,7 @@ from blitzutils import WoTinspector logging.getLogger("asyncio").setLevel(logging.DEBUG) FILE_CONFIG = 'blitzstats.ini' -DB_C_REPLAYS = 'Replays' +# DB_C_REPLAYS = 'Replays' DEBUG = False VERBOSE = False @@ -31,29 +31,29 @@ async def main(argv): ## Read config config = configparser.ConfigParser() config.read(FILE_CONFIG) - configDB = config['DATABASE'] - DB_SERVER = configDB.get('db_server', 'localhost') - DB_PORT = configDB.getint('db_port', 27017) - DB_SSL = configDB.getboolean('db_ssl', False) - DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) - DB_AUTH = configDB.get('db_auth', 'admin') - DB_NAME = configDB.get('db_name', 'BlitzStats') - DB_USER = configDB.get('db_user', 'mongouser') - DB_PASSWD = configDB.get('db_password', "PASSWORD") - DB_CERT = configDB.get('db_ssl_cert_file', None) - DB_CA = configDB.get('db_ssl_ca_file', None) + # configDB = config['DATABASE'] + # DB_SERVER = configDB.get('db_server', 'localhost') + # DB_PORT = configDB.getint('db_port', 27017) + # DB_SSL = configDB.getboolean('db_ssl', False) + # DB_CERT_REQ = configDB.getint('db_ssl_req', ssl.CERT_NONE) + # DB_AUTH = configDB.get('db_auth', 'admin') + # DB_NAME = configDB.get('db_name', 'BlitzStats') + # DB_USER = configDB.get('db_user', 'mongouser') + # DB_PASSWD = configDB.get('db_password', "PASSWORD") + # DB_CERT = configDB.get('db_ssl_cert_file', None) + # DB_CA = configDB.get('db_ssl_ca_file', None) - bu.debug('DB_SERVER: ' + DB_SERVER) - bu.debug('DB_PORT: ' + str(DB_PORT)) - bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") - bu.debug('DB_AUTH: ' + DB_AUTH) - bu.debug('DB_NAME: ' + DB_NAME) + # bu.debug('DB_SERVER: ' + DB_SERVER) + # bu.debug('DB_PORT: ' + str(DB_PORT)) + # bu.debug('DB_SSL: ' + "True" if DB_SSL else "False") + # bu.debug('DB_AUTH: ' + DB_AUTH) + # bu.debug('DB_NAME: ' + DB_NAME) configGeneral = config['GENERAL'] # WG account id of the uploader: # # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/ WG_ID = configGeneral.getint('wg_id', 0) - USE_DB = configGeneral.getint('use_DB', False) +# USE_DB = configGeneral.getint('use_DB', False) parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data') #parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database') @@ -64,7 +64,7 @@ async def main(argv): parser.add_argument('--tasks', dest='N_tasks', type=int, default=5, help='Number of worker threads') parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"') parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"') - parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') +# parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it') parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode') @@ -78,19 +78,19 @@ async def main(argv): wi = WoTinspector() #### Connect to MongoDB (TBD) - client = None - db = None - if args.db: - try: - client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) - db = client[DB_NAME] - bu.debug('Database connection initiated') - except Exception as err: - bu.error("Could no initiate DB connection: Disabling DB", err) - args.db = False - pass - else: - bu.debug('No DB in use') + # client = None + # db = None + # if args.db: + # try: + # client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA) + # db = client[DB_NAME] + # bu.debug('Database connection initiated') + # except Exception as err: + # bu.error("Could no initiate DB connection: Disabling DB", err) + # args.db = False + # pass + # else: + # bu.debug('No DB in use') if args.account != None: args.accountID = await wg.get_account_id(args.account) @@ -107,7 +107,8 @@ async def main(argv): tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title))) # Start tasks to process the Queue for i in range(args.N_tasks): - tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private))) + #tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private))) + tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private))) bu.debug('Task ' + str(i) + ' started') bu.debug('Waiting for the replay scanner to finish') @@ -175,7 +176,8 @@ async def mkQueueItem(filename : str, title : str) -> list: REPLAY_N +=1 return [filename, REPLAY_N, getTitle(filename, title, REPLAY_N)] -async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False): +#async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMotorDatabase, workerID: int, account_id: int, priv = False): +async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False): """Async Worker to process the replay queue""" global SKIPPED_N global ERROR_N @@ -199,7 +201,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto if wi.chk_JSON_replay(replay_json): bu.verbose(msg_str + title + ' has already been posted. Skipping.' ) SKIPPED_N += 1 - await saveReplay2DB(db, replay_json) + #await saveReplay2DB(db, replay_json) queue.task_done() continue else: @@ -221,7 +223,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto else: bu.error(msg_str + 'Error saving replay: ' + filename) ERROR_N += 1 - await saveReplay2DB(db, json_resp) + # await saveReplay2DB(db, json_resp) except Exception as err: bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) ) bu.debug(msg_str + 'Marking task done') @@ -230,24 +232,24 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto return None -async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool: - """Save retriever replay JSON into DB (mongo)""" - try: - if db == None: - return False - dbc = db[DB_C_REPLAYS] - replay_link = replay['data']['download_url'] - replay_id = wi.get_replay_id(replay_link) - replay['_id'] = replay_id - await dbc.insert_one(replay) - bu.debug('Replay added to database') - except pymongo.errors.DuplicateKeyError as err: - bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) - return False - except Exception as err: - bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) - return False - return True +# async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: dict) -> bool: +# """Save retriever replay JSON into DB (mongo)""" +# try: +# if db == None: +# return False +# dbc = db[DB_C_REPLAYS] +# replay_link = replay['data']['download_url'] +# replay_id = wi.get_replay_id(replay_link) +# replay['_id'] = replay_id +# await dbc.insert_one(replay) +# bu.debug('Replay added to database') +# except pymongo.errors.DuplicateKeyError as err: +# bu.verbose_std('Replay already in the DB: ' + replay['data']['summary']['title'] + ' : id=' + replay_id) +# return False +# except Exception as err: +# bu.error('Unexpected Exception: ' + str(type(err)) + ' : ' + str(err)) +# return False +# return True def getTitle(replayfile: str, title: str, i : int) -> str: