Major update: local cache added, naming according to PEP 8

Class WG uses now Sqlite3 DB to cache stats retrieved from WG API. This enables faster rerun of replay analysis

Variable, function and script names changed to PEB-8 standard (lower_case_with_underscores)

extract_tankopedia.py

@@ -21,7 +21,7 @@ async def main(argv):
     os.chdir(os.path.dirname(sys.argv[0]))
     
     parser = argparse.ArgumentParser(description='Extract Tankopedia data from Blitz game files')
-    parser.add_argument('blitzAppBase', type=str,  metavar="BLITZAPP_FOLDER", default=".", help='Base dir of the Blitz App files')
+    parser.add_argument('blitz_app_base', type=str,  metavar="BLITZAPP_FOLDER", default=".", help='Base dir of the Blitz App files')
     parser.add_argument('tanks', type=str, default='tanks.json', nargs='?', metavar="TANKS_FILE", help='File to write Tankopedia')
     parser.add_argument('maps', type=str, default='maps.json', nargs='?', metavar='MAPS_FILE', help='File to write map names')
     parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
@@ -29,19 +29,19 @@ async def main(argv):
         
     args = parser.parse_args()
 
-    bu.setVerbose(args.verbose)
+    bu.set_verbose(args.verbose)
-    bu.setDebug(args.debug)
+    bu.set_debug(args.debug)
     wg = WG()
     
     tasks = []
-    for nation in wg.nations:
+    for nation in wg.NATIONS:
-        tasks.append(asyncio.create_task(extractTanks(args.blitzAppBase, nation)))
+        tasks.append(asyncio.create_task(extract_tanks(args.blitz_app_base, nation)))
 
     tanklist = []
     for tanklist_tmp in await asyncio.gather(*tasks):
         tanklist.extend(tanklist_tmp)
     
-    tank_strs, map_strs = await readUserStrs(args.blitzAppBase)
+    tank_strs, map_strs = await read_user_strs(args.blitz_app_base)
 
     json_data = None
     userStrs = {}
@@ -56,7 +56,7 @@ async def main(argv):
             bu.error('Unexpected error when reading file: ' + args.tanks + ' : ' + str(err))
 
     async with aiofiles.open(args.tanks, 'w', encoding="utf8") as outfile:
-        new_tanks, new_userStrs = await convertTankNames(tanklist, tank_strs)
+        new_tanks, new_userStrs = await convert_tank_names(tanklist, tank_strs)
         # merge old and new tankopedia
         tanks.update(new_tanks)
         userStrs.update(new_userStrs) 
@@ -65,6 +65,7 @@ async def main(argv):
         tankopedia['meta'] = { "count":  len(tanks) }
         tankopedia['data'] = tanks
         tankopedia['userStr'] = userStrs
+        bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
         await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
     
     if args.maps != None:
@@ -78,14 +79,15 @@ async def main(argv):
         # merge old and new map data
         maps.update(map_strs)
         async with aiofiles.open(args.maps, 'w', encoding="utf8") as outfile:
+            bu.verbose_std('New maps file \'' + args.maps + '\' contains ' + str(len(maps)) + ' maps')
             await outfile.write(json.dumps(maps, ensure_ascii=False, indent=4, sort_keys=True))
 
     return None
     
-async def extractTanks(blitzAppBase : str, nation: str):
+async def extract_tanks(blitz_app_base : str, nation: str):
 
     tanks = []
-    list_xml_file = blitzAppBase + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
+    list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
     if not os.path.isfile(list_xml_file): 
         print('ERROR: cannot open ' + list_xml_file)
         return None
@@ -96,24 +98,24 @@ async def extractTanks(blitzAppBase : str, nation: str):
             for data in tankList['root'].keys():
                 tank_xml = tankList['root'][data]
                 tank = {}
-                tank['tank_id'] = await getTankID(nation, int(tank_xml['id']))
+                tank['tank_id'] = await get_tank_id(nation, int(tank_xml['id']))
                 tank['userStr'] = tank_xml['userString']
                 tank['nation'] = nation
                 tank['tier'] = int(tank_xml['level'])
                 #debug(tank_xml['price'])
                 tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
-                tank['type'] = await getTankType(tank_xml['tags'])
+                tank['type'] = await get_tank_type(tank_xml['tags'])
                 tanks.append(tank)
         except Exception as err:
             bu.error(err)
             sys.exit(2)
     return tanks
 
-async def readUserStrs(blitzAppBase : str) -> dict:
+async def read_user_strs(blitz_app_base : str) -> dict:
     """Read user strings to convert map and tank names"""
     tank_strs = {}
     map_strs = {}
-    filename = blitzAppBase + BLITZAPP_STRINGS
+    filename = blitz_app_base + BLITZAPP_STRINGS
     bu.debug('Opening file: ' + filename + ' for reading UserStrings')
     try:
         async with aiofiles.open(filename, 'r', encoding="utf8") as f:
@@ -135,7 +137,7 @@ async def readUserStrs(blitzAppBase : str) -> dict:
 
     return tank_strs, map_strs
     
-async def convertTankNames(tanklist : list, tank_strs: dict) -> dict:
+async def convert_tank_names(tanklist : list, tank_strs: dict) -> dict:
     """Convert tank names for Tankopedia"""
     tankopedia = {}
     userStrs = {}
@@ -162,12 +164,12 @@ async def convertTankNames(tanklist : list, tank_strs: dict) -> dict:
 
     return tankopedia_sorted, userStrs_sorted
 
-async def getTankID(nation: str, tankID : int) -> int:
+async def get_tank_id(nation: str, tank_id : int) -> int:
-    return (tankID << 8) + (wg.nation_id[nation] << 4) + 1 
+    return (tank_id << 8) + (wg.NATION_ID[nation] << 4) + 1 
 
-async def getTankType(tagstr : str):
+async def get_tank_type(tagstr : str):
     tags = tagstr.split(' ')
-    for t_type in wg.tank_type:
+    for t_type in wg.TANK_TYPE:
         if tags[0] == t_type:
             return t_type
     return None

get_tankopedia.py

@@ -4,6 +4,9 @@
 
 import sys, argparse, json, os, inspect, aiohttp, asyncio, re, logging
 from pathlib import Path
+import blitzutils as bu
+from blitzutils import WG
+from blitzutils import WoTinspector
 
 logging.getLogger("asyncio").setLevel(logging.DEBUG)
 
@@ -47,8 +50,7 @@ async def get_tankopedia(filename: str):
             return None
 
 
-def main(argv):
+async def main(argv):
-    global VERBOSE, DEBUG
     # set the directory for the script
     os.chdir(os.path.dirname(sys.argv[0]))
 
@@ -58,12 +60,16 @@ def main(argv):
     parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Verbose mode')
     
     args = parser.parse_args()
-    VERBOSE = args.verbose
+    bu.set_verbose(args.verbose)
-    DEBUG = args.debug
+    bu.set_debug(args.debug)
-    if DEBUG: VERBOSE = True
+    
+    wi = WoTinspector()
 
-    asyncio.run(get_tankopedia(args.outfile))
+    await wi.get_tankopedia(args.outfile)
+
+    await wi.close()
     
 
+### main()
 if __name__ == "__main__":
-   main(sys.argv[1:])
+   asyncio.run(main(sys.argv[1:]), debug=False)

upload_wotb_replays.py

@@ -70,28 +70,30 @@ async def main(argv):
 	parser.add_argument('files', metavar='FILE1 [FILE2 ...]', type=str, nargs='+', help='Files to read. Use \'-\' for STDIN"')
 	args = parser.parse_args()
 
-	bu.setVerbose(args.verbose)
+	bu.set_verbose(args.verbose)
-	bu.setDebug(args.debug)
+	bu.set_debug(args.debug)
-	if args.silent:
+	bu.set_silent(args.silent)   	
-		bu.setVerbose(False)    	
 
 	wg = WG(WG_appID, args.tankopedia, args.mapfile)
 	wi = WoTinspector()
 
-		#### Connect to MongoDB (TBD)
+	#### Connect to MongoDB (TBD)
+	client = None
+	db = None
 	if args.db:
 		try:
 			client = motor.motor_asyncio.AsyncIOMotorClient(DB_SERVER,DB_PORT, authSource=DB_AUTH, username=DB_USER, password=DB_PASSWD, ssl=DB_SSL, ssl_cert_reqs=DB_CERT_REQ, ssl_certfile=DB_CERT, tlsCAFile=DB_CA)
 			db = client[DB_NAME]
-		except:
+			bu.debug('Database connection initiated')
-			db = None
+		except Exception as err: 
+			bu.error("Could no initiate DB connection: Disabling DB", err) 
+			args.db = False
 			pass
 	else:
-		db = None
+		bu.debug('No DB in use')
-
 
 	if args.account != None:
-		args.accountID = await wg.getAccountID(args.account)
+		args.accountID = await wg.get_account_id(args.account)
 		bu.debug('WG  account_id: ' + str(args.accountID))
 
 	if args.accountID == None: 
@@ -194,7 +196,7 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
 				async with aiofiles.open(replay_json_fn) as fp:
 					replay_json = json.loads(await fp.read())
 					#if replay_json['status'] == 'ok':
-					if wi.chkJSONreplay(replay_json):
+					if wi.chk_JSON_replay(replay_json):
 						bu.verbose(msg_str + title + ' has already been posted. Skipping.' )
 						SKIPPED_N += 1
 						queue.task_done()						
@@ -211,9 +213,9 @@ async def replayWorker(queue: asyncio.Queue, db: motor.motor_asyncio.AsyncIOMoto
 			async with aiofiles.open(filename,'rb') as fp:
 				filename = os.path.basename(filename)
 				bu.debug(msg_str + 'File:  ' + filename)
-				json_resp = await wi.postReplay(await fp.read(), filename, account_id, title, priv, N)
+				json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
 				if json_resp != None:
-					if (await bu.saveJSON(replay_json_fn,json_resp)):						
+					if (await bu.save_JSON(replay_json_fn,json_resp)):						
 						bu.debug(msg_str + 'Replay saved OK: ' + filename )
 					else:
 						bu.error(msg_str + 'Error saving replay: ' + filename)
@@ -234,7 +236,7 @@ async def saveReplay2DB(db: motor.motor_asyncio.AsyncIOMotorDatabase, replay: di
 			return False
 		dbc = db[DB_C_REPLAYS]
 		replay_link = replay['data']['download_url'] 
-		replay_id = wi.getReplayID(replay_link)
+		replay_id = wi.get_replay_id(replay_link)
 		replay['_id'] = replay_id
 		await dbc.insert_one(replay)
 		bu.debug('Replay added to database')
@@ -254,8 +256,8 @@ def getTitle(replayfile: str, title: str, i : int) -> str:
 		try:
 			filename = os.path.basename(replayfile)	
 			bu.debug(filename)
-			map_usrStrs = wg.getMapUserStrs()
+			map_usrStrs = wg.get_map_user_strs()
-			tank_userStrs = wg.getTankUserStrs()
+			tank_userStrs = wg.get_tank_user_strs()
 			
 			#p = re.compile('\\d{8}_\\d{4}_(.+)_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
 			# update 6.2 changed the file name format. Bug fixed 2019-09-09 Jylpah