Merge pull request #7 from Jylpah/dev

Config file update, longer cache expiry

analyze_wotb_replays.py

@@ -483,9 +483,10 @@ async def main(argv):
 	# WG account id of the uploader: 
 	# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
 	OPT_DB			= configOptions.getboolean('opt_DB', False)
-	OPT_EXTENDED 	= configOptions.getboolean('opt_extended', False)
-	OPT_HIST		= configOptions.getboolean('opt_hist', False)
-	OPT_STAT_FUNC	= configOptions.get('opt_stat_func', fallback='tank_tier')
+	OPT_EXTENDED 	= configOptions.getboolean('opt_analyzer_extended', False)
+	OPT_HIST		= configOptions.getboolean('opt_analyzer_hist', False)
+	OPT_STAT_FUNC	= configOptions.get('opt_analyzer_stat_func', fallback='player')
+	OPT_WORKERS_N = configOptions.getint('opt_analyzer_workers', 10)
 
 	configWG 		= config['WG']
 	WG_ID			= configWG.getint('wg_id', None)
@@ -503,9 +504,7 @@ async def main(argv):
 	DB_PASSWD 	= configDB.get('db_password', "PASSWORD")
 	DB_CERT 	= configDB.get('db_ssl_cert_file', None)
 	DB_CA 		= configDB.get('db_ssl_ca_file', None)
-
-	TASK_N = 7
-
+	
 	parser = argparse.ArgumentParser(description='Analyze Blitz replay JSONs from WoTinspector.com')
 	parser.add_argument('--output', default='plain', choices=['json', 'plain', 'db'], help='Select output mode: JSON, plain text or database')
 	parser.add_argument('-id', dest='account_id', type=int, default=WG_ID, help='WG account_id to analyze')
@@ -569,7 +568,7 @@ async def main(argv):
 		scanner_task = asyncio.create_task(mk_replayQ(replayQ, args, db))
 		bu.debug('Replay scanner started')
 		# Start tasks to process the Queue
-		for i in range(TASK_N):
+		for i in range(OPT_WORKERS_N):
 			reader_tasks.append(asyncio.create_task(replay_reader(replayQ, i, args)))
 			bu.debug('ReplayReader ' + str(i) + ' started')
 
@@ -589,7 +588,7 @@ async def main(argv):
 			results.extend(res[0])
 			players.update(res[1])
 
-		(player_stats, stat_id_map) = await process_player_stats(players, TASK_N, args, db)
+		(player_stats, stat_id_map) = await process_player_stats(players, OPT_WORKERS_N, args, db)
 		bu.verbose('')
 		bu.debug('Number of player stats: ' + str(len(player_stats)))
 		teamresults = calc_team_stats(results, player_stats, stat_id_map, args)

blitzstats.ini.default

@@ -1,14 +1,13 @@
 [OPTIONS]
 ## You may set your preferred commandline param defaults here
-#opt_extended    = False
-#opt_hist        = False
-#opt_stat_func   = tank_tier
+#opt_analyzer_extended    = False
+#opt_analyzer_hist        = False
+#opt_analyzer_stat_func   = player
 
 [WG]
 ## Find your WG account_id :  https://developers.wargaming.net/reference/all/wotb/account/list/
 #wg_id		    = 0
 ## WG API limits requests / sec to 10 for client apps
 ## Check https://developers.wargaming.net/documentation/guide/getting-started/
-#wg_rate_limit   = 10
 #wg_app_id       = XXXXXXXXXXX
 

blitzutils.py

@@ -437,7 +437,7 @@ class WG:
     URL_WG_ACCOUNT_ID        = 'account/list/?fields=account_id%2Cnickname&application_id='
     URL_WG_PLAYER_STATS      = 'account/info/?application_id='
     CACHE_DB_FILE           = '.blitzutils_cache.sqlite3' 
-    CACHE_GRACE_TIME        =  2*7*24*3600  # 2 weeks cache
+    CACHE_GRACE_TIME        =  30*24*3600  # 30 days cache
 
     # sql_create_player_stats_tbl = """CREATE TABLE IF NOT EXISTS player_stats (
     #                             account_id INTEGER NOT NULL,

upload_wotb_replays.py

@@ -14,7 +14,6 @@ FILE_CONFIG 	= 'blitzstats.ini'
 DEBUG 	= False
 VERBOSE = False
 SLEEP 		= 2
-N_WORKERS	= 5
 MAX_RETRIES = 3
 REPLAY_N 	= 0
 SKIPPED_N 	= 0
@@ -33,6 +32,9 @@ async def main(argv):
 	config = configparser.ConfigParser()
 	config.read(FILE_CONFIG)
 
+	configOptions 	= config['OPTIONS']
+	OPT_WORKERS_N = configOptions.getint('opt_uploader_workers', 5)
+
 	configWG 		= config['WG']
 	# WG account id of the uploader: 
 	# # Find it here: https://developers.wargaming.net/reference/all/wotb/account/list/
@@ -41,15 +43,12 @@ async def main(argv):
 	WG_RATE_LIMIT	= configWG.getint('wg_rate_limit', 10)
 
 	parser = argparse.ArgumentParser(description='Post replays(s) to WoTinspector.com and retrieve battle data')
-	#parser.add_argument('--output', default='file', choices=['file', 'files', 'db'] , help='Select output mode: single/multiple files or database')
 	parser.add_argument('-id', dest='accountID', type=int, default=WG_ID, help='WG account_id')
 	parser.add_argument('-a', '--account', dest='account', type=str, default=None, help='Uploader\'s WG account name. Format: ACCOUNT_NAME@SERVER')
 	parser.add_argument('-t','--title', type=str, default=None, help='Title for replays. Use NN for continous numbering. Default is filename-based numbering')
 	parser.add_argument('-p', '--private', dest="private", action='store_true', default=False, help='Set replays private on WoTinspector.com')
-	parser.add_argument('--tasks', dest='N_tasks', type=int, default=N_WORKERS, help='Number of worker threads')
 	parser.add_argument('--tankopedia', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default: "tanks.json"')
 	parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default: "maps.json"')
-#	parser.add_argument('--db', action='store_true', default=USE_DB, help='Use DB - You are unlikely to have it')
 	parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
 	parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode')
 	parser.add_argument('-s', '--silent', action='store_true', default=False, help='Silent mode')
@@ -76,8 +75,7 @@ async def main(argv):
 		# Make replay Queue
 		tasks.append(asyncio.create_task(mkReplayQ(queue, args.files, args.title)))
 		# Start tasks to process the Queue
-		for i in range(args.N_tasks):
-			#tasks.append(asyncio.create_task(replayWorker(queue, db, i, args.accountID, args.private)))
+		for i in range(OPT_WORKERS_N):
 			tasks.append(asyncio.create_task(replayWorker(queue, i, args.accountID, args.private)))
 			bu.debug('Task ' + str(i) + ' started')
 		
@@ -164,7 +162,6 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
 
 		#bu.debug(msg_str + replay_json_fn)
 		try:
-			#if os.path.exists(replay_json_fn) and os.path.isfile(replay_json_fn):
 			if os.path.isfile(replay_json_fn):
 				async with aiofiles.open(replay_json_fn) as fp:
 					replay_json = json.loads(await fp.read())