Merge pull request #23 from Jylpah/stable-csv

CSV export

analyze_wotb_replays.py

@@ -4,7 +4,7 @@
 # Script Analyze WoT Blitz replays
 
 import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole
-import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections
+import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections, csv
 from datetime import datetime, timedelta, date
 import blitzutils as bu
 from blitzutils import WG
@@ -89,7 +89,6 @@ class StatFunc:
 		return cls.STAT_FUNC[cls._stat_func][3]
 
 
-
 replay_summary_flds = [
 	'battle_result',
 	'battle_type',
@@ -434,6 +433,22 @@ class BattleCategorizationList():
 		return None
 
 
+	def get_results_list(self, urls: bool = False):
+		try:
+			res = list()			
+			for cat in self.categorizations_list:
+				res.append('')
+				res.extend(self.categorizations[cat].get_results_list())			
+			if urls:
+				res.append('')
+				for url in self.urls:
+					res.append(url)
+			return res
+		except Exception as err:
+			bu.error(exception=err)
+		return None
+
+
 class BattleCategorization():
 
 	RESULT_CAT_HEADER_LEN = 15
@@ -479,7 +494,7 @@ class BattleCategorization():
 		print(FORMAT.format(self.category_key, self.title), end=' ')		
 
 
-	def record_result(self, result: dict):
+	def record_result(self, result: dict) -> bool:
 		try:
 			cat = self.get_category(result)
 			self.categories[cat].record_result(result)
@@ -501,6 +516,7 @@ class BattleCategorization():
 
 
 	def print_headers(self):
+		try:
 			if len(self.title) > self.RESULT_CAT_HEADER_LEN:
 				cat_name = self.title[:self.RESULT_CAT_HEADER_LEN]
 			else:
@@ -508,6 +524,17 @@ class BattleCategorization():
 				header = self.RESULT_CAT_HEADER_FRMT.format(cat_name) 
 			BattleCategory.print_headers(header)
 			return None
+		except Exception as err:
+			bu.error(exception = err)
+
+
+	def get_headers(self) -> list:
+		"""Get BattleCategory headers as list"""
+		try:
+			return BattleCategory.get_headers(self.title)		
+		except Exception as err:
+			bu.error(exception = err)
+		return None
 
 
 	def print_results(self):
@@ -528,7 +555,7 @@ class BattleCategorization():
 		return None
 
 
-	def get_results_json(self):
+	def get_results_json(self) -> dict:
 		"""Get results in JSON format"""
 		try:
 			results = dict()
@@ -543,6 +570,19 @@ class BattleCategorization():
 		return None
 
 	
+	def get_results_list(self) -> list:
+		"""Get results in  list format (for CSV printing)"""
+		try:
+			results = list()
+			results.append(self.get_headers())
+			for cat in self.get_categories():
+				results.append( [cat] + self.categories[cat].get_results_list() )
+			return results
+		except KeyError as err:
+			bu.error('Key not found', err)  
+		return None
+
+
 class BattleTotals(BattleCategorization):
 	TOTAL = 'Total'
 
@@ -1008,6 +1048,20 @@ class BattleCategory():
 		return None
 
 
+	@classmethod
+	def get_headers(cls, header: str = None) -> list:
+		try:
+			headers = [ header ]
+			for field in cls.result_fields:
+				headers.append(cls.get_field_name(field))
+			return headers
+		except KeyError as err:
+			bu.error('Key not found', exception=err)  
+		except Exception as err:
+			bu.error(exception=err) 
+		return None
+
+
 	def __init__(self):
 		try:
 			
@@ -1091,6 +1145,22 @@ class BattleCategory():
 		return None
 
 	
+	def get_results_list(self) -> list:
+		"""Return results as a list"""
+		if not self.results_ready:
+			bu.error('Stats have not been calculated yet. call calc_results() before get_results()')
+		try:
+			results = list()
+			for field in self.result_fields:
+				results.append(self._result_fields[field][3].format(self.results[field]))
+			return results
+		except KeyError as err:
+			bu.error('Key not found', err)  
+		except Exception as err:
+			bu.error(exception=err) 
+		return None
+	
+
 	def print_results(self):
 		if not self.results_ready:
 			bu.error('Stats have not been calculated yet. call calc_results() before printing()')
@@ -1121,6 +1191,7 @@ class PlayerHistogram():
 		self.allies 	= [0] * self.ncat
 		self.enemies 	= [0] * self.ncat
 		self.total 		= [0] * self.ncat
+		self.results 	= None
 	
 	def record_ally(self, stat: float):
 		for cat in range(0, self.ncat):
@@ -1150,7 +1221,7 @@ class PlayerHistogram():
 		return self.cat_format.format(float(val * self.cat_factor))
 
 
-	def print(self, ret_json: bool = False):
+	def calc_results(self):
 		N_enemies = 0
 		N_allies = 0
 		res = dict()
@@ -1160,10 +1231,7 @@ class PlayerHistogram():
 			N_allies += self.allies[cat]
 		N_total = N_allies + N_enemies
 	
-		print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL"))
 		for cat in range(0, self.ncat):			
-			print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(self.get_category_name(cat), self.allies[cat], self.allies[cat]/N_allies*100, self.enemies[cat], self.enemies[cat]/N_enemies*100, self.allies[cat] + self.enemies[cat], (self.allies[cat] + self.enemies[cat])/N_total*100 ))
-			if ret_json:
 			stat = dict()
 			stat['allies'] 		= self.allies[cat]
 			stat['allies%'] 	= stat['allies'] / N_allies
@@ -1171,11 +1239,51 @@ class PlayerHistogram():
 			stat['enemies%'] 	= stat['enemies'] / N_enemies
 			stat['total'] 		= self.allies[cat] + self.enemies[cat]
 			stat['total%'] 		= stat['total'] / N_total
+			
 			res[self.get_category_name(cat)] = stat
 
+		self.results = res
 		return res
 
 
+	def get_results(self) -> dict:
+		return self.results
+
+	
+	def get_results_list(self) -> list:
+		try:
+			if (self.results != None):
+				res = list()
+				res.append([self.name, "Allies", "Allies %",  "Enemies", "Enemies %" ,"TOTAL", "TOTAL %"])
+				for cat in self.results:
+					stat = self.results[cat]
+					res.append([ cat, stat['allies'], stat['allies%']*100,  
+							stat['enemies'], stat['enemies%']*100, 
+							stat['total'], stat['total%']*100 ])
+				return res
+			else:
+				bu.error('Results have not been calculated yet.')
+				return None
+		except Exception as err:
+			bu.error(exception=err)
+		return None
+
+
+	def print(self):
+		try:
+			if (self.results != None):
+				print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL"))
+				for cat in self.results:
+					stat = self.results[cat]
+					print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(cat, stat['allies'], stat['allies%']*100, stat['enemies'], stat['enemies%']*100, stat['total'], stat['total%']*100 ))
+			else:
+				bu.error('Results have not been calculated yet.')
+		except Exception as err:
+			bu.error(exception=err)
+		return None
+
+
+
 class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 	def exit(self, status=0, message=None):
 		if status:
@@ -1202,7 +1310,8 @@ async def main(argv):
 	OPT_HIST			= False
 	OPT_STAT_FUNC		= StatFunc.get_default()
 	OPT_WORKERS_N 		= 10
-
+	OPT_EXPORT_CSV_FILE	 = 'export.csv'
+	OPT_EXPORT_JSON_FILE = 'export.json'
 
 	#WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					 	# China is not supported since WG API stats are not available there
@@ -1236,6 +1345,8 @@ async def main(argv):
 					OPT_HIST		= configAnalyzer.getboolean('histograms', OPT_HIST)
 					OPT_STAT_FUNC	= configAnalyzer.get('stat_func', fallback=OPT_STAT_FUNC)
 					OPT_WORKERS_N 	= configAnalyzer.getint('workers', OPT_WORKERS_N)
+					OPT_EXPORT_CSV_FILE  = configAnalyzer.get('csv_file', OPT_EXPORT_CSV_FILE)
+					OPT_EXPORT_JSON_FILE = configAnalyzer.get('json_file', OPT_EXPORT_JSON_FILE)
 					res_categorizations  = configAnalyzer.get('categorizations', None)
 					if res_categorizations != None:
 						res_categorizations.replace(' ','')
@@ -1289,7 +1400,8 @@ async def main(argv):
 		parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs')
 		parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"')
 		parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"')
-		parser.add_argument('-j', '--json', action='store_true', default=False, help='Export data in JSON')
+		parser.add_argument('--json', action='store_true', default=False, help='Export data in JSON format')
+		parser.add_argument('--csv', action='store_true', default=False, help='Export data in CSV format')
 		parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
 		parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
 		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,20] }, { "player_wins" : 5 }]')
@@ -1395,16 +1507,44 @@ async def main(argv):
 			blt_cat_list = process_battle_results(teamresults, args)
 			bu.verbose_std('WR = ' + StatFunc.get_title())
 			blt_cat_list.print_results()
-			hist = None
+
+			histograms = None
 			if args.hist:
-				 hist = process_player_dist(results, player_stats, stat_id_map, args.json)
+				histograms = process_player_dist(results, player_stats, stat_id_map)
+				print_player_dist(histograms)			
+			
 			if args.json:
 				res = blt_cat_list.get_results_json()
-				res['histograms'] = hist
+				if histograms != None:
+					res_hist = dict()
+					for histogram in histograms.keys():
+						res_hist[histogram] = histograms[histogram].get_results()
+					res['histograms'] = res_hist
 				if args.outfile == '-':
-					args.outfile = 'export.json'					
+					export_file = OPT_EXPORT_JSON_FILE
-				await bu.save_JSON(args.outfile, res, pretty=False) ## Excel does not understand pretty JSON
+				else:
-				bu.verbose_std('Data exported to ' + args.outfile)
+					export_file = args.outfile
+				await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
+				bu.verbose_std('Results exported to: ' + export_file)
+			
+			if args.csv:
+				rows = blt_cat_list.get_results_list()
+				if args.hist:
+					rows.append('')
+					rows.append(['Player Histograms'])
+					for histogram in histograms.values():
+						rows.append('')
+						rows.extend(histogram.get_results_list())
+				if args.outfile == '-':
+					export_file = OPT_EXPORT_CSV_FILE
+				else:
+					export_file = args.outfile
+				with open(export_file, 'w') as csvfile:  
+					# creating a csv writer object  
+					csvwriter = csv.writer(csvfile)        					
+					csvwriter.writerows(rows)
+					bu.verbose_std('Results exported to: ' + export_file)			
+			
 			bu.debug('Finished. Cleaning up..................')
 		except Exception as err:
 			bu.error(exception=err)
@@ -1510,7 +1650,7 @@ def filter_min_replays_by_player(results: list, min_replays: int) -> list:
 	return None	
 
 
-def process_player_dist(results: list, player_stats: dict, stat_id_map: dict, res_json: bool = False) -> dict:
+def process_player_dist(results: list, player_stats: dict, stat_id_map: dict) -> dict:
 	"""Process player distribution"""
 	hist_stats = dict()
 	try:
@@ -1528,19 +1668,23 @@ def process_player_dist(results: list, player_stats: dict, stat_id_map: dict, re
 						for stat_field in hist_stats:
 							hist_stats[stat_field].record_enemy(player_stats[player_remapped][stat_field])
 
-		# Print results
-		print('\nPlayer Histograms______', end='', flush=True)
-		res = dict()
 		for stat_field in hist_stats:
-			res[stat_field] = hist_stats[stat_field].print(res_json)
+			hist_stats[stat_field].calc_results()
 		
-		return res	
+		return hist_stats	
 	except Exception as err:
 		bu.error(exception=err)
 
 	return None
 
 
+def print_player_dist(histograms: dict):
+	# Print results
+	print('\nPlayer Histograms______', end='', flush=True)
+	for hist in histograms.values():
+		print('')
+		hist.print()
+
 def process_battle_results(results: dict, args : argparse.Namespace):
 	"""Process replay battle results""" 
 	try:

blitzutils.py

@@ -22,7 +22,7 @@ _log_level  = NORMAL
 LOG = False
 LOGGER = None
 
-## Progress display
+# Progress display
 _progress_N = 100
 _progress_i = 0
 _progress_id = None
@@ -31,9 +31,10 @@ _progress_obj = None
 UMASK = os.umask(0)
 os.umask(UMASK)
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class ThrottledClientSession(aiohttp.ClientSession)
+# Class ThrottledClientSession(aiohttp.ClientSession)
-## -----------------------------------------------------------
+# -----------------------------------------------------------
+
 
 class ThrottledClientSession(aiohttp.ClientSession):
     """Rate-throttled client session class inherited from aiohttp.ClientSession)"""
@@ -55,31 +56,28 @@ class ThrottledClientSession(aiohttp.ClientSession):
             self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
             self._fillerTask = asyncio.create_task(self._filler())
 
-     
-
     def _get_sleep(self) -> list:
         if self.rate_limit == None:
             return None
         else:
             return 1/self.rate_limit
 
-
     def get_rate(self) -> float:
         """Return rate of requests"""
         return self._count / (time.time() - self._start_time)
 
-
     def get_stats(self):
         """Get session statistics"""
-        res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
+        res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
+                                     'count': self._count}
         return res
 
-
     def get_stats_str(self):
         """Print session statistics"""
         return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
-                ' rate: ' +  "{0:.1f}".format(self.get_rate()) + ' requests: ' + str(self._count)
+                ' rate: ' + \
-
+                    "{0:.1f}".format(self.get_rate()) + \
+                                     ' requests: ' + str(self._count)
 
     def reset_counters(self):
         """Reset rate counters and return current results"""
@@ -88,14 +86,12 @@ class ThrottledClientSession(aiohttp.ClientSession):
         self._count = 0
         return res
 
-
     def set_rate_limit(self, rate_limit: float = 1):
         if rate_limit >= 0:
             self.rate_limit = rate_limit
             return self.rate_limit
         return None
 
-
     async def close(self) -> None:
         """Close rate-limiter's "bucket filler" task"""
         # DEBUG
@@ -108,7 +104,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
             pass
         await super().close()
 
-
     async def _filler_simple(self):
         """Filler task to fill the leaky bucket algo"""
         try:
@@ -123,7 +118,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
         except Exception as err:
             error(exception=err)
 
-
     async def _filler(self):
         """Filler task to fill the leaky bucket algo"""
         try:
@@ -142,7 +136,8 @@ class ThrottledClientSession(aiohttp.ClientSession):
                     increment = self.rate_limit * (now - updated_at)
                     fraction += increment % 1
                     extra_increment = fraction // 1
-                    items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
+                    items_2_add = int(
+                        min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
                     fraction = fraction % 1
                     for i in range(0, items_2_add):
                         self._queue.put_nowait(i)
@@ -153,7 +148,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
         except Exception as err:
             error(exception=err)
 
-
     async def _request(self, *args, **kwargs):
         """Throttled _request()"""
         if self._queue != None:
@@ -165,12 +159,12 @@ class ThrottledClientSession(aiohttp.ClientSession):
 
 class AsyncLogger():
     """Async file logger"""
+
     def __init__(self) -> None:
         self._queue = asyncio.Queue()
         self._task = None
         self._file = None
 
-
     async def open(self, logfn: str = None):
         """Set logging to file"""
         if logfn == None:
@@ -184,7 +178,6 @@ class AsyncLogger():
             self._file = None
         return False
 
-
     async def logger(self):
         """Async file logger"""
         if self._file == None:
@@ -200,14 +193,12 @@ class AsyncLogger():
             except Exception as err:
                 error(exception=err)
 
-
     def log(self, msg: str = ''):
         self._queue.put_nowait(msg)
 
-
     async def close(self):
         try:
-            ## empty queue & close
+            # empty queue & close
             await self._queue.join()
             self._task.cancel()
             self._file.close()
@@ -215,9 +206,10 @@ class AsyncLogger():
             error('Error closing log file', err)
         return None
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Utils
+# Utils
-## -----------------------------------------------------------
+# -----------------------------------------------------------
+
 
 def set_debug(debug: bool):
     global _log_level
@@ -301,6 +293,7 @@ async def close_file_logging():
     await LOGGER.close()
     LOGGER = None
 
+
 def _randomword(length):
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(length))
@@ -338,6 +331,21 @@ def log(msg = "", id = None, exception = None) -> bool:
     return _print_log_msg('LOG', msg=msg, exception=exception, id=id, print_msg=(_log_level >= DEBUG))
 
 
+## Copy with pride: https://stackoverflow.com/questions/2203424/python-how-to-retrieve-class-information-from-a-frame-object
+def get_class_from_frame(fr):
+    args, _, _, value_dict = inspect.getargvalues(fr)
+    # we check the first parameter for the frame function is
+    # named 'self'
+    if len(args) and args[0] == 'self':
+        # in that case, 'self' will be referenced in value_dict
+        instance = value_dict.get('self', None)
+        if instance:
+            # return its class
+            return getattr(instance, '__class__', None)
+    # return None otherwise
+    return None
+
+
 def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_msg : bool = True):
     # Use empty prefix to determine standard verbose messages
     if not (print_msg or LOG):
@@ -346,8 +354,13 @@ def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_
     if prefix != '':
         curframe = inspect.currentframe()
         calframe = inspect.getouterframes(curframe)
+        class_name = get_class_from_frame(curframe)
         caller = calframe[2].function
-        prefix = prefix + ': ' + caller + '(): '
+        if class_name != None:
+            prefix = prefix + ': ' + class_name + '.' + caller
+        else:
+            prefix = prefix + ': ' + caller
+        prefix = prefix + '(): '
     
     if id != None:
         prefix = prefix + '[' + str(id) + ']: '
@@ -535,7 +548,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
         if url == None:
             return None
         
-        ## To avoid excessive use of servers            
+        # To avoid excessive use of servers            
         for retry in range(1,max_tries+1):
             try:
                 async with session.get(url) as resp:
@@ -613,9 +626,9 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
     raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
 
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class SlowBar 
+# Class SlowBar 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class SlowBar(IncrementalBar):
     suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).0f h %(remaining_mins).0f mins'
@@ -628,9 +641,9 @@ class SlowBar(IncrementalBar):
         return (self.eta - (self.eta // 3600)*3600) // 60
  
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class StatsNotFound 
+# Class StatsNotFound 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class StatsNotFound(Exception):
     pass
@@ -638,9 +651,9 @@ class StatsNotFound(Exception):
 
 
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class WG 
+# Class WG 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class WG:
 
@@ -719,7 +732,7 @@ class WG:
 
     SQL_PRUNE_CACHE             = """DELETE from {} WHERE update_time < {}""" 
 
-## Default data. Please use the latest maps.json
+# Default data. Please use the latest maps.json
 
     maps = {
         "Random": "Random map",
@@ -868,7 +881,7 @@ class WG:
             error(exception=err)
 
         
-    ## Class methods  ----------------------------------------------------------
+    # Class methods  ----------------------------------------------------------
 
     @classmethod
     def get_server(cls, account_id: int) -> str:
@@ -1053,7 +1066,7 @@ class WG:
         return False
 
 
-    ## Methods --------------------------------------------------
+    # Methods --------------------------------------------------
     def load_tanks(self, tankopedia_fn: str):
         """Load tanks from tankopedia JSON"""
         if tankopedia_fn == None:
@@ -1444,7 +1457,7 @@ class WG:
             return None
         try:
             self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE)
-            ## Create cache tables table
+            # Create cache tables table
             await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL)
             await self.cache.execute(WG.SQL_PLAYER_STATS_CREATE_TBL)
             await self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_CREATE_TBL)
@@ -1655,9 +1668,9 @@ class WG:
         return None
 
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class WoTinspector 
+# Class WoTinspector 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class WoTinspector:
     URL_WI          = 'https://replays.wotinspector.com'
@@ -1740,7 +1753,7 @@ class WoTinspector:
             hash.update(data)
             replay_id = hash.hexdigest()
 
-            ##  Testing if the replay has already been posted
+            # Testing if the replay has already been posted
             json_resp = await self.get_replay_JSON(replay_id)
             if json_resp != None:
                 debug('Already uploaded: ' + title, id=N)