--CSV: CSV export added

analyze_wotb_replays.py

@@ -4,7 +4,7 @@
 # Script Analyze WoT Blitz replays
 
 import sys, argparse, json, os, concurrent, inspect, aiohttp, asyncio, aiofiles, aioconsole
-import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections
+import motor.motor_asyncio, configparser, ssl, re, logging, time, xmltodict, collections, csv
 from datetime import datetime, timedelta, date
 import blitzutils as bu
 from blitzutils import WG
@@ -433,13 +433,12 @@ class BattleCategorizationList():
 		return None
 
 
-	def get_results_csv(self, urls: bool = False):
+	def get_results_list(self, urls: bool = False):
 		try:
 			res = list()			
-			
 			for cat in self.categorizations_list:
 				res.append('')
-				res.append(self.categorizations[cat].get_results_csv())			
+				res.extend(self.categorizations[cat].get_results_list())			
 			if urls:
 				res.append('')
 				for url in self.urls:
@@ -495,7 +494,7 @@ class BattleCategorization():
 		print(FORMAT.format(self.category_key, self.title), end=' ')		
 
 
-	def record_result(self, result: dict):
+	def record_result(self, result: dict) -> bool:
 		try:
 			cat = self.get_category(result)
 			self.categories[cat].record_result(result)
@@ -517,6 +516,7 @@ class BattleCategorization():
 
 
 	def print_headers(self):
+		try:
 			if len(self.title) > self.RESULT_CAT_HEADER_LEN:
 				cat_name = self.title[:self.RESULT_CAT_HEADER_LEN]
 			else:
@@ -524,6 +524,17 @@ class BattleCategorization():
 				header = self.RESULT_CAT_HEADER_FRMT.format(cat_name) 
 			BattleCategory.print_headers(header)
 			return None
+		except Exception as err:
+			bu.error(exception = err)
+
+
+	def get_headers(self) -> list:
+		"""Get BattleCategory headers as list"""
+		try:
+			return BattleCategory.get_headers(self.title)		
+		except Exception as err:
+			bu.error(exception = err)
+		return None
 
 
 	def print_results(self):
@@ -544,7 +555,7 @@ class BattleCategorization():
 		return None
 
 
-	def get_results_json(self):
+	def get_results_json(self) -> dict:
 		"""Get results in JSON format"""
 		try:
 			results = dict()
@@ -559,6 +570,19 @@ class BattleCategorization():
 		return None
 
 	
+	def get_results_list(self) -> list:
+		"""Get results in  list format (for CSV printing)"""
+		try:
+			results = list()
+			results.append(self.get_headers())
+			for cat in self.get_categories():
+				results.append( [cat] + self.categories[cat].get_results_list() )
+			return results
+		except KeyError as err:
+			bu.error('Key not found', err)  
+		return None
+
+
 class BattleTotals(BattleCategorization):
 	TOTAL = 'Total'
 
@@ -1024,6 +1048,20 @@ class BattleCategory():
 		return None
 
 
+	@classmethod
+	def get_headers(cls, header: str = None) -> list:
+		try:
+			headers = [ header ]
+			for field in cls.result_fields:
+				headers.append(cls.get_field_name(field))
+			return headers
+		except KeyError as err:
+			bu.error('Key not found', exception=err)  
+		except Exception as err:
+			bu.error(exception=err) 
+		return None
+
+
 	def __init__(self):
 		try:
 			
@@ -1107,6 +1145,22 @@ class BattleCategory():
 		return None
 
 	
+	def get_results_list(self) -> list:
+		"""Return results as a list"""
+		if not self.results_ready:
+			bu.error('Stats have not been calculated yet. call calc_results() before get_results()')
+		try:
+			results = list()
+			for field in self.result_fields:
+				results.append(self._result_fields[field][3].format(self.results[field]))
+			return results
+		except KeyError as err:
+			bu.error('Key not found', err)  
+		except Exception as err:
+			bu.error(exception=err) 
+		return None
+	
+
 	def print_results(self):
 		if not self.results_ready:
 			bu.error('Stats have not been calculated yet. call calc_results() before printing()')
@@ -1189,32 +1243,34 @@ class PlayerHistogram():
 			res[self.get_category_name(cat)] = stat
 
 		self.results = res
-
 		return res
 
 
-	def get_json(self):
+	def get_results(self) -> dict:
 		return self.results
 
 	
-	def get_csv(self):
+	def get_results_list(self) -> list:
+		try:
 			if (self.results != None):
 				res = list()
-			res.append(['{:12s}'.format(self.name), '{:13s}'.format("Allies"), '{:13s}'.format("Enemies"), '{:13s}'.format( "TOTAL")])
+				res.append([self.name, "Allies", "Allies %",  "Enemies", "Enemies %" ,"TOTAL", "TOTAL %"])
 				for cat in self.results:
 					stat = self.results[cat]
-				res.append([ '{:12s}'.format(cat), 
+					res.append([ cat, stat['allies'], stat['allies%']*100,  
-					'{:5d}'.format(stat['allies']), '({:4.1f}%)'.format(stat['allies%']*100), 
+							stat['enemies'], stat['enemies%']*100, 
-					'{:5d}'.format(stat['enemies']), '({:4.1f}%)'.format(stat['enemies%']*100), 
+							stat['total'], stat['total%']*100 ])
-						'{:5d}'.format(stat['total']), '({:4.1f}%)'.format(stat['total%']*100) ])
-
 				return res
 			else:
 				bu.error('Results have not been calculated yet.')
 				return None
+		except Exception as err:
+			bu.error(exception=err)
+		return None
 
 
 	def print(self):
+		try:
 			if (self.results != None):
 				print("\n{:12s} | {:13s} | {:13s} | {:13s}".format(self.name, "Allies", "Enemies", "TOTAL"))
 				for cat in self.results:
@@ -1222,6 +1278,11 @@ class PlayerHistogram():
 					print("{:12s} | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%) | {:5d} ({:4.1f}%)".format(cat, stat['allies'], stat['allies%']*100, stat['enemies'], stat['enemies%']*100, stat['total'], stat['total%']*100 ))
 			else:
 				bu.error('Results have not been calculated yet.')
+		except Exception as err:
+			bu.error(exception=err)
+		return None
+
+
 
 class ErrorCatchingArgumentParser(argparse.ArgumentParser):
 	def exit(self, status=0, message=None):
@@ -1249,7 +1310,8 @@ async def main(argv):
 	OPT_HIST			= False
 	OPT_STAT_FUNC		= StatFunc.get_default()
 	OPT_WORKERS_N 		= 10
-
+	OPT_EXPORT_CSV_FILE	 = 'export.csv'
+	OPT_EXPORT_JSON_FILE = 'export.json'
 
 	#WG_ACCOUNT 		= None 	# format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'. 
 	  					 	# China is not supported since WG API stats are not available there
@@ -1283,6 +1345,8 @@ async def main(argv):
 					OPT_HIST		= configAnalyzer.getboolean('histograms', OPT_HIST)
 					OPT_STAT_FUNC	= configAnalyzer.get('stat_func', fallback=OPT_STAT_FUNC)
 					OPT_WORKERS_N 	= configAnalyzer.getint('workers', OPT_WORKERS_N)
+					OPT_EXPORT_CSV_FILE  = configAnalyzer.get('csv_file', OPT_EXPORT_CSV_FILE)
+					OPT_EXPORT_JSON_FILE = configAnalyzer.get('json_file', OPT_EXPORT_JSON_FILE)
 					res_categorizations  = configAnalyzer.get('categorizations', None)
 					if res_categorizations != None:
 						res_categorizations.replace(' ','')
@@ -1336,8 +1400,8 @@ async def main(argv):
 		parser.add_argument('-u', '--url', action='store_true', default=False, help='Print replay URLs')
 		parser.add_argument('--tankfile', type=str, default='tanks.json', help='JSON file to read Tankopedia from. Default is "tanks.json"')
 		parser.add_argument('--mapfile', type=str, default='maps.json', help='JSON file to read Blitz map names from. Default is "maps.json"')
-		parser.add_argument('--json', action='store_true', default=False, help='Export data in JSON')
+		parser.add_argument('--json', action='store_true', default=False, help='Export data in JSON format')
-		parser.add_argument('--csv', action='store_true', default=False, help='Export data in CSV')
+		parser.add_argument('--csv', action='store_true', default=False, help='Export data in CSV format')
 		parser.add_argument('-o','--outfile', type=str, default='-', metavar="OUTPUT", help='File to write results. Default STDOUT')
 		parser.add_argument('--db', action='store_true', default=OPT_DB, help='Use DB - You are unlikely to have it')
 		parser.add_argument('--filters', type=str, default=None, help='Filter replays based on categories. Filters given in JSON format.\nUse array "[]" for multiple filters/values. see --mode help.\nExample: : [ {"tier" : [8,9,20] }, { "player_wins" : 5 }]')
@@ -1443,28 +1507,43 @@ async def main(argv):
 			blt_cat_list = process_battle_results(teamresults, args)
 			bu.verbose_std('WR = ' + StatFunc.get_title())
 			blt_cat_list.print_results()
+
 			histograms = None
 			if args.hist:
 				histograms = process_player_dist(results, player_stats, stat_id_map)
+				print_player_dist(histograms)			
+			
 			if args.json:
 				res = blt_cat_list.get_results_json()
-				if args.hist:
+				if histograms != None:
 					res_hist = dict()
 					for histogram in histograms.keys():
-						res_hist[histogram] = histograms[histogram].get_json()
+						res_hist[histogram] = histograms[histogram].get_results()
 					res['histograms'] = res_hist
 				if args.outfile == '-':
-					args.outfile = 'export.json'					
+					export_file = OPT_EXPORT_JSON_FILE
-				await bu.save_JSON(args.outfile, res, pretty=False) ## Excel does not understand pretty JSON
+				else:
-				bu.verbose_std('Data exported to ' + args.outfile)
+					export_file = args.outfile
+				await bu.save_JSON(export_file, res, pretty=False) ## Excel does not understand pretty JSON
+				bu.verbose_std('Results exported to: ' + export_file)
+			
 			if args.csv:
-				res = blt_cat_list.get_results_csv()
+				rows = blt_cat_list.get_results_list()
 				if args.hist:
-					res.append('')
+					rows.append('')
-					res.append('Player Histograms')
+					rows.append(['Player Histograms'])
 					for histogram in histograms.values():
-						res.append('')
+						rows.append('')
-						res.extend(histogram.get_csv())				
+						rows.extend(histogram.get_results_list())
+				if args.outfile == '-':
+					export_file = OPT_EXPORT_CSV_FILE
+				else:
+					export_file = args.outfile
+				with open(export_file, 'w') as csvfile:  
+					# creating a csv writer object  
+					csvwriter = csv.writer(csvfile)        					
+					csvwriter.writerows(rows)
+					bu.verbose_std('Results exported to: ' + export_file)			
 			
 			bu.debug('Finished. Cleaning up..................')
 		except Exception as err:

blitzutils.py

@@ -22,7 +22,7 @@ _log_level  = NORMAL
 LOG = False
 LOGGER = None
 
-## Progress display
+# Progress display
 _progress_N = 100
 _progress_i = 0
 _progress_id = None
@@ -31,9 +31,10 @@ _progress_obj = None
 UMASK = os.umask(0)
 os.umask(UMASK)
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class ThrottledClientSession(aiohttp.ClientSession)
+# Class ThrottledClientSession(aiohttp.ClientSession)
-## -----------------------------------------------------------
+# -----------------------------------------------------------
+
 
 class ThrottledClientSession(aiohttp.ClientSession):
     """Rate-throttled client session class inherited from aiohttp.ClientSession)"""
@@ -55,31 +56,28 @@ class ThrottledClientSession(aiohttp.ClientSession):
             self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
             self._fillerTask = asyncio.create_task(self._filler())
 
-     
-
     def _get_sleep(self) -> list:
         if self.rate_limit == None:
             return None
         else:
             return 1/self.rate_limit
 
-
     def get_rate(self) -> float:
         """Return rate of requests"""
         return self._count / (time.time() - self._start_time)
 
-
     def get_stats(self):
         """Get session statistics"""
-        res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
+        res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
+                                     'count': self._count}
         return res
 
-
     def get_stats_str(self):
         """Print session statistics"""
         return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
-                ' rate: ' +  "{0:.1f}".format(self.get_rate()) + ' requests: ' + str(self._count)
+                ' rate: ' + \
-
+                    "{0:.1f}".format(self.get_rate()) + \
+                                     ' requests: ' + str(self._count)
 
     def reset_counters(self):
         """Reset rate counters and return current results"""
@@ -88,14 +86,12 @@ class ThrottledClientSession(aiohttp.ClientSession):
         self._count = 0
         return res
 
-
     def set_rate_limit(self, rate_limit: float = 1):
         if rate_limit >= 0:
             self.rate_limit = rate_limit
             return self.rate_limit
         return None
 
-
     async def close(self) -> None:
         """Close rate-limiter's "bucket filler" task"""
         # DEBUG
@@ -108,7 +104,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
             pass
         await super().close()
 
-
     async def _filler_simple(self):
         """Filler task to fill the leaky bucket algo"""
         try:
@@ -123,7 +118,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
         except Exception as err:
             error(exception=err)
 
-
     async def _filler(self):
         """Filler task to fill the leaky bucket algo"""
         try:
@@ -142,7 +136,8 @@ class ThrottledClientSession(aiohttp.ClientSession):
                     increment = self.rate_limit * (now - updated_at)
                     fraction += increment % 1
                     extra_increment = fraction // 1
-                    items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
+                    items_2_add = int(
+                        min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
                     fraction = fraction % 1
                     for i in range(0, items_2_add):
                         self._queue.put_nowait(i)
@@ -153,7 +148,6 @@ class ThrottledClientSession(aiohttp.ClientSession):
         except Exception as err:
             error(exception=err)
 
-
     async def _request(self, *args, **kwargs):
         """Throttled _request()"""
         if self._queue != None:
@@ -165,12 +159,12 @@ class ThrottledClientSession(aiohttp.ClientSession):
 
 class AsyncLogger():
     """Async file logger"""
+
     def __init__(self) -> None:
         self._queue = asyncio.Queue()
         self._task = None
         self._file = None
 
-
     async def open(self, logfn: str = None):
         """Set logging to file"""
         if logfn == None:
@@ -184,7 +178,6 @@ class AsyncLogger():
             self._file = None
         return False
 
-
     async def logger(self):
         """Async file logger"""
         if self._file == None:
@@ -200,14 +193,12 @@ class AsyncLogger():
             except Exception as err:
                 error(exception=err)
 
-
     def log(self, msg: str = ''):
         self._queue.put_nowait(msg)
 
-
     async def close(self):
         try:
-            ## empty queue & close
+            # empty queue & close
             await self._queue.join()
             self._task.cancel()
             self._file.close()
@@ -215,9 +206,10 @@ class AsyncLogger():
             error('Error closing log file', err)
         return None
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Utils
+# Utils
-## -----------------------------------------------------------
+# -----------------------------------------------------------
+
 
 def set_debug(debug: bool):
     global _log_level
@@ -301,6 +293,7 @@ async def close_file_logging():
     await LOGGER.close()
     LOGGER = None
 
+
 def _randomword(length):
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(length))
@@ -338,6 +331,21 @@ def log(msg = "", id = None, exception = None) -> bool:
     return _print_log_msg('LOG', msg=msg, exception=exception, id=id, print_msg=(_log_level >= DEBUG))
 
 
+## Copy with pride: https://stackoverflow.com/questions/2203424/python-how-to-retrieve-class-information-from-a-frame-object
+def get_class_from_frame(fr):
+    args, _, _, value_dict = inspect.getargvalues(fr)
+    # we check the first parameter for the frame function is
+    # named 'self'
+    if len(args) and args[0] == 'self':
+        # in that case, 'self' will be referenced in value_dict
+        instance = value_dict.get('self', None)
+        if instance:
+            # return its class
+            return getattr(instance, '__class__', None)
+    # return None otherwise
+    return None
+
+
 def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_msg : bool = True):
     # Use empty prefix to determine standard verbose messages
     if not (print_msg or LOG):
@@ -346,8 +354,13 @@ def _print_log_msg(prefix = 'LOG', msg = '', exception = None, id = None, print_
     if prefix != '':
         curframe = inspect.currentframe()
         calframe = inspect.getouterframes(curframe)
+        class_name = get_class_from_frame(curframe)
         caller = calframe[2].function
-        prefix = prefix + ': ' + caller + '(): '
+        if class_name != None:
+            prefix = prefix + ': ' + class_name + '.' + caller
+        else:
+            prefix = prefix + ': ' + caller
+        prefix = prefix + '(): '
     
     if id != None:
         prefix = prefix + '[' + str(id) + ']: '
@@ -535,7 +548,7 @@ async def get_url_JSON(session: aiohttp.ClientSession, url: str, chk_JSON_func =
         if url == None:
             return None
         
-        ## To avoid excessive use of servers            
+        # To avoid excessive use of servers            
         for retry in range(1,max_tries+1):
             try:
                 async with session.get(url) as resp:
@@ -613,9 +626,9 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
     raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
 
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class SlowBar 
+# Class SlowBar 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class SlowBar(IncrementalBar):
     suffix = '%(index)d/%(max)d %(percent)d%% ETA %(remaining_hours).0f h %(remaining_mins).0f mins'
@@ -628,9 +641,9 @@ class SlowBar(IncrementalBar):
         return (self.eta - (self.eta // 3600)*3600) // 60
  
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class StatsNotFound 
+# Class StatsNotFound 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class StatsNotFound(Exception):
     pass
@@ -638,9 +651,9 @@ class StatsNotFound(Exception):
 
 
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class WG 
+# Class WG 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class WG:
 
@@ -719,7 +732,7 @@ class WG:
 
     SQL_PRUNE_CACHE             = """DELETE from {} WHERE update_time < {}""" 
 
-## Default data. Please use the latest maps.json
+# Default data. Please use the latest maps.json
 
     maps = {
         "Random": "Random map",
@@ -868,7 +881,7 @@ class WG:
             error(exception=err)
 
         
-    ## Class methods  ----------------------------------------------------------
+    # Class methods  ----------------------------------------------------------
 
     @classmethod
     def get_server(cls, account_id: int) -> str:
@@ -1053,7 +1066,7 @@ class WG:
         return False
 
 
-    ## Methods --------------------------------------------------
+    # Methods --------------------------------------------------
     def load_tanks(self, tankopedia_fn: str):
         """Load tanks from tankopedia JSON"""
         if tankopedia_fn == None:
@@ -1444,7 +1457,7 @@ class WG:
             return None
         try:
             self.cache = await aiosqlite.connect(WG.CACHE_DB_FILE)
-            ## Create cache tables table
+            # Create cache tables table
             await self.cache.execute(WG.SQL_TANK_STATS_CREATE_TBL)
             await self.cache.execute(WG.SQL_PLAYER_STATS_CREATE_TBL)
             await self.cache.execute(WG.SQL_PLAYER_ACHIEVEMENTS_CREATE_TBL)
@@ -1655,9 +1668,9 @@ class WG:
         return None
 
 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
-#### Class WoTinspector 
+# Class WoTinspector 
-## -----------------------------------------------------------
+# -----------------------------------------------------------
 
 class WoTinspector:
     URL_WI          = 'https://replays.wotinspector.com'
@@ -1740,7 +1753,7 @@ class WoTinspector:
             hash.update(data)
             replay_id = hash.hexdigest()
 
-            ##  Testing if the replay has already been posted
+            # Testing if the replay has already been posted
             json_resp = await self.get_replay_JSON(replay_id)
             if json_resp != None:
                 debug('Already uploaded: ' + title, id=N)