Author SHA1 Message Date
Jylpah 5f84ec4724 Merge branch 'dev-datetime-cats' into merge-master-dev-datetime 2021-05-03 19:08:42 +03:00
Jylpah 170e6a287d Update analyze_wotb_replays.py 2021-02-22 22:06:52 +02:00
Jylpah 4e9032d7ba Merge pull request #26 from Jylpah/master
Backport extract_tankopedia.py tank strings fix
2021-02-16 12:07:25 +02:00
Jylpah 9c0949748e Update analyze_wotb_replays.py 2021-02-13 17:08:51 +02:00
Jylpah 5b2ef816f5 Update analyze_wotb_replays.py 2021-02-10 23:25:53 +02:00
Jylpah 448b91a866 Update analyze_wotb_replays.py 2021-02-10 21:42:49 +02:00
Jylpah cddc7dd947 --fields, datetime fields/categorizations 2021-02-09 01:02:59 +02:00
10 changed files with 1035 additions and 2017 deletions

No files matched your search

-15
View File
@@ -1,15 +0,0 @@
__pycache__/
BlitzApp
tmp/
ssl/
*.ini
*.swp
*.code-workspace
*.sqlite3
.vscode/
*.sqlite3-journal
.gitattributes
export*
*.csv
!/stats/*.csv
!*.py
+23 -25
View File
@@ -1,5 +1,5 @@
# blitz-tools
Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or later.
Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or higher. I am using 3.8 myself so you may have to change the Python interpreter version in the scripts' first line.
### Replays
* [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader.
@@ -13,48 +13,46 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
* [blitzutils.py](blitzutils.py): A utils class for various Blitz related to functions. *Other scripts use this*, so you need to place it to the same folder with the other scripts
### Data
* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py](extractTankopedia.py). Contains only limited information required by the scripts.
* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py](extract_tankopedia.py).
* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py]](extractTankopedia.py]). Contains only limited information required by the scripts.
* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py]](extract_tankopedia.py]).
### Quick instructions
1. Install [Python 3.8](https://www.python.org/downloads/) or later to your computer/laptop
2. Install the required Python modules (see [requirements.txt](requirements.txt): motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
```
python -m pip install -r requirements.txt
# Remember to run the command with a Python interpreter >= 3.7
python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress
```
3. Clone the repository. [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for `git`. This makes it easy to receive the updates via `git pull`.
3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. **The recommened way** is to install the scipts via `git clone`: [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for git. This makes it easy to receive the updates via `git pull`.
4. Upload your replays.
```
git clone https://github.com/Jylpah/blitz-tools.git
```
4. Upload your replays with `upload_wotb_replays.py`
```
## Check the options
# Windows Command Prompt or Powershell
# Windows, run directly from Command Prompt or Powershell
# Check the options
python.exe .\upload_wotb_replays.py --help
# Linux or Windows Subsystem for Linux (WSL)
./upload_wotb_replays.py --help
# Upload replays (Windows)
python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay DIR_WITH_REPLAYS
# Syntax
python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
# example
python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
# Linux
./upload_wotb_replays.py --help
```
5. Analyze replays with `analyze_wotb_replays.py`
5. Analyze replays.
```
## Check options
# Windows Command Prompt or Powershell
# Windows, run directly from Command Prompt or Powershell
python.exe .\analyze_wotb_replays.py --help
# Syntax (Windows)
python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json [DIR_WITH_REPLAYS ...]
# Syntax
python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
# Example (Windows)
# Example
python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
# Linux or Windows Subsystem for Linux (WSL)
# Linux
./analyze_wotb_replays.py --help
```
+654 -501
View File
File diff suppressed because it is too large. Load diff
+124 -218
View File
@@ -36,163 +36,132 @@ _progress_obj = None
UMASK = os.umask(0)
os.umask(UMASK)
## -----------------------------------------------------------
#### Class ThrottledClientSession(aiohttp.ClientSession)
#
# Rate-limited async http client session
#
# Inherits aiohttp.ClientSession
## -----------------------------------------------------------
# -----------------------------------------------------------
# Class ThrottledClientSession(aiohttp.ClientSession)
# -----------------------------------------------------------
from typing import Optional, Union
import aiohttp
import asyncio
import time
import logging
import re
class ThrottledClientSession(aiohttp.ClientSession):
"""Rate-throttled client session class inherited from aiohttp.ClientSession)"""
"""Rate-throttled client session class inherited from aiohttp.ClientSession)"""
# MIN_SLEEP = 0.1
def __init__(self, rate_limit: float = 0, filters: list[str] = list() ,
limit_filtered: bool = False, re_filter: bool = False, *args,**kwargs) -> None:
assert isinstance(rate_limit, (int, float)), "rate_limit has to be float"
assert isinstance(filters, list), "filters has to be list"
assert isinstance(limit_filtered, bool),"limit_filtered has to be bool"
assert isinstance(re_filter, bool), "re_filter has to be bool"
def __init__(self, rate_limit: float = None, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.rate_limit = None
self._fillerTask = None
self._queue = None
self._start_time = None
self._count = 0
self._start_time = time.time()
super().__init__(*args,**kwargs)
self.rate_limit : float
self._fillerTask : Optional[asyncio.Task] = None
self._queue : Optional[asyncio.Queue] = None
self._start_time : float = time.time()
self._count : int = 0
self._limit_filtered: bool = limit_filtered
self._re_filter : bool = re_filter
self._filters : list[Union[str, re.Pattern]] = list()
if rate_limit != None:
if rate_limit <= 0:
raise ValueError('rate_limit must be positive')
self.rate_limit = rate_limit
self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
self._fillerTask = asyncio.create_task(self._filler())
if re_filter:
for filter in filters:
self._filters.append(re.compile(filter))
def _get_sleep(self) -> list:
if self.rate_limit == None:
return None
else:
for filter in filters:
self._filters.append(filter)
self.set_rate_limit(rate_limit)
def _get_sleep(self) -> float:
if self.rate_limit > 0:
return 1/self.rate_limit
return 0
def get_rate(self) -> float:
"""Return rate of requests"""
return self._count / (time.time() - self._start_time)
def get_stats(self) -> dict[str, float]:
def get_stats(self):
"""Get session statistics"""
res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
'count': self._count}
return res
def get_stats_str(self) -> str:
def get_stats_str(self):
"""Print session statistics"""
return f"rate limit: {str(self.rate_limit if self.rate_limit != None else '-')} \
rate: {0:.1f}.format(self.get_rate()) requests: {str(self._count)}"
return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
' rate: ' + \
"{0:.1f}".format(self.get_rate()) + \
' requests: ' + str(self._count)
def reset_counters(self) -> dict[str, float]:
def reset_counters(self):
"""Reset rate counters and return current results"""
res = self.get_stats()
self._start_time = time.time()
self._count = 0
return res
def set_rate_limit(self, rate_limit: float = 0) -> float:
assert rate_limit is not None, "rate_limit must not be None"
assert isinstance(rate_limit, (int,float)) and rate_limit >= 0, "rate_limit has to be type of 'float' >= 0"
self.rate_limit = rate_limit
if rate_limit > 0:
self._queue = asyncio.Queue(int(rate_limit)+1)
if self._fillerTask is not None:
self._fillerTask.cancel()
self._fillerTask = asyncio.create_task(self._filler())
return self.rate_limit
def set_rate_limit(self, rate_limit: float = 1):
if rate_limit >= 0:
self.rate_limit = rate_limit
return self.rate_limit
return None
async def close(self) -> None:
"""Close rate-limiter's "bucket filler" task"""
# DEBUG
logging.debug(self.get_stats_str())
# DEBUG
debug(self.get_stats_str())
try:
if self._fillerTask is not None:
if self._fillerTask != None:
self._fillerTask.cancel()
await asyncio.wait_for(self._fillerTask, timeout=0.5)
except asyncio.TimeoutError as err:
logging.error(str(err))
await asyncio.wait_for(self._fillerTask, timeout=0.5)
except (asyncio.CancelledError, asyncio.TimeoutError):
pass
await super().close()
async def _filler(self) -> None:
async def _filler_simple(self):
"""Filler task to fill the leaky bucket algo"""
try:
if self._queue is None:
return None
logging.debug('SLEEP: ' + str(self._get_sleep()))
if self._queue == None:
return
sleep = self._get_sleep()
while True:
self._queue.put(None)
await asyncio.sleep(sleep)
except asyncio.CancelledError:
debug('Cancelled')
except Exception as err:
error(exception=err)
async def _filler(self):
"""Filler task to fill the leaky bucket algo"""
try:
if self._queue == None:
return
sleep = self._get_sleep()
# debug('SLEEP: ' + str(sleep))
updated_at = time.monotonic()
extra_increment : float = 0
fraction = 0
extra_increment = 0
for i in range(0, self._queue.maxsize):
await self._queue.put(i)
self._queue.put_nowait(i)
while True:
if not self._queue.full():
now = time.monotonic()
increment = self.rate_limit * (now - updated_at)
items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment + extra_increment)))
extra_increment = (increment + extra_increment) % 1
for i in range(0,items_2_add):
fraction += increment % 1
extra_increment = fraction // 1
items_2_add = int(
min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
fraction = fraction % 1
for i in range(0, items_2_add):
self._queue.put_nowait(i)
updated_at = now
await asyncio.sleep(self._get_sleep())
await asyncio.sleep(sleep)
except asyncio.CancelledError:
logging.debug('Cancelled')
debug('Cancelled')
except Exception as err:
logging.error(str(err))
return None
error(exception=err)
async def _request(self, *args,**kwargs) -> aiohttp.ClientResponse:
async def _request(self, *args, **kwargs):
"""Throttled _request()"""
if self._queue is not None and self.is_limited(*args):
if self._queue != None:
await self._queue.get()
self._queue.task_done()
self._count += 1
return await super()._request(*args,**kwargs)
return await super()._request(*args, **kwargs)
def is_limited(self, *args: str) -> bool:
"""Check wether the rate limit should be applied"""
try:
url: str = args[1]
for filter in self._filters:
if isinstance(filter, re.Pattern) and filter.match(url) is not None:
return self._limit_filtered
elif isinstance(filter, str) and url.startswith(filter):
return self._limit_filtered
return not self._limit_filtered
except Exception as err:
logging.error(str(err))
return True
# -----------------------------------------------------------
# Class AsyncLogger()
# -----------------------------------------------------------
class AsyncLogger():
"""Async file logger"""
@@ -517,27 +486,18 @@ def NOW() -> int:
return int(time.time())
def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
"""Return YYYYMMDD_HHmm date string"""
try:
ts = datetime.fromtimestamp(timestamp)
return ts.strftime(date_format)
except Exception as err:
error(exception=err)
return None
def rebase_file_args(current_dir, files):
"""REbase file command line params after moving working dir to the script's dir"""
if isinstance(files, list):
if (files[0] == '-') or (files[0] == 'db:'):
return files
else:
return [ os.path.join(current_dir, fn) for fn in files ]
elif isinstance(files, str):
return os.path.join(current_dir, files)
def rebase_file_args(current_dir : str, files: list[str]) -> list[str]:
"""Rebase file command line params after moving working dir to the script's dir"""
assert isinstance(files, list), "files has to be a list"
if (files[0] == '-') or (files[0] == 'db:'):
return [ files[0] ]
else:
return [ os.path.join(current_dir, fn) for fn in files ]
async def read_int_list(filename: str) -> list[int]:
async def read_int_list(filename: str) -> list():
"""Read file to a list and return list of integers in the input file"""
input_list = []
@@ -673,14 +633,6 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
def sort_dict(d: dict, number: bool = False) -> dict:
"""Sort a dict by keys"""
if number:
return dict(sorted(d.items(), key=lambda item: int(item[0])))
else:
return dict(sorted(d.items(), key=lambda item: item[0]))
# -----------------------------------------------------------
# Class SlowBar
# -----------------------------------------------------------
@@ -964,18 +916,33 @@ class WG:
@classmethod
def get_map(self, map_str: str) -> str:
def get_map(cls, map_str: str) -> str:
"""Return map name from short map string in replays"""
try:
return self.maps[map_str]
return cls.maps[map_str]
except:
error('Map ' + map_str + ' not found')
return None
@classmethod
def get_tank_user_strs(self) -> str:
return self.tanks["userStr"].keys()
def get_map_user_strs(cls) -> str:
return cls.maps.keys()
@classmethod
def tank_str2name(cls, tank_str: str) -> str:
"""Return tank name from short tank string in replays"""
try:
return cls.tanks["userStr"][tank_str]
except:
error('Tank ' + tank_str + ' not found')
return tank_str
@classmethod
def get_tank_user_strs(cls) -> str:
return cls.tanks["userStr"].keys()
@classmethod
@@ -1111,15 +1078,11 @@ class WG:
def load_tanks(self, tankopedia_fn: str):
"""Load tanks from tankopedia JSON"""
if tankopedia_fn == None:
return False
return False
try:
with open(tankopedia_fn, 'rt', encoding='utf8') as f:
self.tanks = json.loads(f.read())
self.tanks['tankStr'] = dict()
p = re.compile('.+_short$')
for usr_str in self.tanks['userStr']:
if p.match(usr_str) == None:
self.tanks['tankStr'][self.tanks['userStr'][usr_str]] = usr_str
self.tanks_by_tier = dict()
for tier in range(1,11):
self.tanks_by_tier[str(tier)] = list()
@@ -1129,44 +1092,7 @@ class WG:
except Exception as err:
error('Could not read tankopedia: ' + tankopedia_fn, err)
return False
def get_replay_filename(self, replay: dict):
try:
summary = replay['data']['summary']
timestamp = int(summary['battle_start_timestamp'])
timestamp = get_date_str(timestamp, date_format='%Y%m%d_%H%M')
player = summary['player_name']
arena_id = int(summary['arena_unique_id'])
vehicle = self.name2tank_str(summary['vehicle'])
vehicle.replace(' ','_')
return '{:s}_{:s}_{:s}_{:d}.wotbreplay.json'.format(timestamp, player, vehicle, arena_id)
except Exception as err:
error('Unexpected Exception', err)
return None
def get_map_user_strs(self) -> str:
return self.maps.keys()
def tank_str2name(self, tank_str: str) -> str:
"""Return tank name from short tank string in replays"""
try:
return self.tanks["userStr"][tank_str]
except:
error('Tank ' + tank_str + ' not found')
return tank_str
def name2tank_str(self, name: str) -> str:
"""Return shorttank string from tank string in replays"""
try:
return self.tanks["tankStr"][name]
except:
debug('Tank ' + name + ' not found')
return name
def get_tanks_by_tier(self, tier: int) -> list():
"""Returns tank_ids by tier"""
@@ -1626,8 +1552,6 @@ class WG:
async def store_tank_stats(self, key: list , stats_data: list, update_time: int):
"""Save tank stats into cache"""
assert self.cache is not None, "cache must be initiated"
try:
account_id = key[0]
tank_ids = set(key[1])
@@ -1649,8 +1573,6 @@ class WG:
async def store_player_stats(self, key: list , stats_data: list, update_time: int):
"""Save player stats into cache"""
assert self.cache is not None, "cache must be initiated"
try:
account_id = key[0]
if stats_data != None:
@@ -1667,8 +1589,6 @@ class WG:
async def store_player_achievements(self, key: list , stats_data: list, update_time: int):
"""Save player stats into cache"""
assert self.cache is not None, "cache must be initiated"
try:
account_id = key[0]
if stats_data != None:
@@ -1684,8 +1604,6 @@ class WG:
async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ):
assert self.cache is not None, "cache must be initiated"
try:
# test for cacheDB existence
debug('Trying cached stats first')
@@ -1799,27 +1717,17 @@ class WoTinspector:
URL_REPLAY_VIEW = URL_WI +'/en/view/'
URL_REPLAY_UL = 'https://api.wotinspector.com/replay/upload?'
URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
URL_TANK_DB = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
REPLAY_N = 1
DEFAULT_RATE_LIMIT : float = 10/60 # 10 requests / min
DEFAULT_TOKEN : str = 'a5428af49bf7495986577d59b0e5bcff'
def __init__(self, rate_limit: float = DEFAULT_RATE_LIMIT, auth_token: Optional[str] = None):
headers : Optional[dict[str, str]] = None
if auth_token is None:
auth_token = self.DEFAULT_TOKEN
headers = dict()
headers['Authorization'] = 'Token ' + auth_token
self.session = ThrottledClientSession(rate_limit=rate_limit,
filters=[self.URL_REPLAY_LIST, self.URL_REPLAY_INFO],
re_filter=False, limit_filtered=True, headers = headers)
def __init__(self, rate_limit: int = 30):
self.session = ThrottledClientSession(rate_limit=rate_limit)
async def close(self) -> None:
if self.session is not None:
async def close(self):
if self.session != None:
debug('Closing aiohttp session')
await self.session.close()
@@ -1883,7 +1791,7 @@ class WoTinspector:
hash.update(data)
replay_id = hash.hexdigest()
## Testing if the replay has already been posted
# Testing if the replay has already been posted
json_resp = await self.get_replay_JSON(replay_id)
if json_resp != None:
debug('Already uploaded: ' + title, id=N)
@@ -1898,7 +1806,7 @@ class WoTinspector:
}
url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
#debug('URL: ' + url)
# debug('URL: ' + url)
headers ={'Content-type': 'application/x-www-form-urlencoded'}
payload = { 'file' : (filename, base64.b64encode(data)) }
except Exception as err:
@@ -1960,6 +1868,11 @@ class WoTinspector:
return replay_links
@classmethod
def get_replay_id(cls, url):
return url.rsplit('/', 1)[-1]
@classmethod
def read_replay_id(cls, json_replay):
"""Read replay_id from replay JSON file"""
@@ -1970,18 +1883,14 @@ class WoTinspector:
error(exception=err)
return None
@classmethod
def get_replay_id(cls, url):
return url.rsplit('/', 1)[-1]
@classmethod
def chk_JSON_replay(cls, json_resp) -> bool:
def chk_JSON_replay(cls, json_resp):
""""Check String for being a valid JSON file"""
try:
# FIX The replay can have allies/enemies i.e. be useful for
# team composition analysis, but does not have player contribution
if ('status' in json_resp) and json_resp['status'] == 'ok' and \
(get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
(get_JSON_value(json_resp, key='data.summary.winner_team') != None) :
debug("JSON check OK")
return True
except KeyError as err:
@@ -1990,9 +1899,6 @@ class WoTinspector:
debug("Replay JSON check failed: " + str(json_resp))
return False
# -----------------------------------------------------------
# Class BlitzStars
# -----------------------------------------------------------
class BlitzStars:
+8 -14
View File
@@ -3,7 +3,6 @@
# Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
from os import path
import sys, argparse, json, os, inspect, aiohttp, asyncio, aiofiles, re, logging, time, xmltodict, collections, configparser
import blitzutils as bu
from blitzutils import WG
@@ -12,11 +11,9 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG)
FILE_CONFIG = 'blitzstats.ini'
# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
BLITZAPP_STRINGS='Data/Strings/en.yaml'
BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
BLITZAPP_VEHICLE_FILE='list.xml'
BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
BLITZAPP_VEHICLE_FILE='/list.xml'
wg = None
@@ -82,8 +79,8 @@ async def main(argv):
tankopedia = collections.OrderedDict()
tankopedia['status'] = 'ok'
tankopedia['meta'] = { "count": len(tanks) }
tankopedia['data'] = bu.sort_dict(tanks, number=True)
tankopedia['userStr'] = bu.sort_dict(userStrs)
tankopedia['data'] = tanks
tankopedia['userStr'] = userStrs
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')
await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
@@ -107,11 +104,8 @@ async def main(argv):
async def extract_tanks(blitz_app_base : str, nation: str):
tanks = []
# WG has changed the location of Data directory - at least in steam client
if path.isdir(path.join(blitz_app_base, 'assets')):
blitz_app_base = path.join(blitz_app_base, 'assets')
list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
if not path.isfile(list_xml_file):
list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
if not os.path.isfile(list_xml_file):
print('ERROR: cannot open ' + list_xml_file)
return None
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
@@ -138,7 +132,7 @@ async def read_user_strs(blitz_app_base : str) -> dict:
"""Read user strings to convert map and tank names"""
tank_strs = {}
map_strs = {}
filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
filename = blitz_app_base + BLITZAPP_STRINGS
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
try:
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
+1 -1
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/local/bin/python3.7
# Get Tankpedia from WoTinspector.com
+1 -3
View File
@@ -18,7 +18,6 @@
"fort": "Fort Despair",
"grossberg": "Dynasty's Pearl",
"himmelsdorf": "Himmelsdorf",
"holland": "Molendijk",
"holmeisk": "Wasteland",
"iceworld": "Everfrost",
"idle": "Yukon",
@@ -44,7 +43,6 @@
"neptune_02": "Normandy - Beach",
"north": "North",
"ordeal": "Trial by Fire",
"plant": "Ghost Factory",
"pliego": "Castilla",
"port": "Port Bay",
"rift": "Hellas",
@@ -57,4 +55,4 @@
"skit": "Naval Frontier",
"test": "World of Ducks",
"tutorial": "Proving Grounds"
}
}
+81 -94
View File
@@ -1,96 +1,83 @@
# WoT Blitz releases
| Release | Date | Notes |
|:-------:|:--------:|-------------|
| 9.1 | 2022-07-14 | Releases updated via DLC, graphics updates for Object 140, Patton 46 |
| 9.0 | 2022-06-08 | Japanies heavies, Castilla graphics update, new garage |
| 8.10 | 2022-04-28 | New battle results screen, new "Operation" mechanics |
| 8.9 | 2022-03-23 | "Free" camo resupply, VK 72.01 supplies |
| 8.8 | 2022-02-16 | Molendijk map |
| 8.7 | 2022-01-19 | Soviet Light Tanks |
| 8.6 | 2021-12-15 | Joining Light and Medium Tank classes in MM |
| 8.5 | 2021-11-17 | New national crew voices, economy update |
| 8.4 | 2021-10-20 | Polish heavy tanks |
| 8.3 | 2021-09-16 | FV4005, FV4022, VK 91.01 (P) and WZ-121 legendary camos, -5 deg gun depression to WZ-120 |
| 8.2 | 2021-08-18 | Ghost Factory map, XP/credits nerf for inactive players, new store UI |
| 8.1 | 2021-07-15 | Armor highlighting updated |
| 8.0 | 2021-06-08 | American experimental heavies (Yoh-line), PBR camos |
| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |
| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables |
| 7.7 | 2021-02-24 | Czechoslovakian medium tanks |
| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera |
| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |
| 7.4 | 2020-11-12 | TD buffs, Clan missions |
| 7.3 | 2020-10-08 | Swedish HT line |
| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles |
| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff |
| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes |
| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes |
| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff |
| 6.8 | 2020-02-19 | ATGM nerf, Hellas map |
| 6.7 | 2020-01-15 | ATGM US Lights |
| 6.6 | 2019-12-18 | T-62A + Leo buffs |
| 6.5 | 2019-11-21 | New UI |
| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... |
| 6.3 | 2019-09-18 | European line |
| 6.2 | 2019-08-14 | New MM |
| 6.1 | 2019-07-10 | Autoloader buff |
| 6.0 | 2019-06-05 | UK TD line #2 |
| 5.10 | 2019-04-24 | new missions |
| 5.9 | 2019-03-27 | New shell tracers, minor fixes |
| 5.8 | 2019-02-21 | Chinese TD line |
| 5.7 | 2019-01-16 | Avatars |
| 5.6 | 2018-12-05 | Normandy map |
| 5.5 | 2018-11-13 | Low-tier clean-up :-( |
| 5.4 | 2018-10-10 | MM "rigging" adjusted |
| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added |
| 5.2 | 2018-08-22 | Faust map |
| 5.1 | 2018-07-31 | New MM: vehicle class matching |
| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts removed ! |
| 4.10 | 2018-05-24 | Alpenstadt map |
| 4.9 | 2018-04-25 | Chinese heavy tanks |
| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator |
| 4.7 | 2018-02-28 | New minimap |
| 4.6 | 2018-01-31 | Chinese nation |
| 4.5 | 2017-12-19 | Dynasty's Pearl map |
| 4.4 | 2017-11-09 | New British tanks |
| 4.3 | 2017-10-10 | Level 7 screw skills |
| 4.2 | 2017-09-12 | French LTs, Naval Frontier map |
| 4.1 | 2017-08-16 | New markers |
| 4.0 | 2017-07-13 | Rating battles (date uncertain) |
| 3.10 | 2017-06-15 | Fixes (date uncertain) |
| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins (date uncertain) |
| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system (date uncertain) |
| 3.7 | 2017-03-22 | Balance changes, Tournaments (date uncertain) |
| 3.6 | 2017-02-16 | Vehicle Enrichment (date uncertain) |
| 3.5 | 2017-01-13 | Boosters, Dead rail update (date uncertain) |
| 3.4 | 2016-11-28 | Canyon map, USA T57 line |
| 3.3 | 2016-11-01 | Updated Mines map |
| 3.2 | 2016-10-04 | Copperfield update |
| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map (date uncertain) |
| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update (date uncertain) |
| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs |
| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 |
| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker (date uncertain) |
| 2.8 | 2016-04-04 | New Russian TD line, new consumables system (date uncertain) |
| 2.7 | 2016-03-07 | Mirage map (date uncertain) |
| 2.6 | 2016-02-10 | German LTs, shooting through obstacles (date uncertain) |
| 2.5 | 2016-01-11 | Camouflages, Clan supply (date uncertain) |
| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version (date uncertain) |
| 2.3 | 2015-11-12 | Canal map (date uncertain) |
| 2.2 | 2015-10-05 | Desert Sands map update (date uncertain) |
| 2.1 | 2015-09-07 | UK TDs, Clan Chat (date uncertain) |
| 2.0 | 2015-08-05 | Clans, Oasis Palms updated |
| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo (date uncertain) |
| 1.10 | 2015-06-11 | Port Bay map (date uncertain) |
| 1.9 | 2015-05-13 | New German medium tanks (Leo) (date uncertain) |
| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players (date uncertain) |
| 1.7 | 2015-02-16 | US Turreted TDs (date uncertain) |
| 1.6 | 2015-01-19 | Castilla map, UK HT line (date uncertain) |
| 1.5 | 2014-12-01 | UK Nation (date uncertain) |
| 1.4 | 2014-10-29 | German HT line (Maus) (date uncertain) |
| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps (date uncertain) |
| 1.2 | 2014-08-20 | Oasis Palms map (date uncertain) |
| 1.1.1 | 2014-07-23 | Fixes :-) |
| 1.0 | 2014-06-26 | Worldwide launch |
| 0.9 | 2014-05-05 | Nordic soft launch |
| Release | Date | Notes | Inaccurate? |
|:-------:|:--------:|-------------|-----|
| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo | |
| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables | |
| 7.7 | 2021-02-24 | Czechoslovakian medium tanks | |
| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera | |
| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff | |
| 7.4 | 2020-11-12 | TD buffs, Clan missions | |
| 7.3 | 2020-10-08 | Swedish HT line | |
| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles | |
| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff | |
| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes | |
| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes | |
| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff | |
| 6.8 | 2020-02-19 | ATGM nerf, Hellas map | |
| 6.7 | 2020-01-15 | ATGM US Lights | |
| 6.6 | 2019-12-18 | T-62A + Leo buffs | |
| 6.5 | 2019-11-21 | New UI | |
| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... | |
| 6.3 | 2019-09-18 | European line | |
| 6.2 | 2019-08-14 | New MM | |
| 6.1 | 2019-07-10 | Autoloader buff | |
| 6.0 | 2019-06-05 | UK TD line #2 | |
| 5.10 | 2019-04-24 | new missions | |
| 5.9 | 2019-03-27 | New shell tracers, minor fixes | |
| 5.8 | 2019-02-21 | Chinese TD line | |
| 5.7 | 2019-01-16 | Avatars | |
| 5.6 | 2018-12-05 | Normandy map | |
| 5.5 | 2018-11-13 | Low-tier clean-up | |
| 5.4 | 2018-10-10 | MM "rigging" adjusted | |
| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added | |
| 5.2 | 2018-08-22 | Faust map | |
| 5.1 | 2018-07-31 | New MM: vehicle class matching | |
| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts killed | |
| 4.10 | 2018-05-24 | Alpenstadt map | |
| 4.9 | 2018-04-25 | Chinese heavy tanks | |
| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator | |
| 4.7 | 2018-02-28 | New minimap | |
| 4.6 | 2018-01-31 | Chinese nation | |
| 4.5 | 2017-12-19 | Dynasty's Pearl map | |
| 4.4 | 2017-11-09 | New British tanks | |
| 4.3 | 2017-10-10 | Level 7 screw skills | |
| 4.2 | 2017-09-12 | French LTs, Naval Frontier map | |
| 4.1 | 2017-08-16 | New markers | |
| 4.0 | 2017-07-13 | Rating battles | Y |
| 3.10 | 2017-06-15 | Fixes | Y |
| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins | Y |
| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system | Y |
| 3.7 | 2017-03-22 | Balance changes, Tournaments | Y |
| 3.6 | 2017-02-16 | Vehicle Enrichment | Y |
| 3.5 | 2017-01-13 | Boosters, Dead rail update | Y |
| 3.4 | 2016-11-28 | Canyon map, USA T57 line | |
| 3.3 | 2016-11-01 | Updated Mines map | |
| 3.2 | 2016-10-04 | Copperfield update | |
| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map | Y |
| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update | Y |
| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs | |
| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 | |
| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker | Y |
| 2.8 | 2016-04-04 | New Russian TD line, new consumables system | Y |
| 2.7 | 2016-03-07 | Mirage map | Y |
| 2.6 | 2016-02-10 | German LTs, shooting through obstacles | Y |
| 2.5 | 2016-01-11 | Camouflages, Clan supply | Y |
| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version | Y |
| 2.3 | 2015-11-12 | Canal map | Y |
| 2.2 | 2015-10-05 | Desert Sands map update | Y |
| 2.1 | 2015-09-07 | UK TDs, Clan Chat | Y |
| 2.0 | 2015-08-05 | Clans, Oasis Palms updated | |
| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo | Y |
| 1.10 | 2015-06-11 | Port Bay map | Y |
| 1.9 | 2015-05-13 | New German medium tanks (Leo) | Y |
| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players | Y |
| 1.7 | 2015-02-16 | US Turreted TDs | Y |
| 1.6 | 2015-01-19 | Castilla map, UK HT line | Y |
| 1.5 | 2014-12-01 | UK Nation | Y |
| 1.4 | 2014-10-29 | German HT line (Maus) | Y |
| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps | Y |
| 1.2 | 2014-08-20 | Oasis Palms map | Y |
| 1.1.1 | 2014-07-23 | Fixes :-) | |
| 1.0 | 2014-06-26 | Worldwide launch | |
| 0.9 | 2014-05-05 | Nordic soft launch | |
+109 -1112
View File
File diff suppressed because it is too large. Load diff
+34 -34
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
## PYTHON VERSION MUST BE 3.7 OR HIGHER
import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
import logging, re, configparser, zipfile
import json, argparse, inspect, sys, os, io, base64, aiohttp, urllib, asyncio, aiofiles, aioconsole
import logging, re, concurrent.futures, configparser, motor.motor_asyncio, ssl, zipfile
import blitzutils as bu
from blitzutils import WG
from blitzutils import WoTinspector
@@ -21,17 +21,17 @@ SKIPPED_N = 0
ERROR_N = 0
WIurl='https://wotinspector.com/api/replay/upload?'
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
wg : WG | None = None
wi : WoTinspector | None = None
wg = None
wi = None
async def main(argv) -> int:
async def main(argv):
global wg, wi
current_dir = os.getcwd()
# set the directory for the script
os.chdir(os.path.dirname(sys.argv[0]))
# options defaults
OPT_WORKERS_N = 2
OPT_WORKERS_N = 5
WG_ID = None
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
# China is not supported since WG API stats are not available there
@@ -43,7 +43,13 @@ async def main(argv) -> int:
bu.debug('Reading config file: ' + FILE_CONFIG)
config = configparser.ConfigParser()
config.read(FILE_CONFIG)
try:
if 'UPLOADER' in config.sections():
configUploader = config['UPLOADER']
OPT_WORKERS_N = configUploader.getint('opt_workers', OPT_WORKERS_N)
except (KeyError, configparser.NoSectionError) as err:
bu.error(exception=err)
try:
if 'WG' in config.sections():
configWG = config['WG']
@@ -76,19 +82,18 @@ async def main(argv) -> int:
bu.set_log_level(args.silent, args.verbose, args.debug)
wg = WG(WG_appID, args.tankopedia, args.mapfile)
wi = WoTinspector()
wi = WoTinspector(rate_limit=WG_RATE_LIMIT)
if args.account is not None:
if args.account != None:
args.accountID = await wg.get_account_id(args.account)
bu.debug('WG account_id: ' + str(args.accountID))
if args.accountID is None:
if args.accountID == None:
args.accountID = 0
try:
queue : asyncio.Queue = asyncio.Queue()
queue = asyncio.Queue()
# rebase file arguments due to moving the working directory to the script location
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
args.files = bu.rebase_file_args(current_dir, args.files)
tasks = []
@@ -118,7 +123,7 @@ async def main(argv) -> int:
await wg.session.close()
await wi.close()
return 0
return None
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
@@ -132,13 +137,13 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
if not line:
break
else:
if (p_replayfile.match(line) is not None):
if (p_replayfile.match(line) != None):
await queue.put(await mkQueueItem(line, title))
else:
for fn in files:
if fn.endswith('"'):
fn = fn[:-1]
if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
await queue.put(await mkQueueItem(fn, title))
bu.debug('File added to queue: ' + fn)
elif os.path.isdir(fn):
@@ -146,7 +151,7 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
for entry in dirEntry:
try:
bu.debug('Found: ' + entry.name)
if entry.is_file() and (p_replayfile.match(entry.name) is not None):
if entry.is_file() and (p_replayfile.match(entry.name) != None):
bu.debug(entry.name)
await queue.put(await mkQueueItem(entry.path, title))
bu.debug('File added to queue: ' + entry.path)
@@ -170,9 +175,6 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
"""Async Worker to process the replay queue"""
global SKIPPED_N
global ERROR_N
global wi
assert wi is not None, "wi cannot be None"
while True:
item = await queue.get()
@@ -193,7 +195,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
else:
os.remove(replay_json_fn)
bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID)
bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
SKIPPED_N += 1
queue.task_done()
continue
@@ -208,19 +210,19 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
filename = os.path.basename(filename)
bu.debug(msg_str + 'File: ' + filename)
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
if json_resp is not None:
if json_resp != None:
if (await bu.save_JSON(replay_json_fn,json_resp)):
if wi.chk_JSON_replay(json_resp):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
bu.verbose_std(msg_str + title + ' posted')
else:
bu.warning(msg_str +'Replay file is not valid/complete: ' + filename)
bu.warning(msg_str +'Replay file is not valid: ' + filename)
ERROR_N += 1
else:
bu.error(msg_str + 'Error saving replay: ' + filename)
ERROR_N += 1
else:
bu.error(msg_str + 'Replay file is not valid/complete: ' + filename)
bu.error(msg_str + 'Replay file is not valid: ' + filename)
ERROR_N += 1
except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
@@ -233,7 +235,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
def getTitle_old(replayfile: str, title: str, i : int) -> str:
global wg
if title is None:
if title == None:
try:
filename = os.path.basename(replayfile)
bu.debug(filename)
@@ -245,8 +247,8 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
m = p.match(filename)
if (m is not None):
if wg.tanks is not None:
if (m != None):
if wg.tanks != None:
tank = m.group(1)
if tank in wg.tanks['userStr']:
tank = wg.tanks['userStr'][tank]
@@ -271,9 +273,7 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
global wg
assert wg is not None, "wg must not be None"
if title is None:
if title == None:
try:
filename = os.path.basename(replayfile)
bu.debug(filename)
@@ -286,14 +286,14 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
# bu.debug('Replay file\'s metadata: opened')
try:
metadata_json = json.load(meta)
#player = metadata_json['playerName']
tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
#player = metadata_json['playerName']
tank = wg.tank_str2name(metadata_json['playerVehicleName'])
map_name = wg.get_map(metadata_json['mapName'])
bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name)
bu.debug('Tank: ' + tank + ' Map: ' + map_name)
except Exception as err:
bu.error(exception = err)
if (tank_name is not None) and (map_name is not None):
title = tank_name + ' @ ' + map_name
if (tank != None) and (map_name != None):
title = tank + ' @ ' + map_name
else:
title = re.sub('\\.wotbreplay$', '', filename)
except Exception as err: