mirror of
https://github.com/Jylpah/blitz-tools.git
synced 2026-10-24 19:49:52 +02:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5f84ec4724 | ||
|
|
170e6a287d | ||
|
|
4e9032d7ba | ||
|
|
9c0949748e | ||
|
|
5b2ef816f5 | ||
|
|
448b91a866 | ||
|
|
cddc7dd947 |
No files matched your search
-15
@@ -1,15 +0,0 @@
|
|||||||
__pycache__/
|
|
||||||
BlitzApp
|
|
||||||
tmp/
|
|
||||||
ssl/
|
|
||||||
*.ini
|
|
||||||
*.swp
|
|
||||||
*.code-workspace
|
|
||||||
*.sqlite3
|
|
||||||
.vscode/
|
|
||||||
*.sqlite3-journal
|
|
||||||
.gitattributes
|
|
||||||
export*
|
|
||||||
*.csv
|
|
||||||
!/stats/*.csv
|
|
||||||
!*.py
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
# blitz-tools
|
# blitz-tools
|
||||||
Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or later.
|
Python scripts to analyze WoT Blitz replays and stats. All the scripts require [Python 3.7](https://www.python.org/downloads/) or higher. I am using 3.8 myself so you may have to change the Python interpreter version in the scripts' first line.
|
||||||
|
|
||||||
### Replays
|
### Replays
|
||||||
* [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader.
|
* [upload_wotb_replays.py](upload_wotb_replays.py): An asynchronous replay uploader.
|
||||||
@@ -13,48 +13,46 @@ Python scripts to analyze WoT Blitz replays and stats. All the scripts require [
|
|||||||
* [blitzutils.py](blitzutils.py): A utils class for various Blitz related to functions. *Other scripts use this*, so you need to place it to the same folder with the other scripts
|
* [blitzutils.py](blitzutils.py): A utils class for various Blitz related to functions. *Other scripts use this*, so you need to place it to the same folder with the other scripts
|
||||||
|
|
||||||
### Data
|
### Data
|
||||||
* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py](extractTankopedia.py). Contains only limited information required by the scripts.
|
* [tanks.json](tanks.json): Tankopedia in WG JSON format extracted from the Blitz app files with [extract_tankopedia.py]](extractTankopedia.py]). Contains only limited information required by the scripts.
|
||||||
* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py](extract_tankopedia.py).
|
* [maps.json](maps.json): Blitz map names extracted from the Blitz app files with [extract_tankopedia.py]](extract_tankopedia.py]).
|
||||||
|
|
||||||
### Quick instructions
|
### Quick instructions
|
||||||
|
|
||||||
1. Install [Python 3.8](https://www.python.org/downloads/) or later to your computer/laptop
|
1. Install [Python 3.8](https://www.python.org/downloads/) on to your computer/laptop
|
||||||
2. Install the required Python modules (see [requirements.txt](requirements.txt): motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
|
2. Install the required Python modules: motor, aiohttp, aioconsole, aiofiles, aiosqlite, beautifulsoup4, lxml, xmltodict, progress
|
||||||
```
|
```
|
||||||
python -m pip install -r requirements.txt
|
# Remember to run the command with a Python interpreter >= 3.7
|
||||||
|
python -m pip install motor aiohttp aioconsole aiofiles aiosqlite beautifulsoup4 lxml xmltodict progress
|
||||||
```
|
```
|
||||||
3. Clone the repository. [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for `git`. This makes it easy to receive the updates via `git pull`.
|
3. Download all the files including the [maps.json](maps.json) and [tanks.json](tanks.json) files. **The recommened way** is to install the scipts via `git clone`: [GitHub Desktop](https://desktop.github.com/) is an excellent GUI for git. This makes it easy to receive the updates via `git pull`.
|
||||||
|
|
||||||
|
4. Upload your replays.
|
||||||
```
|
```
|
||||||
git clone https://github.com/Jylpah/blitz-tools.git
|
# Windows, run directly from Command Prompt or Powershell
|
||||||
```
|
# Check the options
|
||||||
4. Upload your replays with `upload_wotb_replays.py`
|
|
||||||
```
|
|
||||||
## Check the options
|
|
||||||
# Windows Command Prompt or Powershell
|
|
||||||
python.exe .\upload_wotb_replays.py --help
|
python.exe .\upload_wotb_replays.py --help
|
||||||
|
|
||||||
# Linux or Windows Subsystem for Linux (WSL)
|
# Syntax
|
||||||
./upload_wotb_replays.py --help
|
python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay
|
||||||
|
|
||||||
# Upload replays (Windows)
|
|
||||||
python.exe .\upload_wotb_replays.py REPLAY_FILE.wotbreplay DIR_WITH_REPLAYS
|
|
||||||
|
|
||||||
# example
|
# example
|
||||||
python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
|
python.exe .\upload_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay
|
||||||
|
|
||||||
|
# Linux
|
||||||
|
./upload_wotb_replays.py --help
|
||||||
```
|
```
|
||||||
5. Analyze replays with `analyze_wotb_replays.py`
|
5. Analyze replays.
|
||||||
```
|
```
|
||||||
## Check options
|
# Windows, run directly from Command Prompt or Powershell
|
||||||
# Windows Command Prompt or Powershell
|
|
||||||
python.exe .\analyze_wotb_replays.py --help
|
python.exe .\analyze_wotb_replays.py --help
|
||||||
|
|
||||||
# Syntax (Windows)
|
# Syntax
|
||||||
python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json [DIR_WITH_REPLAYS ...]
|
python.exe .\analyze_wotb_replays.py REPLAY_FILE.wotbreplay.json
|
||||||
|
|
||||||
# Example (Windows)
|
# Example
|
||||||
python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
|
python.exe .\analyze_wotb_replays.py 20200223_0122__jylpah_T-54_grossberg.wotbreplay.json
|
||||||
|
|
||||||
# Linux or Windows Subsystem for Linux (WSL)
|
# Linux
|
||||||
./analyze_wotb_replays.py --help
|
./analyze_wotb_replays.py --help
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
+571
-418
File diff suppressed because it is too large.
Load diff
+114
-208
@@ -36,163 +36,132 @@ _progress_obj = None
|
|||||||
UMASK = os.umask(0)
|
UMASK = os.umask(0)
|
||||||
os.umask(UMASK)
|
os.umask(UMASK)
|
||||||
|
|
||||||
## -----------------------------------------------------------
|
# -----------------------------------------------------------
|
||||||
#### Class ThrottledClientSession(aiohttp.ClientSession)
|
# Class ThrottledClientSession(aiohttp.ClientSession)
|
||||||
#
|
# -----------------------------------------------------------
|
||||||
# Rate-limited async http client session
|
|
||||||
#
|
|
||||||
# Inherits aiohttp.ClientSession
|
|
||||||
## -----------------------------------------------------------
|
|
||||||
|
|
||||||
from typing import Optional, Union
|
|
||||||
import aiohttp
|
|
||||||
import asyncio
|
|
||||||
import time
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
class ThrottledClientSession(aiohttp.ClientSession):
|
class ThrottledClientSession(aiohttp.ClientSession):
|
||||||
"""Rate-throttled client session class inherited from aiohttp.ClientSession)"""
|
"""Rate-throttled client session class inherited from aiohttp.ClientSession)"""
|
||||||
|
# MIN_SLEEP = 0.1
|
||||||
|
|
||||||
def __init__(self, rate_limit: float = 0, filters: list[str] = list() ,
|
def __init__(self, rate_limit: float = None, *args, **kwargs) -> None:
|
||||||
limit_filtered: bool = False, re_filter: bool = False, *args,**kwargs) -> None:
|
super().__init__(*args, **kwargs)
|
||||||
assert isinstance(rate_limit, (int, float)), "rate_limit has to be float"
|
self.rate_limit = None
|
||||||
assert isinstance(filters, list), "filters has to be list"
|
self._fillerTask = None
|
||||||
assert isinstance(limit_filtered, bool),"limit_filtered has to be bool"
|
self._queue = None
|
||||||
assert isinstance(re_filter, bool), "re_filter has to be bool"
|
self._start_time = None
|
||||||
|
self._count = 0
|
||||||
|
self._start_time = time.time()
|
||||||
|
|
||||||
super().__init__(*args,**kwargs)
|
if rate_limit != None:
|
||||||
|
if rate_limit <= 0:
|
||||||
|
raise ValueError('rate_limit must be positive')
|
||||||
|
self.rate_limit = rate_limit
|
||||||
|
self._queue = asyncio.Queue(min(2, int(rate_limit)+1))
|
||||||
|
self._fillerTask = asyncio.create_task(self._filler())
|
||||||
|
|
||||||
self.rate_limit : float
|
def _get_sleep(self) -> list:
|
||||||
self._fillerTask : Optional[asyncio.Task] = None
|
if self.rate_limit == None:
|
||||||
self._queue : Optional[asyncio.Queue] = None
|
return None
|
||||||
self._start_time : float = time.time()
|
|
||||||
self._count : int = 0
|
|
||||||
self._limit_filtered: bool = limit_filtered
|
|
||||||
self._re_filter : bool = re_filter
|
|
||||||
self._filters : list[Union[str, re.Pattern]] = list()
|
|
||||||
|
|
||||||
if re_filter:
|
|
||||||
for filter in filters:
|
|
||||||
self._filters.append(re.compile(filter))
|
|
||||||
else:
|
else:
|
||||||
for filter in filters:
|
|
||||||
self._filters.append(filter)
|
|
||||||
self.set_rate_limit(rate_limit)
|
|
||||||
|
|
||||||
|
|
||||||
def _get_sleep(self) -> float:
|
|
||||||
if self.rate_limit > 0:
|
|
||||||
return 1/self.rate_limit
|
return 1/self.rate_limit
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def get_rate(self) -> float:
|
def get_rate(self) -> float:
|
||||||
"""Return rate of requests"""
|
"""Return rate of requests"""
|
||||||
return self._count / (time.time() - self._start_time)
|
return self._count / (time.time() - self._start_time)
|
||||||
|
|
||||||
|
def get_stats(self):
|
||||||
def get_stats(self) -> dict[str, float]:
|
|
||||||
"""Get session statistics"""
|
"""Get session statistics"""
|
||||||
res = {'rate' : self.get_rate(), 'rate_limit': self.rate_limit, 'count' : self._count }
|
res = {'rate': self.get_rate(), 'rate_limit': self.rate_limit,
|
||||||
|
'count': self._count}
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
def get_stats_str(self):
|
||||||
def get_stats_str(self) -> str:
|
|
||||||
"""Print session statistics"""
|
"""Print session statistics"""
|
||||||
return f"rate limit: {str(self.rate_limit if self.rate_limit != None else '-')} \
|
return 'rate limit: ' + str(self.rate_limit if self.rate_limit != None else '-') + \
|
||||||
rate: {0:.1f}.format(self.get_rate()) requests: {str(self._count)}"
|
' rate: ' + \
|
||||||
|
"{0:.1f}".format(self.get_rate()) + \
|
||||||
|
' requests: ' + str(self._count)
|
||||||
|
|
||||||
|
def reset_counters(self):
|
||||||
def reset_counters(self) -> dict[str, float]:
|
|
||||||
"""Reset rate counters and return current results"""
|
"""Reset rate counters and return current results"""
|
||||||
res = self.get_stats()
|
res = self.get_stats()
|
||||||
self._start_time = time.time()
|
self._start_time = time.time()
|
||||||
self._count = 0
|
self._count = 0
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
def set_rate_limit(self, rate_limit: float = 1):
|
||||||
def set_rate_limit(self, rate_limit: float = 0) -> float:
|
if rate_limit >= 0:
|
||||||
assert rate_limit is not None, "rate_limit must not be None"
|
|
||||||
assert isinstance(rate_limit, (int,float)) and rate_limit >= 0, "rate_limit has to be type of 'float' >= 0"
|
|
||||||
|
|
||||||
self.rate_limit = rate_limit
|
self.rate_limit = rate_limit
|
||||||
if rate_limit > 0:
|
|
||||||
self._queue = asyncio.Queue(int(rate_limit)+1)
|
|
||||||
if self._fillerTask is not None:
|
|
||||||
self._fillerTask.cancel()
|
|
||||||
self._fillerTask = asyncio.create_task(self._filler())
|
|
||||||
return self.rate_limit
|
return self.rate_limit
|
||||||
|
return None
|
||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self) -> None:
|
||||||
"""Close rate-limiter's "bucket filler" task"""
|
"""Close rate-limiter's "bucket filler" task"""
|
||||||
# DEBUG
|
# DEBUG
|
||||||
logging.debug(self.get_stats_str())
|
debug(self.get_stats_str())
|
||||||
try:
|
try:
|
||||||
if self._fillerTask is not None:
|
if self._fillerTask != None:
|
||||||
self._fillerTask.cancel()
|
self._fillerTask.cancel()
|
||||||
await asyncio.wait_for(self._fillerTask, timeout=0.5)
|
await asyncio.wait_for(self._fillerTask, timeout=0.5)
|
||||||
except asyncio.TimeoutError as err:
|
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||||
logging.error(str(err))
|
pass
|
||||||
await super().close()
|
await super().close()
|
||||||
|
|
||||||
|
async def _filler_simple(self):
|
||||||
async def _filler(self) -> None:
|
|
||||||
"""Filler task to fill the leaky bucket algo"""
|
"""Filler task to fill the leaky bucket algo"""
|
||||||
try:
|
try:
|
||||||
if self._queue is None:
|
if self._queue == None:
|
||||||
return None
|
return
|
||||||
logging.debug('SLEEP: ' + str(self._get_sleep()))
|
sleep = self._get_sleep()
|
||||||
|
while True:
|
||||||
|
self._queue.put(None)
|
||||||
|
await asyncio.sleep(sleep)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
debug('Cancelled')
|
||||||
|
except Exception as err:
|
||||||
|
error(exception=err)
|
||||||
|
|
||||||
|
async def _filler(self):
|
||||||
|
"""Filler task to fill the leaky bucket algo"""
|
||||||
|
try:
|
||||||
|
if self._queue == None:
|
||||||
|
return
|
||||||
|
sleep = self._get_sleep()
|
||||||
|
# debug('SLEEP: ' + str(sleep))
|
||||||
updated_at = time.monotonic()
|
updated_at = time.monotonic()
|
||||||
extra_increment : float = 0
|
fraction = 0
|
||||||
|
extra_increment = 0
|
||||||
for i in range(0, self._queue.maxsize):
|
for i in range(0, self._queue.maxsize):
|
||||||
await self._queue.put(i)
|
self._queue.put_nowait(i)
|
||||||
while True:
|
while True:
|
||||||
if not self._queue.full():
|
if not self._queue.full():
|
||||||
now = time.monotonic()
|
now = time.monotonic()
|
||||||
increment = self.rate_limit * (now - updated_at)
|
increment = self.rate_limit * (now - updated_at)
|
||||||
items_2_add = int(min(self._queue.maxsize - self._queue.qsize(), int(increment + extra_increment)))
|
fraction += increment % 1
|
||||||
extra_increment = (increment + extra_increment) % 1
|
extra_increment = fraction // 1
|
||||||
for i in range(0,items_2_add):
|
items_2_add = int(
|
||||||
|
min(self._queue.maxsize - self._queue.qsize(), int(increment) + extra_increment))
|
||||||
|
fraction = fraction % 1
|
||||||
|
for i in range(0, items_2_add):
|
||||||
self._queue.put_nowait(i)
|
self._queue.put_nowait(i)
|
||||||
updated_at = now
|
updated_at = now
|
||||||
await asyncio.sleep(self._get_sleep())
|
await asyncio.sleep(sleep)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
logging.debug('Cancelled')
|
debug('Cancelled')
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
logging.error(str(err))
|
error(exception=err)
|
||||||
return None
|
|
||||||
|
|
||||||
|
async def _request(self, *args, **kwargs):
|
||||||
async def _request(self, *args,**kwargs) -> aiohttp.ClientResponse:
|
|
||||||
"""Throttled _request()"""
|
"""Throttled _request()"""
|
||||||
if self._queue is not None and self.is_limited(*args):
|
if self._queue != None:
|
||||||
await self._queue.get()
|
await self._queue.get()
|
||||||
self._queue.task_done()
|
self._queue.task_done()
|
||||||
self._count += 1
|
self._count += 1
|
||||||
return await super()._request(*args,**kwargs)
|
return await super()._request(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def is_limited(self, *args: str) -> bool:
|
|
||||||
"""Check wether the rate limit should be applied"""
|
|
||||||
try:
|
|
||||||
url: str = args[1]
|
|
||||||
for filter in self._filters:
|
|
||||||
if isinstance(filter, re.Pattern) and filter.match(url) is not None:
|
|
||||||
return self._limit_filtered
|
|
||||||
elif isinstance(filter, str) and url.startswith(filter):
|
|
||||||
return self._limit_filtered
|
|
||||||
|
|
||||||
return not self._limit_filtered
|
|
||||||
except Exception as err:
|
|
||||||
logging.error(str(err))
|
|
||||||
return True
|
|
||||||
|
|
||||||
# -----------------------------------------------------------
|
|
||||||
# Class AsyncLogger()
|
|
||||||
# -----------------------------------------------------------
|
|
||||||
|
|
||||||
class AsyncLogger():
|
class AsyncLogger():
|
||||||
"""Async file logger"""
|
"""Async file logger"""
|
||||||
|
|
||||||
@@ -517,27 +486,18 @@ def NOW() -> int:
|
|||||||
return int(time.time())
|
return int(time.time())
|
||||||
|
|
||||||
|
|
||||||
def get_date_str(timestamp:int = NOW(), date_format = '%Y%m%d_%H%M%S'):
|
def rebase_file_args(current_dir, files):
|
||||||
"""Return YYYYMMDD_HHmm date string"""
|
"""REbase file command line params after moving working dir to the script's dir"""
|
||||||
try:
|
if isinstance(files, list):
|
||||||
ts = datetime.fromtimestamp(timestamp)
|
|
||||||
return ts.strftime(date_format)
|
|
||||||
except Exception as err:
|
|
||||||
error(exception=err)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def rebase_file_args(current_dir : str, files: list[str]) -> list[str]:
|
|
||||||
"""Rebase file command line params after moving working dir to the script's dir"""
|
|
||||||
assert isinstance(files, list), "files has to be a list"
|
|
||||||
|
|
||||||
if (files[0] == '-') or (files[0] == 'db:'):
|
if (files[0] == '-') or (files[0] == 'db:'):
|
||||||
return [ files[0] ]
|
return files
|
||||||
else:
|
else:
|
||||||
return [ os.path.join(current_dir, fn) for fn in files ]
|
return [ os.path.join(current_dir, fn) for fn in files ]
|
||||||
|
elif isinstance(files, str):
|
||||||
|
return os.path.join(current_dir, files)
|
||||||
|
|
||||||
|
|
||||||
async def read_int_list(filename: str) -> list[int]:
|
async def read_int_list(filename: str) -> list():
|
||||||
"""Read file to a list and return list of integers in the input file"""
|
"""Read file to a list and return list of integers in the input file"""
|
||||||
|
|
||||||
input_list = []
|
input_list = []
|
||||||
@@ -673,14 +633,6 @@ def get_JSON_value(json, key : str = None, keys : list = None, keypath = None):
|
|||||||
raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
|
raise KeyError('Key not found: ' + get_JSON_keypath(keypath, keys[0]))
|
||||||
|
|
||||||
|
|
||||||
def sort_dict(d: dict, number: bool = False) -> dict:
|
|
||||||
"""Sort a dict by keys"""
|
|
||||||
if number:
|
|
||||||
return dict(sorted(d.items(), key=lambda item: int(item[0])))
|
|
||||||
else:
|
|
||||||
return dict(sorted(d.items(), key=lambda item: item[0]))
|
|
||||||
|
|
||||||
|
|
||||||
# -----------------------------------------------------------
|
# -----------------------------------------------------------
|
||||||
# Class SlowBar
|
# Class SlowBar
|
||||||
# -----------------------------------------------------------
|
# -----------------------------------------------------------
|
||||||
@@ -964,18 +916,33 @@ class WG:
|
|||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_map(self, map_str: str) -> str:
|
def get_map(cls, map_str: str) -> str:
|
||||||
"""Return map name from short map string in replays"""
|
"""Return map name from short map string in replays"""
|
||||||
try:
|
try:
|
||||||
return self.maps[map_str]
|
return cls.maps[map_str]
|
||||||
except:
|
except:
|
||||||
error('Map ' + map_str + ' not found')
|
error('Map ' + map_str + ' not found')
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_tank_user_strs(self) -> str:
|
def get_map_user_strs(cls) -> str:
|
||||||
return self.tanks["userStr"].keys()
|
return cls.maps.keys()
|
||||||
|
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tank_str2name(cls, tank_str: str) -> str:
|
||||||
|
"""Return tank name from short tank string in replays"""
|
||||||
|
try:
|
||||||
|
return cls.tanks["userStr"][tank_str]
|
||||||
|
except:
|
||||||
|
error('Tank ' + tank_str + ' not found')
|
||||||
|
return tank_str
|
||||||
|
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_tank_user_strs(cls) -> str:
|
||||||
|
return cls.tanks["userStr"].keys()
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -1112,14 +1079,10 @@ class WG:
|
|||||||
"""Load tanks from tankopedia JSON"""
|
"""Load tanks from tankopedia JSON"""
|
||||||
if tankopedia_fn == None:
|
if tankopedia_fn == None:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(tankopedia_fn, 'rt', encoding='utf8') as f:
|
with open(tankopedia_fn, 'rt', encoding='utf8') as f:
|
||||||
self.tanks = json.loads(f.read())
|
self.tanks = json.loads(f.read())
|
||||||
self.tanks['tankStr'] = dict()
|
|
||||||
p = re.compile('.+_short$')
|
|
||||||
for usr_str in self.tanks['userStr']:
|
|
||||||
if p.match(usr_str) == None:
|
|
||||||
self.tanks['tankStr'][self.tanks['userStr'][usr_str]] = usr_str
|
|
||||||
self.tanks_by_tier = dict()
|
self.tanks_by_tier = dict()
|
||||||
for tier in range(1,11):
|
for tier in range(1,11):
|
||||||
self.tanks_by_tier[str(tier)] = list()
|
self.tanks_by_tier[str(tier)] = list()
|
||||||
@@ -1131,43 +1094,6 @@ class WG:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def get_replay_filename(self, replay: dict):
|
|
||||||
try:
|
|
||||||
summary = replay['data']['summary']
|
|
||||||
timestamp = int(summary['battle_start_timestamp'])
|
|
||||||
timestamp = get_date_str(timestamp, date_format='%Y%m%d_%H%M')
|
|
||||||
player = summary['player_name']
|
|
||||||
arena_id = int(summary['arena_unique_id'])
|
|
||||||
vehicle = self.name2tank_str(summary['vehicle'])
|
|
||||||
vehicle.replace(' ','_')
|
|
||||||
|
|
||||||
return '{:s}_{:s}_{:s}_{:d}.wotbreplay.json'.format(timestamp, player, vehicle, arena_id)
|
|
||||||
except Exception as err:
|
|
||||||
error('Unexpected Exception', err)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def get_map_user_strs(self) -> str:
|
|
||||||
return self.maps.keys()
|
|
||||||
|
|
||||||
|
|
||||||
def tank_str2name(self, tank_str: str) -> str:
|
|
||||||
"""Return tank name from short tank string in replays"""
|
|
||||||
try:
|
|
||||||
return self.tanks["userStr"][tank_str]
|
|
||||||
except:
|
|
||||||
error('Tank ' + tank_str + ' not found')
|
|
||||||
return tank_str
|
|
||||||
|
|
||||||
|
|
||||||
def name2tank_str(self, name: str) -> str:
|
|
||||||
"""Return shorttank string from tank string in replays"""
|
|
||||||
try:
|
|
||||||
return self.tanks["tankStr"][name]
|
|
||||||
except:
|
|
||||||
debug('Tank ' + name + ' not found')
|
|
||||||
return name
|
|
||||||
|
|
||||||
|
|
||||||
def get_tanks_by_tier(self, tier: int) -> list():
|
def get_tanks_by_tier(self, tier: int) -> list():
|
||||||
"""Returns tank_ids by tier"""
|
"""Returns tank_ids by tier"""
|
||||||
try:
|
try:
|
||||||
@@ -1626,8 +1552,6 @@ class WG:
|
|||||||
|
|
||||||
async def store_tank_stats(self, key: list , stats_data: list, update_time: int):
|
async def store_tank_stats(self, key: list , stats_data: list, update_time: int):
|
||||||
"""Save tank stats into cache"""
|
"""Save tank stats into cache"""
|
||||||
assert self.cache is not None, "cache must be initiated"
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
account_id = key[0]
|
account_id = key[0]
|
||||||
tank_ids = set(key[1])
|
tank_ids = set(key[1])
|
||||||
@@ -1649,8 +1573,6 @@ class WG:
|
|||||||
|
|
||||||
async def store_player_stats(self, key: list , stats_data: list, update_time: int):
|
async def store_player_stats(self, key: list , stats_data: list, update_time: int):
|
||||||
"""Save player stats into cache"""
|
"""Save player stats into cache"""
|
||||||
assert self.cache is not None, "cache must be initiated"
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
account_id = key[0]
|
account_id = key[0]
|
||||||
if stats_data != None:
|
if stats_data != None:
|
||||||
@@ -1667,8 +1589,6 @@ class WG:
|
|||||||
|
|
||||||
async def store_player_achievements(self, key: list , stats_data: list, update_time: int):
|
async def store_player_achievements(self, key: list , stats_data: list, update_time: int):
|
||||||
"""Save player stats into cache"""
|
"""Save player stats into cache"""
|
||||||
assert self.cache is not None, "cache must be initiated"
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
account_id = key[0]
|
account_id = key[0]
|
||||||
if stats_data != None:
|
if stats_data != None:
|
||||||
@@ -1684,8 +1604,6 @@ class WG:
|
|||||||
|
|
||||||
|
|
||||||
async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ):
|
async def get_cached_tank_stats(self, account_id: int, tank_ids: list, fields: list ):
|
||||||
assert self.cache is not None, "cache must be initiated"
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# test for cacheDB existence
|
# test for cacheDB existence
|
||||||
debug('Trying cached stats first')
|
debug('Trying cached stats first')
|
||||||
@@ -1799,27 +1717,17 @@ class WoTinspector:
|
|||||||
URL_REPLAY_VIEW = URL_WI +'/en/view/'
|
URL_REPLAY_VIEW = URL_WI +'/en/view/'
|
||||||
URL_REPLAY_UL = 'https://api.wotinspector.com/replay/upload?'
|
URL_REPLAY_UL = 'https://api.wotinspector.com/replay/upload?'
|
||||||
URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
|
URL_REPLAY_INFO = 'https://api.wotinspector.com/replay/upload?details=full&key='
|
||||||
|
|
||||||
URL_TANK_DB = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
|
URL_TANK_DB = "https://wotinspector.com/static/armorinspector/tank_db_blitz.js"
|
||||||
|
|
||||||
REPLAY_N = 1
|
REPLAY_N = 1
|
||||||
DEFAULT_RATE_LIMIT : float = 10/60 # 10 requests / min
|
|
||||||
DEFAULT_TOKEN : str = 'a5428af49bf7495986577d59b0e5bcff'
|
|
||||||
|
|
||||||
def __init__(self, rate_limit: float = DEFAULT_RATE_LIMIT, auth_token: Optional[str] = None):
|
def __init__(self, rate_limit: int = 30):
|
||||||
|
self.session = ThrottledClientSession(rate_limit=rate_limit)
|
||||||
headers : Optional[dict[str, str]] = None
|
|
||||||
|
|
||||||
if auth_token is None:
|
|
||||||
auth_token = self.DEFAULT_TOKEN
|
|
||||||
headers = dict()
|
|
||||||
headers['Authorization'] = 'Token ' + auth_token
|
|
||||||
self.session = ThrottledClientSession(rate_limit=rate_limit,
|
|
||||||
filters=[self.URL_REPLAY_LIST, self.URL_REPLAY_INFO],
|
|
||||||
re_filter=False, limit_filtered=True, headers = headers)
|
|
||||||
|
|
||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self):
|
||||||
if self.session is not None:
|
if self.session != None:
|
||||||
debug('Closing aiohttp session')
|
debug('Closing aiohttp session')
|
||||||
await self.session.close()
|
await self.session.close()
|
||||||
|
|
||||||
@@ -1883,7 +1791,7 @@ class WoTinspector:
|
|||||||
hash.update(data)
|
hash.update(data)
|
||||||
replay_id = hash.hexdigest()
|
replay_id = hash.hexdigest()
|
||||||
|
|
||||||
## Testing if the replay has already been posted
|
# Testing if the replay has already been posted
|
||||||
json_resp = await self.get_replay_JSON(replay_id)
|
json_resp = await self.get_replay_JSON(replay_id)
|
||||||
if json_resp != None:
|
if json_resp != None:
|
||||||
debug('Already uploaded: ' + title, id=N)
|
debug('Already uploaded: ' + title, id=N)
|
||||||
@@ -1898,7 +1806,7 @@ class WoTinspector:
|
|||||||
}
|
}
|
||||||
|
|
||||||
url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
|
url = self.URL_REPLAY_UL + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
|
||||||
#debug('URL: ' + url)
|
# debug('URL: ' + url)
|
||||||
headers ={'Content-type': 'application/x-www-form-urlencoded'}
|
headers ={'Content-type': 'application/x-www-form-urlencoded'}
|
||||||
payload = { 'file' : (filename, base64.b64encode(data)) }
|
payload = { 'file' : (filename, base64.b64encode(data)) }
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
@@ -1960,6 +1868,11 @@ class WoTinspector:
|
|||||||
return replay_links
|
return replay_links
|
||||||
|
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_replay_id(cls, url):
|
||||||
|
return url.rsplit('/', 1)[-1]
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def read_replay_id(cls, json_replay):
|
def read_replay_id(cls, json_replay):
|
||||||
"""Read replay_id from replay JSON file"""
|
"""Read replay_id from replay JSON file"""
|
||||||
@@ -1970,18 +1883,14 @@ class WoTinspector:
|
|||||||
error(exception=err)
|
error(exception=err)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_replay_id(cls, url):
|
def chk_JSON_replay(cls, json_resp):
|
||||||
return url.rsplit('/', 1)[-1]
|
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def chk_JSON_replay(cls, json_resp) -> bool:
|
|
||||||
""""Check String for being a valid JSON file"""
|
""""Check String for being a valid JSON file"""
|
||||||
try:
|
try:
|
||||||
|
# FIX The replay can have allies/enemies i.e. be useful for
|
||||||
|
# team composition analysis, but does not have player contribution
|
||||||
if ('status' in json_resp) and json_resp['status'] == 'ok' and \
|
if ('status' in json_resp) and json_resp['status'] == 'ok' and \
|
||||||
(get_JSON_value(json_resp, key='data.summary.exp_base') != None) :
|
(get_JSON_value(json_resp, key='data.summary.winner_team') != None) :
|
||||||
debug("JSON check OK")
|
debug("JSON check OK")
|
||||||
return True
|
return True
|
||||||
except KeyError as err:
|
except KeyError as err:
|
||||||
@@ -1990,9 +1899,6 @@ class WoTinspector:
|
|||||||
debug("Replay JSON check failed: " + str(json_resp))
|
debug("Replay JSON check failed: " + str(json_resp))
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# -----------------------------------------------------------
|
|
||||||
# Class BlitzStars
|
|
||||||
# -----------------------------------------------------------
|
|
||||||
|
|
||||||
class BlitzStars:
|
class BlitzStars:
|
||||||
|
|
||||||
|
|||||||
+8
-14
@@ -3,7 +3,6 @@
|
|||||||
|
|
||||||
# Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
|
# Extract tankopedia data in WG API JSON format from Blitz app files (Android APK unzipped)
|
||||||
|
|
||||||
from os import path
|
|
||||||
import sys, argparse, json, os, inspect, aiohttp, asyncio, aiofiles, re, logging, time, xmltodict, collections, configparser
|
import sys, argparse, json, os, inspect, aiohttp, asyncio, aiofiles, re, logging, time, xmltodict, collections, configparser
|
||||||
import blitzutils as bu
|
import blitzutils as bu
|
||||||
from blitzutils import WG
|
from blitzutils import WG
|
||||||
@@ -12,11 +11,9 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG)
|
|||||||
|
|
||||||
FILE_CONFIG = 'blitzstats.ini'
|
FILE_CONFIG = 'blitzstats.ini'
|
||||||
|
|
||||||
# BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
|
BLITZAPP_STRINGS='/assets/Data/Strings/en.yaml'
|
||||||
# BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
|
BLITZAPP_VEHICLES_DIR='/assets/Data/XML/item_defs/vehicles/'
|
||||||
BLITZAPP_STRINGS='Data/Strings/en.yaml'
|
BLITZAPP_VEHICLE_FILE='/list.xml'
|
||||||
BLITZAPP_VEHICLES_DIR='Data/XML/item_defs/vehicles/'
|
|
||||||
BLITZAPP_VEHICLE_FILE='list.xml'
|
|
||||||
|
|
||||||
wg = None
|
wg = None
|
||||||
|
|
||||||
@@ -82,8 +79,8 @@ async def main(argv):
|
|||||||
tankopedia = collections.OrderedDict()
|
tankopedia = collections.OrderedDict()
|
||||||
tankopedia['status'] = 'ok'
|
tankopedia['status'] = 'ok'
|
||||||
tankopedia['meta'] = { "count": len(tanks) }
|
tankopedia['meta'] = { "count": len(tanks) }
|
||||||
tankopedia['data'] = bu.sort_dict(tanks, number=True)
|
tankopedia['data'] = tanks
|
||||||
tankopedia['userStr'] = bu.sort_dict(userStrs)
|
tankopedia['userStr'] = userStrs
|
||||||
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
|
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(tanks)) + ' tanks')
|
||||||
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')
|
bu.verbose_std('New tankopedia \'' + args.tanks + '\' contains ' + str(len(userStrs)) + ' tank strings')
|
||||||
await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
|
await outfile.write(json.dumps(tankopedia, ensure_ascii=False, indent=4, sort_keys=False))
|
||||||
@@ -107,11 +104,8 @@ async def main(argv):
|
|||||||
async def extract_tanks(blitz_app_base : str, nation: str):
|
async def extract_tanks(blitz_app_base : str, nation: str):
|
||||||
|
|
||||||
tanks = []
|
tanks = []
|
||||||
# WG has changed the location of Data directory - at least in steam client
|
list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
|
||||||
if path.isdir(path.join(blitz_app_base, 'assets')):
|
if not os.path.isfile(list_xml_file):
|
||||||
blitz_app_base = path.join(blitz_app_base, 'assets')
|
|
||||||
list_xml_file = path.join(blitz_app_base, BLITZAPP_VEHICLES_DIR ,nation ,BLITZAPP_VEHICLE_FILE)
|
|
||||||
if not path.isfile(list_xml_file):
|
|
||||||
print('ERROR: cannot open ' + list_xml_file)
|
print('ERROR: cannot open ' + list_xml_file)
|
||||||
return None
|
return None
|
||||||
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
|
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
|
||||||
@@ -138,7 +132,7 @@ async def read_user_strs(blitz_app_base : str) -> dict:
|
|||||||
"""Read user strings to convert map and tank names"""
|
"""Read user strings to convert map and tank names"""
|
||||||
tank_strs = {}
|
tank_strs = {}
|
||||||
map_strs = {}
|
map_strs = {}
|
||||||
filename = path.join(blitz_app_base, BLITZAPP_STRINGS)
|
filename = blitz_app_base + BLITZAPP_STRINGS
|
||||||
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
|
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
|
||||||
try:
|
try:
|
||||||
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
|
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/local/bin/python3.7
|
||||||
|
|
||||||
# Get Tankpedia from WoTinspector.com
|
# Get Tankpedia from WoTinspector.com
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,6 @@
|
|||||||
"fort": "Fort Despair",
|
"fort": "Fort Despair",
|
||||||
"grossberg": "Dynasty's Pearl",
|
"grossberg": "Dynasty's Pearl",
|
||||||
"himmelsdorf": "Himmelsdorf",
|
"himmelsdorf": "Himmelsdorf",
|
||||||
"holland": "Molendijk",
|
|
||||||
"holmeisk": "Wasteland",
|
"holmeisk": "Wasteland",
|
||||||
"iceworld": "Everfrost",
|
"iceworld": "Everfrost",
|
||||||
"idle": "Yukon",
|
"idle": "Yukon",
|
||||||
@@ -44,7 +43,6 @@
|
|||||||
"neptune_02": "Normandy - Beach",
|
"neptune_02": "Normandy - Beach",
|
||||||
"north": "North",
|
"north": "North",
|
||||||
"ordeal": "Trial by Fire",
|
"ordeal": "Trial by Fire",
|
||||||
"plant": "Ghost Factory",
|
|
||||||
"pliego": "Castilla",
|
"pliego": "Castilla",
|
||||||
"port": "Port Bay",
|
"port": "Port Bay",
|
||||||
"rift": "Hellas",
|
"rift": "Hellas",
|
||||||
|
|||||||
+81
-94
@@ -1,96 +1,83 @@
|
|||||||
# WoT Blitz releases
|
# WoT Blitz releases
|
||||||
|
|
||||||
| Release | Date | Notes |
|
| Release | Date | Notes | Inaccurate? |
|
||||||
|:-------:|:--------:|-------------|
|
|:-------:|:--------:|-------------|-----|
|
||||||
| 9.1 | 2022-07-14 | Releases updated via DLC, graphics updates for Object 140, Patton 46 |
|
| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo | |
|
||||||
| 9.0 | 2022-06-08 | Japanies heavies, Castilla graphics update, new garage |
|
| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables | |
|
||||||
| 8.10 | 2022-04-28 | New battle results screen, new "Operation" mechanics |
|
| 7.7 | 2021-02-24 | Czechoslovakian medium tanks | |
|
||||||
| 8.9 | 2022-03-23 | "Free" camo resupply, VK 72.01 supplies |
|
| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera | |
|
||||||
| 8.8 | 2022-02-16 | Molendijk map |
|
| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff | |
|
||||||
| 8.7 | 2022-01-19 | Soviet Light Tanks |
|
| 7.4 | 2020-11-12 | TD buffs, Clan missions | |
|
||||||
| 8.6 | 2021-12-15 | Joining Light and Medium Tank classes in MM |
|
| 7.3 | 2020-10-08 | Swedish HT line | |
|
||||||
| 8.5 | 2021-11-17 | New national crew voices, economy update |
|
| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles | |
|
||||||
| 8.4 | 2021-10-20 | Polish heavy tanks |
|
| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff | |
|
||||||
| 8.3 | 2021-09-16 | FV4005, FV4022, VK 91.01 (P) and WZ-121 legendary camos, -5 deg gun depression to WZ-120 |
|
| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes | |
|
||||||
| 8.2 | 2021-08-18 | Ghost Factory map, XP/credits nerf for inactive players, new store UI |
|
| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes | |
|
||||||
| 8.1 | 2021-07-15 | Armor highlighting updated |
|
| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff | |
|
||||||
| 8.0 | 2021-06-08 | American experimental heavies (Yoh-line), PBR camos |
|
| 6.8 | 2020-02-19 | ATGM nerf, Hellas map | |
|
||||||
| 7.9 | 2021-04-27 | Mines rework, new tutorial, colorblind mode, JgPz E100 legendary camo |
|
| 6.7 | 2020-01-15 | ATGM US Lights | |
|
||||||
| 7.8 | 2021-03-24 | Mid-tier balance changes, Grille line new consumables |
|
| 6.6 | 2019-12-18 | T-62A + Leo buffs | |
|
||||||
| 7.7 | 2021-02-24 | Czechoslovakian medium tanks |
|
| 6.5 | 2019-11-21 | New UI | |
|
||||||
| 7.6 | 2021-01-27 | AMC 13 90, TOG II buffs, free replay camera |
|
| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... | |
|
||||||
| 7.5 | 2020-12-09 | ATGM removal, Batchat, Kpz70, T-2020 buff |
|
| 6.3 | 2019-09-18 | European line | |
|
||||||
| 7.4 | 2020-11-12 | TD buffs, Clan missions |
|
| 6.2 | 2019-08-14 | New MM | |
|
||||||
| 7.3 | 2020-10-08 | Swedish HT line |
|
| 6.1 | 2019-07-10 | Autoloader buff | |
|
||||||
| 7.2 | 2020-08-19 | Balance changes, added tier X ammo capa, new aiming reticles |
|
| 6.0 | 2019-06-05 | UK TD line #2 | |
|
||||||
| 7.1 | 2020-07-15 | New Yukon map, IS-6, KV-4, Patton 48 armor buff, Grille DPM buff, SP IC, AMZ 13 75, M41 Bulldog mobility buff |
|
| 5.10 | 2019-04-24 | new missions | |
|
||||||
| 7.0 | 2020-06-17 | British LTs, graphics improvements, ATGM nerf, Sheridan nerf, map fixes |
|
| 5.9 | 2019-03-27 | New shell tracers, minor fixes | |
|
||||||
| 6.10 | 2020-04-29 | T92E1 nerf, Tier V-IX heavy HP buff, Battle Pass, map fixes |
|
| 5.8 | 2019-02-21 | Chinese TD line | |
|
||||||
| 6.9 | 2020-03-25 | Japanese TDs, Tier X heavy HP buff |
|
| 5.7 | 2019-01-16 | Avatars | |
|
||||||
| 6.8 | 2020-02-19 | ATGM nerf, Hellas map |
|
| 5.6 | 2018-12-05 | Normandy map | |
|
||||||
| 6.7 | 2020-01-15 | ATGM US Lights |
|
| 5.5 | 2018-11-13 | Low-tier clean-up | |
|
||||||
| 6.6 | 2019-12-18 | T-62A + Leo buffs |
|
| 5.4 | 2018-10-10 | MM "rigging" adjusted | |
|
||||||
| 6.5 | 2019-11-21 | New UI |
|
| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added | |
|
||||||
| 6.4 | 2019-10-16 | Halloween map, mid-tier HT buff... |
|
| 5.2 | 2018-08-22 | Faust map | |
|
||||||
| 6.3 | 2019-09-18 | European line |
|
| 5.1 | 2018-07-31 | New MM: vehicle class matching | |
|
||||||
| 6.2 | 2019-08-14 | New MM |
|
| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts killed | |
|
||||||
| 6.1 | 2019-07-10 | Autoloader buff |
|
| 4.10 | 2018-05-24 | Alpenstadt map | |
|
||||||
| 6.0 | 2019-06-05 | UK TD line #2 |
|
| 4.9 | 2018-04-25 | Chinese heavy tanks | |
|
||||||
| 5.10 | 2019-04-24 | new missions |
|
| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator | |
|
||||||
| 5.9 | 2019-03-27 | New shell tracers, minor fixes |
|
| 4.7 | 2018-02-28 | New minimap | |
|
||||||
| 5.8 | 2019-02-21 | Chinese TD line |
|
| 4.6 | 2018-01-31 | Chinese nation | |
|
||||||
| 5.7 | 2019-01-16 | Avatars |
|
| 4.5 | 2017-12-19 | Dynasty's Pearl map | |
|
||||||
| 5.6 | 2018-12-05 | Normandy map |
|
| 4.4 | 2017-11-09 | New British tanks | |
|
||||||
| 5.5 | 2018-11-13 | Low-tier clean-up :-( |
|
| 4.3 | 2017-10-10 | Level 7 screw skills | |
|
||||||
| 5.4 | 2018-10-10 | MM "rigging" adjusted |
|
| 4.2 | 2017-09-12 | French LTs, Naval Frontier map | |
|
||||||
| 5.3 | 2018-09-12 | MM "rigging" started, Newbie queue added |
|
| 4.1 | 2017-08-16 | New markers | |
|
||||||
| 5.2 | 2018-08-22 | Faust map |
|
| 4.0 | 2017-07-13 | Rating battles | Y |
|
||||||
| 5.1 | 2018-07-31 | New MM: vehicle class matching |
|
| 3.10 | 2017-06-15 | Fixes | Y |
|
||||||
| 5.0 | 2018-06-30 | Vineyeards map update, Spare Parts removed ! |
|
| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins | Y |
|
||||||
| 4.10 | 2018-05-24 | Alpenstadt map |
|
| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system | Y |
|
||||||
| 4.9 | 2018-04-25 | Chinese heavy tanks |
|
| 3.7 | 2017-03-22 | Balance changes, Tournaments | Y |
|
||||||
| 4.8 | 2018-03-28 | Choice of battle modes, crew skills accelerator |
|
| 3.6 | 2017-02-16 | Vehicle Enrichment | Y |
|
||||||
| 4.7 | 2018-02-28 | New minimap |
|
| 3.5 | 2017-01-13 | Boosters, Dead rail update | Y |
|
||||||
| 4.6 | 2018-01-31 | Chinese nation |
|
| 3.4 | 2016-11-28 | Canyon map, USA T57 line | |
|
||||||
| 4.5 | 2017-12-19 | Dynasty's Pearl map |
|
| 3.3 | 2016-11-01 | Updated Mines map | |
|
||||||
| 4.4 | 2017-11-09 | New British tanks |
|
| 3.2 | 2016-10-04 | Copperfield update | |
|
||||||
| 4.3 | 2017-10-10 | Level 7 screw skills |
|
| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map | Y |
|
||||||
| 4.2 | 2017-09-12 | French LTs, Naval Frontier map |
|
| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update | Y |
|
||||||
| 4.1 | 2017-08-16 | New markers |
|
| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs | |
|
||||||
| 4.0 | 2017-07-13 | Rating battles (date uncertain) |
|
| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 | |
|
||||||
| 3.10 | 2017-06-15 | Fixes (date uncertain) |
|
| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker | Y |
|
||||||
| 3.9 | 2017-05-19 | Military Honor, Mayan Ruins (date uncertain) |
|
| 2.8 | 2016-04-04 | New Russian TD line, new consumables system | Y |
|
||||||
| 3.8 | 2017-04-24 | Spare Parts, New 9-slot Equipment system (date uncertain) |
|
| 2.7 | 2016-03-07 | Mirage map | Y |
|
||||||
| 3.7 | 2017-03-22 | Balance changes, Tournaments (date uncertain) |
|
| 2.6 | 2016-02-10 | German LTs, shooting through obstacles | Y |
|
||||||
| 3.6 | 2017-02-16 | Vehicle Enrichment (date uncertain) |
|
| 2.5 | 2016-01-11 | Camouflages, Clan supply | Y |
|
||||||
| 3.5 | 2017-01-13 | Boosters, Dead rail update (date uncertain) |
|
| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version | Y |
|
||||||
| 3.4 | 2016-11-28 | Canyon map, USA T57 line |
|
| 2.3 | 2015-11-12 | Canal map | Y |
|
||||||
| 3.3 | 2016-11-01 | Updated Mines map |
|
| 2.2 | 2015-10-05 | Desert Sands map update | Y |
|
||||||
| 3.2 | 2016-10-04 | Copperfield update |
|
| 2.1 | 2015-09-07 | UK TDs, Clan Chat | Y |
|
||||||
| 3.1 | 2016-09-03 | Clan tournaments, Yamato Harbor map (date uncertain) |
|
| 2.0 | 2015-08-05 | Clans, Oasis Palms updated | |
|
||||||
| 3.0 | 2016-07-20 | Supremacy mode, Falls Creek update (date uncertain) |
|
| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo | Y |
|
||||||
| 2.11 | 2016-06-28 | Vineyeards map, Oasis Palms update, German paper TDs |
|
| 1.10 | 2015-06-11 | Port Bay map | Y |
|
||||||
| 2.10 | 2016-06-07 | Rockfield map update, Tank birthdays, high-pen gun for IS-6 |
|
| 1.9 | 2015-05-13 | New German medium tanks (Leo) | Y |
|
||||||
| 2.9 | 2016-05-03 | Training rooms, +/-1 Matchmaker (date uncertain) |
|
| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players | Y |
|
||||||
| 2.8 | 2016-04-04 | New Russian TD line, new consumables system (date uncertain) |
|
| 1.7 | 2015-02-16 | US Turreted TDs | Y |
|
||||||
| 2.7 | 2016-03-07 | Mirage map (date uncertain) |
|
| 1.6 | 2015-01-19 | Castilla map, UK HT line | Y |
|
||||||
| 2.6 | 2016-02-10 | German LTs, shooting through obstacles (date uncertain) |
|
| 1.5 | 2014-12-01 | UK Nation | Y |
|
||||||
| 2.5 | 2016-01-11 | Camouflages, Clan supply (date uncertain) |
|
| 1.4 | 2014-10-29 | German HT line (Maus) | Y |
|
||||||
| 2.4 | 2015-12-09 | Additional Japanese tanks, windows version (date uncertain) |
|
| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps | Y |
|
||||||
| 2.3 | 2015-11-12 | Canal map (date uncertain) |
|
| 1.2 | 2014-08-20 | Oasis Palms map | Y |
|
||||||
| 2.2 | 2015-10-05 | Desert Sands map update (date uncertain) |
|
| 1.1.1 | 2014-07-23 | Fixes :-) | |
|
||||||
| 2.1 | 2015-09-07 | UK TDs, Clan Chat (date uncertain) |
|
| 1.0 | 2014-06-26 | Worldwide launch | |
|
||||||
| 2.0 | 2015-08-05 | Clans, Oasis Palms updated |
|
| 0.9 | 2014-05-05 | Nordic soft launch | |
|
||||||
| 1.11 | 2015-07-13 | KV-2, KV-13, Sherman Jumbo (date uncertain) |
|
|
||||||
| 1.10 | 2015-06-11 | Port Bay map (date uncertain) |
|
|
||||||
| 1.9 | 2015-05-13 | New German medium tanks (Leo) (date uncertain) |
|
|
||||||
| 1.8 | 2015-04-14 | Game tutorial, Bot map for new players (date uncertain) |
|
|
||||||
| 1.7 | 2015-02-16 | US Turreted TDs (date uncertain) |
|
|
||||||
| 1.6 | 2015-01-19 | Castilla map, UK HT line (date uncertain) |
|
|
||||||
| 1.5 | 2014-12-01 | UK Nation (date uncertain) |
|
|
||||||
| 1.4 | 2014-10-29 | German HT line (Maus) (date uncertain) |
|
|
||||||
| 1.3 | 2014-10-02 | USSR IS-4 line, Lost Temple, Black Goldville maps (date uncertain) |
|
|
||||||
| 1.2 | 2014-08-20 | Oasis Palms map (date uncertain) |
|
|
||||||
| 1.1.1 | 2014-07-23 | Fixes :-) |
|
|
||||||
| 1.0 | 2014-06-26 | Worldwide launch |
|
|
||||||
| 0.9 | 2014-05-05 | Nordic soft launch |
|
|
||||||
+109
-1112
File diff suppressed because it is too large.
Load diff
+32
-32
@@ -1,8 +1,8 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
## PYTHON VERSION MUST BE 3.7 OR HIGHER
|
## PYTHON VERSION MUST BE 3.7 OR HIGHER
|
||||||
|
|
||||||
import json, argparse, sys, os, io, asyncio, aiofiles, aioconsole
|
import json, argparse, inspect, sys, os, io, base64, aiohttp, urllib, asyncio, aiofiles, aioconsole
|
||||||
import logging, re, configparser, zipfile
|
import logging, re, concurrent.futures, configparser, motor.motor_asyncio, ssl, zipfile
|
||||||
import blitzutils as bu
|
import blitzutils as bu
|
||||||
from blitzutils import WG
|
from blitzutils import WG
|
||||||
from blitzutils import WoTinspector
|
from blitzutils import WoTinspector
|
||||||
@@ -21,17 +21,17 @@ SKIPPED_N = 0
|
|||||||
ERROR_N = 0
|
ERROR_N = 0
|
||||||
WIurl='https://wotinspector.com/api/replay/upload?'
|
WIurl='https://wotinspector.com/api/replay/upload?'
|
||||||
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
|
WG_appID = '81381d3f45fa4aa75b78a7198eb216ad'
|
||||||
wg : WG | None = None
|
wg = None
|
||||||
wi : WoTinspector | None = None
|
wi = None
|
||||||
|
|
||||||
async def main(argv) -> int:
|
async def main(argv):
|
||||||
global wg, wi
|
global wg, wi
|
||||||
current_dir = os.getcwd()
|
current_dir = os.getcwd()
|
||||||
# set the directory for the script
|
# set the directory for the script
|
||||||
os.chdir(os.path.dirname(sys.argv[0]))
|
os.chdir(os.path.dirname(sys.argv[0]))
|
||||||
|
|
||||||
# options defaults
|
# options defaults
|
||||||
OPT_WORKERS_N = 2
|
OPT_WORKERS_N = 5
|
||||||
WG_ID = None
|
WG_ID = None
|
||||||
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
|
WG_ACCOUNT = None # format: nick@server, where server is either 'eu', 'ru', 'na', 'asia' or 'china'.
|
||||||
# China is not supported since WG API stats are not available there
|
# China is not supported since WG API stats are not available there
|
||||||
@@ -43,6 +43,12 @@ async def main(argv) -> int:
|
|||||||
bu.debug('Reading config file: ' + FILE_CONFIG)
|
bu.debug('Reading config file: ' + FILE_CONFIG)
|
||||||
config = configparser.ConfigParser()
|
config = configparser.ConfigParser()
|
||||||
config.read(FILE_CONFIG)
|
config.read(FILE_CONFIG)
|
||||||
|
try:
|
||||||
|
if 'UPLOADER' in config.sections():
|
||||||
|
configUploader = config['UPLOADER']
|
||||||
|
OPT_WORKERS_N = configUploader.getint('opt_workers', OPT_WORKERS_N)
|
||||||
|
except (KeyError, configparser.NoSectionError) as err:
|
||||||
|
bu.error(exception=err)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if 'WG' in config.sections():
|
if 'WG' in config.sections():
|
||||||
@@ -76,19 +82,18 @@ async def main(argv) -> int:
|
|||||||
bu.set_log_level(args.silent, args.verbose, args.debug)
|
bu.set_log_level(args.silent, args.verbose, args.debug)
|
||||||
|
|
||||||
wg = WG(WG_appID, args.tankopedia, args.mapfile)
|
wg = WG(WG_appID, args.tankopedia, args.mapfile)
|
||||||
wi = WoTinspector()
|
wi = WoTinspector(rate_limit=WG_RATE_LIMIT)
|
||||||
|
|
||||||
if args.account is not None:
|
if args.account != None:
|
||||||
args.accountID = await wg.get_account_id(args.account)
|
args.accountID = await wg.get_account_id(args.account)
|
||||||
bu.debug('WG account_id: ' + str(args.accountID))
|
bu.debug('WG account_id: ' + str(args.accountID))
|
||||||
|
|
||||||
if args.accountID is None:
|
if args.accountID == None:
|
||||||
args.accountID = 0
|
args.accountID = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
queue : asyncio.Queue = asyncio.Queue()
|
queue = asyncio.Queue()
|
||||||
# rebase file arguments due to moving the working directory to the script location
|
# rebase file arguments due to moving the working directory to the script location
|
||||||
bu.verbose_std("replays.wotinspector.com rate limits replays to 6 / minute")
|
|
||||||
args.files = bu.rebase_file_args(current_dir, args.files)
|
args.files = bu.rebase_file_args(current_dir, args.files)
|
||||||
|
|
||||||
tasks = []
|
tasks = []
|
||||||
@@ -118,7 +123,7 @@ async def main(argv) -> int:
|
|||||||
await wg.session.close()
|
await wg.session.close()
|
||||||
await wi.close()
|
await wi.close()
|
||||||
|
|
||||||
return 0
|
return None
|
||||||
|
|
||||||
|
|
||||||
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
|
async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
|
||||||
@@ -132,13 +137,13 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
|
|||||||
if not line:
|
if not line:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
if (p_replayfile.match(line) is not None):
|
if (p_replayfile.match(line) != None):
|
||||||
await queue.put(await mkQueueItem(line, title))
|
await queue.put(await mkQueueItem(line, title))
|
||||||
else:
|
else:
|
||||||
for fn in files:
|
for fn in files:
|
||||||
if fn.endswith('"'):
|
if fn.endswith('"'):
|
||||||
fn = fn[:-1]
|
fn = fn[:-1]
|
||||||
if os.path.isfile(fn) and (p_replayfile.match(fn) is not None):
|
if os.path.isfile(fn) and (p_replayfile.match(fn) != None):
|
||||||
await queue.put(await mkQueueItem(fn, title))
|
await queue.put(await mkQueueItem(fn, title))
|
||||||
bu.debug('File added to queue: ' + fn)
|
bu.debug('File added to queue: ' + fn)
|
||||||
elif os.path.isdir(fn):
|
elif os.path.isdir(fn):
|
||||||
@@ -146,7 +151,7 @@ async def mkReplayQ(queue : asyncio.Queue, files : list, title : str):
|
|||||||
for entry in dirEntry:
|
for entry in dirEntry:
|
||||||
try:
|
try:
|
||||||
bu.debug('Found: ' + entry.name)
|
bu.debug('Found: ' + entry.name)
|
||||||
if entry.is_file() and (p_replayfile.match(entry.name) is not None):
|
if entry.is_file() and (p_replayfile.match(entry.name) != None):
|
||||||
bu.debug(entry.name)
|
bu.debug(entry.name)
|
||||||
await queue.put(await mkQueueItem(entry.path, title))
|
await queue.put(await mkQueueItem(entry.path, title))
|
||||||
bu.debug('File added to queue: ' + entry.path)
|
bu.debug('File added to queue: ' + entry.path)
|
||||||
@@ -170,9 +175,6 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
|
|||||||
"""Async Worker to process the replay queue"""
|
"""Async Worker to process the replay queue"""
|
||||||
global SKIPPED_N
|
global SKIPPED_N
|
||||||
global ERROR_N
|
global ERROR_N
|
||||||
global wi
|
|
||||||
|
|
||||||
assert wi is not None, "wi cannot be None"
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
item = await queue.get()
|
item = await queue.get()
|
||||||
@@ -193,7 +195,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
|
|||||||
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
|
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
|
||||||
else:
|
else:
|
||||||
os.remove(replay_json_fn)
|
os.remove(replay_json_fn)
|
||||||
bu.debug(msg_str + "Replay JSON not valid/complete: Deleting " + replay_json_fn, id=workerID)
|
bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
|
||||||
SKIPPED_N += 1
|
SKIPPED_N += 1
|
||||||
queue.task_done()
|
queue.task_done()
|
||||||
continue
|
continue
|
||||||
@@ -208,19 +210,19 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
|
|||||||
filename = os.path.basename(filename)
|
filename = os.path.basename(filename)
|
||||||
bu.debug(msg_str + 'File: ' + filename)
|
bu.debug(msg_str + 'File: ' + filename)
|
||||||
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
|
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
|
||||||
if json_resp is not None:
|
if json_resp != None:
|
||||||
if (await bu.save_JSON(replay_json_fn,json_resp)):
|
if (await bu.save_JSON(replay_json_fn,json_resp)):
|
||||||
if wi.chk_JSON_replay(json_resp):
|
if wi.chk_JSON_replay(json_resp):
|
||||||
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
|
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
|
||||||
bu.verbose_std(msg_str + title + ' posted')
|
bu.verbose_std(msg_str + title + ' posted')
|
||||||
else:
|
else:
|
||||||
bu.warning(msg_str +'Replay file is not valid/complete: ' + filename)
|
bu.warning(msg_str +'Replay file is not valid: ' + filename)
|
||||||
ERROR_N += 1
|
ERROR_N += 1
|
||||||
else:
|
else:
|
||||||
bu.error(msg_str + 'Error saving replay: ' + filename)
|
bu.error(msg_str + 'Error saving replay: ' + filename)
|
||||||
ERROR_N += 1
|
ERROR_N += 1
|
||||||
else:
|
else:
|
||||||
bu.error(msg_str + 'Replay file is not valid/complete: ' + filename)
|
bu.error(msg_str + 'Replay file is not valid: ' + filename)
|
||||||
ERROR_N += 1
|
ERROR_N += 1
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
|
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
|
||||||
@@ -233,7 +235,7 @@ async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, pri
|
|||||||
def getTitle_old(replayfile: str, title: str, i : int) -> str:
|
def getTitle_old(replayfile: str, title: str, i : int) -> str:
|
||||||
global wg
|
global wg
|
||||||
|
|
||||||
if title is None:
|
if title == None:
|
||||||
try:
|
try:
|
||||||
filename = os.path.basename(replayfile)
|
filename = os.path.basename(replayfile)
|
||||||
bu.debug(filename)
|
bu.debug(filename)
|
||||||
@@ -245,8 +247,8 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
|
|||||||
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
|
p = re.compile('\\d{8}_\\d{4}_.*?(' + '|'.join(tank_userStrs) + ')_(' + '|'.join(map_usrStrs) + ')(?:-\\d)?\\.wotbreplay$')
|
||||||
|
|
||||||
m = p.match(filename)
|
m = p.match(filename)
|
||||||
if (m is not None):
|
if (m != None):
|
||||||
if wg.tanks is not None:
|
if wg.tanks != None:
|
||||||
tank = m.group(1)
|
tank = m.group(1)
|
||||||
if tank in wg.tanks['userStr']:
|
if tank in wg.tanks['userStr']:
|
||||||
tank = wg.tanks['userStr'][tank]
|
tank = wg.tanks['userStr'][tank]
|
||||||
@@ -271,9 +273,7 @@ def getTitle_old(replayfile: str, title: str, i : int) -> str:
|
|||||||
|
|
||||||
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
|
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
|
||||||
global wg
|
global wg
|
||||||
assert wg is not None, "wg must not be None"
|
if title == None:
|
||||||
|
|
||||||
if title is None:
|
|
||||||
try:
|
try:
|
||||||
filename = os.path.basename(replayfile)
|
filename = os.path.basename(replayfile)
|
||||||
bu.debug(filename)
|
bu.debug(filename)
|
||||||
@@ -287,13 +287,13 @@ def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
|
|||||||
try:
|
try:
|
||||||
metadata_json = json.load(meta)
|
metadata_json = json.load(meta)
|
||||||
#player = metadata_json['playerName']
|
#player = metadata_json['playerName']
|
||||||
tank_name = wg.get_tank_name(metadata_json['vehicleCompDescriptor'])
|
tank = wg.tank_str2name(metadata_json['playerVehicleName'])
|
||||||
map_name = wg.get_map(metadata_json['mapName'])
|
map_name = wg.get_map(metadata_json['mapName'])
|
||||||
bu.debug('Tank: ' + tank_name if tank_name is not None else "NOT FOUND IN tanks.json" + ' Map: ' + map_name)
|
bu.debug('Tank: ' + tank + ' Map: ' + map_name)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
bu.error(exception = err)
|
bu.error(exception = err)
|
||||||
if (tank_name is not None) and (map_name is not None):
|
if (tank != None) and (map_name != None):
|
||||||
title = tank_name + ' @ ' + map_name
|
title = tank + ' @ ' + map_name
|
||||||
else:
|
else:
|
||||||
title = re.sub('\\.wotbreplay$', '', filename)
|
title = re.sub('\\.wotbreplay$', '', filename)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
|
|||||||
Reference in new issue
Block a user