💥 Hide raw payloads behind the feature flag, and improve the performance

.github/FUNDING.yml

@@ -1,5 +1 @@
 github: [eigenein]
-custom:
-- 'https://www.paypal.me/eigenein'
-- 'https://bunq.me/eigenein'
-- 'https://money.yandex.ru/to/410011693851843'

.idea/wotbreplay-parser.iml

@@ -4,6 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/benches" isTestSource="true" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />

Cargo.toml

@@ -12,7 +12,8 @@ categories = ["parsing"]
 
 [features]
 default = []
-meta = ["dep:serde_json"]
+meta = ["dep:serde_json"] # `meta.json` parsing
+raw-payload = [] # save raw packet payloads, enabling increases memory usage and decreases performance
 
 [dependencies]
 bytes = "1.4.0"
@@ -28,3 +29,8 @@ zip = { version = "0.6.3", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 anyhow = "1.0.66"
+criterion = "0.4.0"
+
+[[bench]]
+name = "data"
+harness = false

README.md

@@ -18,8 +18,7 @@ use wotbreplay_parser::models::battle_results::TeamNumber;
 use wotbreplay_parser::replay::Replay;
 
 fn main() -> Result<()> {
-    let battle_results = Replay::open(File::open("tests/replays/player_results.wotbreplay")?)?
-        .read_battle_results()?;
+    let battle_results = Replay::open(File::open("replays/20221203_player_results.wotbreplay")?)?.read_battle_results()?;
 
     assert_eq!(battle_results.timestamp, 1670083956);
     assert_eq!(battle_results.players.len(), 14);
@@ -36,26 +35,3 @@ fn main() -> Result<()> {
     Ok(())
 }
 ```
-
-## Replay structure
-
-`*.wotbreplay` is a ZIP-archive containing:
-- `battle_results.dat`
-- `meta.json`
-- `data.wotreplay`
-
-### `battle_results.dat`
-
-This is a [pickled](https://docs.python.org/3/library/pickle.html) 2-tuple:
-- Some big number
-- Battle results serialized with [Protocol Buffers](https://developers.google.com/protocol-buffers)
-
-#### Useful tools
-
-- [Protobuf Decoder](https://protobuf-decoder.netlify.app/)
-- [`protobuf-inspector`](https://github.com/mildsunrise/protobuf-inspector)
-- [Hex Fiend](https://hexfiend.com/)
-
-### `meta.json`
-
-### `data.wotreplay`

benches/data.rs

@@ -0,0 +1,20 @@
+use std::fs::File;
+
+use criterion::{criterion_group, criterion_main, Criterion};
+use wotbreplay_parser::replay::Replay;
+
+fn bench_read_data(criterion: &mut Criterion) {
+    let mut group = criterion.benchmark_group("read_data");
+    for path in [
+        "replays/20230429_0126__helaas_pindakaas.wotbreplay",
+        "replays/20230503_0027__helaas_pindakaas.wotbreplay",
+        "replays/20230429_1682194799_malinovka.wotbreplay",
+    ] {
+        let mut replay = Replay::open(File::open(path).unwrap()).unwrap();
+        group.bench_function(path, |bencher| bencher.iter(|| replay.read_data().unwrap()));
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_read_data);
+criterion_main!(benches);

src/models/battle_results.rs

@@ -7,7 +7,10 @@ use crate::result::Result;
 impl BattleResults {
     /// Parse battle results from the buffer.
     ///
-    /// Note, that this does NOT parse `battle_results.dat` itself, but the un-pickled tuple element.
+    /// # Note
+    ///
+    /// This does **not** parse `battle_results.dat` itself, but the un-pickled tuple element.
+    /// To parse `battle_results.dat`, use [`crate::models::battle_results_dat::BattleResultsDat`].
     pub fn from_buffer(buffer: impl Buf) -> Result<Self> {
         Ok(Self::decode(buffer)?)
     }

src/models/battle_results_dat.rs

@@ -8,7 +8,14 @@ use crate::error::Error;
 use crate::models::battle_results::BattleResults;
 use crate::result::Result;
 
-/// Represents un-pickled `battle_results.dat`.
+/// Un-pickled `battle_results.dat`.
+///
+/// # `battle_results.dat` structure
+///
+/// Entire file is a [pickled](https://docs.python.org/3/library/pickle.html) 2-tuple:
+///
+/// - Arena unique ID
+/// - Battle results serialized with [Protocol Buffers](https://developers.google.com/protocol-buffers)
 #[derive(Debug, Deserialize)]
 #[serde(from = "(u64, ByteBuf)")]
 pub struct BattleResultsDat {
@@ -20,15 +27,12 @@ pub struct BattleResultsDat {
 
 impl From<(u64, ByteBuf)> for BattleResultsDat {
     fn from((arena_unique_id, buffer): (u64, ByteBuf)) -> Self {
-        Self {
-            arena_unique_id,
-            buffer,
-        }
+        Self { arena_unique_id, buffer }
     }
 }
 
 impl BattleResultsDat {
-    /// Parses the pickled structure from the reader which contains `battle_results.dat`.
+    /// Parse the pickled structure from the reader which contains `battle_results.dat`.
     ///
     /// It can be used either on the `battle_results.dat` for a replay archive,
     /// or directly on `**/DAVAProject/battle_results/*/*_full.dat`.
@@ -40,7 +44,7 @@ impl BattleResultsDat {
 impl TryInto<BattleResults> for BattleResultsDat {
     type Error = Error;
 
-    /// Decodes the battle results from the internal buffer.
+    /// Decode the battle results from the internal buffer.
     fn try_into(self) -> Result<BattleResults> {
         Ok(BattleResults::decode(self.buffer.as_ref())?)
     }

src/models/data/base_player_create.rs

@@ -4,8 +4,8 @@ use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct BasePlayerCreate {
-    #[serde(rename(deserialize = "playersBattleCategoriesIds"))]
-    pub players_battle_categories_ids: HashMap<u32, (u8, u32)>,
+    #[serde(default, rename(deserialize = "playersBattleCategoriesIds"))]
+    pub players_battle_categories_ids: Option<HashMap<u32, (u8, u32)>>,
 
     #[serde(rename(deserialize = "battleLevel"))]
     pub battle_level: u8,
@@ -22,17 +22,17 @@ pub struct BasePlayerCreate {
     #[serde(rename(deserialize = "camouflageSlot"))]
     pub camouflage_slot: u8,
 
-    #[serde(rename(deserialize = "avgMmr"))]
-    pub average_mmr: Vec<Option<f64>>,
+    #[serde(default, rename(deserialize = "avgMmr"))]
+    pub average_mmr: Option<Vec<Option<f64>>>,
 
-    #[serde(rename(deserialize = "playerWaitTimes"))]
-    pub player_wait_times: HashMap<u32, f64>,
+    #[serde(default, rename(deserialize = "playerWaitTimes"))]
+    pub player_wait_times: Option<HashMap<u32, f64>>,
 
     #[serde(rename(deserialize = "accountDatabaseIds"))]
     pub account_database_ids: Vec<u32>,
 
-    #[serde(rename(deserialize = "turboBattlesStats"))]
-    pub turbo_battles_statistics: TurboBattlesStatistics,
+    #[serde(default, rename(deserialize = "turboBattlesStats"))]
+    pub turbo_battles_statistics: Option<TurboBattlesStatistics>,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

src/models/data/entity_method.rs

@@ -1,3 +1,5 @@
+//! Structures for «entity method» call packets.
+
 use byteorder::{LittleEndian, ReadBytesExt};
 use prost::encoding::decode_varint;
 use prost::Message;
@@ -13,48 +15,37 @@ pub mod update_arena;
 #[serde_as]
 #[derive(Debug, Serialize)]
 pub enum EntityMethod {
-    /// Subtype `0x2F`.
+    /// Subtype 47.
     UpdateArena {
         field_number: u64,
         arguments: UpdateArena,
-
-        /// Original payload for investigation.
-        #[serde_as(as = "serde_with::hex::Hex")]
-        payload: Vec<u8>,
     },
 
     /// Default variant when subtype is not known by the parser.
-    Unknown {
-        sub_type: u32,
-
-        /// Whole packet payload (including the sub-type).
-        #[serde_as(as = "serde_with::hex::Hex")]
-        payload: Vec<u8>,
-    },
+    Unknown { sub_type: u32 },
 }
 
 impl EntityMethod {
     /// Parse the entity method payload.
-    pub fn new(payload: Vec<u8>) -> Result<Self> {
-        let mut reader = payload.as_slice();
+    pub fn new(payload: &[u8]) -> Result<Self> {
+        let mut payload = payload;
 
-        reader.read_u32::<LittleEndian>()?;
-        let sub_type = reader.read_u32::<LittleEndian>()?;
+        payload.read_u32::<LittleEndian>()?;
+        let sub_type = payload.read_u32::<LittleEndian>()?;
 
         let this = match sub_type {
-            0x2F => {
-                let _inner_length = reader.read_u32::<LittleEndian>()?;
+            47 => {
+                let _inner_length = payload.read_u32::<LittleEndian>()?;
 
-                let field_number = decode_varint(&mut reader)?;
-                let message_length = read_quirky_length(&mut reader)?;
+                let field_number = decode_varint(&mut payload)?;
+                let message_length = read_quirky_length(&mut payload)?;
                 Self::UpdateArena {
                     field_number,
-                    arguments: UpdateArena::decode(&reader[..message_length])?,
-                    payload,
+                    arguments: UpdateArena::decode(&payload[..message_length])?,
                 }
             }
 
-            _ => Self::Unknown { sub_type, payload },
+            _ => Self::Unknown { sub_type },
         };
         Ok(this)
     }

src/models/data/packet.rs

@@ -2,14 +2,20 @@ use std::io::Read;
 
 use byteorder::{LittleEndian, ReadBytesExt};
 use serde::Serialize;
+use serde_with::serde_as;
 
 use crate::models::data::payload::Payload;
 use crate::result::Result;
 
+#[serde_as]
 #[derive(Debug, Serialize)]
 pub struct Packet {
     pub clock: f32,
     pub payload: Payload,
+
+    #[cfg(feature = "raw-payload")]
+    #[serde_as(as = "serde_with::hex::Hex")]
+    pub raw_payload: Vec<u8>,
 }
 
 impl Packet {
@@ -18,8 +24,13 @@ impl Packet {
         let type_ = reader.read_u32::<LittleEndian>()?;
         let clock = reader.read_f32::<LittleEndian>()?;
         let payload = Self::read_payload(reader, length as usize)?;
-        let payload = Payload::new(type_, payload)?;
-        let this = Self { clock, payload };
+        let this = Self {
+            clock,
+            payload: Payload::new(type_, &payload)?,
+
+            #[cfg(feature = "raw-payload")]
+            raw_payload: payload,
+        };
         Ok(Some(this))
     }
 

src/models/data/payload.rs

@@ -9,6 +9,7 @@ use crate::models::data::entity_method::EntityMethod;
 use crate::models::data::{read_pickled, read_quirky_length, read_string};
 use crate::result::Result;
 
+/// Packet payload.
 #[serde_as]
 #[derive(Debug, Serialize)]
 pub enum Payload {
@@ -24,31 +25,27 @@ pub enum Payload {
     EntityMethod(EntityMethod),
 
     /// Default payload when type is not known by the parser.
-    Unknown {
-        type_: u32,
-
-        /// Whole packet payload.
-        #[serde_as(as = "serde_with::hex::Hex")]
-        raw: Vec<u8>,
-    },
+    Unknown { packet_type: u32 },
 }
 
 impl Payload {
-    pub fn new(type_: u32, payload: Vec<u8>) -> Result<Self> {
-        let this = match type_ {
-            0 => {
-                let mut reader = payload.as_slice();
-                reader.read_exact(&mut [0; 10])?;
+    /// Parse the packet payload.
+    pub fn new(packet_type: u32, payload: &[u8]) -> Result<Self> {
+        let mut payload = payload;
 
-                let author_nickname = read_string(&mut reader)?;
-                let arena_unique_id = reader.read_u64::<LittleEndian>()?;
-                let arena_type_id = reader.read_u32::<LittleEndian>()?;
-                let type_0 = {
-                    let pickled_length = read_quirky_length(&mut reader)?;
-                    read_pickled(&mut reader, pickled_length)?
+        let this = match packet_type {
+            0 => {
+                payload.read_exact(&mut [0; 10])?;
+
+                let author_nickname = read_string(&mut payload)?;
+                let arena_unique_id = payload.read_u64::<LittleEndian>()?;
+                let arena_type_id = payload.read_u32::<LittleEndian>()?;
+                let arguments = {
+                    let pickled_length = read_quirky_length(&mut payload)?;
+                    read_pickled(&mut payload, pickled_length)?
                 };
                 Self::BasePlayerCreate {
-                    arguments: type_0,
+                    arguments,
                     author_nickname,
                     arena_unique_id,
                     arena_type_id,
@@ -57,7 +54,7 @@ impl Payload {
 
             8 => Self::EntityMethod(EntityMethod::new(payload)?),
 
-            _ => Self::Unknown { raw: payload, type_ },
+            _ => Self::Unknown { packet_type },
         };
         Ok(this)
     }

src/replay.rs

@@ -9,7 +9,15 @@ use crate::models::data::Data;
 use crate::models::meta::Meta;
 use crate::result::Result;
 
-/// Opened (but not yet parsed) replay.
+/// World of Tanks Blitz replay.
+///
+/// # Replay structure
+///
+/// `*.wotbreplay` is a ZIP-archive containing:
+///
+/// - `battle_results.dat`
+/// - `meta.json`
+/// - `data.wotreplay`
 pub struct Replay<R>(pub(crate) ZipArchive<R>);
 
 impl<R: Read + Seek> Replay<R> {
@@ -18,22 +26,24 @@ impl<R: Read + Seek> Replay<R> {
         Ok(Self(ZipArchive::new(reader)?))
     }
 
-    /// Reads and parses the battle results from the replay.
+    /// Read and parse the full battle results structure from `battle_results.dat`.
     pub fn read_battle_results(&mut self) -> Result<BattleResults> {
         self.read_battle_results_dat()?.try_into()
     }
 
-    /// Reads and parses the included `battle_results.dat`.
+    /// Read and parse the root 2-tuple from `battle_results.dat`.
     pub fn read_battle_results_dat(&mut self) -> Result<BattleResultsDat> {
         let pickled_battle_results = self.0.by_name("battle_results.dat")?;
         BattleResultsDat::from_reader(pickled_battle_results)
     }
 
+    /// Read and parse the included `meta.json`.
     #[cfg(feature = "meta")]
     pub fn read_meta(&mut self) -> Result<Meta> {
         Meta::from_reader(self.0.by_name("meta.json")?)
     }
 
+    /// Read and parse the included `data.wotreplay`.
     pub fn read_data(&mut self) -> Result<Data> {
         Data::from_reader(self.0.by_name("data.wotreplay")?)
     }

tests/battle_results.rs

@@ -6,8 +6,9 @@ use wotbreplay_parser::replay::Replay;
 
 #[test]
 fn player_results_ok() -> Result<()> {
-    let battle_results_dat = Replay::open(File::open("tests/replays/player_results.wotbreplay")?)?
-        .read_battle_results_dat()?;
+    let battle_results_dat =
+        Replay::open(File::open("replays/20221203_player_results.wotbreplay")?)?
+            .read_battle_results_dat()?;
     assert_eq!(battle_results_dat.arena_unique_id, 1661909200500084);
     let battle_results: BattleResults = battle_results_dat.try_into()?;
 
@@ -80,7 +81,7 @@ fn player_results_ok() -> Result<()> {
 
 #[test]
 fn player_info_ok() -> Result<()> {
-    let battle_results = Replay::open(File::open("tests/replays/player_results.wotbreplay")?)?
+    let battle_results = Replay::open(File::open("replays/20221203_player_results.wotbreplay")?)?
         .read_battle_results()?;
 
     assert_eq!(battle_results.players[1].info.clan_tag.as_deref(), Some("AN0NY"));
@@ -103,7 +104,7 @@ fn player_info_ok() -> Result<()> {
 
 #[test]
 fn victory_points_ok() -> Result<()> {
-    let battle_results = Replay::open(File::open("tests/replays/victory_points.wotbreplay")?)?
+    let battle_results = Replay::open(File::open("replays/20221203_victory_points.wotbreplay")?)?
         .read_battle_results()?;
 
     assert_eq!(battle_results.player_results[1].info.victory_points_seized, 40);
@@ -120,8 +121,9 @@ fn victory_points_ok() -> Result<()> {
 
 #[test]
 fn draw_ok() -> Result<()> {
-    let battle_results = Replay::open(File::open("tests/replays/20230503_0027__helaas_pindakaas_A140_ASTRON_REX_105_2319058483532631656.wotbreplay")?)?
-        .read_battle_results()?;
+    let battle_results =
+        Replay::open(File::open("replays/20230503_0027__helaas_pindakaas.wotbreplay")?)?
+            .read_battle_results()?;
     assert_eq!(battle_results.winner_team_number, None);
     Ok(())
 }

tests/data.rs

@@ -7,9 +7,8 @@ use wotbreplay_parser::replay::Replay;
 
 #[test]
 fn parse_ok() -> Result<()> {
-    let mut replay = Replay::open(File::open(
-        "tests/replays/20230429_0126__helaas_pindakaas_A140_ASTRON_REX_105_16114615898101270.wotbreplay",
-    )?)?;
+    let mut replay =
+        Replay::open(File::open("replays/20230429_0126__helaas_pindakaas.wotbreplay")?)?;
     let data = replay.read_data()?;
 
     assert_eq!(data.client_version, "9.8.5_apple");
@@ -19,13 +18,12 @@ fn parse_ok() -> Result<()> {
 }
 
 #[test]
-fn type0_ok() -> Result<()> {
-    let mut replay = Replay::open(File::open(
-        "tests/replays/20230429_0126__helaas_pindakaas_A140_ASTRON_REX_105_16114615898101270.wotbreplay",
-    )?)?;
+fn base_player_create_ok() -> Result<()> {
+    let mut replay =
+        Replay::open(File::open("replays/20230429_0126__helaas_pindakaas.wotbreplay")?)?;
     let data = replay.read_data()?;
 
-    let (author_nickname, arena_unique_id, arena_type_id, type_0) = data
+    let (author_nickname, arena_unique_id, arena_type_id, arguments) = data
         .packets
         .iter()
         .filter_map(|packet| {
@@ -46,14 +44,17 @@ fn type0_ok() -> Result<()> {
     assert_eq!(author_nickname, "helaas_pindakaas");
     assert_eq!(*arena_unique_id, 16114615898101270);
     assert_eq!(*arena_type_id, 43);
-    assert_eq!(type_0.battle_level, 9);
-    assert_eq!(type_0.battle_category_id, 13);
-    assert!(type_0.is_mouse_enabled);
-    assert_eq!(type_0.matchmaker_type, 2);
-    assert_eq!(type_0.camouflage_slot, 1);
-    assert_eq!(type_0.average_mmr, [Some(59.29457753400008), Some(99.34182121604681)]);
+    assert_eq!(arguments.battle_level, 9);
+    assert_eq!(arguments.battle_category_id, 13);
+    assert!(arguments.is_mouse_enabled);
+    assert_eq!(arguments.matchmaker_type, 2);
+    assert_eq!(arguments.camouflage_slot, 1);
     assert_eq!(
-        type_0.account_database_ids,
+        arguments.average_mmr,
+        Some(vec![Some(59.29457753400008), Some(99.34182121604681)])
+    );
+    assert_eq!(
+        arguments.account_database_ids,
         [
             517033644, 537867068, 597583401, 601893865, 576810156, 531053956, 599580824, 558025185,
             526927603, 594778041, 505587490, 597422303, 531923111, 587617355
@@ -64,18 +65,17 @@ fn type0_ok() -> Result<()> {
 }
 
 #[test]
-fn subtype_2f_players_ok() -> Result<()> {
-    let mut replay = Replay::open(File::open(
-        "tests/replays/20230429_0126__helaas_pindakaas_A140_ASTRON_REX_105_16114615898101270.wotbreplay",
-    )?)?;
+fn update_arena_ok() -> Result<()> {
+    let mut replay =
+        Replay::open(File::open("replays/20230429_0126__helaas_pindakaas.wotbreplay")?)?;
     let data = replay.read_data()?;
 
     let players = data
         .packets
         .iter()
         .filter_map(|packet| match &packet.payload {
-            Payload::EntityMethod(EntityMethod::UpdateArena { arguments: sub_type_2f, .. }) => {
-                sub_type_2f.players.as_ref()
+            Payload::EntityMethod(EntityMethod::UpdateArena { arguments, .. }) => {
+                arguments.players.as_ref()
             }
             _ => None,
         })
@@ -91,3 +91,10 @@ fn subtype_2f_players_ok() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn training_room_ok() -> Result<()> {
+    let mut replay = Replay::open(File::open("replays/20230503_training_room.wotbreplay")?)?;
+    replay.read_data()?;
+    Ok(())
+}

tests/meta.rs

@@ -7,7 +7,8 @@ use wotbreplay_parser::replay::Replay;
 
 #[test]
 fn meta_ok() -> Result<()> {
-    let meta = Replay::open(File::open("tests/replays/player_results.wotbreplay")?)?.read_meta()?;
+    let meta =
+        Replay::open(File::open("replays/20221203_player_results.wotbreplay")?)?.read_meta()?;
 
     assert_eq!(meta.player_name, "zeekrab");
     assert_eq!(meta.arena_unique_id, "1661909200500084");