new deku (#747)

* Update to deku v0.18.0

* rust: Fix bit-endianness issue with new-deku

This was the cause of all of our WoW rendering failures with new-deku. Unfortunate we can't (yet?) use Deku for this part, but honestly I think the shift + mask is better than a loop anyway.

* rust: Map seek errors to deku errors

* rust: Fix Unity loading

The deku::byte_offset trick we do for alignment requires reader.bits_read to be correct, as that's the number we pad by. In our PackedVec implementations, we read a u8, and then later skip over it, which double-counts from the perspective of reader.bits_read. Fix this by adjusting after we read it.

* rust: Use std::io instead of deku::no_std_io

* rust: Update polymorph

Author: magcius

rust/Cargo.lock

@@ -25,7 +25,7 @@ opt-level = "s"
 [dependencies]
 byteorder = "1.4.3"
 console_error_panic_hook = "0.1.7"
-deku = { version = "0.16.0", features = ["logging"] }
+deku = { version = "0.18.1", features = ["logging"] }
 env_logger = "0.10.1"
 inflate = "0.4.5"
 js-sys = "0.3.60"

rust/Cargo.toml

@@ -1,5 +1,7 @@
-use deku::bitvec::BitSlice;
-use deku::{DekuContainerRead, DekuRead};
+use std::io::Cursor;
+
+use deku::reader::Reader;
+use deku::{DekuContainerRead, DekuReader};
 use wasm_bindgen::prelude::*;
 
 use crate::unity::types::wasm::WasmFriendlyPPtr;
@@ -39,11 +41,12 @@ impl AssetFile {
 
     pub fn append_metadata_chunk(&mut self, data: &[u8]) -> Result<(), String> {
         // data will be the file from bytes 0..data_offset, so skip to where the metadata starts
-        let bitslice = BitSlice::from_slice(data);
-        let (rest, _) = SerializedFileHeader::read(&bitslice, ())
+        let mut cursor = Cursor::new(data);
+        let mut reader = Reader::new(&mut cursor);
+        let _header = SerializedFileHeader::from_reader_with_ctx(&mut reader, ())
             .map_err(|err| format!("failed to parse metadata file header: {:?}", err))?;
-        match SerializedFileMetadata::read(rest, self.header.version) {
-            Ok((_, metadata)) => self.metadata = Some(metadata),
+        match SerializedFileMetadata::from_reader_with_ctx(&mut reader, self.header.version) {
+            Ok(metadata) => self.metadata = Some(metadata),
             Err(err) => return Err(format!("failed to parse metadata: {:?}", err)),
         }
         Ok(())

rust/src/unity/asset_file.rs

@@ -1,3 +1,4 @@
+
 use deku::prelude::*;
 
 // https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2019.4.39f1.dump
@@ -153,14 +154,14 @@ pub struct Mesh {
 }
 
 #[derive(DekuRead, Clone, Copy, Debug)]
-#[deku(type = "i32")]
+#[deku(id_type = "i32")]
 pub enum IndexFormat {
     UInt16 = 0,
     UInt32 = 1,
 }
 
 #[derive(DekuRead, Clone, Copy, Debug)]
-#[deku(type = "u8")]
+#[deku(id_type = "u8")]
 pub enum MeshCompression {
     Off = 0,
     Low = 1,
@@ -220,21 +221,12 @@ pub struct ByteArray {
     pub data: Vec<u8>,
 }
 
-impl<'a> DekuRead<'a> for ByteArray {
-    fn read(
-        input: &'a deku::bitvec::BitSlice<u8, deku::bitvec::Msb0>,
-        ctx: (),
-    ) -> Result<(&'a deku::bitvec::BitSlice<u8, deku::bitvec::Msb0>, Self), DekuError>
-    where Self: Sized {
-        let (rest, count) = i32::read(input, ctx)?;
-        let (data_bits, rest) = rest.split_at(count as usize * 8);
-        let bytes = data_bits.domain().region().unwrap().1;
-        Ok((
-            rest,
-            Self {
-                data: bytes.to_vec(),
-            },
-        ))
+impl<'a, Ctx> DekuReader<'a, Ctx> for ByteArray where Ctx: Copy {
+    fn from_reader_with_ctx<R: std::io::Read + std::io::Seek>(reader: &mut Reader<R>, _ctx: Ctx) -> Result<Self, DekuError> {
+        let count = i32::from_reader_with_ctx(reader, ())? as usize;
+        let mut buf = vec![0x00; count];
+        reader.read_bytes(count, &mut buf)?;
+        Ok(ByteArray{ data: buf })
     }
 }
 
@@ -271,7 +263,7 @@ pub struct ChannelInfo {
 }
 
 #[derive(DekuRead, Clone, Debug)]
-#[deku(type = "u8")]
+#[deku(id_type = "u8")]
 pub enum VertexFormat {
     #[deku(id = "0")] Float,
     #[deku(id = "1")] Float16,
@@ -390,15 +382,15 @@ pub struct GLTextureSettings {
 }
 
 #[derive(DekuRead, Clone, Debug)]
-#[deku(type = "i32")]
+#[deku(id_type = "i32")]
 pub enum TextureFilterMode {
     Nearest = 0,
     Bilinear = 1,
     Trilinear = 2,
 }
 
 #[derive(DekuRead, Clone, Debug)]
-#[deku(type = "i32")]
+#[deku(id_type = "i32")]
 pub enum TextureWrapMode {
     Repeat = 0,
     Clamp = 1,
@@ -408,7 +400,7 @@ pub enum TextureWrapMode {
 
 // copied from https://github.com/Unity-Technologies/UnityCsReference/blob/129a67089d125df5b95b659d3535deaf9968e86c/Editor/Mono/AssetPipeline/TextureImporterEnums.cs#L37
 #[derive(DekuRead, Clone, Debug)]
-#[deku(type = "i32")]
+#[deku(id_type = "i32")]
 pub enum TextureFormat {
     // Alpha 8 bit texture format.
     Alpha8 = 1,
@@ -523,7 +515,7 @@ pub enum TextureFormat {
 }
 
 #[derive(DekuRead, Clone, Debug)]
-#[deku(type = "i32")]
+#[deku(id_type = "i32")]
 pub enum ColorSpace {
     Linear = 0x00,
     SRGB   = 0x01,

rust/src/unity/types/binary.rs

@@ -3,7 +3,7 @@ use deku::prelude::*;
 
 #[wasm_bindgen(js_name = "UnityClassID")]
 #[derive(DekuRead, Debug, Copy, Clone, PartialEq)]
-#[deku(type = "i32")]
+#[deku(id_type = "i32")]
 #[repr(i32)]
 pub enum ClassID {
     #[deku(id = "-1")] UnknownType,