unknown fields in conan exiles inline data record!

Author: panzi

README.md

@@ -145,27 +145,19 @@ if version <= 1
 end
      ?    20  uint8_t[20]  data sha1 hash
 if version >= 3
- if compression method != 0x00
+ if compression_method != 0x00
   ?+20     4  uint32_t     block count (M)
   ?+24  M*16  CB[M]        compression blocks
  end
      ?     1  uint8_t      is encrypted
    ?+1     4  uint32_t     The uncompressed size of each compression block.
 end                        The last block can be smaller, of course.
-if variant == "Conan Exiles" or (version >= 4 and is data record)
-     ?     4  uint32_t     Unknown field. For Conan Exiles only seen it to have
-                           the value 0.
+if variant == "Conan Exiles"
+     ?     4  uint32_t     Unknown field. For Conan Exiles index record only
+                           seen it to have the value 0.
 end
 ```
 
-**NOTE:** Starting with version 4 there is an additional 4 bytes in the repeated
-*inline* record copy (the record that precedes the actual file date, not the
-record in the index). I don't know what that is. It is not always the same value.
-E.g. it is the same for some files, but different for others. The last 2 bytes
-are more often the same than the whole 4 bytes.
-
-This is why I've deactivated packing for versions > 3.
-
 ### Compression Block (CB)
 
 Size: 16 bytes
@@ -188,16 +180,35 @@ Offset  Size  Type         Description
 ```plain
 Offset  Size  Type            Description
      0     ?  Record          file metadata (offset field is 0, N = compressed_size)
+if variant == "Conan Exiles"
+     ?    20  ?               Unknown. Maybe another SHA-1 sum of something?
+                              The first 4 bytes have values other than the extra
+                              4 bytes in the index record, which is why I didn't
+                              put those into the general record structure.
+else if version >= 4
+     ?     4  uint32_t        Unknown.
+end
      ?     N  uint8_t[N]      file data
 ```
 
+**NOTE:** Starting with version 4 there is an additional 4 bytes in the repeated
+*data* record copy (the record that precedes the actual file date, not the
+record in the index). I don't know what that is. It is not always the same value.
+E.g. it is the same for some files, but different for others. The last 2 bytes
+are more often the same than the whole 4 bytes.
+
+This is why I've deactivated packing for versions > 3.
+
 ### Index Record
 
 ```plain
 Offset  Size  Type            Description
      0     4  uint32_t        file name size (S)
      4     S  char[S]         file name (includes terminating null byte)
    4+S     ?  Record          file metadata
+if variant == "Conan Exiles"
+     ?     4  ?               Unknown. Only saw all 0 so far.
+end
 ```
 
 ### Index

src/check.rs

@@ -22,6 +22,7 @@ use openssl::sha::Sha1 as OpenSSLSha1;
 use crate::{Error, Filter, Pak, pak::{BUFFER_SIZE, COMPR_METHODS, COMPR_NONE, HexDisplay, Sha1, Variant}};
 use crate::reopen::Reopen;
 use crate::{Record, Result};
+//use crate::{decode, decode::Decode};
 
 pub const NULL_SHA1: Sha1 = [0u8; 20];
 
@@ -218,6 +219,14 @@ pub fn check<'a>(pak: &'a Pak, in_file: &mut File, options: CheckOptions) -> Res
                                 check_error!(ok, result_sender, abort_on_error, error);
                             }
                         };
+                        //if version >= 4 {
+                        //    let mut f = || -> Result<()> {
+                        //        decode!(&mut reader, unknown: u32);
+                        //        println!(">>> {:20} {:15x} {}", unknown, unknown, record.filename());
+                        //        Ok(())
+                        //    };
+                        //    let _ = f();
+                        //}
                     }
 
                     if let Some(blocks) = record.compression_blocks() {
@@ -228,8 +237,12 @@ pub fn check<'a>(pak: &'a Pak, in_file: &mut File, options: CheckOptions) -> Res
                             let base_offset;
                             let mut next_start_offset;
 
-                            // + 4 for unknown extra field in inline record
-                            if version >= 7 {
+                            if variant == Variant::ConanExiles {
+                                // only version 4 is correctly supported
+                                base_offset = 0;
+                                next_start_offset = record.offset() + header_size + 20;
+                            } else if version >= 7 {
+                                // + 4 for unknown extra field in inline record
                                 base_offset = record.offset();
                                 next_start_offset = header_size + 4;
                             } else if version >= 4 {

src/pack.rs

@@ -160,13 +160,18 @@ impl Default for PackOptions<'_> {
 pub fn pack(pak_path: impl AsRef<Path>, paths: &[PackPath], options: PackOptions) -> Result<Pak> {
     let write_record_inline = match options.variant {
         Variant::ConanExiles => {
-            if options.version != 4 {
-                return Err(Error::new(format!(
-                    "Only know how to handle Conan Exile paks of version 4, but version was {}.",
-                    options.version)).
-                    with_path(pak_path));
-            }
-            Record::write_conan_exiles_inline
+            return Err(Error::new("Writing of Conan Exile paks is not supported.".to_string()).
+                with_path(pak_path));
+            // XXX: There a are 20 unknown bytes after the inline record information if compressed.
+            //      That is 16 extra to the already 4 extra bytes in standard version >= 4.
+            //      In the index record there are only 4 extra bytes that are always 0.
+            //if options.version != 4 {
+            //    return Err(Error::new(format!(
+            //        "Only know how to handle Conan Exile paks of version 4, but version was {}.",
+            //        options.version)).
+            //        with_path(pak_path));
+            //}
+            //Record::write_conan_exiles_inline
         }
         Variant::Standard => match options.version {
             1 => Record::write_v1_inline,

src/pak.rs

@@ -354,6 +354,7 @@ impl Pak {
     //    filter.filter(self.records.iter())
     //}
 
+    // FIXME: inline header has different size in some versions/variants!
     pub fn header_size(version: u32, variant: Variant, record: &Record) -> u64 {
         match variant {
             Variant::ConanExiles => {

src/record.rs

@@ -16,7 +16,7 @@
 use std::io::{Read, Write};
 use std::fmt::Write as FmtWrite;
 
-use crate::pak::{COMPR_NONE, HexDisplay, Sha1};
+use crate::{check::NULL_SHA1, pak::{COMPR_NONE, HexDisplay, Sha1}};
 use crate::decode;
 use crate::decode::Decode;
 use crate::encode;
@@ -346,7 +346,8 @@ impl Record {
             }
             self.encrypted as u8,
             self.compression_block_size,
-            0u32,
+            // there are suppodes to be 20 more bytes of something that I don't know:
+            NULL_SHA1,
         );
         Ok(())
     }