Implemented updated level.cform format with integrated CDB cache

Externals/OPCODE/OPC_AABBTree.cpp

@@ -432,9 +432,9 @@ udword AABBTree::ComputeDepth() const
  *	\return		number of bytes used
  */
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-udword AABBTree::GetUsedBytes() const
+size_t AABBTree::GetUsedBytes() const
 {
-    udword TotalSize = mTotalNbNodes * GetNodeSize();
+    size_t TotalSize = static_cast<size_t>(mTotalNbNodes) * GetNodeSize();
     if (mIndices)
         TotalSize += mNbPrimitives * sizeof(udword);
     return TotalSize;

Externals/OPCODE/OPC_AABBTree.h

@@ -77,7 +77,7 @@ class OPCODE_API AABBTree : public AABBTreeNode
     bool IsComplete() const;
     // Stats
     udword ComputeDepth() const;
-    udword GetUsedBytes() const;
+    size_t GetUsedBytes() const;
 
 private:
     udword* mIndices; //!< Indices in the app list. Indices are reorganized during build.

Externals/OPCODE/OPC_Model.cpp

@@ -276,7 +276,7 @@ bool OPCODE_Model::Build(const OPCODECREATE& create)
     return true;
 }
 
-bool OPCODE_Model::Load(IReader* stream, bool skipCrc32Check /*= false*/)
+bool OPCODE_Model::Load(IReader* stream, bool skipCrc32Check /*= false*/, bool readCrc32 /*= true*/)
 {
     mNoLeaf = stream->r_u32();
     mQuantized = stream->r_u32();
@@ -285,14 +285,17 @@ bool OPCODE_Model::Load(IReader* stream, bool skipCrc32Check /*= false*/)
 
     if (mNoLeaf && !mQuantized)
     {
-        const u32 size = nodesNum * sizeof(AABBNoLeafNode);
+        const size_t size = static_cast<size_t>(nodesNum) * sizeof(AABBNoLeafNode);
         if (size > stream->elapsed())
             return false;
 
-        const auto crc = stream->r_u32();
-        auto actualCrc = skipCrc32Check ? crc : crc32(stream->pointer(), size);
-        if (crc != actualCrc)
-            return false;
+        if (readCrc32)
+        {
+            const auto crc = stream->r_u32();
+            auto actualCrc = skipCrc32Check ? crc : crc32(stream->pointer(), size);
+            if (crc != actualCrc)
+                return false;
+        }
 
         mTree = xr_new<AABBNoLeafTree>();
         auto* ptr = xr_alloc<AABBNoLeafNode>(nodesNum);
@@ -325,7 +328,7 @@ void OPCODE_Model::Save(IWriter* stream) const
     void* pData = nullptr;
     if (mNoLeaf && !mQuantized)
     {
-        const u32 size = nodesNum * sizeof(AABBNoLeafNode);
+        const size_t size = static_cast<size_t>(nodesNum) * sizeof(AABBNoLeafNode);
         R_ASSERT(size == mTree->GetUsedBytes());
 
         auto* ptr = xr_alloc<AABBNoLeafNode>(nodesNum);

Externals/OPCODE/OPC_Model.h

@@ -46,7 +46,7 @@ class OPCODE_API OPCODE_Model
     OPCODE_Model();
     ~OPCODE_Model();
 
-    bool Load(IReader* stream, bool skipCrc32Check = false);
+    bool Load(IReader* stream, bool skipCrc32Check = false, bool readCrc32 = true);
     void Save(IWriter* stream) const;
 
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -110,7 +110,7 @@ class OPCODE_API OPCODE_Model
      *	\return		amount of bytes used
      */
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-    udword GetUsedBytes() const { return mTree->GetUsedBytes(); }
+    size_t GetUsedBytes() const { return mTree->GetUsedBytes(); }
 private:
     AABBTree* mSource; //!< Original source tree
     AABBOptimizedTree* mTree; //!< Optimized tree

Externals/OPCODE/OPC_OptimizedTree.h

@@ -124,7 +124,7 @@ public:\
     void* GetData() const override { return static_cast<void*>(mNodes); }\
     void SetData(void* ptr, udword nbNodes) override { mNodes = static_cast<volume*>(ptr); mNbNodes = nbNodes; }\
     /* Stats */\
-    virtual udword GetUsedBytes() const { return mNbNodes * sizeof(volume); }\
+    virtual size_t GetUsedBytes() const { return static_cast<size_t>(mNbNodes) * sizeof(volume); }\
     \
 private:\
     volume* mNodes;
@@ -139,7 +139,7 @@ class OPCODE_API AABBOptimizedTree
     inline_ udword GetNbNodes() const { return mNbNodes; }
     virtual void* GetData() const = 0;
     virtual void SetData(void* ptr, udword nbNodes) = 0;
-    virtual udword GetUsedBytes() const = 0;
+    virtual size_t GetUsedBytes() const = 0;
     virtual bool Build(AABBTree* tree) = 0;
 
 protected:

src/Common/LevelStructure.hpp

@@ -555,6 +555,7 @@ using NodeCompressed = NodeCompressed13;
 const u32 XRCL_CURRENT_VERSION = 18; // input
 const u32 XRCL_PRODUCTION_VERSION = 14; // output
 const u32 CFORM_CURRENT_VERSION = 4;
+const u32 CFORM_CACHE_CURRENT_VERSION = 1; // SkyLoader: Added CDB cache, physical material list
 
 enum xrAI_Versions : u8
 {

src/xrCDB/xrCDB.cpp

@@ -83,7 +83,7 @@ void MODEL::build_internal(Fvector* V, u32 Vcnt, TRI* T, u32 Tcnt, build_callbac
 
     xr_free(verts);
     xr_free(tris);
-    xr_free(tree);
+    xr_delete(tree);
 
     // verts
     verts_count = Vcnt;
@@ -141,6 +141,22 @@ void MODEL::build_internal(Fvector* V, u32 Vcnt, TRI* T, u32 Tcnt, build_callbac
     xr_free(temp_tris);
 }
 
+void MODEL::load_geom(Fvector* V, u32 Vcnt, TRI* T, u32 Tcnt)
+{
+    xr_free(verts);
+    xr_free(tris);
+
+    // verts
+    verts_count = Vcnt;
+    verts = xr_alloc<Fvector>(verts_count);
+    CopyMemory(verts, V, static_cast<size_t>(verts_count) * sizeof(Fvector));
+
+    // tris
+    tris_count = Tcnt;
+    tris = xr_alloc<TRI>(tris_count);
+    CopyMemory(tris, T, static_cast<size_t>(tris_count) * sizeof(TRI));
+}
+
 /*
     Serialization/Deserialization
 
@@ -218,9 +234,9 @@ bool MODEL::deserialize(pcstr fileName, bool skipCrc32Check /*= false*/, deseria
     verts_count = rstream->r_u32();
     tris_count = rstream->r_u32();
 
-    const u32 vertsSize = verts_count * sizeof(Fvector);
-    const u32 trisSize = tris_count * sizeof(TRI);
-    const u32 treeSize = sizeof(verts_count) + sizeof(tris_count) + vertsSize + trisSize;
+    const size_t vertsSize = static_cast<size_t>(verts_count) * sizeof(Fvector);
+    const size_t trisSize = static_cast<size_t>(tris_count) * sizeof(TRI);
+    const size_t treeSize = sizeof(verts_count) + sizeof(tris_count) + vertsSize + trisSize;
     if (treeSize > rstream->elapsed())
     {
         FS.r_close(rstream);
@@ -236,7 +252,7 @@ bool MODEL::deserialize(pcstr fileName, bool skipCrc32Check /*= false*/, deseria
 
     xr_free(verts);
     xr_free(tris);
-    xr_free(tree);
+    xr_delete(tree);
 
     verts = xr_alloc<Fvector>(verts_count);
     tris = xr_alloc<TRI>(tris_count);
@@ -257,15 +273,29 @@ bool MODEL::deserialize(pcstr fileName, bool skipCrc32Check /*= false*/, deseria
     return success;
 }
 
-u32 MODEL::memory()
+void MODEL::deserialize_tree(IReader* rstream)
+{
+    R_ASSERT(rstream);
+
+    xr_delete(tree);
+
+    tree = xr_new<OPCODE_Model>();
+
+    // Load the OPCODE tree
+    const bool success = tree->Load(rstream, true, false);
+    if (success)
+        status = S_READY;
+}
+
+size_t MODEL::memory()
 {
     if (S_BUILD == status)
     {
         Msg("! xrCDB: model still isn't ready");
         return 0;
     }
-    u32 V = verts_count * sizeof(Fvector);
-    u32 T = tris_count * sizeof(TRI);
+    size_t V = static_cast<size_t>(verts_count) * sizeof(Fvector);
+    size_t T = static_cast<size_t>(tris_count) * sizeof(TRI);
     return tree->GetUsedBytes() + V + T + sizeof(*this) + sizeof(*tree);
 }
 

src/xrCDB/xrCDB.h

@@ -60,6 +60,7 @@ static_assert(sizeof(TRI) == 16, "TRI always should be 16 bytes on any architect
 using build_callback = void(Fvector* V, u32 Vcnt, TRI* T, u32 Tcnt, void* params);
 using serialize_callback = void(IWriter& writer);
 using deserialize_callback = bool(IReader& reader);
+using remapping_materials_callback = void(TRI* T, u32 Tcnt, xr_map<u16, shared_str>& gameMtls);
 
 // Model definition
 class XRCDB_API MODEL : Noncopyable
@@ -113,10 +114,12 @@ class XRCDB_API MODEL : Noncopyable
     void build(Fvector* V, u32 Vcnt, TRI* T, u32 Tcnt, build_callback* bc = nullptr, void* bcp = nullptr);
 
     void set_model_crc32(u32 value) { model_crc32 = value; }
+    void load_geom(Fvector* V, u32 Vcnt, TRI* T, u32 Tcnt);
     bool serialize(pcstr fileName, serialize_callback callback = nullptr) const;
     bool deserialize(pcstr fileName, bool skipCrc32Check = false, deserialize_callback callback = nullptr);
+    void deserialize_tree(IReader* rstream);
 
-    u32 memory();
+    size_t memory();
 
 private:
     void syncronize_impl() const;

src/xrCDB/xr_area.cpp

@@ -87,25 +87,28 @@ int CObjectSpace::GetNearest(xr_vector<IGameObject*>& q_nearest, ICollisionForm*
 
 void CObjectSpace::Load(CDB::build_callback build_callback,
     CDB::serialize_callback serialize_callback,
-    CDB::deserialize_callback deserialize_callback)
+    CDB::deserialize_callback deserialize_callback,
+    CDB::remapping_materials_callback remapping_materials_callback)
 {
-    Load("$level$", "level.cform", build_callback, serialize_callback, deserialize_callback);
+    Load("$level$", "level.cform", build_callback, serialize_callback, deserialize_callback, remapping_materials_callback);
 }
 
 void CObjectSpace::Load(LPCSTR path, LPCSTR fname,
     CDB::build_callback build_callback,
     CDB::serialize_callback serialize_callback,
-    CDB::deserialize_callback deserialize_callback)
+    CDB::deserialize_callback deserialize_callback,
+    CDB::remapping_materials_callback remapping_materials_callback)
 {
     IReader* F = FS.r_open(path, fname);
     R_ASSERT(F);
-    Load(F, build_callback, serialize_callback, deserialize_callback);
+    Load(F, build_callback, serialize_callback, deserialize_callback, remapping_materials_callback);
 }
 
 void CObjectSpace::Load(IReader* F,
     CDB::build_callback build_callback,
     CDB::serialize_callback serialize_callback,
-    CDB::deserialize_callback deserialize_callback)
+    CDB::deserialize_callback deserialize_callback,
+    CDB::remapping_materials_callback remapping_materials_callback)
 {
     ZoneScoped;
 
@@ -119,14 +122,27 @@ void CObjectSpace::Load(IReader* F,
     Fvector* verts = (Fvector*)F->pointer();
     CDB::TRI* tris = (CDB::TRI*)(verts + H.vertcount);
 
-    Create(verts, tris, H, build_callback, serialize_callback, deserialize_callback);
+    // SkyLoader: Check for the new format
+    IReader* cacheStream = nullptr;
+    size_t totalGeomSize = (static_cast<size_t>(H.vertcount) * sizeof(Fvector)) + (H.facecount * sizeof(CDB::TRI));
+    F->advance(totalGeomSize);
+    if (F->elapsed() > sizeof(u32))
+    {
+        u32 version = F->r_u32();
+        if (version == CFORM_CACHE_CURRENT_VERSION)
+            cacheStream = F;
+    }
+
+    Create(verts, tris, H, build_callback, serialize_callback, deserialize_callback, remapping_materials_callback, cacheStream);
     FS.r_close(F);
 }
 
 void CObjectSpace::Create(Fvector* verts, CDB::TRI* tris, const hdrCFORM& H,
     CDB::build_callback build_callback,
     CDB::serialize_callback serialize_callback,
-    CDB::deserialize_callback deserialize_callback)
+    CDB::deserialize_callback deserialize_callback,
+    CDB::remapping_materials_callback remapping_materials_callback,
+    IReader* cacheStream /*= nullptr*/)
 {
     ZoneScoped;
 
@@ -139,7 +155,33 @@ void CObjectSpace::Create(Fvector* verts, CDB::TRI* tris, const hdrCFORM& H,
     strconcat(file_name, "cdb_cache" DELIMITER, FS.get_path("$level$")->m_Add, "objspace.bin");
     FS.update_path(file_name, "$app_data_root$", file_name);
 
-    if (use_cache && FS.exist(file_name) && Static.deserialize(file_name, skip_crc32_check, deserialize_callback))
+    if (use_cache && cacheStream)
+    {
+#ifndef MASTER_GOLD
+        Msg("* Loading ObjectSpace cache from level.cform...");
+#endif
+
+        // Load geometry
+        Static.load_geom(verts, H.vertcount, tris, H.facecount);
+
+        // Read game material list
+        xr_map<u16, shared_str> gameMtls;
+        u32 cnt = cacheStream->r_u32();
+        for (u32 i = 0; i < cnt; i++)
+        {
+            u16 idx = cacheStream->r_u16();
+            shared_str mtlName;
+            cacheStream->r_stringZ(mtlName);
+            gameMtls[idx] = mtlName;
+        }
+
+        if (remapping_materials_callback)
+            remapping_materials_callback(Static.get_tris(), Static.get_tris_count(), gameMtls);
+
+        // Load OPCODE tree
+        Static.deserialize_tree(cacheStream);
+    }
+    else if (use_cache && FS.exist(file_name) && Static.deserialize(file_name, skip_crc32_check, deserialize_callback))
     {
 #ifndef MASTER_GOLD
         Msg("* Loaded ObjectSpace cache (%s)...", file_name);

src/xrCDB/xr_area.h

@@ -62,20 +62,25 @@ class XRCDB_API CObjectSpace : protected CObjectSpaceData, public Noncopyable
 
     void Load  (CDB::build_callback build_callback,
                 CDB::serialize_callback serialize_callback,
-                CDB::deserialize_callback deserialize_callback);
+                CDB::deserialize_callback deserialize_callback,
+                CDB::remapping_materials_callback remapping_materials_callback);
 
     void Load  (LPCSTR path, LPCSTR fname, CDB::build_callback build_callback,
                 CDB::serialize_callback serialize_callback,
-                CDB::deserialize_callback deserialize_callback);
+                CDB::deserialize_callback deserialize_callback,
+                CDB::remapping_materials_callback remapping_materials_callback);
 
     void Load  (IReader* R, CDB::build_callback build_callback,
                 CDB::serialize_callback serialize_callback,
-                CDB::deserialize_callback deserialize_callback);
+                CDB::deserialize_callback deserialize_callback,
+                CDB::remapping_materials_callback remapping_materials_callback);
 
     void Create(Fvector* verts, CDB::TRI* tris, const hdrCFORM& H,
                 CDB::build_callback build_callback,
                 CDB::serialize_callback serialize_callback,
-                CDB::deserialize_callback deserialize_callback);
+                CDB::deserialize_callback deserialize_callback,
+                CDB::remapping_materials_callback remapping_materials_callback,
+                IReader* cacheStream = nullptr);
 
     // Occluded/No
     bool RayTest(const Fvector& start, const Fvector& dir, float range, collide::rq_target tgt,

src/xrCore/FS.cpp

@@ -503,8 +503,10 @@ CVirtualFileRW::CVirtualFileRW(pcstr cFileName)
     // Open the file
     hSrcFile = CreateFile(cFileName, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, 0, OPEN_EXISTING, 0, 0);
     R_ASSERT3(hSrcFile != INVALID_HANDLE_VALUE, cFileName, xrDebug::ErrorToString(GetLastError()));
-    Size = GetFileSize(hSrcFile, NULL);
-    R_ASSERT3(Size, cFileName, xrDebug::ErrorToString(GetLastError()));
+    LARGE_INTEGER size;
+    BOOL result = GetFileSizeEx(hSrcFile, &size);
+    Size = static_cast<size_t>(size.QuadPart);
+    R_ASSERT3(result && Size, cFileName, xrDebug::ErrorToString(GetLastError()));
 
     hSrcMap = CreateFileMapping(hSrcFile, 0, PAGE_READWRITE, 0, 0, 0);
     R_ASSERT3(hSrcMap != INVALID_HANDLE_VALUE, cFileName, xrDebug::ErrorToString(GetLastError()));

src/xrCore/file_stream_reader.cpp

@@ -11,7 +11,10 @@ void CFileStreamReader::construct(pcstr file_name, const size_t& window_size)
     m_file_handle = CreateFile(file_name, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
 
     VERIFY(m_file_handle != INVALID_HANDLE_VALUE);
-    const auto file_size = static_cast<size_t>(GetFileSize(m_file_handle, nullptr));
+    LARGE_INTEGER size;
+    BOOL result = GetFileSizeEx(m_file_handle, &size);
+    const auto file_size = static_cast<size_t>(size.QuadPart);
+    R_ASSERT2(result && file_size, xrDebug::ErrorToString(GetLastError()));
 
     const auto file_mapping_handle = CreateFileMapping(m_file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr);
     VERIFY(file_mapping_handle != INVALID_HANDLE_VALUE);

src/xrEngine/IGame_Level.cpp

@@ -93,6 +93,11 @@ bool deserialize_callback(IReader& reader)
 {
     return g_pGameLevel->Load_GameSpecific_CFORM_Deserialize(reader);
 }
+
+void remapping_materials_callback(CDB::TRI* T, u32 Tcnt, xr_map<u16, shared_str>& gameMtls)
+{
+    g_pGameLevel->Load_GameSpecific_CFORM_SetMaterials(T, Tcnt, gameMtls);
+}
 } // namespace
 
 bool IGame_Level::Load(u32 dwNum)
@@ -119,7 +124,7 @@ bool IGame_Level::Load(u32 dwNum)
     // CForms
     g_pGamePersistent->LoadTitle("st_loading_cform");
 
-    ObjectSpace.Load(build_callback, serialize_callback, deserialize_callback);
+    ObjectSpace.Load(build_callback, serialize_callback, deserialize_callback, remapping_materials_callback);
     g_pGamePersistent->SpatialSpace.initialize(ObjectSpace.GetBoundingVolume());
     g_pGamePersistent->SpatialSpacePhysic.initialize(ObjectSpace.GetBoundingVolume());
 

src/xrEngine/IGame_Level.h

@@ -113,6 +113,7 @@ class ENGINE_API IGame_Level : public IInputReceiver,
     virtual void Load_GameSpecific_CFORM(CDB::TRI* T, u32 count) = 0;
     virtual void Load_GameSpecific_CFORM_Serialize(IWriter& writer) = 0;
     virtual bool Load_GameSpecific_CFORM_Deserialize(IReader& reader) = 0;
+    virtual void Load_GameSpecific_CFORM_SetMaterials(CDB::TRI* tris, u32 count, xr_map<u16, shared_str>& gameMtls) = 0;
 
     virtual void OnFrame(void);
     virtual void OnRender(void);