Merge pull request #4349 from rouault/EMBED_GRIDS_DIRECTORY

Add a CMake EMBED_GRIDS_DIRECTORY option to embed .tif/.json files in to libproj

.github/workflows/fedora_rawhide/start.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils
+dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils wget
 
 cd "$WORK_DIR"
 
@@ -18,10 +18,34 @@ ccache -s
 
 mkdir build
 cd build
+echo "Build with -DEMBED_RESOURCE_FILES=ON"
 CC=clang CXX=clang++ cmake .. \
-    -DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
+    -DEMBED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
 make -j$(nproc)
 ctest -j$(nproc)
+
+# Try EMBED_GRIDS_DIRECTORY option
+wget https://raw.githubusercontent.com/OSGeo/PROJ-data/refs/heads/master/us_nga/us_nga_egm96_15.tif
+mkdir grids
+mv us_nga_egm96_15.tif grids
+echo "Build with -DEMBED_RESOURCE_FILES=ON -DEMBED_GRIDS_DIRECTORY=$PWD/grids"
+CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids
+make -j$(nproc)
+rm -rf data
+echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979
+echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979  | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false)
+echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96
+echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 | grep 0.0067 >/dev/null || (echo "Expected 0.0067 0.0061 -0.0185 as a result" && /bin/false)
+
+echo "Build with -DEMBED_RESOURCE_FILES=ON -DEMBED_GRIDS_DIRECTORY=$PWD/grids -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON"
+CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON
+make -j$(nproc)
+rm -rf data
+echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979
+echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979  | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false)
+echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96
+echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 | grep 0.0067 >/dev/null || (echo "Expected 0.0067 0.0061 -0.0185 as a result" && /bin/false)
+
 cd ..
 
 ccache -s

CMakeLists.txt

@@ -384,6 +384,21 @@ endif()
 ################################################################################
 # Build configured components
 ################################################################################
+
+set(PROJ_DICTIONARY
+  world
+  other.extra
+  nad27
+  GL27
+  nad83
+  nad.lst
+  CH
+  ITRF2000
+  ITRF2008
+  ITRF2014
+  ITRF2020
+)
+
 include_directories(${PROJ_SOURCE_DIR}/src)
 
 add_subdirectory(data)

data/CMakeLists.txt

@@ -15,20 +15,6 @@ set(PROJ_INI
   proj.ini
 )
 
-set(PROJ_DICTIONARY
-  world
-  other.extra
-  nad27
-  GL27
-  nad83
-  nad.lst
-  CH
-  ITRF2000
-  ITRF2008
-  ITRF2014
-  ITRF2020
-)
-
 #
 # gridshift file
 #
@@ -50,7 +36,7 @@ set(PROJ_DB_SQL_EXPECTED_MD5 "179a5d10e801cb758db7aaa925b97a8f")
 add_custom_command(
   OUTPUT ${PROJ_DB}
   COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJ_DB}
-  COMMAND ${CMAKE_COMMAND} "-DALL_SQL_IN=${ALL_SQL_IN}" "-DEXE_SQLITE3=${EXE_SQLITE3}" "-DPROJ_DB=${PROJ_DB}" "-DPROJ_VERSION=${PROJ_VERSION}" "-DPROJ_DB_CACHE_DIR=${PROJ_DB_CACHE_DIR}" "-DPROJ_DB_SQL_EXPECTED_MD5=${PROJ_DB_SQL_EXPECTED_MD5}"
+  COMMAND ${CMAKE_COMMAND} "-DALL_SQL_IN=${ALL_SQL_IN}" "-DEXE_SQLITE3=${EXE_SQLITE3}" "-DPROJ_DB=${PROJ_DB}" "-DPROJ_VERSION=${PROJ_VERSION}" "-DPROJ_DB_CACHE_DIR=${PROJ_DB_CACHE_DIR}" "-DPROJ_DB_SQL_EXPECTED_MD5=${PROJ_DB_SQL_EXPECTED_MD5}" "-DDATA_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}"
     -P "${CMAKE_CURRENT_SOURCE_DIR}/generate_proj_db.cmake"
   COMMAND ${CMAKE_COMMAND} -E copy ${PROJ_DB} ${CMAKE_CURRENT_BINARY_DIR}/for_tests
   DEPENDS ${SQL_FILES} "${CMAKE_CURRENT_SOURCE_DIR}/generate_proj_db.cmake"
@@ -111,14 +97,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_D
 #install
 #
 set(ALL_DATA_FILE
-  ${PROJ_DICTIONARY}
   ${GRIDSHIFT_FILES}
   ${SCHEMA_FILES}
   ${GEOTIFF_FILES}
 )
 
 if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES)
-    list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB})
+    list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB} ${PROJ_DICTIONARY})
 endif()
 
 install(

data/generate_proj_db.cmake

@@ -28,6 +28,8 @@ endfunction()
 
 generate_all_sql_in("${ALL_SQL_IN}" OFF PROJ_DB_SQL_MD5)
 
+file(WRITE "${DATA_BINARY_DIR}/PROJ_DB_SQL_MD5.h" "const char* PROJ_DB_SQL_MD5=\"${PROJ_DB_SQL_MD5}\";\n")
+
 if (NOT "${PROJ_DB_SQL_MD5}" STREQUAL "${PROJ_DB_SQL_EXPECTED_MD5}")
     message(WARNING "all.sql.in content has changed. Running extra validation checks when building proj.db...")
 

docs/source/install.rst

@@ -450,19 +450,35 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory.
 
     .. versionadded:: 9.6
 
+    When ON, :file:`proj.db`, :file:`proj.ini` and ITRF resource files will be
+    embedded into the PROJ library.
     Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON
     for static library builds (BUILD_SHARED_LIBS=OFF).
-    When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library.
+
+.. option:: EMBED_GRIDS_DIRECTORY=<directory>
+
+    .. versionadded:: 9.6
+
+    Embed files from <directory> ending with .tif or .json in the PROJ library itself.
+
+    The pointed directory can potentially be the full PROJ-data package (uncompressed).
+    In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ.
+
+    When using this parameter, EMBED_RESOURCE_FILES must be set to ON.
+
+    If the content of the directory changes, you need to run CMake again to
+    update the list of files.
 
 .. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF
 
     .. versionadded:: 9.6
 
     Even if EMBED_RESOURCE_FILES=ON, by default PROJ will still try to locate
-    :file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the
-    embedded version if not found.
+    :file:`proj.db`, :file:`proj.ini`, ITRF resource files or grid files on the
+    file system, and fallback to the embedded version if not found.
     By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating
-    those files on the file system is made. Default is OFF.
+    those files on the file system is made.
+    Default is OFF.
     Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting
     USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF.
 

src/embedded_resources.c

@@ -1,22 +1,21 @@
+#include <stdint.h>
+#include <string.h>
+
 #include "embedded_resources.h"
 
 #if USE_SHARP_EMBED
+
+#include "PROJ_DB_SQL_MD5.h"
+
 const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
+    (void)PROJ_DB_SQL_MD5;
     static const unsigned char proj_db[] = {
 #embed PROJ_DB
     };
     *pnSize = (unsigned int)sizeof(proj_db);
     return proj_db;
 }
 
-const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
-    static const char proj_ini[] = {
-#embed PROJ_INI
-    };
-    *pnSize = (unsigned int)sizeof(proj_ini);
-    return proj_ini;
-}
-
 #else
 
 #include "file_embed/proj_db.h"
@@ -25,10 +24,6 @@ const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
     return proj_db_data;
 }
 
-#include "file_embed/proj_ini.h"
-const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
-    *pnSize = proj_ini_size;
-    return (const char *)proj_ini_data;
-}
-
 #endif
+
+#include "file_embed/embedded_resources.c"

src/embedded_resources.h

@@ -6,7 +6,9 @@ extern "C" {
 #endif
 
 const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize);
-const char *pj_get_embedded_proj_ini(unsigned int *pnSize);
+
+const unsigned char *pj_get_embedded_resource(const char *filename,
+                                              unsigned int *pnSize);
 
 #ifdef __cplusplus
 }

src/filemanager.cpp

@@ -143,6 +143,8 @@ std::string File::read_line(size_t maxLen, bool &maxLenReached,
 
 // ---------------------------------------------------------------------------
 
+#if !USE_ONLY_EMBEDDED_RESOURCE_FILES
+
 #ifdef _WIN32
 
 /* The bulk of utf8towc()/utf8fromwc() is derived from the utf.c module from
@@ -805,6 +807,8 @@ std::unique_ptr<File> FileStdio::open(PJ_CONTEXT *ctx, const char *filename,
 
 #endif // _WIN32
 
+#endif // !USE_ONLY_EMBEDDED_RESOURCE_FILES
+
 // ---------------------------------------------------------------------------
 
 class FileApiAdapter : public File {
@@ -893,6 +897,80 @@ std::unique_ptr<File> FileApiAdapter::open(PJ_CONTEXT *ctx,
 
 // ---------------------------------------------------------------------------
 
+#if EMBED_RESOURCE_FILES
+
+class FileMemory : public File {
+    PJ_CONTEXT *m_ctx;
+    const unsigned char *const m_data;
+    const size_t m_size;
+    size_t m_pos = 0;
+
+    FileMemory(const FileMemory &) = delete;
+    FileMemory &operator=(const FileMemory &) = delete;
+
+  protected:
+    FileMemory(const std::string &filename, PJ_CONTEXT *ctx,
+               const unsigned char *data, size_t size)
+        : File(filename), m_ctx(ctx), m_data(data), m_size(size) {}
+
+  public:
+    size_t read(void *buffer, size_t sizeBytes) override;
+    size_t write(const void *, size_t) override;
+    bool seek(unsigned long long offset, int whence = SEEK_SET) override;
+    unsigned long long tell() override { return m_pos; }
+    void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
+
+    bool hasChanged() const override { return false; }
+
+    static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename,
+                                      FileAccess access,
+                                      const unsigned char *data, size_t size) {
+        if (access != FileAccess::READ_ONLY)
+            return nullptr;
+        return std::unique_ptr<File>(new FileMemory(filename, ctx, data, size));
+    }
+};
+
+size_t FileMemory::read(void *buffer, size_t sizeBytes) {
+    if (m_pos >= m_size)
+        return 0;
+    if (sizeBytes >= m_size - m_pos) {
+        const size_t bytesToCopy = m_size - m_pos;
+        memcpy(buffer, m_data + m_pos, bytesToCopy);
+        m_pos = m_size;
+        return bytesToCopy;
+    }
+    memcpy(buffer, m_data + m_pos, sizeBytes);
+    m_pos += sizeBytes;
+    return sizeBytes;
+}
+
+size_t FileMemory::write(const void *, size_t) {
+    // shouldn't happen given we have bailed out in open() in non read-only
+    // modes
+    return 0;
+}
+
+bool FileMemory::seek(unsigned long long offset, int whence) {
+    if (whence == SEEK_SET) {
+        m_pos = static_cast<size_t>(offset);
+        return m_pos == offset;
+    } else if (whence == SEEK_CUR) {
+        const unsigned long long newPos = m_pos + offset;
+        m_pos = static_cast<size_t>(newPos);
+        return m_pos == newPos;
+    } else {
+        if (offset != 0)
+            return false;
+        m_pos = m_size;
+        return true;
+    }
+}
+
+#endif
+
+// ---------------------------------------------------------------------------
+
 std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename,
                                         FileAccess access) {
     if (starts_with(filename, "http://") || starts_with(filename, "https://")) {
@@ -909,11 +987,31 @@ std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename,
     if (ctx->fileApi.open_cbk != nullptr) {
         return FileApiAdapter::open(ctx, filename, access);
     }
+
+    std::unique_ptr<File> ret;
+#if !(EMBED_RESOURCE_FILES && USE_ONLY_EMBEDDED_RESOURCE_FILES)
 #ifdef _WIN32
-    return FileWin32::open(ctx, filename, access);
+    ret = FileWin32::open(ctx, filename, access);
 #else
-    return FileStdio::open(ctx, filename, access);
+    ret = FileStdio::open(ctx, filename, access);
 #endif
+#endif
+
+#if EMBED_RESOURCE_FILES
+#if USE_ONLY_EMBEDDED_RESOURCE_FILES
+    if (!ret)
+#endif
+    {
+        unsigned int size = 0;
+        const unsigned char *in_memory_data =
+            pj_get_embedded_resource(filename, &size);
+        if (in_memory_data) {
+            ret = FileMemory::open(ctx, filename, access, in_memory_data, size);
+        }
+    }
+#endif
+
+    return ret;
 }
 
 // ---------------------------------------------------------------------------
@@ -1574,6 +1672,24 @@ static void *pj_open_lib_internal(
             errno = 0;
         }
 
+#if EMBED_RESOURCE_FILES
+        if (!fid && fname != name && name[0] != '.' && name[0] != '/' &&
+            name[0] != '~' && !starts_with(name, "http://") &&
+            !starts_with(name, "https://")) {
+            fid = open_file(ctx, name, mode);
+            if (fid) {
+                if (out_full_filename != nullptr &&
+                    out_full_filename_size > 0) {
+                    // cppcheck-suppress nullPointer
+                    strncpy(out_full_filename, name, out_full_filename_size);
+                    out_full_filename[out_full_filename_size - 1] = '\0';
+                }
+                fname = name;
+                errno = 0;
+            }
+        }
+#endif
+
         if (ctx->last_errno == 0 && errno != 0)
             proj_context_errno_set(ctx, errno);
 
@@ -1870,20 +1986,9 @@ void pj_load_ini(PJ_CONTEXT *ctx) {
 
     ctx->iniFileLoaded = true;
     std::string content;
-    std::unique_ptr<NS_PROJ::File> file;
-#ifndef USE_ONLY_EMBEDDED_RESOURCE_FILES
-    file.reset(reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal(
-        ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0)));
-#endif
-    if (!file) {
-#ifdef EMBED_RESOURCE_FILES
-        unsigned int content_size = 0;
-        const char *c_content = pj_get_embedded_proj_ini(&content_size);
-        content.assign(c_content, content_size);
-#else
-        return;
-#endif
-    }
+    auto file = std::unique_ptr<NS_PROJ::File>(
+        reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal(
+            ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0)));
     if (file) {
         file->seek(0, SEEK_END);
         const auto filesize = file->tell();