Skip to content

Commit cc0a63c

Browse files
rouaultrouault
authored
Merge pull request #4349 from rouault/EMBED_GRIDS_DIRECTORY
Add a CMake EMBED_GRIDS_DIRECTORY option to embed .tif/.json files in to libproj
2 parents 387f442 + e2129c5 commit cc0a63c

File tree

9 files changed

+277
-74
lines changed

9 files changed

+277
-74
lines changed

‎.github/workflows/fedora_rawhide/start.sh

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -e
44

5-
dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils
5+
dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils wget
66

77
cd "$WORK_DIR"
88

@@ -18,10 +18,34 @@ ccache -s
1818

1919
mkdir build
2020
cd build
21+
echo "Build with -DEMBED_RESOURCE_FILES=ON"
2122
CC=clang CXX=clang++ cmake .. \
22-
-DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
23+
-DEMBED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
2324
make -j$(nproc)
2425
ctest -j$(nproc)
26+
27+
# Try EMBED_GRIDS_DIRECTORY option
28+
wget https://raw.githubusercontent.com/OSGeo/PROJ-data/refs/heads/master/us_nga/us_nga_egm96_15.tif
29+
mkdir grids
30+
mv us_nga_egm96_15.tif grids
31+
echo "Build with -DEMBED_RESOURCE_FILES=ON -DEMBED_GRIDS_DIRECTORY=$PWD/grids"
32+
CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids
33+
make -j$(nproc)
34+
rm -rf data
35+
echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979
36+
echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false)
37+
echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96
38+
echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 | grep 0.0067 >/dev/null || (echo "Expected 0.0067 0.0061 -0.0185 as a result" && /bin/false)
39+
40+
echo "Build with -DEMBED_RESOURCE_FILES=ON -DEMBED_GRIDS_DIRECTORY=$PWD/grids -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON"
41+
CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON
42+
make -j$(nproc)
43+
rm -rf data
44+
echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979
45+
echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false)
46+
echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96
47+
echo 0 0 0 | bin/cct +init=ITRF2000:ITRF96 | grep 0.0067 >/dev/null || (echo "Expected 0.0067 0.0061 -0.0185 as a result" && /bin/false)
48+
2549
cd ..
2650

2751
ccache -s

‎CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,21 @@ endif()
384384
################################################################################
385385
# Build configured components
386386
################################################################################
387+
388+
set(PROJ_DICTIONARY
389+
world
390+
other.extra
391+
nad27
392+
GL27
393+
nad83
394+
nad.lst
395+
CH
396+
ITRF2000
397+
ITRF2008
398+
ITRF2014
399+
ITRF2020
400+
)
401+
387402
include_directories(${PROJ_SOURCE_DIR}/src)
388403

389404
add_subdirectory(data)

‎data/CMakeLists.txt

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,6 @@ set(PROJ_INI
1515
proj.ini
1616
)
1717

18-
set(PROJ_DICTIONARY
19-
world
20-
other.extra
21-
nad27
22-
GL27
23-
nad83
24-
nad.lst
25-
CH
26-
ITRF2000
27-
ITRF2008
28-
ITRF2014
29-
ITRF2020
30-
)
31-
3218
#
3319
# gridshift file
3420
#
@@ -50,7 +36,7 @@ set(PROJ_DB_SQL_EXPECTED_MD5 "179a5d10e801cb758db7aaa925b97a8f")
5036
add_custom_command(
5137
OUTPUT ${PROJ_DB}
5238
COMMAND ${CMAKE_COMMAND} -E remove -f ${PROJ_DB}
53-
COMMAND ${CMAKE_COMMAND} "-DALL_SQL_IN=${ALL_SQL_IN}" "-DEXE_SQLITE3=${EXE_SQLITE3}" "-DPROJ_DB=${PROJ_DB}" "-DPROJ_VERSION=${PROJ_VERSION}" "-DPROJ_DB_CACHE_DIR=${PROJ_DB_CACHE_DIR}" "-DPROJ_DB_SQL_EXPECTED_MD5=${PROJ_DB_SQL_EXPECTED_MD5}"
39+
COMMAND ${CMAKE_COMMAND} "-DALL_SQL_IN=${ALL_SQL_IN}" "-DEXE_SQLITE3=${EXE_SQLITE3}" "-DPROJ_DB=${PROJ_DB}" "-DPROJ_VERSION=${PROJ_VERSION}" "-DPROJ_DB_CACHE_DIR=${PROJ_DB_CACHE_DIR}" "-DPROJ_DB_SQL_EXPECTED_MD5=${PROJ_DB_SQL_EXPECTED_MD5}" "-DDATA_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}"
5440
-P "${CMAKE_CURRENT_SOURCE_DIR}/generate_proj_db.cmake"
5541
COMMAND ${CMAKE_COMMAND} -E copy ${PROJ_DB} ${CMAKE_CURRENT_BINARY_DIR}/for_tests
5642
DEPENDS ${SQL_FILES} "${CMAKE_CURRENT_SOURCE_DIR}/generate_proj_db.cmake"
@@ -111,14 +97,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_D
11197
#install
11298
#
11399
set(ALL_DATA_FILE
114-
${PROJ_DICTIONARY}
115100
${GRIDSHIFT_FILES}
116101
${SCHEMA_FILES}
117102
${GEOTIFF_FILES}
118103
)
119104

120105
if (NOT USE_ONLY_EMBEDDED_RESOURCE_FILES)
121-
list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB})
106+
list(APPEND ALL_DATA_FILE ${PROJ_INI} ${PROJ_DB} ${PROJ_DICTIONARY})
122107
endif()
123108

124109
install(

‎data/generate_proj_db.cmake

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ endfunction()
2828

2929
generate_all_sql_in("${ALL_SQL_IN}" OFF PROJ_DB_SQL_MD5)
3030

31+
file(WRITE "${DATA_BINARY_DIR}/PROJ_DB_SQL_MD5.h" "const char* PROJ_DB_SQL_MD5=\"${PROJ_DB_SQL_MD5}\";\n")
32+
3133
if (NOT "${PROJ_DB_SQL_MD5}" STREQUAL "${PROJ_DB_SQL_EXPECTED_MD5}")
3234
message(WARNING "all.sql.in content has changed. Running extra validation checks when building proj.db...")
3335

‎docs/source/install.rst

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -450,19 +450,35 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory.
450450

451451
.. versionadded:: 9.6
452452

453+
When ON, :file:`proj.db`, :file:`proj.ini` and ITRF resource files will be
454+
embedded into the PROJ library.
453455
Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON
454456
for static library builds (BUILD_SHARED_LIBS=OFF).
455-
When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library.
457+
458+
.. option:: EMBED_GRIDS_DIRECTORY=<directory>
459+
460+
.. versionadded:: 9.6
461+
462+
Embed files from <directory> ending with .tif or .json in the PROJ library itself.
463+
464+
The pointed directory can potentially be the full PROJ-data package (uncompressed).
465+
In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ.
466+
467+
When using this parameter, EMBED_RESOURCE_FILES must be set to ON.
468+
469+
If the content of the directory changes, you need to run CMake again to
470+
update the list of files.
456471

457472
.. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF
458473

459474
.. versionadded:: 9.6
460475

461476
Even if EMBED_RESOURCE_FILES=ON, by default PROJ will still try to locate
462-
:file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the
463-
embedded version if not found.
477+
:file:`proj.db`, :file:`proj.ini`, ITRF resource files or grid files on the
478+
file system, and fallback to the embedded version if not found.
464479
By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating
465-
those files on the file system is made. Default is OFF.
480+
those files on the file system is made.
481+
Default is OFF.
466482
Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting
467483
USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF.
468484

‎src/embedded_resources.c

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,21 @@
1+
#include <stdint.h>
2+
#include <string.h>
3+
14
#include "embedded_resources.h"
25

36
#if USE_SHARP_EMBED
7+
8+
#include "PROJ_DB_SQL_MD5.h"
9+
410
const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
11+
(void)PROJ_DB_SQL_MD5;
512
static const unsigned char proj_db[] = {
613
#embed PROJ_DB
714
};
815
*pnSize = (unsigned int)sizeof(proj_db);
916
return proj_db;
1017
}
1118

12-
const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
13-
static const char proj_ini[] = {
14-
#embed PROJ_INI
15-
};
16-
*pnSize = (unsigned int)sizeof(proj_ini);
17-
return proj_ini;
18-
}
19-
2019
#else
2120

2221
#include "file_embed/proj_db.h"
@@ -25,10 +24,6 @@ const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize) {
2524
return proj_db_data;
2625
}
2726

28-
#include "file_embed/proj_ini.h"
29-
const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
30-
*pnSize = proj_ini_size;
31-
return (const char *)proj_ini_data;
32-
}
33-
3427
#endif
28+
29+
#include "file_embed/embedded_resources.c"

‎src/embedded_resources.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ extern "C" {
66
#endif
77

88
const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize);
9-
const char *pj_get_embedded_proj_ini(unsigned int *pnSize);
9+
10+
const unsigned char *pj_get_embedded_resource(const char *filename,
11+
unsigned int *pnSize);
1012

1113
#ifdef __cplusplus
1214
}

‎src/filemanager.cpp

Lines changed: 121 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ std::string File::read_line(size_t maxLen, bool &maxLenReached,
143143

144144
// ---------------------------------------------------------------------------
145145

146+
#if !USE_ONLY_EMBEDDED_RESOURCE_FILES
147+
146148
#ifdef _WIN32
147149

148150
/* The bulk of utf8towc()/utf8fromwc() is derived from the utf.c module from
@@ -805,6 +807,8 @@ std::unique_ptr<File> FileStdio::open(PJ_CONTEXT *ctx, const char *filename,
805807

806808
#endif // _WIN32
807809

810+
#endif // !USE_ONLY_EMBEDDED_RESOURCE_FILES
811+
808812
// ---------------------------------------------------------------------------
809813

810814
class FileApiAdapter : public File {
@@ -893,6 +897,80 @@ std::unique_ptr<File> FileApiAdapter::open(PJ_CONTEXT *ctx,
893897

894898
// ---------------------------------------------------------------------------
895899

900+
#if EMBED_RESOURCE_FILES
901+
902+
class FileMemory : public File {
903+
PJ_CONTEXT *m_ctx;
904+
const unsigned char *const m_data;
905+
const size_t m_size;
906+
size_t m_pos = 0;
907+
908+
FileMemory(const FileMemory &) = delete;
909+
FileMemory &operator=(const FileMemory &) = delete;
910+
911+
protected:
912+
FileMemory(const std::string &filename, PJ_CONTEXT *ctx,
913+
const unsigned char *data, size_t size)
914+
: File(filename), m_ctx(ctx), m_data(data), m_size(size) {}
915+
916+
public:
917+
size_t read(void *buffer, size_t sizeBytes) override;
918+
size_t write(const void *, size_t) override;
919+
bool seek(unsigned long long offset, int whence = SEEK_SET) override;
920+
unsigned long long tell() override { return m_pos; }
921+
void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
922+
923+
bool hasChanged() const override { return false; }
924+
925+
static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename,
926+
FileAccess access,
927+
const unsigned char *data, size_t size) {
928+
if (access != FileAccess::READ_ONLY)
929+
return nullptr;
930+
return std::unique_ptr<File>(new FileMemory(filename, ctx, data, size));
931+
}
932+
};
933+
934+
size_t FileMemory::read(void *buffer, size_t sizeBytes) {
935+
if (m_pos >= m_size)
936+
return 0;
937+
if (sizeBytes >= m_size - m_pos) {
938+
const size_t bytesToCopy = m_size - m_pos;
939+
memcpy(buffer, m_data + m_pos, bytesToCopy);
940+
m_pos = m_size;
941+
return bytesToCopy;
942+
}
943+
memcpy(buffer, m_data + m_pos, sizeBytes);
944+
m_pos += sizeBytes;
945+
return sizeBytes;
946+
}
947+
948+
size_t FileMemory::write(const void *, size_t) {
949+
// shouldn't happen given we have bailed out in open() in non read-only
950+
// modes
951+
return 0;
952+
}
953+
954+
bool FileMemory::seek(unsigned long long offset, int whence) {
955+
if (whence == SEEK_SET) {
956+
m_pos = static_cast<size_t>(offset);
957+
return m_pos == offset;
958+
} else if (whence == SEEK_CUR) {
959+
const unsigned long long newPos = m_pos + offset;
960+
m_pos = static_cast<size_t>(newPos);
961+
return m_pos == newPos;
962+
} else {
963+
if (offset != 0)
964+
return false;
965+
m_pos = m_size;
966+
return true;
967+
}
968+
}
969+
970+
#endif
971+
972+
// ---------------------------------------------------------------------------
973+
896974
std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename,
897975
FileAccess access) {
898976
if (starts_with(filename, "http://") || starts_with(filename, "https://")) {
@@ -909,11 +987,31 @@ std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename,
909987
if (ctx->fileApi.open_cbk != nullptr) {
910988
return FileApiAdapter::open(ctx, filename, access);
911989
}
990+
991+
std::unique_ptr<File> ret;
992+
#if !(EMBED_RESOURCE_FILES && USE_ONLY_EMBEDDED_RESOURCE_FILES)
912993
#ifdef _WIN32
913-
return FileWin32::open(ctx, filename, access);
994+
ret = FileWin32::open(ctx, filename, access);
914995
#else
915-
return FileStdio::open(ctx, filename, access);
996+
ret = FileStdio::open(ctx, filename, access);
916997
#endif
998+
#endif
999+
1000+
#if EMBED_RESOURCE_FILES
1001+
#if USE_ONLY_EMBEDDED_RESOURCE_FILES
1002+
if (!ret)
1003+
#endif
1004+
{
1005+
unsigned int size = 0;
1006+
const unsigned char *in_memory_data =
1007+
pj_get_embedded_resource(filename, &size);
1008+
if (in_memory_data) {
1009+
ret = FileMemory::open(ctx, filename, access, in_memory_data, size);
1010+
}
1011+
}
1012+
#endif
1013+
1014+
return ret;
9171015
}
9181016

9191017
// ---------------------------------------------------------------------------
@@ -1574,6 +1672,24 @@ static void *pj_open_lib_internal(
15741672
errno = 0;
15751673
}
15761674

1675+
#if EMBED_RESOURCE_FILES
1676+
if (!fid && fname != name && name[0] != '.' && name[0] != '/' &&
1677+
name[0] != '~' && !starts_with(name, "http://") &&
1678+
!starts_with(name, "https://")) {
1679+
fid = open_file(ctx, name, mode);
1680+
if (fid) {
1681+
if (out_full_filename != nullptr &&
1682+
out_full_filename_size > 0) {
1683+
// cppcheck-suppress nullPointer
1684+
strncpy(out_full_filename, name, out_full_filename_size);
1685+
out_full_filename[out_full_filename_size - 1] = '\0';
1686+
}
1687+
fname = name;
1688+
errno = 0;
1689+
}
1690+
}
1691+
#endif
1692+
15771693
if (ctx->last_errno == 0 && errno != 0)
15781694
proj_context_errno_set(ctx, errno);
15791695

@@ -1870,20 +1986,9 @@ void pj_load_ini(PJ_CONTEXT *ctx) {
18701986

18711987
ctx->iniFileLoaded = true;
18721988
std::string content;
1873-
std::unique_ptr<NS_PROJ::File> file;
1874-
#ifndef USE_ONLY_EMBEDDED_RESOURCE_FILES
1875-
file.reset(reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal(
1876-
ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0)));
1877-
#endif
1878-
if (!file) {
1879-
#ifdef EMBED_RESOURCE_FILES
1880-
unsigned int content_size = 0;
1881-
const char *c_content = pj_get_embedded_proj_ini(&content_size);
1882-
content.assign(c_content, content_size);
1883-
#else
1884-
return;
1885-
#endif
1886-
}
1989+
auto file = std::unique_ptr<NS_PROJ::File>(
1990+
reinterpret_cast<NS_PROJ::File *>(pj_open_lib_internal(
1991+
ctx, "proj.ini", "rb", pj_open_file_with_manager, nullptr, 0)));
18871992
if (file) {
18881993
file->seek(0, SEEK_END);
18891994
const auto filesize = file->tell();

0 commit comments

Comments
 (0)