Skip to content

Commit 17afae7

Browse files
5cript5cript
authored
Merge pull request #123 from NuiCpp/feat/bin2hpp_compression
Added base64 and compressed index.hpp encoding.
2 parents d85e29e + 967e8ed commit 17afae7

16 files changed

+323
-64
lines changed

‎.github/workflows/macos_13.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ jobs:
2929
- name: Install Brew Dependencies
3030
run: brew install cmake llvm@16 boost git openssl@3 cryptopp curl ninja make
3131

32+
- name: Tool Version Dump
33+
run: |
34+
clang++ --version
35+
cmake --version
36+
ninja --version
37+
brew list --versions boost
38+
3239
- name: Configure CMake
3340
run: cmake -B ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} -G"Ninja" -DNUI_ENABLE_TESTS=on -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DNUI_BUILD_EXAMPLES=on -DCMAKE_CXX_EXTENSIONS=on -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm@16/bin/clang++ -DCMAKE_C_COMPILER=/usr/local/opt/llvm@16/bin/clang -DNUI_NPM=npm -DNUI_NODE=node -DCMAKE_CXX_STANDARD=20
3441
env:

‎.github/workflows/ubuntu_20.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ jobs:
3131
# OPTIONAL: Specify a platform version
3232
platform_version: 20.04
3333

34+
- name: Tool Version Dump
35+
run: |
36+
clang++ --version
37+
cmake --version
38+
ninja --version
39+
3440
- name: Setup clang
3541
uses: egor-tensin/setup-clang@v1
3642
with:
@@ -40,7 +46,7 @@ jobs:
4046
- name: Configure CMake
4147
run: cmake -B ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} -G"Ninja" -DNUI_ENABLE_TESTS=on -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DNUI_BUILD_EXAMPLES=on -DCMAKE_CXX_EXTENSIONS=on -DCMAKE_CXX_COMPILER=c++ -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_STANDARD=20
4248
env:
43-
BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
49+
Boost_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
4450

4551
- name: Build
4652
run: cmake --build ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} --config ${{env.BUILD_TYPE}}

‎.github/workflows/ubuntu_22.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,19 @@ jobs:
3434
# OPTIONAL: Specify a platform version
3535
platform_version: 22.04
3636

37+
- name: Tool Version Dump
38+
run: |
39+
clang++ --version
40+
cmake --version
41+
ninja --version
42+
3743
- name: Symlink xlocale
3844
run: sudo ln -s /usr/include/locale.h /usr/include/xlocale.h
3945

4046
- name: Configure CMake
4147
run: cmake -B ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} -G"Ninja" -DNUI_ENABLE_TESTS=on -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DNUI_BUILD_EXAMPLES=on -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_CXX_EXTENSIONS=on -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -DCMAKE_LINKER=lld -DCMAKE_CXX_STANDARD=20
4248
env:
43-
BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
49+
Boost_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
4450

4551
- name: Build
4652
run: cmake --build ${{github.workspace}}/build/clang_${{env.BUILD_TYPE}} --config ${{env.BUILD_TYPE}}

‎cmake/backend/emscripten.cmake

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ function(nui_add_emscripten_target)
4040
cmake_parse_arguments(
4141
NUI_ADD_EMSCRIPTEN_TARGET_ARGS
4242
"DISABLE_BIN2HPP;DISABLE_PARCEL_ADAPTER;ENABLE_TAILWIND;ENABLE_DOTENV"
43-
"TARGET;PREJS;SOURCE_DIR"
43+
"TARGET;PREJS;SOURCE_DIR;BIN2HPP_ENCODING"
4444
"CMAKE_OPTIONS"
4545
${ARGN}
4646
)
@@ -99,8 +99,14 @@ function(nui_add_emscripten_target)
9999
set(BUILD_COMMAND BUILD_COMMAND cmake -E copy "${SOURCE_DIR}/package.json" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/package.json")
100100
endif()
101101

102+
if (NUI_ADD_EMSCRIPTEN_TARGET_ARGS_BIN2HPP_ENCODING)
103+
set(BIN2HPP_ENCODING ${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_BIN2HPP_ENCODING})
104+
else()
105+
set(BIN2HPP_ENCODING "raw")
106+
endif()
107+
102108
if (ENABLE_BIN2HPP AND ${ENABLE_BIN2HPP})
103-
set(BIN2HPP_COMMAND COMMAND $<TARGET_FILE:bin2hpp> "on" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/bin/index.html" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/include/index.hpp" index)
109+
set(BIN2HPP_COMMAND COMMAND $<TARGET_FILE:bin2hpp> "on" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/bin/index.html" "${CMAKE_BINARY_DIR}/module_${NUI_ADD_EMSCRIPTEN_TARGET_ARGS_TARGET}/include/index.hpp" index ${BIN2HPP_ENCODING})
104110
else()
105111
set(BIN2HPP_COMMAND COMMAND cmake -E true)
106112
endif()

‎cmake/dependencies/boost.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
if (CMAKE_VERSION VERSION_LESS "3.30")
2-
find_package(Boost 1.81.0 REQUIRED COMPONENTS system)
2+
find_package(Boost 1.81.0 REQUIRED COMPONENTS system iostreams)
33
else()
4-
find_package(Boost CONFIG 1.81.0 REQUIRED COMPONENTS system)
4+
find_package(Boost CONFIG 1.81.0 REQUIRED COMPONENTS system iostreams)
55
endif()

‎cmake/dependencies/roar.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
option(NUI_FETCH_ROAR "Fetch roar" ON)
22
set(NUI_ROAR_REPOSITORY "https://github.com/5cript/roar.git" CACHE STRING "roar repository")
3-
set(NUI_ROAR_TAG "4ac9a877b70fbb821de2edbb7dd3fcad1b0a7f08" CACHE STRING "roar tag")
3+
set(NUI_ROAR_TAG "2781a88fcd1fce9af1a697673e26fd3ad55817e9" CACHE STRING "roar tag")
44

55
if(NUI_FETCH_ROAR)
66
include(FetchContent)

‎tools/bin2hpp/CMakeLists.txt

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,14 @@ cmake_minimum_required(VERSION 3.16)
22

33
project(bin2hpp VERSION 0.1.0)
44

5-
add_executable(bin2hpp main.cpp)
5+
add_executable(bin2hpp main.cpp raw_encoder.cpp base64_encoder.cpp compressed_encoder.cpp)
66
target_compile_features(bin2hpp PRIVATE cxx_std_20)
77

8+
include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/dependencies/boost.cmake")
9+
810
set_target_properties(bin2hpp
911
PROPERTIES
1012
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools_bin"
11-
)
13+
)
14+
15+
target_link_libraries(bin2hpp PRIVATE roar Boost::iostreams)

‎tools/bin2hpp/base64_encoder.cpp

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#include "base64_encoder.hpp"
2+
#include "raw_encoder.hpp"
3+
#include "constants.hpp"
4+
5+
#include <roar/utility/base64.hpp>
6+
7+
#include <iostream>
8+
#include <sstream>
9+
10+
std::ostream& Base64Encoder::header(std::ostream& output) const
11+
{
12+
output <<
13+
R"(#pragma once
14+
15+
#include <roar/utility/base64.hpp>
16+
17+
#include <string_view>
18+
#include <string>
19+
20+
static const std::string_view )"
21+
<< name_ << R"(_data[] = {)";
22+
23+
return output;
24+
}
25+
26+
std::ostream& Base64Encoder::content(std::ostream& output, std::istream& input) const
27+
{
28+
std::stringstream readStream;
29+
readStream << input.rdbuf();
30+
31+
auto encoded = Roar::base64Encode(std::move(readStream).str());
32+
33+
std::cout << "Encoded size: " << encoded.size() << "\n";
34+
35+
std::stringstream ss(std::move(encoded));
36+
37+
RawEncoder rawEncoder(name_);
38+
return rawEncoder.content(output, ss);
39+
}
40+
41+
std::ostream& Base64Encoder::index(std::ostream& output) const
42+
{
43+
output << "};\n\n";
44+
output << "static std::string " << name_ << "()\n";
45+
output << "{\n";
46+
output << "\tstatic std::string memo;\n";
47+
output << "\tif(!memo.empty())\n";
48+
output << "\t\treturn memo;\n";
49+
output << "\tfor (std::size_t i = 0; i != sizeof(" << name_ << "_data) / sizeof(std::string_view)"
50+
<< "; ++i) {\n";
51+
output << "\t\tmemo += " << name_ << "_data[i];\n";
52+
output << "\t}\n\n";
53+
output << "\tmemo = Roar::base64Decode(memo);\n";
54+
output << "\treturn memo;\n";
55+
output << "}\n";
56+
57+
return output;
58+
}

‎tools/bin2hpp/base64_encoder.hpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#pragma once
2+
3+
#include "encoder.hpp"
4+
5+
class Base64Encoder : public Encoder
6+
{
7+
public:
8+
using Encoder::Encoder;
9+
10+
std::ostream& header(std::ostream& output) const override;
11+
std::ostream& content(std::ostream& output, std::istream& input) const override;
12+
std::ostream& index(std::ostream& output) const override;
13+
};

‎tools/bin2hpp/compressed_encoder.cpp

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#include "compressed_encoder.hpp"
2+
#include "raw_encoder.hpp"
3+
#include "constants.hpp"
4+
5+
#include <roar/utility/base64.hpp>
6+
#include <boost/iostreams/filtering_stream.hpp>
7+
#include <boost/iostreams/copy.hpp>
8+
#include <boost/iostreams/filter/gzip.hpp>
9+
10+
#include <iostream>
11+
#include <sstream>
12+
13+
std::ostream& CompressedEncoder::header(std::ostream& output) const
14+
{
15+
output <<
16+
R"(#pragma once
17+
18+
#include <roar/utility/base64.hpp>
19+
#include <boost/iostreams/filtering_stream.hpp>
20+
#include <boost/iostreams/copy.hpp>
21+
#include <boost/iostreams/filter/gzip.hpp>
22+
23+
#include <string_view>
24+
#include <string>
25+
#include <sstream>
26+
27+
static const std::string_view )"
28+
<< name_ << R"(_data[] = {)";
29+
30+
return output;
31+
}
32+
33+
std::ostream& CompressedEncoder::content(std::ostream& output, std::istream& input) const
34+
{
35+
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
36+
in.push(boost::iostreams::gzip_compressor());
37+
in.push(input);
38+
39+
std::stringstream compressedRaw;
40+
boost::iostreams::copy(in, compressedRaw);
41+
42+
auto encoded = Roar::base64Encode(std::move(compressedRaw).str());
43+
std::stringstream ss{std::move(encoded)};
44+
45+
RawEncoder rawEncoder(name_);
46+
return rawEncoder.content(output, ss);
47+
}
48+
49+
std::ostream& CompressedEncoder::index(std::ostream& output) const
50+
{
51+
output << "};\n\n";
52+
output << "static std::string " << name_ << "()\n";
53+
output << "{\n";
54+
output << "\tstatic std::string memo;\n";
55+
output << "\tif(!memo.empty())\n";
56+
output << "\t\treturn memo;\n\n";
57+
output << "\tstd::string compressed;\n";
58+
output << "\tfor (std::size_t i = 0; i != sizeof(" << name_ << "_data) / sizeof(std::string_view)"
59+
<< "; ++i) {\n";
60+
output << "\t\tcompressed += " << name_ << "_data[i];\n";
61+
output << "\t}\n\n";
62+
output << "\tcompressed = Roar::base64Decode(compressed);\n";
63+
output << "\tstd::stringstream compressedStream{compressed};\n";
64+
output << "\tboost::iostreams::filtering_streambuf<boost::iostreams::input> in;\n";
65+
output << "\tin.push(boost::iostreams::gzip_decompressor());\n";
66+
output << "\tin.push(compressedStream);\n";
67+
output << "\tstd::stringstream decompressed;\n";
68+
output << "\tboost::iostreams::copy(in, decompressed);\n";
69+
output << "\tmemo = std::move(decompressed).str();\n";
70+
output << "\treturn memo;\n";
71+
output << "}\n";
72+
73+
return output;
74+
}

‎tools/bin2hpp/compressed_encoder.hpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#pragma once
2+
3+
#include "encoder.hpp"
4+
5+
class CompressedEncoder : public Encoder
6+
{
7+
public:
8+
using Encoder::Encoder;
9+
10+
std::ostream& header(std::ostream& output) const override;
11+
std::ostream& content(std::ostream& output, std::istream& input) const override;
12+
std::ostream& index(std::ostream& output) const override;
13+
};

‎tools/bin2hpp/constants.hpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#pragma once
2+
3+
constexpr std::size_t lineWidth = 120;

‎tools/bin2hpp/encoder.hpp

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#pragma once
2+
3+
#include <iosfwd>
4+
#include <string>
5+
6+
class Encoder
7+
{
8+
public:
9+
Encoder(std::string name)
10+
: name_(std::move(name))
11+
{}
12+
13+
virtual std::ostream& header(std::ostream& output) const = 0;
14+
virtual std::ostream& content(std::ostream& output, std::istream& input) const = 0;
15+
virtual std::ostream& index(std::ostream& output) const = 0;
16+
17+
virtual ~Encoder() = default;
18+
Encoder(Encoder const&) = default;
19+
Encoder(Encoder&&) = default;
20+
Encoder& operator=(Encoder const&) = default;
21+
Encoder& operator=(Encoder&&) = default;
22+
23+
protected:
24+
std::string name_;
25+
};

0 commit comments

Comments
 (0)