Skip to content

Commit 398af30

Browse files
sekerseker
committed
file_helper support mmap
1 parent f1d748e commit 398af30

File tree

4 files changed

+683
-3
lines changed

4 files changed

+683
-3
lines changed

‎include/spdlog/details/file_helper-inl.h‎

Lines changed: 258 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,27 @@
1212

1313
#include <cerrno>
1414
#include <cstdio>
15+
#include <cstring>
1516
#include <string>
1617
#include <tuple>
18+
#include <algorithm>
19+
20+
// Platform-specific includes for mmap support
21+
#ifdef _WIN32
22+
#include <spdlog/details/windows_include.h>
23+
#include <io.h>
24+
#include <fcntl.h>
25+
#include <sys/stat.h>
26+
#else
27+
#include <sys/mman.h>
28+
#include <unistd.h>
29+
#endif
1730

1831
namespace spdlog {
1932
namespace details {
2033

21-
SPDLOG_INLINE file_helper::file_helper(const file_event_handlers &event_handlers)
22-
: event_handlers_(event_handlers) {}
34+
SPDLOG_INLINE file_helper::file_helper(const file_event_handlers &event_handlers, bool mmap_enabled)
35+
: event_handlers_(event_handlers), mmap_enabled_(mmap_enabled) {}
2336

2437
SPDLOG_INLINE file_helper::~file_helper() { close(); }
2538

@@ -51,6 +64,13 @@ SPDLOG_INLINE void file_helper::open(const filename_t &fname, bool truncate) {
5164
if (event_handlers_.after_open) {
5265
event_handlers_.after_open(filename_, fd_);
5366
}
67+
68+
// Try to initialize mmap if enabled
69+
if (mmap_enabled_ && !try_init_mmap()) {
70+
// mmap initialization failed, continue with regular file I/O
71+
// No need to throw exception, just log a debug message if needed
72+
}
73+
5474
return;
5575
}
5676

@@ -69,12 +89,40 @@ SPDLOG_INLINE void file_helper::reopen(bool truncate) {
6989
}
7090

7191
SPDLOG_INLINE void file_helper::flush() {
92+
if (mmap_active_) {
93+
// For mmap, flush means sync the memory mapping
94+
#ifdef _WIN32
95+
if (!FlushViewOfFile(mmap_ptr_, mmap_offset_)) {
96+
// If mmap flush fails, fallback to stdio
97+
fallback_to_stdio();
98+
}
99+
#else
100+
if (msync(mmap_ptr_, mmap_offset_, MS_ASYNC) != 0) {
101+
// If mmap flush fails, fallback to stdio
102+
fallback_to_stdio();
103+
}
104+
#endif
105+
}
106+
72107
if (std::fflush(fd_) != 0) {
73108
throw_spdlog_ex("Failed flush to file " + os::filename_to_str(filename_), errno);
74109
}
75110
}
76111

77112
SPDLOG_INLINE void file_helper::sync() {
113+
if (mmap_active_) {
114+
// For mmap, sync means synchronous sync of the memory mapping
115+
#ifdef _WIN32
116+
if (!FlushViewOfFile(mmap_ptr_, mmap_offset_)) {
117+
fallback_to_stdio();
118+
}
119+
#else
120+
if (msync(mmap_ptr_, mmap_offset_, MS_SYNC) != 0) {
121+
fallback_to_stdio();
122+
}
123+
#endif
124+
}
125+
78126
if (!os::fsync(fd_)) {
79127
throw_spdlog_ex("Failed to fsync file " + os::filename_to_str(filename_), errno);
80128
}
@@ -86,6 +134,9 @@ SPDLOG_INLINE void file_helper::close() {
86134
event_handlers_.before_close(filename_, fd_);
87135
}
88136

137+
// Clean up mmap resources before closing file
138+
cleanup_mmap();
139+
89140
std::fclose(fd_);
90141
fd_ = nullptr;
91142

@@ -100,6 +151,31 @@ SPDLOG_INLINE void file_helper::write(const memory_buf_t &buf) {
100151
size_t msg_size = buf.size();
101152
auto data = buf.data();
102153

154+
// Try mmap write first if active
155+
if (mmap_active_) {
156+
size_t required_size = mmap_offset_ + msg_size;
157+
158+
// Check if we need to expand the mapping
159+
if (required_size > mmap_size_) {
160+
if (!expand_mmap(required_size)) {
161+
// Expansion failed, fallback to stdio
162+
goto stdio_write;
163+
}
164+
}
165+
166+
// Write to mmap
167+
try {
168+
std::memcpy(static_cast<char*>(mmap_ptr_) + mmap_offset_, data, msg_size);
169+
mmap_offset_ += msg_size;
170+
return;
171+
} catch (...) {
172+
// mmap write failed, fallback to stdio
173+
fallback_to_stdio();
174+
}
175+
}
176+
177+
stdio_write:
178+
// Standard file I/O fallback
103179
if (!details::os::fwrite_bytes(data, msg_size, fd_)) {
104180
throw_spdlog_ex("Failed writing to file " + os::filename_to_str(filename_), errno);
105181
}
@@ -109,6 +185,13 @@ SPDLOG_INLINE size_t file_helper::size() const {
109185
if (fd_ == nullptr) {
110186
throw_spdlog_ex("Cannot use size() on closed file " + os::filename_to_str(filename_));
111187
}
188+
189+
// If mmap is active, return the current offset (actual written size)
190+
if (mmap_active_) {
191+
// DEBUG: Print mmap_offset_ value
192+
return mmap_offset_;
193+
}
194+
112195
return os::filesize(fd_);
113196
}
114197

@@ -147,5 +230,178 @@ SPDLOG_INLINE std::tuple<filename_t, filename_t> file_helper::split_by_extension
147230
return std::make_tuple(fname.substr(0, ext_index), fname.substr(ext_index));
148231
}
149232

233+
// mmap helper methods implementation
234+
SPDLOG_INLINE void file_helper::set_mmap_enabled(bool enabled) {
235+
mmap_enabled_ = enabled;
236+
if (!enabled && mmap_active_) {
237+
fallback_to_stdio();
238+
}
239+
}
240+
241+
SPDLOG_INLINE bool file_helper::is_mmap_enabled() const {
242+
return mmap_enabled_;
243+
}
244+
245+
SPDLOG_INLINE bool file_helper::try_init_mmap() {
246+
if (!mmap_enabled_ || fd_ == nullptr) {
247+
return false;
248+
}
249+
250+
// Get current file size BEFORE any modifications
251+
size_t current_file_size = os::filesize(fd_);
252+
size_t required_size = (std::max)(current_file_size + initial_mmap_size_, initial_mmap_size_);
253+
254+
#ifdef _WIN32
255+
// Windows implementation
256+
file_handle_ = reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(fd_)));
257+
if (file_handle_ == INVALID_HANDLE_VALUE) {
258+
return false;
259+
}
260+
261+
// Create file mapping
262+
LARGE_INTEGER map_size;
263+
map_size.QuadPart = required_size;
264+
265+
mapping_handle_ = CreateFileMapping(file_handle_, nullptr, PAGE_READWRITE,
266+
map_size.HighPart, map_size.LowPart, nullptr);
267+
if (mapping_handle_ == nullptr) {
268+
return false;
269+
}
270+
271+
// Map view of file
272+
mmap_ptr_ = MapViewOfFile(mapping_handle_, FILE_MAP_WRITE, 0, 0, required_size);
273+
if (mmap_ptr_ == nullptr) {
274+
CloseHandle(mapping_handle_);
275+
mapping_handle_ = nullptr;
276+
return false;
277+
}
278+
279+
#else
280+
// Unix/Linux implementation
281+
file_descriptor_ = fileno(fd_);
282+
if (file_descriptor_ == -1) {
283+
return false;
284+
}
285+
286+
// Create memory mapping first (without extending file)
287+
mmap_ptr_ = mmap(nullptr, required_size, PROT_READ | PROT_WRITE, MAP_SHARED,
288+
file_descriptor_, 0);
289+
if (mmap_ptr_ == MAP_FAILED) {
290+
291+
mmap_ptr_ = nullptr;
292+
return false;
293+
}
294+
295+
// Extend file if necessary (only after mmap succeeds)
296+
if (ftruncate(file_descriptor_, static_cast<off_t>(required_size)) != 0) {
297+
munmap(mmap_ptr_, required_size);
298+
mmap_ptr_ = nullptr;
299+
return false;
300+
}
301+
#endif
302+
303+
mmap_size_ = required_size;
304+
mmap_offset_ = 0; // Always start from beginning for new mmap
305+
mmap_active_ = true;
306+
307+
return true;
308+
}
309+
310+
SPDLOG_INLINE bool file_helper::expand_mmap(size_t required_size) {
311+
if (!mmap_active_ || required_size <= mmap_size_) {
312+
return true;
313+
}
314+
315+
// Calculate new size (double current size or required size, whichever is larger)
316+
size_t new_size = (std::max)(mmap_size_ * 2, required_size);
317+
new_size = (std::min)(new_size, max_mmap_size_);
318+
319+
if (new_size <= mmap_size_) {
320+
// Cannot expand further, fallback to stdio
321+
fallback_to_stdio();
322+
return false;
323+
}
324+
325+
// Save current offset before cleanup
326+
size_t saved_offset = mmap_offset_;
327+
cleanup_mmap();
328+
329+
#ifdef _WIN32
330+
// Windows re-mapping
331+
LARGE_INTEGER map_size;
332+
map_size.QuadPart = new_size;
333+
334+
mapping_handle_ = CreateFileMapping(file_handle_, nullptr, PAGE_READWRITE,
335+
map_size.HighPart, map_size.LowPart, nullptr);
336+
if (mapping_handle_ == nullptr) {
337+
fallback_to_stdio();
338+
return false;
339+
}
340+
341+
mmap_ptr_ = MapViewOfFile(mapping_handle_, FILE_MAP_WRITE, 0, 0, new_size);
342+
if (mmap_ptr_ == nullptr) {
343+
CloseHandle(mapping_handle_);
344+
mapping_handle_ = nullptr;
345+
fallback_to_stdio();
346+
return false;
347+
}
348+
349+
#else
350+
// Unix/Linux re-mapping
351+
if (ftruncate(file_descriptor_, static_cast<off_t>(new_size)) != 0) {
352+
fallback_to_stdio();
353+
return false;
354+
}
355+
356+
mmap_ptr_ = mmap(nullptr, new_size, PROT_READ | PROT_WRITE, MAP_SHARED,
357+
file_descriptor_, 0);
358+
if (mmap_ptr_ == MAP_FAILED) {
359+
mmap_ptr_ = nullptr;
360+
fallback_to_stdio();
361+
return false;
362+
}
363+
#endif
364+
365+
mmap_size_ = new_size;
366+
mmap_offset_ = saved_offset; // Restore the saved offset
367+
mmap_active_ = true;
368+
return true;
369+
}
370+
371+
SPDLOG_INLINE void file_helper::cleanup_mmap() {
372+
if (!mmap_active_) {
373+
return;
374+
}
375+
376+
#ifdef _WIN32
377+
if (mmap_ptr_ != nullptr) {
378+
UnmapViewOfFile(mmap_ptr_);
379+
mmap_ptr_ = nullptr;
380+
}
381+
if (mapping_handle_ != nullptr) {
382+
CloseHandle(mapping_handle_);
383+
mapping_handle_ = nullptr;
384+
}
385+
#else
386+
if (mmap_ptr_ != nullptr) {
387+
munmap(mmap_ptr_, mmap_size_);
388+
mmap_ptr_ = nullptr;
389+
}
390+
#endif
391+
392+
mmap_active_ = false;
393+
mmap_size_ = 0;
394+
mmap_offset_ = 0;
395+
}
396+
397+
SPDLOG_INLINE void file_helper::fallback_to_stdio() {
398+
if (mmap_active_) {
399+
// Sync any pending data
400+
sync();
401+
cleanup_mmap();
402+
}
403+
mmap_enabled_ = false; // Disable mmap for this file
404+
}
405+
150406
} // namespace details
151407
} // namespace spdlog

‎include/spdlog/details/file_helper.h‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,26 @@
66
#include <spdlog/common.h>
77
#include <tuple>
88

9+
// Platform-specific includes for mmap support
10+
#ifdef _WIN32
11+
#include <spdlog/details/windows_include.h>
12+
#else
13+
#include <sys/mman.h>
14+
#include <unistd.h>
15+
#endif
16+
917
namespace spdlog {
1018
namespace details {
1119

1220
// Helper class for file sinks.
1321
// When failing to open a file, retry several times(5) with a delay interval(10 ms).
1422
// Throw spdlog_ex exception on errors.
23+
// Supports mmap mode for improved performance with automatic fallback to regular file I/O.
1524

1625
class SPDLOG_API file_helper {
1726
public:
1827
file_helper() = default;
19-
explicit file_helper(const file_event_handlers &event_handlers);
28+
explicit file_helper(const file_event_handlers &event_handlers, bool mmap_enabled = false);
2029

2130
file_helper(const file_helper &) = delete;
2231
file_helper &operator=(const file_helper &) = delete;
@@ -31,6 +40,10 @@ class SPDLOG_API file_helper {
3140
size_t size() const;
3241
const filename_t &filename() const;
3342

43+
// Enable/disable mmap mode (enabled by default)
44+
void set_mmap_enabled(bool enabled);
45+
bool is_mmap_enabled() const;
46+
3447
//
3548
// return file path and its extension:
3649
//
@@ -47,11 +60,34 @@ class SPDLOG_API file_helper {
4760
static std::tuple<filename_t, filename_t> split_by_extension(const filename_t &fname);
4861

4962
private:
63+
// Regular file I/O members
5064
const int open_tries_ = 5;
5165
const unsigned int open_interval_ = 10;
5266
std::FILE *fd_{nullptr};
5367
filename_t filename_;
5468
file_event_handlers event_handlers_;
69+
70+
// mmap related members
71+
bool mmap_enabled_{false}; // Disable mmap by default
72+
bool mmap_active_{false}; // Whether mmap is currently active
73+
void* mmap_ptr_{nullptr}; // Pointer to mapped memory
74+
size_t mmap_size_{0}; // Current size of mapped region
75+
size_t mmap_offset_{0}; // Current write offset in mapped region
76+
size_t initial_mmap_size_{1024 * 1024}; // Initial mmap size (1MB)
77+
size_t max_mmap_size_{100 * 1024 * 1024}; // Max mmap size (100MB)
78+
79+
#ifdef _WIN32
80+
HANDLE file_handle_{INVALID_HANDLE_VALUE};
81+
HANDLE mapping_handle_{nullptr};
82+
#else
83+
int file_descriptor_{-1};
84+
#endif
85+
86+
// mmap helper methods
87+
bool try_init_mmap();
88+
bool expand_mmap(size_t required_size);
89+
void cleanup_mmap();
90+
void fallback_to_stdio();
5591
};
5692
} // namespace details
5793
} // namespace spdlog

0 commit comments

Comments
 (0)