Skip to content

Commit 22837fe

Browse files
fumiamafumiama
committed
feat: add mmap
1 parent 2138d8f commit 22837fe

File tree

6 files changed

+162
-111
lines changed

6 files changed

+162
-111
lines changed

‎CMakeLists.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ if (${isBigEndian})
1212
endif()
1313

1414
IF(CMAKE_SIZEOF_VOID_P EQUAL 8)
15-
add_library(b1464 STATIC base1464.c base14.c)
16-
add_library(b14 SHARED base1464.c base14.c)
15+
add_library(b1464 STATIC base1464.c)
16+
add_library(b14 SHARED base1464.c)
1717
target_link_libraries(base16384 b1464)
1818
ELSE()
19-
add_library(b1432 STATIC base1432.c base14.c)
20-
add_library(b14 SHARED base1432.c base14.c)
19+
add_library(b1432 STATIC base1432.c)
20+
add_library(b14 SHARED base1432.c)
2121
target_link_libraries(base16384 b1432)
2222
ENDIF()
2323

‎base14.c‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎base14.h‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,45 @@
1+
#ifndef _BASE14_H_
2+
#define _BASE14_H_
3+
14
// base14.h
25
// fumiama 20220319
36

47
// encode_len calc min buf size to fill encode result
5-
int encode_len(int dlen);
8+
int encode_len(int dlen) {
9+
int outlen = dlen / 7 * 8;
10+
int offset = dlen % 7;
11+
switch(offset) { // 算上偏移标志字符占用的2字节
12+
case 0: break;
13+
case 1: outlen += 4; break;
14+
case 2:
15+
case 3: outlen += 6; break;
16+
case 4:
17+
case 5: outlen += 8; break;
18+
case 6: outlen += 10; break;
19+
default: break;
20+
}
21+
return outlen + 8; // 冗余的8B用于可能的结尾的覆盖
22+
}
23+
624
// decode_len calc min buf size to fill decode result
7-
int decode_len(int dlen, int offset);
25+
int decode_len(int dlen, int offset) {
26+
int outlen = dlen;
27+
switch(offset) { // 算上偏移标志字符占用的2字节
28+
case 0: break;
29+
case 1: outlen -= 4; break;
30+
case 2:
31+
case 3: outlen -= 6; break;
32+
case 4:
33+
case 5: outlen -= 8; break;
34+
case 6: outlen -= 10; break;
35+
default: break;
36+
}
37+
return outlen / 8 * 7 + offset + 1; // 多出1字节用于循环覆盖
38+
}
839

940
// encode data and write result into buf
1041
int encode(const char* data, int dlen, char* buf, int blen);
1142
// decode data and write result into buf
1243
int decode(const char* data, int dlen, char* buf, int blen);
44+
45+
#endif

‎base1432.c‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
#endif
4444
#endif
4545
#endif
46-
#include "base14.h"
4746

4847
// #define DEBUG
4948

‎base1464.c‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
# define htobe64(x) _byteswap_uint64(x)
4444
#endif
4545
#endif
46-
#include "base14.h"
4746

4847
// #define DEBUG
4948

‎base16384.c‎

Lines changed: 123 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
#include <stdio.h>
33
#include <stdlib.h>
44
#include <sys/stat.h>
5+
#include <fcntl.h>
56
#include <time.h>
6-
#ifdef __WINNT__
7+
#ifdef _WIN32
78
#include <windows.h>
9+
#else
10+
#include <sys/mman.h>
811
#endif
912
#endif
1013
#include "base14.h"
@@ -18,122 +21,169 @@ static off_t get_file_size(const char* filepath) {
1821
}
1922
#endif
2023

24+
char encbuf[BUFSIZ*1024/7*7];
25+
char decbuf[BUFSIZ*1024/8*8+2];
26+
2127
void encode_file(const char* input, const char* output) {
2228
off_t inputsize = get_file_size(input);
2329
if(inputsize <= 0) {
24-
puts("Get file size error!");
30+
perror("Get file size error: ");
2531
return;
2632
}
27-
FILE* fp = NULL;
28-
fp = fopen(input, "rb");
29-
if(!fp) {
30-
puts("Open input file error!");
33+
if(inputsize > (1u<<31)-1) {
34+
puts("Input file too big!");
3135
return;
3236
}
3337
FILE* fpo = NULL;
3438
fpo = fopen(output, "wb");
3539
if(!fpo) {
36-
puts("Open output file error!");
40+
perror("Fopen output file error: ");
3741
return;
3842
}
39-
if(inputsize > BUFSIZ*1024) inputsize = BUFSIZ*1024/7*7; // big file
40-
char* bufi = (char*)malloc(inputsize);
41-
if(!bufi) {
42-
puts("Allocate input buffer error!");
43-
return;
43+
if(inputsize > BUFSIZ*1024) { // big file, use encbuf & fread
44+
inputsize = BUFSIZ*1024/7*7;
45+
#ifdef _WIN32
4446
}
45-
int outputsize = encode_len(inputsize)+16;
46-
char* bufo = (char*)malloc(outputsize);
47-
if(!bufo) {
48-
puts("Allocate output buffer error!");
49-
return;
50-
}
51-
size_t cnt = 0;
52-
fputc(0xFE, fpo);
53-
fputc(0xFF, fpo);
54-
while((cnt = fread(bufi, sizeof(char), inputsize, fp))) {
55-
int n = encode(bufi, cnt, bufo, outputsize);
56-
if(fwrite(bufo, n, 1, fpo) <= 0) {
57-
puts("Write file error!");
47+
#endif
48+
FILE* fp = NULL;
49+
fp = fopen(input, "rb");
50+
if(!fp) {
51+
perror("Fopen input file error: ");
5852
return;
5953
}
54+
55+
int outputsize = encode_len(inputsize)+16;
56+
size_t cnt = 0;
57+
fputc(0xFE, fpo);
58+
fputc(0xFF, fpo);
59+
while((cnt = fread(encbuf, sizeof(char), inputsize, fp))) {
60+
int n = encode(encbuf, cnt, decbuf, outputsize);
61+
if(fwrite(decbuf, n, 1, fpo) <= 0) {
62+
perror("Write file error: ");
63+
return;
64+
}
65+
}
66+
/* 由操作系统负责释放资源
67+
fclose(fpo);
68+
fclose(fp);
69+
以缩短程序运行时间 */
70+
#ifndef _WIN32
71+
} else { // small file, use mmap & fwrite
72+
int fd = open(input, O_RDONLY);
73+
if(fd <= 0) {
74+
perror("Open input file error: ");
75+
return;
76+
}
77+
char *input_file = mmap(NULL, (size_t)inputsize, PROT_READ, MAP_PRIVATE, fd, 0);
78+
if(input_file == MAP_FAILED) {
79+
perror("Map input file error: ");
80+
return;
81+
}
82+
int outputsize = encode_len(inputsize)+16;
83+
fputc(0xFE, fpo);
84+
fputc(0xFF, fpo);
85+
int n = encode(input_file, (int)inputsize, decbuf, outputsize);
86+
if(fwrite(decbuf, n, 1, fpo) <= 0) {
87+
perror("Write file error: ");
88+
return;
89+
}
90+
munmap(input_file, (size_t)inputsize);
91+
/* 由操作系统负责释放资源
92+
fclose(fpo);
93+
close(fd);
94+
以缩短程序运行时间 */
6095
}
61-
/* 由操作系统负责释放资源
62-
free(bufo);
63-
free(bufi);
64-
fclose(fpo);
65-
fclose(fp);
66-
以缩短程序运行时间 */
96+
#endif
6797
}
6898

69-
void rm_head(FILE* fp) {
70-
int ch = fgetc(fp);
71-
if(ch == 0xFE) fgetc(fp);
72-
else rewind(fp);
99+
#define rm_head(fp) {\
100+
int ch = fgetc(fp);\
101+
if(ch == 0xFE) fgetc(fp);\
102+
else rewind(fp);\
73103
}
74104

105+
#define skip_offset(input_file) ((input_file[0]==(char)0xFE)?2:0)
106+
75107
static int is_next_end(FILE* fp) {
76108
int ch = fgetc(fp);
77109
if(ch == '=') return fgetc(fp);
78-
else {
79-
ungetc(ch, fp);
80-
return 0;
81-
}
110+
ungetc(ch, fp);
111+
return 0;
82112
}
83113

84114
void decode_file(const char* input, const char* output) {
85115
off_t inputsize = get_file_size(input);
86116
if(inputsize <= 0) {
87-
puts("Get file size error!");
117+
perror("Get file size error: ");
88118
return;
89119
}
90-
FILE* fp = NULL;
91-
fp = fopen(input, "rb");
92-
if(!fp) {
93-
puts("Open input file error!");
120+
if(inputsize > (1u<<31)-1) {
121+
puts("Input file too big!");
94122
return;
95123
}
96124
FILE* fpo = NULL;
97125
fpo = fopen(output, "wb");
98126
if(!fpo) {
99-
puts("Open output file error!");
127+
perror("Fopen output file error: ");
100128
return;
101129
}
102-
if(inputsize > BUFSIZ*1024) inputsize = BUFSIZ*1024/8*8; // big file
103-
char* bufi = (char*)malloc(inputsize+2); // +2避免漏检结束偏移标志
104-
if(!bufi) {
105-
puts("Allocate input buffer error!");
106-
return;
130+
if(inputsize > BUFSIZ*1024) { // big file, use decbuf & fread
131+
inputsize = BUFSIZ*1024/8*8;
132+
#ifdef _WIN32
107133
}
108-
int outputsize = decode_len(inputsize, 0)+16;
109-
char* bufo = (char*)malloc(outputsize);
110-
if(!bufo) {
111-
puts("Allocate output buffer error!");
112-
return;
113-
}
114-
int cnt = 0;
115-
int end = 0;
116-
rm_head(fp);
117-
while((cnt = fread(bufi, sizeof(char), inputsize, fp))) {
118-
if((end = is_next_end(fp))) {
119-
bufi[cnt++] = '=';
120-
bufi[cnt++] = end;
134+
#endif
135+
FILE* fp = NULL;
136+
fp = fopen(input, "rb");
137+
if(!fp) {
138+
perror("Fopen input file error: ");
139+
return;
121140
}
122-
int n = decode(bufi, cnt, bufo, outputsize);
123-
if(fwrite(bufo, n, 1, fpo) <= 0) {
124-
puts("Write file error!");
141+
int outputsize = decode_len(inputsize, 0)+16;
142+
int cnt = 0;
143+
int end = 0;
144+
rm_head(fp);
145+
while((cnt = fread(decbuf, sizeof(char), inputsize, fp))) {
146+
if((end = is_next_end(fp))) {
147+
decbuf[cnt++] = '=';
148+
decbuf[cnt++] = end;
149+
}
150+
if(fwrite(encbuf, decode(decbuf, cnt, encbuf, outputsize), 1, fpo) <= 0) {
151+
perror("Write file error: ");
152+
return;
153+
}
154+
}
155+
/* 由操作系统负责释放资源
156+
fclose(fpo);
157+
fclose(fp);
158+
以缩短程序运行时间 */
159+
#ifndef _WIN32
160+
} else { // small file, use mmap & fwrite
161+
int fd = open(input, O_RDONLY);
162+
if(fd <= 0) {
163+
perror("Open input file error: ");
164+
return;
165+
}
166+
char *input_file = mmap(NULL, (size_t)inputsize, PROT_READ, MAP_PRIVATE, fd, 0);
167+
if(input_file == MAP_FAILED) {
168+
perror("Map input file error: ");
125169
return;
126170
}
171+
int outputsize = decode_len(inputsize, 0)+16;
172+
int off = skip_offset(input_file);
173+
if(fwrite(encbuf, decode(input_file+off, inputsize-off, encbuf, outputsize), 1, fpo) <= 0) {
174+
perror("Write file error: ");
175+
return;
176+
}
177+
munmap(input_file, (size_t)inputsize);
178+
/* 由操作系统负责释放资源
179+
fclose(fpo);
180+
close(fd);
181+
以缩短程序运行时间 */
127182
}
128-
/* 由操作系统负责释放资源
129-
free(bufo);
130-
free(bufi);
131-
fclose(fpo);
132-
fclose(fp);
133-
以缩短程序运行时间 */
183+
#endif
134184
}
135185

136-
#ifndef __WINNT__
186+
#ifndef _WIN32
137187
unsigned long get_start_ms() {
138188
struct timespec ts;
139189
clock_gettime(CLOCK_MONOTONIC, &ts);

0 commit comments

Comments
 (0)