bump 7z #517
Merged
bump 7z #517
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Frenzie
approved these changes
Oct 2, 2024
|
If you want faster, I have a patch for ZSTD support in my meson branch. For 19.00, but I think latest has builtin support. |
I was happy with fast as before. I'm even happier now with "faster". I wouldn't mind "even faster" but I care much more about the compression ratio. So, if we can squeeze the same data in ZSTD as in XZ I'm all in for the change :) |
|
Keep in mind memory usage could also be an issue. Those Android 4.x ereaders aren't very well endowed, and zstd is known to use rather a lot more while decompressing. |
|
Well, we have reduced our APKs size, compared to past versions:
So we have a little headroom. |
|
On modern devices the difference on speed while uncompressing should be below 1s, because that's what should take to extract our current 7z payload.
You did. Nice work. These few MB sum up on the monthtly bandwidth, I guess. Anyhow, feel free to do it. I'm curious enough to give it a benchmark on a few emulators and a couple of VMs ;) |
benoit-pierre
added a commit
to benoit-pierre/koreader
that referenced
this pull request
Oct 12, 2024
Frenzie
pushed a commit
to koreader/koreader
that referenced
this pull request
Oct 12, 2024
|
I ran some tests, with that patch: diff --git i/jni/lzma/7z/C/7zDec.c w/jni/lzma/7z/C/7zDec.c
index c9b4064e..e16bdcd2 100755
--- i/jni/lzma/7z/C/7zDec.c
+++ w/jni/lzma/7z/C/7zDec.c
@@ -3,29 +3,46 @@
#include "Precomp.h"
+#include <android/log.h>
#include <string.h>
+#include <stdio.h>
/* #define Z7_PPMD_SUPPORT */
+/* #define Z7_ZSTD_SUPPORT */
+#define Z7_NO_METHOD_BCJ2
+#define Z7_NO_METHODS_FILTERS
+/* #define Z7_NO_METHOD_LZMA */
+/* #define Z7_NO_METHOD_LZMA2 */
#include "7z.h"
#include "7zCrc.h"
+#include "Alloc.h"
+#ifndef Z7_NO_METHOD_BCJ2
#include "Bcj2.h"
+#endif
#include "Bra.h"
#include "CpuArch.h"
#include "Delta.h"
+#ifndef Z7_NO_METHOD_LZMA
#include "LzmaDec.h"
+#endif
+#ifndef Z7_NO_METHOD_LZMA2
#include "Lzma2Dec.h"
+#endif
#ifdef Z7_PPMD_SUPPORT
#include "Ppmd7.h"
#endif
+#ifdef Z7_ZSTD_SUPPORT
+#include "ZstdDec.h"
+#endif
#define k_Copy 0
-#ifndef Z7_NO_METHOD_LZMA2
#define k_LZMA2 0x21
-#endif
#define k_LZMA 0x30101
#define k_BCJ2 0x303011B
+#define k_PPMD 0x30401
+#define k_ZSTD 0x4f71101
#if !defined(Z7_NO_METHODS_FILTERS)
#define Z7_USE_BRANCH_FILTER
@@ -61,8 +78,6 @@
#ifdef Z7_PPMD_SUPPORT
-#define k_PPMD 0x30401
-
typedef struct
{
IByteIn vt;
@@ -160,6 +175,75 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, I
#endif
+#ifdef Z7_ZSTD_SUPPORT
+
+static SRes SzDecodeZstd(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStreamPtr inStream,
+ Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain)
+{
+ void *inBuf = NULL;
+ CZstdDecHandle dec = NULL;
+ SRes res;
+
+ inBuf = ISzAlloc_Alloc(allocMain, inSize);
+ if (inBuf == NULL) {
+ res = SZ_ERROR_MEM;
+ goto end;
+ }
+
+ {
+ SizeT size = inSize;
+
+ res = ILookInStream_Read(inStream, inBuf, &size);
+ if (res != SZ_OK || size != inSize) {
+ res = SZ_ERROR_READ;
+ goto end;
+ }
+ }
+
+ dec = ZstdDec_Create(&g_AlignedAlloc, &g_BigAlloc);
+ if (!dec) {
+ res = SZ_ERROR_MEM;
+ goto end;
+ }
+
+ ZstdDec_Init(dec);
+
+ {
+ CZstdDecState state;
+
+ ZstdDecState_Clear(&state);
+
+ state.inBuf = inBuf;
+ state.inPos = 0;
+ state.inLim = inSize;
+ state.outSize_Defined = True;
+ state.outSize = outSize;
+ state.outBuf_fromCaller = outBuffer;
+ state.outBufSize_fromCaller = outSize;
+
+ do {
+ res = ZstdDec_Decode(dec, &state);
+ if (res != SZ_OK)
+ {
+ CZstdDecResInfo status;
+ ZstdDec_GetResInfo(dec, &state, res, &status);
+ res = status.decode_SRes;
+ break;
+ }
+ } while (state.outProcessed < outSize);
+ }
+
+end:
+ if (dec)
+ ZstdDec_Destroy(dec);
+ if (inBuf != NULL)
+ ISzAlloc_Free(allocMain, inBuf);
+ return res;
+}
+
+#endif
+
+#ifndef Z7_NO_METHOD_LZMA
static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStreamPtr inStream,
Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain)
@@ -218,6 +302,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I
return res;
}
+#endif
#ifndef Z7_NO_METHOD_LZMA2
@@ -304,12 +389,17 @@ static BoolInt IS_MAIN_METHOD(UInt32 m)
switch (m)
{
case k_Copy:
+ #ifndef Z7_NO_METHOD_LZMA
case k_LZMA:
+ #endif
#ifndef Z7_NO_METHOD_LZMA2
case k_LZMA2:
#endif
#ifdef Z7_PPMD_SUPPORT
case k_PPMD:
+ #endif
+ #ifdef Z7_ZSTD_SUPPORT
+ case k_ZSTD:
#endif
return True;
}
@@ -465,16 +555,50 @@ static SRes SzFolder_Decode2(const CSzFolder *folder,
inSize = packPositions[(size_t)si + 1] - offset;
RINOK(LookInStream_SeekTo(inStream, startPos + offset))
+ {
+ const char *method;
+ switch (coder->MethodID) {
+ case k_Copy:
+ method = "copy";
+ break;
+ case k_BCJ2:
+ method = "BCJ2";
+ break;
+ case k_LZMA:
+ method = "LZMA";
+ break;
+ case k_LZMA2:
+ method = "LZMA2";
+ break;
+ case k_PPMD:
+ method = "PPMD";
+ break;
+ case k_ZSTD:
+ method = "ZSTD";
+ break;
+ default:
+ {
+ static char _method_buf[9];
+ snprintf(_method_buf, sizeof (_method_buf), "%x", coder->MethodID);
+ method = _method_buf;
+ }
+ break;
+ }
+ __android_log_print(ANDROID_LOG_INFO, "7z", "method=%s, size=%llu/%u", method, inSize, outSizeCur);
+ }
+
if (coder->MethodID == k_Copy)
{
if (inSize != outSizeCur) /* check it */
return SZ_ERROR_DATA;
RINOK(SzDecodeCopy(inSize, inStream, outBufCur))
}
+ #ifndef Z7_NO_METHOD_LZMA
else if (coder->MethodID == k_LZMA)
{
RINOK(SzDecodeLzma(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain))
}
+ #endif
#ifndef Z7_NO_METHOD_LZMA2
else if (coder->MethodID == k_LZMA2)
{
@@ -486,10 +610,17 @@ static SRes SzFolder_Decode2(const CSzFolder *folder,
{
RINOK(SzDecodePpmd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain))
}
+ #endif
+ #ifdef Z7_ZSTD_SUPPORT
+ else if (coder->MethodID == k_ZSTD)
+ {
+ RINOK(SzDecodeZstd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain));
+ }
#endif
else
return SZ_ERROR_UNSUPPORTED;
}
+ #ifndef Z7_NO_METHOD_BCJ2
else if (coder->MethodID == k_BCJ2)
{
const UInt64 offset = packPositions[1];
@@ -537,6 +668,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder,
}
}
}
+#endif
#if defined(Z7_USE_BRANCH_FILTER)
else if (ci == 1)
{
diff --git i/jni/lzma/Android.mk w/jni/lzma/Android.mk
index 130f3f7..5309c37 100644
--- i/jni/lzma/Android.mk
+++ w/jni/lzma/Android.mk
@@ -3,12 +3,28 @@ include $(CLEAR_VARS)
LOCAL_MODULE := 7z
-lzma_SOURCES := \
- 7zAlloc.c 7zArcIn.c 7zBuf.c 7zBuf2.c \
- 7zCrc.c 7zCrcOpt.c 7zDec.c 7zFile.c \
- 7zStream.c Bcj2.c Bra.c Bra86.c BraIA64.c \
- CpuArch.c Delta.c Lzma2Dec.c LzmaDec.c \
- Ppmd7.c Ppmd7Dec.c
+sevenzip_SOURCES := \
+ 7zAlloc.c 7zArcIn.c 7zBuf.c 7zCrc.c \
+ 7zCrcOpt.c 7zDec.c 7zFile.c 7zStream.c \
+ CpuArch.c Delta.c \
+
+# For BCJ2 support.
+# sevenzip_SOURCES += Bcj2.c
+
+# For LZMA support.
+sevenzip_SOURCES += LzmaDec.c
+
+# For LZMA2 support (need LZMA).
+sevenzip_SOURCES += Lzma2Dec.c
+
+# For PPMD support.
+# sevenzip_SOURCES += Ppmd7.c Ppmd7Dec.c
+
+# For ZSTD support.
+# sevenzip_SOURCES += Alloc.c Xxh64.c ZstdDec.c
+
+# For method filters support.
+# sevenzip_SOURCES += Bra.c Bra86.c BraIA64.c
un7zip_SOURCES := \
7zAssetFile.cpp \
@@ -20,7 +36,7 @@ LOCAL_C_INCLUDES := \
$(LOCAL_PATH)/un7zip
LOCAL_SRC_FILES := \
- $(addprefix 7z/C/, $(lzma_SOURCES)) \
+ $(addprefix 7z/C/, $(sevenzip_SOURCES)) \
$(addprefix un7zip/, $(un7zip_SOURCES)) \
un7z.cpp
Running on my Nexus 7 (2013):
Note: Best choice, IMHO, stick with lzma2, but update our Docker images 7z version so we can use |
|
It's a pity the flzma2 savings are mainly in compression speed. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
from 19.00 to 24.06
It seems faster uncompressing on arm64 but I didn't benchmark.
This change is