Skip to content

Commit 4347189

Browse files
kjkkjk
committed
test various compression algos
1 parent 0022edc commit 4347189

File tree

4 files changed

+174
-29
lines changed

4 files changed

+174
-29
lines changed

‎do/build.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,17 @@ func revertBuildConfig() {
226226
runExeMust("git", "checkout", buildConfigPath())
227227
}
228228

229+
func addZipDataStoreMust(w *zip.Writer, data []byte, nameInZip string) {
230+
fih := &zip.FileHeader{
231+
Name: nameInZip,
232+
Method: zip.Store,
233+
}
234+
fw, err := w.CreateHeader(fih)
235+
must(err)
236+
_, err = fw.Write(data)
237+
must(err)
238+
}
239+
229240
func addZipFileWithNameMust(w *zip.Writer, path, nameInZip string) {
230241
fi, err := os.Stat(path)
231242
must(err)
@@ -308,9 +319,9 @@ func createPdbZipMust(dir string) {
308319
must(err)
309320
}
310321

311-
func createPdbLzsaMust(dir string) {
312-
args := []string{"SumatraPDF.pdb.lzsa"}
313-
args = append(args, pdbFiles...)
322+
func createLzsaFromFiles(lzsaPath string, files []string, dir string) {
323+
args := []string{lzsaPath}
324+
args = append(args, files...)
314325
curDir, err := os.Getwd()
315326
must(err)
316327
makeLzsaPath := filepath.Join(curDir, "bin", "MakeLZSA.exe")
@@ -319,6 +330,10 @@ func createPdbLzsaMust(dir string) {
319330
runCmdLoggedMust(cmd)
320331
}
321332

333+
func createPdbLzsaMust(dir string) {
334+
createLzsaFromFiles("SumatraPDF.pdb.lzsa", pdbFiles, dir)
335+
}
336+
322337
// manifest is build for pre-release builds and contains information about file sizes
323338
func createManifestMust() {
324339
var lines []string

‎do/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ require (
3030
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
3131
github.com/modern-go/reflect2 v1.0.2 // indirect
3232
github.com/rs/xid v1.6.0 // indirect
33+
github.com/ulikunitz/xz v0.5.12 // indirect
3334
golang.org/x/crypto v0.28.0 // indirect
3435
golang.org/x/net v0.30.0 // indirect
3536
golang.org/x/sys v0.26.0 // indirect

‎do/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
7070
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
7171
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
7272
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
73+
github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
74+
github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
7375
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
7476
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
7577
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

‎do/main.go‎

Lines changed: 153 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,27 @@
11
package main
22

33
import (
4+
"archive/zip"
5+
"bufio"
6+
"bytes"
47
"flag"
58
"io"
69
"os"
710
"os/exec"
811
"path/filepath"
12+
"runtime"
913
"strconv"
1014
"strings"
15+
"sync"
1116
"time"
1217

18+
"github.com/andybalholm/brotli"
1319
"github.com/kjk/common/u"
20+
"github.com/klauspost/compress/zstd"
21+
"github.com/ulikunitz/xz/lzma"
1422
)
1523

1624
var (
17-
flgSkipSign bool
1825
r2Access string
1926
r2Secret string
2027
b2Access string
@@ -171,7 +178,6 @@ func runCppCheck(all bool) {
171178
}
172179

173180
type BuildOptions struct {
174-
sign bool
175181
upload bool
176182
verifyTranslationUpToDate bool
177183
doCleanCheck bool
@@ -180,7 +186,6 @@ type BuildOptions struct {
180186

181187
func ensureBuildOptionsPreRequesites(opts *BuildOptions) {
182188
logf("upload: %v\n", opts.upload)
183-
logf("sign: %v\n", opts.sign)
184189
logf("verifyTranslationUpToDate: %v\n", opts.verifyTranslationUpToDate)
185190

186191
if opts.upload {
@@ -197,10 +202,6 @@ func ensureBuildOptionsPreRequesites(opts *BuildOptions) {
197202
verifyOnReleaseBranchMust()
198203
os.RemoveAll("out")
199204
}
200-
201-
if !opts.sign {
202-
flgSkipSign = true
203-
}
204205
}
205206

206207
func main() {
@@ -370,17 +371,13 @@ func main() {
370371
detectVersions()
371372

372373
if false {
373-
testGenUpdateTxt()
374-
return
375-
}
376-
377-
if false {
378-
//buildPreRelease()
379-
return
380-
}
381-
382-
if false {
383-
deleteFilesOneOff()
374+
testCompressOneOff()
375+
if false {
376+
// make them reachable
377+
testGenUpdateTxt()
378+
buildPreRelease(kPlatformIntel64, true)
379+
deleteFilesOneOff()
380+
}
384381
return
385382
}
386383

@@ -414,16 +411,8 @@ func main() {
414411

415412
opts := &BuildOptions{}
416413

417-
if flgCIBuild {
418-
// triggered via -ci from .github workflow file
419-
// only sign if this is my repo (not a fork)
420-
// master branch (not work branches) and on push (not pull requests etc.)
421-
opts.sign = isGithubMyMasterBranch()
422-
}
423-
424414
if flgUpload {
425415
// given by me from cmd-line
426-
opts.sign = true
427416
opts.upload = true
428417
}
429418

@@ -610,3 +599,141 @@ func printBuildNoInfo(buildNo int) {
610599
s := lines[n]
611600
logf("%d: %s\n", buildNo, s)
612601
}
602+
603+
func compressFileWithBrMust(path string) []byte {
604+
buf := bytes.Buffer{}
605+
w := brotli.NewWriterLevel(&buf, brotli.BestCompression)
606+
f := u.Must2(os.Open(path))
607+
u.Must2(io.Copy(w, f))
608+
must(f.Close())
609+
must(w.Close())
610+
return buf.Bytes()
611+
}
612+
613+
func compressWithZstdMust(path string) []byte {
614+
buf := bytes.Buffer{}
615+
w := u.Must2(zstd.NewWriter(&buf, zstd.WithEncoderLevel(zstd.SpeedBestCompression)))
616+
f := u.Must2(os.Open(path))
617+
u.Must2(io.Copy(w, f))
618+
must(f.Close())
619+
must(w.Close())
620+
return buf.Bytes()
621+
}
622+
623+
func compressWithLzma2Must(path string) []byte {
624+
buf := bytes.Buffer{}
625+
bw := bufio.NewWriter(&buf)
626+
w := u.Must2(lzma.NewWriter2(bw))
627+
f := u.Must2(os.Open(path))
628+
u.Must2(io.Copy(w, f))
629+
must(f.Close())
630+
must(w.Close())
631+
must(bw.Flush())
632+
return buf.Bytes()
633+
}
634+
635+
func compressWithLzma2BetterMust(path string) []byte {
636+
buf := bytes.Buffer{}
637+
bw := bufio.NewWriter(&buf)
638+
var c lzma.Writer2Config
639+
c.DictCap = (8 * 1024 * 1024) * 16
640+
must(c.Verify())
641+
w := u.Must2(c.NewWriter2(bw))
642+
f := u.Must2(os.Open(path))
643+
u.Must2(io.Copy(w, f))
644+
must(f.Close())
645+
must(w.Close())
646+
must(bw.Flush())
647+
return buf.Bytes()
648+
}
649+
650+
func creaZipWithCompressFunction(zipPath string, files []string, dir string, compressFunc func(string) []byte, comprSuffix string) {
651+
os.Remove(zipPath)
652+
w := u.Must2(os.Create(zipPath))
653+
zw := zip.NewWriter(w)
654+
defer zw.Close()
655+
var wg sync.WaitGroup
656+
nConcurrent := runtime.NumCPU()
657+
sem := make(chan bool, nConcurrent)
658+
for _, f := range files {
659+
path := filepath.Join(dir, f)
660+
wg.Add(1)
661+
sem <- true
662+
go func() {
663+
data := compressFunc(path)
664+
addZipDataStoreMust(zw, data, f+comprSuffix)
665+
<-sem
666+
wg.Done()
667+
}()
668+
}
669+
wg.Wait()
670+
must(zw.Close())
671+
}
672+
673+
func testCompressOneOff() {
674+
dir := filepath.Join("out", "rel64")
675+
files := []string{"SumatraPDF.exe", "SumatraPDF-dll.exe", "libmupdf.pdb", "SumatraPDF.pdb", "SumatraPDF-dll.pdb"}
676+
{
677+
}
678+
origSize := int64(0)
679+
for _, f := range files {
680+
origSize += u.FileSize(filepath.Join(dir, f))
681+
}
682+
logf("origSize: %s\n", u.FormatSize(origSize))
683+
684+
{
685+
archivePath := filepath.Join(dir, "rel64.lzma2.better.zip")
686+
os.Remove(archivePath)
687+
logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU())
688+
printDur := measureDuration()
689+
creaZipWithCompressFunction(archivePath, files, dir, compressWithLzma2BetterMust, ".lzma2")
690+
printDur()
691+
compressedSize := u.FileSize(archivePath)
692+
ratio := float64(origSize) / float64(compressedSize)
693+
logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio)
694+
}
695+
{
696+
archivePath := filepath.Join(dir, "rel64.lzma2.zip")
697+
os.Remove(archivePath)
698+
logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU())
699+
printDur := measureDuration()
700+
creaZipWithCompressFunction(archivePath, files, dir, compressWithLzma2Must, ".lzma2")
701+
printDur()
702+
compressedSize := u.FileSize(archivePath)
703+
ratio := float64(origSize) / float64(compressedSize)
704+
logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio)
705+
}
706+
{
707+
archivePath := filepath.Join(dir, "rel64.br.zip")
708+
os.Remove(archivePath)
709+
logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU())
710+
printDur := measureDuration()
711+
creaZipWithCompressFunction(archivePath, files, dir, compressFileWithBrMust, ".br")
712+
printDur()
713+
compressedSize := u.FileSize(archivePath)
714+
ratio := float64(origSize) / float64(compressedSize)
715+
logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio)
716+
}
717+
{
718+
archivePath := filepath.Join(dir, "rel64.zstd.zip")
719+
os.Remove(archivePath)
720+
logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU())
721+
printDur := measureDuration()
722+
creaZipWithCompressFunction(archivePath, files, dir, compressWithZstdMust, ".zstd")
723+
printDur()
724+
compressedSize := u.FileSize(archivePath)
725+
ratio := float64(origSize) / float64(compressedSize)
726+
logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio)
727+
}
728+
{
729+
logf("\nCreating rel64.lzsa using\n")
730+
printDur := measureDuration()
731+
archivePath := filepath.Join(dir, "rel64.lzsa")
732+
os.Remove(archivePath)
733+
createLzsaFromFiles("rel64.lzsa", files, dir)
734+
printDur()
735+
compressedSize := u.FileSize(archivePath)
736+
ratio := float64(origSize) / float64(compressedSize)
737+
logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio)
738+
}
739+
}

0 commit comments

Comments
 (0)