Switch exports to Zstd, from bzip2

The main benefit of this format for our users is that it decompresses much faster than bzip2, even at high compression levels. At level 19 it compresses even better than bzip2 for our files, hopefully the compression time is still acceptable, if not we can reduce it as to not overwork the server, at the price of some slightly bigger files. On my i7-8700K, unarchiving sentences.tar.bz2 takes 15.5s, compared to 994ms for sentences.csv.zst compressed at level 19. The file is 183 MiB compared to 197 MiB with bzip2. We could go down to 167 MiB with level 22 (which decompresses in 941ms), but compression time starts to get much higher, not sure this is worth it. The only downside I see to this change is that user automation will have to be changed, so perhaps announce it somehow before deploying it.
Tatoeba · Dec 9, 2024 · e41635e · e41635e
1 parent 3206771
commit e41635e
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 41 deletions.
diff --git a/ansible/roles/setup_database/tasks/import_csv.yml b/ansible/roles/setup_database/tasks/import_csv.yml
@@ -19,10 +19,10 @@
     url: "{{ download_url }}{{ item }}"
     dest: "/tmp/{{ item }}"
   with_items:
-  - sentences.tar.bz2
-  - links.tar.bz2
+  - sentences.csv.zst
+  - links.csv.zst
   - tag_metadata.csv
-  - tags_detailed.tar.bz2
+  - tags_detailed.csv.zst
   when: import_csv == 'download'
 
 - name: Unpacking csv's
@@ -34,9 +34,9 @@
     owner: mysql
     group: mysql
   with_items:
-  - sentences.tar.bz2
-  - links.tar.bz2
-  - tags_detailed.tar.bz2
+  - sentences.csv.zst
+  - links.csv.zst
+  - tags_detailed.csv.zst
   when: import_csv == 'download'
 
 - name: Import sentences in the tatoeba database

diff --git a/docs/cron/export.sh b/docs/cron/export.sh
@@ -19,26 +19,26 @@ mv /var/tmp/*csv "$DL_DIR"
 
 echo "Starting tarring at $(date -Iseconds)"
 cd "$DL_DIR"
-tar -cjf sentences_base.tar.bz2 sentences_base.csv
-tar -cjf sentences_detailed.tar.bz2 sentences_detailed.csv
-tar -cjf links.tar.bz2 links.csv
-tar -cjf sentences.tar.bz2 sentences.csv
-tar -cjf contributions.tar.bz2 contributions.csv
+zstd -19 sentences_base.csv
+zstd -19 sentences_detailed.csv
+zstd -19 links.csv
+zstd -19 sentences.csv
+zstd -19 contributions.csv
 rm contributions.csv
-tar -cjf comments.tar.bz2 sentence_comments.csv
+zstd -19 sentence_comments.csv
 rm sentence_comments.csv
-tar -cjf wall.tar.bz2 wall_posts.csv
+zstd -19 wall_posts.csv
 rm wall_posts.csv
-tar -cjf tags.tar.bz2 tags.csv
-tar -cjf user_lists.tar.bz2 user_lists.csv
-tar -cjf sentences_in_lists.tar.bz2 sentences_in_lists.csv
-tar -cjf jpn_indices.tar.bz2 jpn_indices.csv
-tar -cjf sentences_with_audio.tar.bz2 sentences_with_audio.csv
-tar -cjf user_languages.tar.bz2 user_languages.csv
-tar -cjf tags_detailed.tar.bz2 tags_detailed.csv
-tar -cjf sentences_CC0.tar.bz2 sentences_CC0.csv
-tar -cjf transcriptions.tar.bz2 transcriptions.csv
-tar -cjf sentences_base.tar.bz2 sentences_base.csv
+zstd -19 tags.csv
+zstd -19 user_lists.csv
+zstd -19 sentences_in_lists.csv
+zstd -19 jpn_indices.csv
+zstd -19 sentences_with_audio.csv
+zstd -19 user_languages.csv
+zstd -19 tags_detailed.csv
+zstd -19 sentences_CC0.csv
+zstd -19 transcriptions.csv
+zstd -19 sentences_base.csv
 
 echo "Starting language splitting for sentences at $(date -Iseconds)"
 # Create per-language files for the different sentences files
@@ -173,7 +173,7 @@ mysql --skip-column-names --batch tatoeba -e \
   }'    
 
 echo "Starting cleanup at $(date -Iseconds)"
-find $TEMP_DIR -path '*tsv' -exec bzip2 -qf '{}' +
+find $TEMP_DIR -path '*tsv' -exec zstd -19 -qf '{}' +
 rm -rf $DL_DIR/per_language
 rm transcriptions.csv
 mv -f $TEMP_DIR $DL_DIR

diff --git a/src/Template/Pages/downloads.ctp b/src/Template/Pages/downloads.ctp
@@ -273,7 +273,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>sentences_base.tar.bz2">sentences_base.tar.bz2</a>
+                        <a href="<?= $download_url ?>sentences_base.csv.zst">sentences_base.csv.zst</a>
                     </dd>
                     <dt><?= $description ?></dt>
                     <dd>
@@ -323,7 +323,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>links.tar.bz2">links.tar.bz2</a>
+                        <a href="<?= $download_url ?>links.csv.zst">links.csv.zst</a>
                     </dd>
                     <dt><?= $description ?></dt>
                     <dd>
@@ -350,7 +350,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>tags.tar.bz2">tags.tar.bz2</a>
+                        <a href="<?= $download_url ?>tags.csv.zst">tags.csv.zst</a>
                     </dd>
                     <dt><?= $description ?></dt>
                     <dd>
@@ -376,7 +376,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>user_lists.tar.bz2">user_lists.tar.bz2</a>
+                        <a href="<?= $download_url ?>user_lists.csv.zst">user_lists.csv.zst</a>
                     </dd>
                     <dt><?= $description ?></dt>
                     <dd>
@@ -400,8 +400,8 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>sentences_in_lists.tar.bz2">
-                                sentences_in_lists.tar.bz2
+                        <a href="<?= $download_url ?>sentences_in_lists.csv.zst">
+                                sentences_in_lists.csv.zst
                         </a>
                     </dd>
                     <dt><?= $description ?></dt>
@@ -429,7 +429,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>jpn_indices.tar.bz2">jpn_indices.tar.bz2</a>
+                        <a href="<?= $download_url ?>jpn_indices.csv.zst">jpn_indices.csv.zst</a>
                     </dd>
                     <dt><?= $description ?></dt>
                     <dd>
@@ -460,8 +460,8 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>sentences_with_audio.tar.bz2">
-                            sentences_with_audio.tar.bz2
+                        <a href="<?= $download_url ?>sentences_with_audio.csv.zst">
+                            sentences_with_audio.csv.zst
                         </a>
                     </dd>
                     <dt><?= $description ?></dt>
@@ -501,7 +501,7 @@ $transcriptionsOptions = $this->Downloads->createOptions('transcriptions');
                 <dl>
                     <dt><?= $filename ?></dt>
                     <dd>
-                        <a href="<?= $download_url ?>user_languages.tar.bz2">user_languages.tar.bz2</a>
+                        <a href="<?= $download_url ?>user_languages.csv.zst">user_languages.csv.zst</a>
                     </dd>
                     <dt><?= $description ?></dt>
                     <dd>

diff --git a/src/View/Helper/DownloadsHelper.php b/src/View/Helper/DownloadsHelper.php
@@ -48,7 +48,7 @@ private function availableFiles($basename) {
         );
 
         $dir = new Folder($perLanguageDir);
-        $paths = $dir->findRecursive(".*$basename\.tsv\.bz2$");
+        $paths = $dir->findRecursive(".*$basename\.tsv\.zst$");
         $map = [];
         foreach ($paths as $path) {
             $path = substr($path, strlen($perLanguageDir) + 1);
@@ -74,7 +74,7 @@ private function availableFiles($basename) {
     public function createOptions($basename) {
         $urlForAll = Folder::addPathElement(
             Configure::read('Downloads.url'),
-            "$basename.tar.bz2"
+            "$basename.csv.zst"
         );
         $options[0] = [
             'language' => __('All languages'),

diff --git a/tests/TestCase/View/Helper/DownloadsHelperTest.php b/tests/TestCase/View/Helper/DownloadsHelperTest.php
@@ -14,7 +14,7 @@ class DownloadsHelperTest extends TestCase {
 
     private static function createTempTree() {
         $languages = ['eng', 'fra', 'jpn', 'unknown'];
-        $files = ['sentences.tsv.bz2', 'sentences_detailed.tsv.bz2', 'sentences_CC0.tsv.bz2'];
+        $files = ['sentences.tsv.zst', 'sentences_detailed.tsv.zst', 'sentences_CC0.tsv.zst'];
         foreach ($languages as $lang) {
             $path = Folder::addPathElement(TMP, ['exports', 'per_language', $lang]);
             $subdir = new Folder($path, true);
@@ -54,7 +54,7 @@ public function testCreateOptions_InvalidBasename() {
 
         $this->assertEquals(1, count($options));
         $this->assertEquals(
-            Folder::addPathElement(Configure::read('Downloads.url'), "foobar.tar.bz2"),
+            Folder::addPathElement(Configure::read('Downloads.url'), "foobar.csv.zst"),
             $options[0]['url']
         );
     }
@@ -77,14 +77,14 @@ public function testCreateOptions_ValidBasename($basename) {
         $this->assertEquals(
             Folder::addPathElement(
                 Configure::read('Downloads.url'),
-                "$basename.tar.bz2"
+                "$basename.csv.zst"
             ),
             $options[0]['url']
         );
         $this->assertEquals(
             Folder::addPathElement(
                 Configure::read('Downloads.url'),
-                ['per_language', 'eng', "eng_$basename.tsv.bz2"]
+                ['per_language', 'eng', "eng_$basename.tsv.zst"]
             ),
             $options[1]['url']
         );
@@ -103,7 +103,7 @@ public function testCreateOptions_NoPerLanguageFilesAvailable($basename) {
         $this->assertEquals(
             Folder::addPathElement(
                 Configure::read('Downloads.url'),
-                "$basename.tar.bz2"
+                "$basename.csv.zst"
             ),
             $options[0]['url']
         );