Add zstandard as a compression option

.github/workflows/rust.yml

@@ -5,26 +5,24 @@ on:
     branches: [ main, dev ]
   pull_request:
     branches: [ main, dev ]
-
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Build
-      run: cargo build --verbose
-      
-    - name: Run tests
-      run: cargo test --verbose
-      
-    - name: Run cargo-tarpaulin
-      uses: actions-rs/[email protected]
-      continue-on-error: true
-      
-    - name: Upload to codecov.io
-      uses: codecov/codecov-action@v1
+      - uses: actions/checkout@v2
+      - name: Build
+        run: cargo build --verbose
+
+      - name: Run tests
+        run: cargo test --verbose --features zstd
+
+      - name: Run cargo-tarpaulin
+        uses: actions-rs/[email protected]
+        continue-on-error: true
+
+      - name: Upload to codecov.io
+        uses: codecov/codecov-action@v1

Cargo.toml

@@ -3,6 +3,10 @@ authors = ["Pedro J. Ortiz <[email protected]>"]
 edition = "2021"
 name = "oscar-tools"
 version = "0.1.0"
+
+[features]
+zstd = ["dep:zstd"]
+
 [dependencies]
 env_logger = "0.9.0"
 flate2 = "1.0.22"
@@ -14,6 +18,7 @@ rayon = "1.5.1"
 runiq-lib = "1.2.2"
 serde_json = "1.0.78"
 sha2 = "0.10.1"
+zstd = {version="0.11.2", optional=true}
 
 [dependencies.clap]
 features = ["derive"]
@@ -24,4 +29,5 @@ oscar-io = "0.1.3"
 tempfile = "3.3.0"
 
 [profile.release]
-debug = true
+debug = true
+

src/error.rs

@@ -1,4 +1,5 @@
 //! Errors
+
 #[derive(Debug)]
 #[cfg(not(tarpaulin_include))]
 pub enum Error {

src/impls/oscar_doc/compress.rs

@@ -0,0 +1,65 @@
+use std::path::PathBuf;
+
+use clap::{arg, ArgMatches};
+
+use crate::{cli::Command, error::Error, ops::Compress};
+
+/// internal struct for compression op implementation
+pub struct CompressDoc;
+impl Compress for CompressDoc {}
+impl Command for CompressDoc {
+    fn subcommand() -> clap::App<'static>
+    where
+        Self: Sized,
+    {
+        clap::App::new("compress")
+        .about("Compress provided file and/or files in provided folder, up to a depth of 2.")
+        .long_about("Compression of corpus files and folders.
+
+This command can be used to compress a single file (by specifying a source and destination file path) or a set of files (by specifying a source and destination folder path).
+
+If a file path is specified, oscar-tools will compress the given file and write it in the destination file path.
+If a folder is specified, oscar-tools will compress files in subfolders and write the compressed files in the destination folder path.
+
+Only one thread is used if a file is provided. If a folder is provided, takes all threads available. Use -J to specify a different number of threads.
+
+Only provide a folder (resp. file) as a destination if a folder (resp. file) has been provided.
+")
+            .arg(arg!([SOURCE] "File/folder to compress. If a folder is provided, keeps arborescence and compresses up to a depth of 2.").required(true))
+            .arg(arg!([DESTINATION] "File/folder to write to.").required(true))
+            .arg(arg!(--del_src "If set, deletes source files as they are being compressed.").required(false))
+            .arg(arg!(--compression <COMP> "Compression to use (gzip, zstd)").required(false).default_value("zstd"))
+            .arg(arg!(-J --num_threads <NUM_THREADS> "Number of threads to use (iif source is a folder). If 0, take all available").default_value("0").required(false))
+    }
+
+    fn run(matches: &ArgMatches) -> Result<(), Error>
+    where
+        Self: Sized,
+    {
+        let src: PathBuf = matches
+            .value_of("SOURCE")
+            .expect("Value of 'SOURCE' is required.")
+            .into();
+        let dst: PathBuf = matches
+            .value_of("DESTINATION")
+            .expect("Value of 'DESTINATION' is required.")
+            .into();
+        let del_src = matches.is_present("del_src");
+        let compression = matches.value_of("compression").unwrap();
+        let num_threads: usize = matches
+            .value_of("num_threads")
+            .unwrap()
+            .parse()
+            .expect("'num_threads' has to be a number.");
+        if src.is_file() {
+            CompressDoc::compress_file(&src, &dst, del_src, compression)?;
+        } else if src.is_dir() {
+            CompressDoc::compress_corpus(&src, &dst, del_src, compression, num_threads)?;
+        } else {
+            return Err(
+                std::io::Error::new(std::io::ErrorKind::NotFound, format!("{:?}", src)).into(),
+            );
+        }
+        Ok(())
+    }
+}

src/impls/oscar_doc/filter_tags.rs

@@ -1,12 +1,7 @@
 /*! The goal is to filter the documents based on the annotation ["short s", "header"]
  * take a document
  */
-use std::{
-    borrow::Cow,
-    collections::HashSet,
-    fs::File,
-    io::{BufWriter},
-};
+use std::{borrow::Cow, collections::HashSet, fs::File, io::BufWriter};
 
 use oscar_io::oscar_doc::{Document, SplitFolderReader, Writer};
 
@@ -195,7 +190,6 @@ mod test {
         lang::Lang,
         oscar_doc::{Document, Metadata, Reader, Writer},
     };
-
 
     use super::FilterTagDoc;
 

src/impls/oscar_doc/mod.rs

@@ -1,4 +1,5 @@
 /*! OSCAR v2 (22.01) operation implementations!*/
+mod compress;
 mod filter_tags;
 mod oscar_doc;
 pub(crate) use oscar_doc::*;

src/impls/oscar_doc/oscar_doc.rs

@@ -1,6 +1,7 @@
 //! OSCAR Schema v2 (See [oscar-corpus.com](https://oscar-corpus.com)) operation implementations.
 //!
 //! Implementations mostly use default trait implementations, as the format is simple.
+use crate::impls::oscar_doc::compress::CompressDoc;
 use crate::ops::FilterTags;
 use crate::{
     cli::Command,
@@ -228,64 +229,6 @@ if SOURCE is a folder, DESTINATION must be an empty folder. Subfolders will be c
     }
 }
 
-/// internal struct for compression op implementation
-struct CompressDoc;
-impl Compress for CompressDoc {}
-impl Command for CompressDoc {
-    fn subcommand() -> clap::App<'static>
-    where
-        Self: Sized,
-    {
-        clap::App::new("compress")
-        .about("Compress provided file and/or files in provided folder, up to a depth of 2.")
-        .long_about("Compression of corpus files and folders.
-
-This command can be used to compress a single file (by specifying a source and destination file path) or a set of files (by specifying a source and destination folder path).
-
-If a file path is specified, oscar-tools will compress the given file and write it in the destination file path.
-If a folder is specified, oscar-tools will compress files in subfolders and write the compressed files in the destination folder path.
-
-Only one thread is used if a file is provided. If a folder is provided, takes all threads available. Use -J to specify a different number of threads.
-
-Only provide a folder (resp. file) as a destination if a folder (resp. file) has been provided.
-")
-            .arg(arg!([SOURCE] "File/folder to compress. If a folder is provided, keeps arborescence and compresses up to a depth of 2.").required(true))
-            .arg(arg!([DESTINATION] "File/folder to write to.").required(true))
-            .arg(arg!(--del_src "If set, deletes source files as they are being compressed.").required(false))
-            .arg(arg!(-J --num_threads <NUM_THREADS> "Number of threads to use (iif source is a folder). If 0, take all available").default_value("0").required(false))
-    }
-
-    fn run(matches: &ArgMatches) -> Result<(), Error>
-    where
-        Self: Sized,
-    {
-        let src: PathBuf = matches
-            .value_of("SOURCE")
-            .expect("Value of 'SOURCE' is required.")
-            .into();
-        let dst: PathBuf = matches
-            .value_of("DESTINATION")
-            .expect("Value of 'DESTINATION' is required.")
-            .into();
-        let del_src = matches.is_present("del_src");
-        let num_threads: usize = matches
-            .value_of("num_threads")
-            .unwrap()
-            .parse()
-            .expect("'num_threads' has to be a number.");
-        if src.is_file() {
-            CompressDoc::compress_file(&src, &dst, del_src)?;
-        } else if src.is_dir() {
-            CompressDoc::compress_corpus(&src, &dst, del_src, num_threads)?;
-        } else {
-            return Err(
-                std::io::Error::new(std::io::ErrorKind::NotFound, format!("{:?}", src)).into(),
-            );
-        }
-        Ok(())
-    }
-}
-
 /// impl block for helper functions related to [ExtractText].
 //TODO: move into a proper op
 impl OscarDoc {
@@ -333,8 +276,8 @@ impl OscarDoc {
 #[cfg(test)]
 mod tests {
 
-    use super::CompressDoc;
     use super::SplitDoc;
+    use crate::impls::oscar_doc::compress::CompressDoc;
     use crate::ops::Split;
     use crate::{impls::OscarDoc, ops::Compress};
     use std::{
@@ -470,7 +413,7 @@ quux
 
         // create destination path and compress
         let tmpdst = tempfile::tempdir().unwrap();
-        CompressDoc::compress_folder(tmpdir.path(), tmpdst.path(), false).unwrap();
+        CompressDoc::compress_folder(tmpdir.path(), tmpdst.path(), false, "gzip").unwrap();
 
         println!(
             "{:?}",

src/ops/checksum.rs

@@ -129,7 +129,7 @@ pub trait Checksum {
 #[cfg(test)]
 mod tests {
     use sha2::Digest;
-    
+
     use std::fs::File;
     use std::io::Write;
     use std::path::PathBuf;