diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa9e62002..8302716d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,66 +19,98 @@ jobs: - beta - nightly - 1.56.1 + env: + RUSTFLAGS: "-C target-cpu=native -C opt-level=3" + ROARINGRS_BENCH_OFFLINE: "true" steps: - - uses: actions/checkout@v2 + - name: Checkout roaring-rs + uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - name: Checkout benchmark datasets + uses: actions/checkout@v2 + with: + repository: "RoaringBitmap/real-roaring-datasets" + path: "benchmarks/real-roaring-datasets" + + - name: Initialize rust toolchain + uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: ${{ matrix.rust }} override: true components: rustfmt, clippy - - uses: actions-rs/cargo@v1 + - name: Fetch + uses: actions-rs/cargo@v1 + with: + command: fetch + + - name: Fetch benchmarks + uses: actions-rs/cargo@v1 + with: + command: fetch + args: --manifest-path benchmarks/Cargo.toml + + - name: Build + uses: actions-rs/cargo@v1 with: command: build + args: --all-targets - - uses: actions-rs/cargo@v1 + - name: Build benchmarks + uses: actions-rs/cargo@v1 with: - command: test + command: build + args: --manifest-path benchmarks/Cargo.toml --all-targets - - uses: actions-rs/cargo@v1 + - name: Check + uses: actions-rs/cargo@v1 with: - command: test - args: --benches --manifest-path benchmarks/Cargo.toml + command: clippy + args: --all-targets -- -D warnings - - uses: actions-rs/cargo@v1 + - name: Check benchmarks + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --manifest-path benchmarks/Cargo.toml --all-targets -- -D warnings + + - name: Check formatting + uses: actions-rs/cargo@v1 with: command: fmt args: -- --check - - uses: actions-rs/cargo@v1 + - name: Check benchmark formatting + uses: actions-rs/cargo@v1 with: command: fmt args: --manifest-path benchmarks/Cargo.toml -- --check - - uses: actions-rs/cargo@v1 + - name: Test + uses: actions-rs/cargo@v1 with: - command: clippy - args: --all-targets -- -D warnings - simd: - name: SIMD Feature - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 + command: test - - uses: actions-rs/toolchain@v1 + - name: Test benchmarks + uses: actions-rs/cargo@v1 with: - profile: minimal - toolchain: nightly - override: true - components: rustfmt, clippy + command: test + args: --manifest-path benchmarks/Cargo.toml --benches - - uses: actions-rs/cargo@v1 + - name: SIMD test + if: matrix.rust == 'nightly' + uses: actions-rs/cargo@v1 with: toolchain: nightly command: test args: --features "simd" - - uses: actions-rs/cargo@v1 + - name: SIMD test benchmarks + if: matrix.rust == 'nightly' + uses: actions-rs/cargo@v1 with: toolchain: nightly command: test - args: --features "simd" --benches --manifest-path benchmarks/Cargo.toml \ No newline at end of file + args: --manifest-path benchmarks/Cargo.toml --features "simd" --benches diff --git a/.gitignore b/.gitignore index 75d4b9447..4fffb2f89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,2 @@ /target /Cargo.lock - -# This is generated by the benchmarks crate build script, do not version with git. -/benchmarks/benches/datasets_paths.rs -/benchmarks/target -/benchmarks/Cargo.lock diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 000000000..4c312bb62 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,3 @@ +/target +/Cargo.lock +/real-roaring-datasets diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 924794ecb..5fa81514e 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -10,16 +10,11 @@ publish = false roaring = { path = ".." } [dev-dependencies] +once_cell = "1.9" +git2 = { version = "0.13", default-features = false, features = ["vendored-openssl"] } +zip = { version = "0.5", default-features = false, features = ["deflate"] } +indicatif = "0.16" criterion = { version = "0.3", features = ["html_reports"] } -quickcheck = "0.9" -quickcheck_macros = "0.9" - -[build-dependencies] -anyhow = "1.0" -bytes = "1.0" -convert_case = "0.4" -reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false } -zip = "0.5.12" [features] simd = ["roaring/simd"] diff --git a/benchmarks/benches/datasets.rs b/benchmarks/benches/datasets.rs new file mode 100644 index 000000000..3dd6d9a67 --- /dev/null +++ b/benchmarks/benches/datasets.rs @@ -0,0 +1,203 @@ +use std::env; +use std::fs::File; +use std::io::BufReader; +use std::path::{Path, PathBuf}; + +use git2::FetchOptions; +use once_cell::sync::OnceCell as SyncOnceCell; + +use roaring::RoaringBitmap; + +static INSTANCE: SyncOnceCell> = SyncOnceCell::new(); + +pub struct Datasets; + +pub struct DatasetsIter { + iter: std::slice::Iter<'static, Dataset>, +} + +impl Iterator for DatasetsIter { + type Item = &'static Dataset; + + fn next(&mut self) -> Option { + self.iter.next() + } +} + +impl IntoIterator for Datasets { + type Item = &'static Dataset; + type IntoIter = DatasetsIter; + + fn into_iter(self) -> Self::IntoIter { + DatasetsIter { + iter: INSTANCE + .get_or_init(|| { + init_datasets().and_then(parse_datasets).expect("a collection of datasets") + }) + .iter(), + } + } +} + +pub struct Dataset { + pub name: String, + pub bitmaps: Vec, +} + +fn init_datasets() -> Result> { + let out_dir = env::var_os("CARGO_MANIFEST_DIR").ok_or(env::VarError::NotPresent)?; + + let out_path = Path::new(&out_dir); + let repo_path = out_path.join("real-roaring-datasets"); + + // Check if in offline mode + + let offline = env::var("ROARINGRS_BENCH_OFFLINE"); + match offline { + Ok(value) => { + if value.parse::()? { + return Ok(repo_path); + } + } + Err(ref err) => match err { + env::VarError::NotPresent => (), + _ => { + offline?; + } + }, + }; + + // Setup progress callbacks + + let pb_cell = once_cell::unsync::OnceCell::new(); + let mut cb = git2::RemoteCallbacks::new(); + + cb.transfer_progress(|progress| { + let pb = pb_cell.get_or_init(|| { + indicatif::ProgressBar::new(progress.total_objects() as u64) + .with_style( + indicatif::ProgressStyle::default_bar() + .template(&format!( + "{{prefix}}{{msg:.cyan/blue}} [{{bar}}] {{pos}}/{}", + progress.total_objects() + )) + .progress_chars("#> "), + ) + .with_prefix(" ") + .with_message("Receiving objects") + }); + + pb.set_position((progress.local_objects() + progress.received_objects()) as u64); + true + }); + + let mut fetch_opts = FetchOptions::new(); + fetch_opts.remote_callbacks(cb); + + // Do update + + if !Path::new(&repo_path).exists() { + git2::build::RepoBuilder::new() + .fetch_options(fetch_opts) + .clone("git://github.com/RoaringBitmap/real-roaring-datasets.git", &repo_path)?; + } else { + let repo = git2::Repository::open(&repo_path)?; + repo.find_remote("origin")?.fetch(&["master"], Some(&mut fetch_opts), None)?; + + let head = repo.head()?.peel_to_commit()?; + let origin_master_head = repo + .find_branch("origin/master", git2::BranchType::Remote)? + .into_reference() + .peel_to_commit()?; + + if head.id() != origin_master_head.id() { + repo.reset(origin_master_head.as_object(), git2::ResetType::Hard, None)?; + } + } + + if let Some(pb) = pb_cell.get() { + pb.finish() + } + + Ok(repo_path) +} + +fn parse_datasets>(path: P) -> Result, Box> { + const DATASET_FILENAME_WHITELIST: &[&str] = &[ + "census-income.zip", + "census-income_srt.zip", + "census1881.zip", + "census1881_srt.zip", + "weather_sept_85.zip", + "weather_sept_85_srt.zip", + "wikileaks-noquotes.zip", + "wikileaks-noquotes_srt.zip", + ]; + + use indicatif::{ProgressBar, ProgressStyle}; + use std::io::BufRead; + use zip::ZipArchive; + + let dir = path.as_ref().read_dir()?; + + let mut datasets = Vec::new(); + + // Future work: Reuse this buffer to parse croaring bitmaps for comparison + let mut numbers = Vec::new(); + + for dir_entry_result in dir { + let dir_entry = dir_entry_result?; + let metadata = dir_entry.metadata()?; + let file_name = dir_entry.file_name(); + // TODO dont panic + let file_name_str = file_name.to_str().expect("utf-8 filename"); + + if metadata.is_file() && DATASET_FILENAME_WHITELIST.contains(&file_name_str) { + let file = File::open(dir_entry.path())?; + let name = file_name_str.split_at(file_name_str.len() - ".zip".len()).0.to_string(); + + let mut zip = ZipArchive::new(file)?; + + let mut total_size = 0; + for i in 0..zip.len() { + let file = zip.by_index(i)?; + total_size += file.size(); + } + + let pb = ProgressBar::new(total_size) + .with_style( + ProgressStyle::default_bar() + .template(" {prefix:.green} [{bar}] {msg}") + .progress_chars("#> "), + ) + .with_prefix("Parsing") + .with_message(name.clone()); + + let mut bitmaps = Vec::with_capacity(zip.len()); + for i in 0..zip.len() { + let file = zip.by_index(i)?; + let size = file.size(); + let buf = BufReader::new(file); + + for bytes in buf.split(b',') { + let bytes = bytes?; + let str = String::from_utf8(bytes)?; + let n = str.trim().parse::()?; + numbers.push(n); + } + + let bitmap = RoaringBitmap::from_sorted_iter(numbers.iter().copied())?; + numbers.clear(); + bitmaps.push(bitmap); + + pb.set_position(pb.position() + size); + } + + pb.finish(); + datasets.push(Dataset { name, bitmaps }); + } + } + datasets.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + println!(); + Ok(datasets) +} diff --git a/benchmarks/benches/lib.rs b/benchmarks/benches/lib.rs index 25e52e2c7..a87ab9c6f 100644 --- a/benchmarks/benches/lib.rs +++ b/benchmarks/benches/lib.rs @@ -1,14 +1,13 @@ -mod datasets_paths; - use std::cmp::Reverse; -use std::convert::TryInto; -use std::num::ParseIntError; -use std::path::{Path, PathBuf}; -use std::{fs, io}; -use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion}; +use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; + use roaring::RoaringBitmap; +use crate::datasets::Datasets; + +mod datasets; + fn create(c: &mut Criterion) { c.bench_function("create", |b| { b.iter(|| { @@ -96,34 +95,24 @@ fn len(c: &mut Criterion) { } fn rank(c: &mut Criterion) { - let files = self::datasets_paths::WIKILEAKS_NOQUOTES_SRT; - let parsed_numbers = parse_dir_files(files).unwrap(); - - // Cache len to prevent len calculation from effecting benchmark - let bitmaps: Vec<_> = parsed_numbers - .into_iter() - .map(|(_, r)| { - r.map(|iter| { - let bitmap = RoaringBitmap::from_sorted_iter(iter).unwrap(); - let len: u32 = bitmap.len().try_into().expect("len <= u32::MAX"); - (bitmap, len) - }) - .unwrap() - }) - .collect(); - - // Rank all multiples of 100 < bitmap.len() - // Mupliplier chosen arbitrarily, but should be sure not to rank many values > len() - // Doing so would degenerate into benchmarking len() - c.bench_function("rank", |b| { - b.iter(|| { - for (bitmap, len) in bitmaps.iter() { - for i in (0..*len).step_by(100) { - black_box(bitmap.rank(i)); + let mut group = c.benchmark_group("rank"); + for dataset in Datasets { + let bitmaps = + dataset.bitmaps.iter().map(|bitmap| (bitmap, bitmap.len() as u32)).collect::>(); + + // Rank all multiples of 100 < bitmap.len() + // Mupliplier chosen arbitrarily, but should be sure not to rank many values > len() + // Doing so would degenerate into benchmarking len() + group.bench_function(BenchmarkId::new("rank", &dataset.name), |b| { + b.iter(|| { + for (bitmap, len) in bitmaps.iter() { + for i in (0..*len).step_by(100) { + black_box(bitmap.rank(i)); + } } - } + }); }); - }); + } } fn and(c: &mut Criterion) { @@ -277,66 +266,18 @@ fn insert_range_bitmap(c: &mut Criterion) { } fn iter(c: &mut Criterion) { - c.bench_function("iter bitmap 1..10_000", |b| { - let bitmap: RoaringBitmap = (1..10_000).collect(); - b.iter(|| { - bitmap.iter().for_each(|i| { - black_box(i); - }); - }); - }); - - c.bench_function("iter bitmap sparse", |b| { - let bitmap: RoaringBitmap = (0..1 << 16).step_by(61).collect(); - b.iter(|| { - bitmap.iter().for_each(|i| { - black_box(i); - }); - }); - }); - - c.bench_function("iter bitmap dense", |b| { - let bitmap: RoaringBitmap = (0..1 << 16).step_by(2).collect(); - b.iter(|| { - bitmap.iter().for_each(|i| { - black_box(i); + let mut group = c.benchmark_group("iter"); + for dataset in Datasets { + group.bench_function(BenchmarkId::new("iter", &dataset.name), |b| { + b.iter(|| { + dataset.bitmaps.iter().flat_map(|bitmap| bitmap.iter()).for_each(|i| { + black_box(i); + }); }); }); - }); - - c.bench_function("iter bitmap minimal", |b| { - let bitmap: RoaringBitmap = (0..4096).collect(); - b.iter(|| { - bitmap.iter().for_each(|i| { - black_box(i); - }); - }); - }); - - c.bench_function("iter bitmap full", |b| { - let bitmap: RoaringBitmap = (0..1 << 16).collect(); - b.iter(|| { - bitmap.iter().for_each(|i| { - black_box(i); - }); - }); - }); - - c.bench_function("iter parsed", |b| { - let files = self::datasets_paths::WIKILEAKS_NOQUOTES_SRT; - let parsed_numbers = parse_dir_files(files).unwrap(); - - let bitmaps: Vec<_> = parsed_numbers - .into_iter() - .map(|(_, r)| r.map(|iter| RoaringBitmap::from_sorted_iter(iter).unwrap()).unwrap()) - .collect(); + } - b.iter(|| { - bitmaps.iter().flat_map(|bitmap| bitmap.iter()).for_each(|i| { - black_box(i); - }); - }); - }); + group.finish(); } fn is_empty(c: &mut Criterion) { @@ -402,127 +343,111 @@ fn serialized_size(c: &mut Criterion) { }); } -fn extract_integers>(content: A) -> Result, ParseIntError> { - content.as_ref().split(',').map(|s| s.trim().parse()).collect() -} - -// Parse every file into a vector of integer. -fn parse_dir_files>( - files: A, -) -> io::Result, ParseIntError>)>> { - fs::read_dir(files)? - .map(|r| r.and_then(|e| fs::read_to_string(e.path()).map(|r| (e.path(), r)))) - .map(|r| r.map(|(p, c)| (p, extract_integers(c)))) - .collect() -} - fn from_sorted_iter(c: &mut Criterion) { - let files = self::datasets_paths::WIKILEAKS_NOQUOTES_SRT; - let parsed_numbers = parse_dir_files(files).unwrap(); + let mut group = c.benchmark_group("from_sorted_iter"); + + for dataset in Datasets { + let dataset_numbers = dataset + .bitmaps + .iter() + .map(|bitmap| bitmap.iter().collect::>()) + .collect::>(); + + group.bench_function(BenchmarkId::new("from_sorted_iter", &dataset.name), |b| { + b.iter(|| { + for bitmap_numbers in &dataset_numbers { + RoaringBitmap::from_sorted_iter(bitmap_numbers.iter().copied()).unwrap(); + } + }) + }); + } - c.bench_function("from_sorted_iter", |b| { - b.iter(|| { - for (_, numbers) in &parsed_numbers { - let numbers = numbers.as_ref().unwrap(); - RoaringBitmap::from_sorted_iter(numbers.iter().copied()).unwrap(); - } - }) - }); + group.finish(); } fn successive_and(c: &mut Criterion) { - let files = self::datasets_paths::WIKILEAKS_NOQUOTES_SRT; - let parsed_numbers = parse_dir_files(files).unwrap(); - - let mut bitmaps: Vec<_> = parsed_numbers - .into_iter() - .map(|(_, r)| r.map(|iter| RoaringBitmap::from_sorted_iter(iter).unwrap()).unwrap()) - .collect(); - - // biggest bitmaps first. - bitmaps.sort_unstable_by_key(|b| Reverse(b.len())); - let mut group = c.benchmark_group("Successive And"); - group.bench_function("Successive And Assign Ref", |b| { - b.iter_batched( - || bitmaps.clone(), - |bitmaps| { - let mut iter = bitmaps.into_iter(); - let mut first = iter.next().unwrap().clone(); - for bitmap in iter { - first &= bitmap; - } - }, - BatchSize::LargeInput, - ); - }); + for dataset in Datasets { + // biggest bitmaps first. + let mut sorted_bitmaps = dataset.bitmaps.clone(); + sorted_bitmaps.sort_unstable_by_key(|b| Reverse(b.len())); - group.bench_function("Successive And Assign Owned", |b| { - b.iter_batched( - || bitmaps.clone(), - |bitmaps| { - black_box(bitmaps.into_iter().reduce(|a, b| a & b).unwrap()); - }, - BatchSize::LargeInput, - ); - }); + group.bench_function(BenchmarkId::new("Successive And Assign Ref", &dataset.name), |b| { + b.iter_batched( + || sorted_bitmaps.clone(), + |bitmaps| { + let mut iter = bitmaps.into_iter(); + let mut first = iter.next().unwrap(); + for bitmap in iter { + first &= bitmap; + } + }, + BatchSize::LargeInput, + ); + }); - group.bench_function("Successive And Ref Ref", |b| { - b.iter_batched( - || bitmaps.clone(), - |bitmaps| { - let mut iter = bitmaps.iter(); - let first = iter.next().unwrap().clone(); - black_box(iter.fold(first, |acc, x| (&acc) & x)); - }, - BatchSize::LargeInput, - ); - }); + group.bench_function(BenchmarkId::new("Successive And Assign Owned", &dataset.name), |b| { + b.iter_batched( + || sorted_bitmaps.clone(), + |bitmaps| { + black_box(bitmaps.into_iter().reduce(|a, b| a & b).unwrap()); + }, + BatchSize::LargeInput, + ); + }); + + group.bench_function(BenchmarkId::new("Successive And Ref Ref", &dataset.name), |b| { + b.iter_batched( + || sorted_bitmaps.clone(), + |bitmaps| { + let mut iter = bitmaps.iter(); + let first = iter.next().unwrap().clone(); + black_box(iter.fold(first, |acc, x| (&acc) & x)); + }, + BatchSize::LargeInput, + ); + }); + } group.finish(); } fn successive_or(c: &mut Criterion) { - let files = self::datasets_paths::WIKILEAKS_NOQUOTES_SRT; - let parsed_numbers = parse_dir_files(files).unwrap(); - - let bitmaps: Vec<_> = parsed_numbers - .into_iter() - .map(|(_, r)| r.map(|iter| RoaringBitmap::from_sorted_iter(iter).unwrap()).unwrap()) - .collect(); - let mut group = c.benchmark_group("Successive Or"); - group.bench_function("Successive Or Assign Ref", |b| { - b.iter(|| { - let mut output = RoaringBitmap::new(); - for bitmap in &bitmaps { - output |= bitmap; - } - }); - }); - group.bench_function("Successive Or Assign Owned", |b| { - b.iter_batched( - || bitmaps.clone(), - |bitmaps: Vec| { + for dataset in Datasets { + group.bench_function(BenchmarkId::new("Successive Or Assign Ref", &dataset.name), |b| { + b.iter(|| { let mut output = RoaringBitmap::new(); - for bitmap in bitmaps { + for bitmap in &dataset.bitmaps { output |= bitmap; } - }, - BatchSize::LargeInput, - ); - }); + }); + }); - group.bench_function("Successive Or Ref Ref", |b| { - b.iter(|| { - let mut output = RoaringBitmap::new(); - for bitmap in &bitmaps { - output = (&output) | bitmap; - } + group.bench_function(BenchmarkId::new("Successive Or Assign Owned", &dataset.name), |b| { + b.iter_batched( + || dataset.bitmaps.clone(), + |bitmaps: Vec| { + let mut output = RoaringBitmap::new(); + for bitmap in bitmaps { + output |= bitmap; + } + }, + BatchSize::LargeInput, + ); }); - }); + + group.bench_function(BenchmarkId::new("Successive Or Ref Ref", &dataset.name), |b| { + b.iter(|| { + let mut output = RoaringBitmap::new(); + for bitmap in &dataset.bitmaps { + output = (&output) | bitmap; + } + }); + }); + } group.finish(); } diff --git a/benchmarks/build.rs b/benchmarks/build.rs deleted file mode 100644 index 4622b3c9c..000000000 --- a/benchmarks/build.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::fs::File; -use std::io::{Cursor, Read, Seek, Write}; -use std::path::{Path, PathBuf}; -use std::{env, fs}; - -use bytes::Bytes; -use convert_case::{Case, Casing}; -use reqwest::{blocking::get, IntoUrl}; -use zip::read::ZipArchive; - -const BASE_URL: &str = "https://github.com/RoaringBitmap/real-roaring-datasets/raw/master/"; - -// const DATASET_CENSUS_INCOME: &str = "census-income"; -// const DATASET_CENSUS_INCOME_SRT: &str = "census-income_srt"; -// const DATASET_CENSUS1881: &str = "census1881"; -// const DATASET_CENSUS1881_SRT: &str = "census1881_srt"; -// const DATASET_DIMENSION_003: &str = "dimension_003"; -// const DATASET_DIMENSION_008: &str = "dimension_008"; -// const DATASET_DIMENSION_033: &str = "dimension_033"; -// const DATASET_USCENSUS2000: &str = "uscensus2000"; -// const DATASET_WEATHER_SEPT_85: &str = "weather_sept_85"; -// const DATASET_WEATHER_SEPT_85_SRT: &str = "weather_sept_85_srt"; -// const DATASET_WIKILEAKS_NOQUOTES: &str = "wikileaks-noquotes"; -const DATASET_WIKILEAKS_NOQUOTES_SRT: &str = "wikileaks-noquotes_srt"; - -// const DATASETS: &[&str] = &[ -// DATASET_CENSUS_INCOME, -// DATASET_CENSUS_INCOME_SRT, -// DATASET_CENSUS1881, -// DATASET_CENSUS1881_SRT, -// DATASET_DIMENSION_003, -// DATASET_DIMENSION_008, -// DATASET_DIMENSION_033, -// DATASET_USCENSUS2000, -// DATASET_WEATHER_SEPT_85, -// DATASET_WEATHER_SEPT_85_SRT, -// DATASET_WIKILEAKS_NOQUOTES, -// DATASET_WIKILEAKS_NOQUOTES_SRT, -// ]; - -fn main() -> anyhow::Result<()> { - let out_dir = PathBuf::from(env::var("OUT_DIR")?); - - let benches_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("benches"); - let mut manifest_paths_file = File::create(benches_dir.join("datasets_paths.rs"))?; - - writeln!( - &mut manifest_paths_file, - "// This file is generated by the build script.\n// Do not modify by hand, use the build.rs file.\n" - )?; - - #[allow(clippy::single_element_loop)] - for dataset in &[DATASET_WIKILEAKS_NOQUOTES_SRT] { - let out_path = out_dir.join(dataset); - let url = format!("{}/{}.zip", BASE_URL, dataset); - let bytes = download_dataset(url)?; - unzip_in_folder(bytes, &out_path)?; - - writeln!( - &mut manifest_paths_file, - r#"pub const {}: &str = {:?};"#, - dataset.to_case(Case::ScreamingSnake), - out_path.display(), - )?; - } - - Ok(()) -} - -fn download_dataset(url: U) -> anyhow::Result> { - let bytes = get(url)?.bytes()?; - Ok(Cursor::new(bytes)) -} - -fn unzip_in_folder>(bytes: R, path: P) -> anyhow::Result<()> { - let path = path.as_ref(); - fs::create_dir_all(path).unwrap(); - let mut zip = ZipArchive::new(bytes)?; - zip.extract(path)?; - Ok(()) -} diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs deleted file mode 100644 index 829adafa0..000000000 --- a/benchmarks/src/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! This library is only used to isolate the benchmarks -//! from the original roaring library. -//! -//! It does not include interesting functions for roaring library -//! users only for roaring contributors.