Need help understanding why this does not work

[sanketkedia]

#[tokio::test]
    async fn test_writing_only_to_disk_works() {
        let dir = tempfile::tempdir()
            .expect("To be able to create temp path")
            .path()
            .to_str()
            .expect("To be able to parse path")
            .to_string();
        let cache = FoyerCacheConfig {
            dir: Some(dir.clone()),
            flushers: 1,
            file_size: 1,
            flush: true,
            buffer_pool: 1,
            ..Default::default()
        }
        .build_hybrid_test::<String, String>()
        .await
        .unwrap();
        // Insert a 512KB string value generated at random.
        let large_value = "val1".repeat(512 * 1024);
        self.cache.storage_writer("key1".to_string()).insert(large_value.clone());
        for i in 0..100 {
            // Insert another 512KB string value generated at random.
            // This should trigger a flush.
            let another_large_value = format!("val{}", i).repeat(512 * 1024);
            self.cache.storage_writer(format!("key{}", i).to_string()).insert(another_large_value);
        }
        // Sleep for 5 secs.
        // This should give the cache enough time to flush the data to disk.
        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
        let value = cache
            .get(&"key1".to_string())
            .await
            .expect("Expected to be able to get value")
            .expect("Value should not be None");
        assert_eq!(value, large_value);
    }

What am I doing:

1. Inserting data to a hybrid cache bypassing the in-memory layer. I use the HybridCacheStorageWriter to directly write to disk
2. I fill the buffer pool completely and go past it. As per my understanding this should be enough data to trigger a flush. I am also doing a fsync on every flush.
3. I then try to get() the first key that I inserted. Ideally it should be present in the disk and should return the value but instead it returns None

Please help me debug this. Specifically I have two questions:

1. When does the hybrid cache flush to disk?
2. If I flush the same key twice to disk then will it cause duplication on the disk or will the second entry overwrite the first?

[sanketkedia]

@MrCroxx any help on this will be greatly appreciated

[MrCroxx]

Hi, @sanketkedia .

I found the disk cache is misconfigured.

        let cache = FoyerCacheConfig {
            dir: Some(dir.clone()),
            flushers: 1,
            file_size: 1, // !!! This is the file size in bytes.
            flush: true,
            buffer_pool: 1,
            ..Default::default()
        }

[sanketkedia]

Hi @MrCroxx, Thank you for responding. I checked the code and I am multiplying it by 1024 * 1024 later when I build the cache. So I am setting it to 1MB and not 1 byte

pub async fn hybrid(
        config: &FoyerCacheConfig,
    ) -> Result<FoyerHybridCache<K, V>, Box<dyn ChromaError>> {
        let tracing_options = TracingOptions::new()
            .with_record_hybrid_insert_threshold(Duration::from_micros(config.trace_insert_us as _))
            .with_record_hybrid_get_threshold(Duration::from_micros(config.trace_get_us as _))
            .with_record_hybrid_obtain_threshold(Duration::from_micros(config.trace_obtain_us as _))
            .with_record_hybrid_remove_threshold(Duration::from_micros(config.trace_remove_us as _))
            .with_record_hybrid_fetch_threshold(Duration::from_micros(config.trace_fetch_us as _));

        let otel_0_27_metrics = Box::new(
            mixtrics::registry::opentelemetry_0_27::OpenTelemetryMetricsRegistry::new(
                global::meter("chroma"),
            ),
        );
        let builder = HybridCacheBuilder::<K, V>::new()
            .with_metrics_registry(otel_0_27_metrics)
            .with_tracing_options(tracing_options)
            .memory(config.mem)
            .with_shards(config.shards);

        let builder = match config.eviction.as_str() {
            "lru" => builder.with_eviction_config(LruConfig::default()),
            "lfu" => builder.with_eviction_config(LfuConfig::default()),
            "fifo" => builder.with_eviction_config(FifoConfig::default()),
            "s3fifo" => builder.with_eviction_config(S3FifoConfig::default()),
            _ => {
                return Err(Box::new(CacheError::InvalidCacheConfig(format!(
                    "eviction: {}",
                    config.eviction
                ))));
            }
        };

        let builder = match config.deterministic_hashing {
            true => {
                // These are generated from a good RNG.
                let rs = RandomState::with_seeds(
                    18408126631592559320,
                    14098607199905812554,
                    3530350452151671086,
                    4042281453092388365,
                );
                builder.with_hash_builder(rs)
            }
            false => builder,
        };

        let Some(dir) = config.dir.as_ref() else {
            return Err(Box::new(CacheError::InvalidCacheConfig(
                "missing dir".to_string(),
            )));
        };

        let mut builder = builder
            .with_weighter(|_, v| v.weight())
            .storage(Engine::Large)
            .with_device_options(
                DirectFsDeviceOptions::new(dir)
                    .with_capacity(config.disk * 1024 * 1024)
                    .with_file_size(config.file_size * 1024 * 1024),
            )
            .with_flush(config.flush)
            .with_recover_mode(foyer::RecoverMode::Strict)
            .with_large_object_disk_cache_options(
                LargeEngineOptions::new()
                    .with_indexer_shards(config.shards)
                    .with_recover_concurrency(config.recover_concurrency)
                    .with_flushers(config.flushers)
                    .with_reclaimers(config.reclaimers)
                    .with_buffer_pool_size(config.buffer_pool * 1024 * 1024)
                    .with_eviction_pickers(vec![
                        Box::new(InvalidRatioPicker::new(config.invalid_ratio)),
                        Box::new(FifoPicker::default()),
                    ]),
            );

        if config.admission_rate_limit > 0 {
            builder = builder.with_admission_picker(Arc::new(RateLimitPicker::new(
                config.admission_rate_limit * 1024 * 1024,
            )));
        }
        let cache = builder.build().await.map_err(|e| {
            CacheError::InvalidCacheConfig(format!("builder failed: {:?}", e)).boxed()
        })?;
        cache.enable_tracing();
        cache.update_tracing_options(
            TracingOptions::new().with_record_hybrid_get_threshold(Duration::from_millis(10)),
        );
        let meter = global::meter("chroma");
        let cache_hit = meter.u64_counter("cache_hit").build();
        let cache_miss = meter.u64_counter("cache_miss").build();
        let get_latency = meter.u64_histogram("get_latency").build();
        let insert_latency = meter.u64_histogram("insert_latency").build();
        let remove_latency = meter.u64_histogram("remove_latency").build();
        let clear_latency = meter.u64_histogram("clear_latency").build();
        Ok(FoyerHybridCache {
            cache,
            cache_hit,
            cache_miss,
            get_latency,
            insert_latency,
            remove_latency,
            clear_latency,
        })
    }

[sanketkedia]

@MrCroxx any help on this? Sorry this is time sensitive on our side as this blocks release hence the followup

[MrCroxx]

Hi, @sanketkedia .

I found you are already using foyer v0.13.1

Please use the builder to build a cache with tuned default arguments instead of filling in EVERY argument.

Please refer to the document: https://docs.rs/foyer/0.13.1/foyer/struct.HybridCacheBuilder.html

[MrCroxx]

https://github.com/risingwavelabs/risingwave/blob/main/src/storage/src/store_impl.rs#L719-L767

If you are not sure about how to use the builder, here is a real-world case in RisingWave.

[sanketkedia]

The test passes when I use the default config. So I am going to close this issue and mark as answer the suggestion of using default values. I still don't understand what is wrong in my custom config but that's for another day