feat: Quilt client part ii

crates/walrus-core/src/encoding/quilt_encoding.rs

@@ -125,7 +125,7 @@ pub trait QuiltIndexApi<V: QuiltVersion>: Clone + Into<QuiltIndex> {
     -> Result<&V::QuiltPatch, QuiltError>;
 
     /// Returns the sliver indices of the quilt patches stored in.
-    fn get_sliver_indices_by_identifiers(
+    fn get_sliver_indices_for_identifiers(
         &self,
         identifiers: &[&str],
     ) -> Result<Vec<SliverIndex>, QuiltError>;
@@ -1878,7 +1878,7 @@ mod tests {
         );
 
         let required_indices = quilt_index_v1
-            .get_sliver_indices_by_identifiers(&[identifier])
+            .get_sliver_indices_for_identifiers(&[identifier])
             .expect("Should get sliver indices by identifiers");
         assert_eq!(required_indices, missing_indices);
 

crates/walrus-core/src/metadata.rs

@@ -124,13 +124,13 @@ pub enum QuiltIndex {
 
 impl QuiltIndex {
     /// Returns the sliver indices of the quilt patch with the given identifiers.
-    pub fn get_sliver_indices_by_identifiers(
+    pub fn get_sliver_indices_for_identifiers(
         &self,
         identifiers: &[&str],
     ) -> Result<Vec<SliverIndex>, QuiltError> {
         match self {
             QuiltIndex::V1(quilt_index) => {
-                quilt_index.get_sliver_indices_by_identifiers(identifiers)
+                quilt_index.get_sliver_indices_for_identifiers(identifiers)
             }
         }
     }
@@ -228,7 +228,7 @@ impl QuiltIndexApi<QuiltVersionV1> for QuiltIndexV1 {
     }
 
     /// If the quilt contains duplicate identifiers, all matching patches are returned.
-    fn get_sliver_indices_by_identifiers(
+    fn get_sliver_indices_for_identifiers(
         &self,
         identifiers: &[&str],
     ) -> Result<Vec<SliverIndex>, QuiltError> {

crates/walrus-e2e-tests/tests/test_client.rs

@@ -19,7 +19,7 @@ use std::{
 };
 
 use indicatif::MultiProgress;
-use rand::random;
+use rand::{Rng, random, seq::SliceRandom, thread_rng};
 #[cfg(msim)]
 use sui_macros::{clear_fail_point, register_fail_point_if};
 use sui_types::base_types::{SUI_ADDRESS_LENGTH, SuiAddress};
@@ -32,14 +32,25 @@ use walrus_core::{
     EpochCount,
     ShardIndex,
     SliverPairIndex,
-    encoding::{EncodingConfigTrait as _, Primary},
+    encoding::{
+        EncodingConfigTrait as _,
+        Primary,
+        quilt_encoding::{QuiltApi, QuiltStoreBlob, QuiltStoreBlobOwned, QuiltVersionV1},
+    },
     merkle::Node,
     messages::BlobPersistenceType,
-    metadata::VerifiedBlobMetadataWithId,
+    metadata::{QuiltMetadata, VerifiedBlobMetadataWithId},
 };
 use walrus_proc_macros::walrus_simtest;
 use walrus_sdk::{
-    client::{Blocklist, Client, WalrusStoreBlob, WalrusStoreBlobApi, responses::BlobStoreResult},
+    client::{
+        Blocklist,
+        Client,
+        WalrusStoreBlob,
+        WalrusStoreBlobApi,
+        quilt_client::QuiltClientConfig,
+        responses::{BlobStoreResult, QuiltStoreResult},
+    },
     error::{
         ClientError,
         ClientErrorKind::{
@@ -977,6 +988,117 @@ async fn test_storage_nodes_delete_data_for_deleted_blobs() -> TestResult {
     Ok(())
 }
 
+fn group_identifiers_randomly<'a>(identifiers: &'a mut [&str]) -> Vec<Vec<&'a str>> {
+    identifiers.shuffle(&mut thread_rng());
+
+    let mut groups = Vec::new();
+    let mut current_pos = 0;
+
+    while current_pos < identifiers.len() {
+        let end_index = thread_rng().gen_range(current_pos..=identifiers.len());
+        let group = identifiers[current_pos..end_index].to_vec();
+        groups.push(group);
+        current_pos = end_index;
+    }
+
+    groups
+}
+
+async_param_test! {
+    #[ignore = "ignore E2E tests by default"]
+    #[walrus_simtest]
+    test_store_quilt -> TestResult : [
+        one_blob: (1),
+        two_blobs: (2),
+        seven_blobs: (10),
+    ]
+}
+/// Tests that a quilt can be stored.
+async fn test_store_quilt(blobs_to_create: u32) -> TestResult {
+    telemetry_subscribers::init_for_testing();
+
+    let test_nodes_config = TestNodesConfig {
+        node_weights: vec![7, 7, 7, 7, 7],
+        ..Default::default()
+    };
+    let test_cluster_builder =
+        test_cluster::E2eTestSetupBuilder::new().with_test_nodes_config(test_nodes_config);
+    let (_sui_cluster_handle, _cluster, client, _) = test_cluster_builder.build().await?;
+    let client = client.as_ref();
+    let blobs = walrus_test_utils::random_data_list(314, blobs_to_create as usize);
+    let encoding_type = DEFAULT_ENCODING;
+    let quilt_store_blobs = blobs
+        .iter()
+        .enumerate()
+        .map(|(i, blob)| QuiltStoreBlob::new(blob, format!("test-blob-{}", i + 1)))
+        .collect::<Vec<_>>();
+
+    // Store the quilt.
+    let quilt_client = client.quilt_client(QuiltClientConfig::new(4, Duration::from_secs(30)));
+    let quilt = quilt_client
+        .construct_quilt::<QuiltVersionV1>(&quilt_store_blobs, encoding_type)
+        .await?;
+    let store_operation_result = quilt_client
+        .reserve_and_store_quilt::<QuiltVersionV1>(
+            &quilt,
+            encoding_type,
+            2,
+            StoreWhen::Always,
+            BlobPersistence::Permanent,
+            PostStoreAction::Keep,
+        )
+        .await?;
+
+    let QuiltStoreResult {
+        blob_store_result,
+        stored_quilt_blobs,
+    } = store_operation_result;
+    let blob_object = match blob_store_result {
+        BlobStoreResult::NewlyCreated { blob_object, .. } => blob_object,
+        _ => panic!("Expected NewlyCreated, got {:?}", blob_store_result),
+    };
+
+    // Read the blobs in the quilt.
+    let id_blob_map = quilt_store_blobs
+        .iter()
+        .map(|b| (b.identifier(), b))
+        .collect::<HashMap<_, _>>();
+
+    let blob_id = blob_object.blob_id;
+    let quilt_metadata = quilt_client.get_quilt_metadata(&blob_id).await?;
+    let QuiltMetadata::V1(metadata_v1) = quilt_metadata;
+    assert_eq!(&metadata_v1.index, quilt.quilt_index()?);
+
+    let mut identifiers = stored_quilt_blobs
+        .iter()
+        .map(|b| b.identifier.as_str())
+        .collect::<Vec<_>>();
+    let groups = group_identifiers_randomly(&mut identifiers);
+
+    tracing::info!(groups = ?groups, "test retrieving quilts by groups");
+
+    for group in groups {
+        let retrieved_quilt_blobs: Vec<QuiltStoreBlobOwned> = quilt_client
+            .get_blobs_by_identifiers(&blob_id, &group)
+            .await?;
+
+        assert_eq!(
+            retrieved_quilt_blobs.len(),
+            group.len(),
+            "Mismatch in number of blobs retrieved from quilt"
+        );
+
+        for retrieved_quilt_blob in &retrieved_quilt_blobs {
+            let original_blob = id_blob_map
+                .get(retrieved_quilt_blob.identifier())
+                .expect("identifier should be present");
+            assert_eq!(&retrieved_quilt_blob, original_blob);
+        }
+    }
+
+    Ok(())
+}
+
 #[ignore = "ignore E2E tests by default"]
 #[walrus_simtest]
 async fn test_blocklist() -> TestResult {

crates/walrus-sdk/src/client.rs

@@ -75,6 +75,7 @@ use self::{
 pub(crate) use crate::utils::{CompletedReasonWeight, WeightedFutures};
 use crate::{
     active_committees::ActiveCommittees,
+    client::quilt_client::{QuiltClient, QuiltClientConfig},
     config::CommunicationLimits,
     error::{ClientError, ClientErrorKind, ClientResult},
     store_when::StoreWhen,
@@ -88,6 +89,7 @@ pub use crate::{
 pub mod client_types;
 pub mod communication;
 pub mod metrics;
+pub mod quilt_client;
 pub mod refresh;
 pub mod resource;
 pub mod responses;
@@ -1445,6 +1447,11 @@ impl<T> Client<T> {
         self
     }
 
+    /// Returns a [`QuiltClient`] for storing and retrieving quilts.
+    pub fn quilt_client(&self, config: QuiltClientConfig) -> QuiltClient<'_, T> {
+        QuiltClient::new(self, config)
+    }
+
     /// Stores the already-encoded metadata and sliver pairs for a blob into Walrus, by sending
     /// sliver pairs to at least 2f+1 shards.
     ///

crates/walrus-sdk/src/client/client_types.rs

@@ -6,11 +6,13 @@
 use std::{fmt::Debug, sync::Arc};
 
 use enum_dispatch::enum_dispatch;
+use serde::{Deserialize, Serialize};
 use sui_types::base_types::ObjectID;
 use tracing::{Level, Span, field};
 use walrus_core::{
     BlobId,
-    encoding::SliverPair,
+    QuiltPatchId,
+    encoding::{SliverPair, quilt_encoding::QuiltPatchInternalIdApi},
     messages::ConfirmationCertificate,
     metadata::{BlobMetadataApi as _, VerifiedBlobMetadataWithId},
 };
@@ -27,6 +29,25 @@ use super::{
 /// The log level for all WalrusStoreBlob spans.
 pub(crate) const BLOB_SPAN_LEVEL: Level = Level::DEBUG;
 
+/// Identifies a stored quilt patch.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StoredQuiltPatch {
+    /// The identifier of the quilt patch.
+    pub identifier: String,
+    /// The quilt patch id.
+    pub quilt_patch_id: String,
+}
+
+impl StoredQuiltPatch {
+    /// Create a new stored quilt patch.
+    pub fn new<T: QuiltPatchInternalIdApi>(blob_id: BlobId, identifier: &str, patch_id: T) -> Self {
+        Self {
+            identifier: identifier.to_string(),
+            quilt_patch_id: QuiltPatchId::new(blob_id, patch_id.to_bytes()).to_string(),
+        }
+    }
+}
+
 /// API for a blob that is being stored to Walrus.
 #[enum_dispatch]
 pub trait WalrusStoreBlobApi<'a, T: Debug + Clone + Send + Sync> {