diff --git a/client/blockchain-service/src/commands.rs b/client/blockchain-service/src/commands.rs index 5cc5dea3c..4650058d2 100644 --- a/client/blockchain-service/src/commands.rs +++ b/client/blockchain-service/src/commands.rs @@ -10,7 +10,8 @@ use sp_api::ApiError; use pallet_file_system_runtime_api::{ IsStorageRequestOpenToVolunteersError, QueryBspConfirmChunksToProveForFileError, - QueryFileEarliestVolunteerTickError, QueryMspConfirmChunksToProveForFileError, + QueryBspsVolunteeredForFileError, QueryFileEarliestVolunteerTickError, + QueryMspConfirmChunksToProveForFileError, }; use pallet_payment_streams_runtime_api::GetUsersWithDebtOverThresholdError; use pallet_proofs_dealer_runtime_api::{ @@ -92,6 +93,11 @@ pub enum BlockchainServiceCommand { msp_id: ProofsDealerProviderId, file_key: Runtime::Hash, }, + #[command(success_type = bool, error_type = QueryBspsVolunteeredForFileError)] + QueryBspVolunteeredForFile { + bsp_id: BackupStorageProviderId, + file_key: Runtime::Hash, + }, #[command(success_type = Vec, error_type = QueryProviderMultiaddressesError)] QueryProviderMultiaddresses { provider_id: ProviderId }, QueueSubmitProofRequest { diff --git a/client/blockchain-service/src/handler.rs b/client/blockchain-service/src/handler.rs index 61ed84a07..c1a73ed5b 100644 --- a/client/blockchain-service/src/handler.rs +++ b/client/blockchain-service/src/handler.rs @@ -17,7 +17,8 @@ use sp_runtime::{traits::Header, SaturatedConversion, Saturating}; use pallet_file_system_runtime_api::{ FileSystemApi, IsStorageRequestOpenToVolunteersError, QueryBspConfirmChunksToProveForFileError, - QueryFileEarliestVolunteerTickError, QueryMspConfirmChunksToProveForFileError, + QueryBspsVolunteeredForFileError, QueryFileEarliestVolunteerTickError, + QueryMspConfirmChunksToProveForFileError, }; use pallet_payment_streams_runtime_api::{GetUsersWithDebtOverThresholdError, PaymentStreamsApi}; use pallet_proofs_dealer_runtime_api::{ @@ -617,6 +618,30 @@ where } } } + BlockchainServiceCommand::QueryBspVolunteeredForFile { + bsp_id, + file_key, + callback, + } => { + let current_block_hash = self.client.info().best_hash; + + let bsps_volunteered = self + .client + .runtime_api() + .query_bsps_volunteered_for_file(current_block_hash, file_key) + .unwrap_or_else(|_| Err(QueryBspsVolunteeredForFileError::InternalError)); + + let volunteered = bsps_volunteered.map(|bsps| bsps.contains(&bsp_id)); + + match callback.send(volunteered) { + Ok(_) => { + trace!(target: LOG_TARGET, "BSP volunteered status sent successfully"); + } + Err(e) => { + error!(target: LOG_TARGET, "Failed to send BSP volunteered status: {:?}", e); + } + } + } BlockchainServiceCommand::QueryProviderMultiaddresses { provider_id, callback, diff --git a/client/src/tasks/bsp_submit_proof.rs b/client/src/tasks/bsp_submit_proof.rs index 9a7555826..49eb0c54d 100644 --- a/client/src/tasks/bsp_submit_proof.rs +++ b/client/src/tasks/bsp_submit_proof.rs @@ -302,9 +302,9 @@ where // - If the proof is outdated. // - If the Forest root of the BSP has changed. Box::pin(Self::should_retry_submit_proof( - cloned_sh_handler.clone(), - cloned_event.clone(), - cloned_forest_root.clone(), + cloned_sh_handler, + cloned_event, + cloned_forest_root, error, )) as Pin + Send>> }; diff --git a/client/src/tasks/bsp_upload_file.rs b/client/src/tasks/bsp_upload_file.rs index d56f87f43..7568cd0f4 100644 --- a/client/src/tasks/bsp_upload_file.rs +++ b/client/src/tasks/bsp_upload_file.rs @@ -1,6 +1,9 @@ use std::{ collections::{HashMap, HashSet}, + future::Future, + pin::Pin, str::FromStr, + sync::Arc, time::Duration, }; @@ -16,13 +19,13 @@ use shc_blockchain_service::{ capacity_manager::CapacityRequestData, commands::{BlockchainServiceCommandInterface, BlockchainServiceCommandInterfaceExt}, events::{NewStorageRequest, ProcessConfirmStoringRequest}, - types::{ConfirmStoringRequest, RetryStrategy, SendExtrinsicOptions}, + types::{ConfirmStoringRequest, RetryStrategy, SendExtrinsicOptions, WatchTransactionError}, }; use shc_common::{ consts::CURRENT_FOREST_KEY, traits::StorageEnableRuntime, types::{ - FileKey, FileKeyWithProof, FileMetadata, HashT, StorageProofsMerkleTrieLayout, + FileKey, FileKeyWithProof, FileMetadata, HashT, ProviderId, StorageProofsMerkleTrieLayout, StorageProviderId, BATCH_CHUNK_FILE_TRANSFER_MAX_SIZE, }, }; @@ -76,7 +79,7 @@ pub struct BspUploadFileTask where NT: ShNodeType, NT::FSH: BspForestStorageHandlerT, - Runtime: StorageEnableRuntime, + Runtime: StorageEnableRuntime + 'static, { storage_hub_handler: StorageHubHandler, file_key_cleanup: Option, @@ -101,7 +104,7 @@ where impl BspUploadFileTask where - NT: ShNodeType, + NT: ShNodeType + 'static, NT::FSH: BspForestStorageHandlerT, Runtime: StorageEnableRuntime, { @@ -428,8 +431,8 @@ where impl BspUploadFileTask where - NT: ShNodeType, - NT::FSH: BspForestStorageHandlerT, + NT: ShNodeType + 'static, + NT::FSH: BspForestStorageHandlerT + 'static, Runtime: StorageEnableRuntime, { async fn handle_new_storage_request_event( @@ -671,12 +674,36 @@ where } .into(); - // Send extrinsic and wait for it to be included in the block. - let result = self + // Clone necessary data for the retry check. + let cloned_sh_handler = Arc::new(self.storage_hub_handler.clone()); + let cloned_own_bsp_id = Arc::new(own_bsp_id.clone()); + let cloned_file_key: Arc = Arc::new(file_key.clone().into()); + + let should_retry = move |error| { + let cloned_sh_handler = Arc::clone(&cloned_sh_handler); + let cloned_own_bsp_id = Arc::clone(&cloned_own_bsp_id); + let cloned_file_key = Arc::clone(&cloned_file_key); + + // Check: + // - If we've already successfully volunteered for the file. + // - If the storage request is no longer open to volunteers. + // Also waits for the tick to be able to volunteer for the file has actually been reached, + // not the tick before the BSP can volunteer for the file. To make sure the chain wasn't + // spammed just before the BSP could volunteer for the file. + Box::pin(Self::should_retry_volunteer( + cloned_sh_handler, + cloned_own_bsp_id, + cloned_file_key, + error, + )) as Pin + Send>> + }; + + // Try to send the volunteer extrinsic + if let Err(e) = self .storage_hub_handler .blockchain - .send_extrinsic( - call.clone().into(), + .submit_extrinsic_with_retry( + call.clone(), SendExtrinsicOptions::new( Duration::from_secs( self.storage_hub_handler @@ -687,57 +714,43 @@ where Some("fileSystem".to_string()), Some("bspVolunteer".to_string()), ), + RetryStrategy::default() + .with_max_retries(self.config.max_try_count) + .with_max_tip(self.config.max_tip.saturated_into()) + .with_should_retry(Some(Box::new(should_retry))), + false, ) - .await? - .watch_for_success(&self.storage_hub_handler.blockchain) - .await; + .await + { + error!(target: LOG_TARGET, "Failed to volunteer for file {:x}: {:?}", file_key, e); + } - if let Err(e) = result { + // Check if the BSP has been registered as a volunteer for the file. + let volunteer_result = self + .storage_hub_handler + .blockchain + .query_bsp_volunteered_for_file(own_bsp_id, file_key.into()) + .await + .map_err(|e| anyhow!("Failed to query BSP volunteered for file: {:?}", e))?; + + // Handle the volunteer result. + if volunteer_result { + info!( + target: LOG_TARGET, + "🍾 BSP successfully volunteered for file {:x}", + file_key + ); + } else { error!( target: LOG_TARGET, - "Failed to volunteer for file {:?}: {:?}", - file_key, - e + "BSP not registered as a volunteer for file {:x}", + file_key ); - - // If the initial call errored out, it could mean the chain was spammed so the tick did not advance. - // Wait until the actual earliest volunteer tick to occur and retry volunteering. - self.storage_hub_handler - .blockchain - .wait_for_tick(earliest_volunteer_tick) - .await?; - - // Send extrinsic and wait for it to be included in the block. - let result = self - .storage_hub_handler - .blockchain - .send_extrinsic( - call, - SendExtrinsicOptions::new( - Duration::from_secs( - self.storage_hub_handler - .provider_config - .blockchain_service - .extrinsic_retry_timeout, - ), - Some("fileSystem".to_string()), - Some("bspVolunteer".to_string()), - ), - ) - .await? - .watch_for_success(&self.storage_hub_handler.blockchain) - .await; - - if let Err(e) = result { - error!( - target: LOG_TARGET, - "Failed to volunteer for file {:?} after retry in volunteer tick: {:?}", - file_key, - e - ); - - self.unvolunteer_file(file_key.into()).await; - } + self.unvolunteer_file(file_key.into()).await; + return Err(anyhow!( + "BSP not registered as a volunteer for file {:x}", + file_key + )); } Ok(()) @@ -750,6 +763,8 @@ where &mut self, event: RemoteUploadRequest, ) -> anyhow::Result { + debug!(target: LOG_TARGET, "Handling remote upload request for file key {:x}", event.file_key); + let file_key = event.file_key.into(); let mut write_file_storage = self.storage_hub_handler.file_storage.write().await; @@ -997,6 +1012,85 @@ where return Ok(true); } + /// Function to determine if a volunteer request should be retried, + /// sending the same request again. + /// + /// This function will return `true` if and only if the following conditions are met: + /// 1. If the storage request is no longer open to volunteers. + /// 2. If we've already successfully volunteered for the file. + /// + /// Also waits for the tick to be able to volunteer for the file has actually been reached, + /// not the tick before the BSP can volunteer for the file. To make sure the chain wasn't + /// spammed just before the BSP could volunteer for the file. + async fn should_retry_volunteer( + sh_handler: Arc>, + bsp_id: Arc>, + file_key: Arc, + _error: WatchTransactionError, + ) -> bool { + // Wait for the tick to be able to volunteer for the file has actually been reached. + let earliest_volunteer_tick = match sh_handler + .blockchain + .query_file_earliest_volunteer_tick(*bsp_id, *file_key) + .await + { + Ok(tick) => tick, + Err(e) => { + error!(target: LOG_TARGET, "Failed to query file earliest volunteer block: {:?}", e); + return false; + } + }; + match sh_handler + .blockchain + .wait_for_tick(earliest_volunteer_tick) + .await + { + Ok(_) => {} + Err(e) => { + error!(target: LOG_TARGET, "Failed to wait for tick: {:?}", e); + return false; + } + } + + // Check if the storage request is no longer open to volunteers. + let can_volunteer = match sh_handler + .blockchain + .is_storage_request_open_to_volunteers(*file_key) + .await + { + Ok(can_volunteer) => can_volunteer, + Err(e) => { + error!(target: LOG_TARGET, "Failed to query file can volunteer: {:?}", e); + return false; + } + }; + + if !can_volunteer { + warn!(target: LOG_TARGET, "Storage request is no longer open to volunteers. Stop retrying."); + return false; + } + + // Check if we've already successfully volunteered for the file. + let volunteered = match sh_handler + .blockchain + .query_bsp_volunteered_for_file(*bsp_id, *file_key) + .await + { + Ok(volunteered) => volunteered, + Err(e) => { + error!(target: LOG_TARGET, "Failed to query file volunteered: {:?}", e); + return false; + } + }; + + if volunteered { + info!(target: LOG_TARGET, "Already successfully volunteered for the file. Stop retrying."); + return false; + } + + return true; + } + async fn unvolunteer_file(&self, file_key: Runtime::Hash) { warn!(target: LOG_TARGET, "Unvolunteering file {:?}", file_key); diff --git a/client/src/tasks/msp_upload_file.rs b/client/src/tasks/msp_upload_file.rs index 397d46efa..7af098bdd 100644 --- a/client/src/tasks/msp_upload_file.rs +++ b/client/src/tasks/msp_upload_file.rs @@ -358,16 +358,51 @@ where .watch_for_success(&self.storage_hub_handler.blockchain) .await?; - // Remove the files that were rejected from the File Storage. + // Log accepted and rejected files, and remove rejected files from File Storage. // Accepted files will be added to the Bucket's Forest Storage by the BlockchainService. for storage_request_msp_bucket_response in storage_request_msp_response { - let mut fs = self.storage_hub_handler.file_storage.write().await; + // Log accepted file keys + if let Some(ref accepted) = storage_request_msp_bucket_response.accept { + let accepted_file_keys: Vec<_> = accepted + .file_keys_and_proofs + .iter() + .map(|fk| fk.file_key) + .collect(); + + if !accepted_file_keys.is_empty() { + info!( + target: LOG_TARGET, + "✅ Accepted {} file(s) for bucket {:?}: {:?}", + accepted_file_keys.len(), + storage_request_msp_bucket_response.bucket_id, + accepted_file_keys + ); + } + } - for RejectedStorageRequest { file_key, .. } in - &storage_request_msp_bucket_response.reject - { - if let Err(e) = fs.delete_file(&file_key) { - error!(target: LOG_TARGET, "Failed to delete file {:?}: {:?}", file_key, e); + // Log and delete rejected file keys + if !storage_request_msp_bucket_response.reject.is_empty() { + let rejected_file_keys: Vec<_> = storage_request_msp_bucket_response + .reject + .iter() + .map(|r| (r.file_key, &r.reason)) + .collect(); + + info!( + target: LOG_TARGET, + "❌ Rejected {} file(s) for bucket {:?}: {:?}", + rejected_file_keys.len(), + storage_request_msp_bucket_response.bucket_id, + rejected_file_keys + ); + + let mut fs = self.storage_hub_handler.file_storage.write().await; + for RejectedStorageRequest { file_key, .. } in + &storage_request_msp_bucket_response.reject + { + if let Err(e) = fs.delete_file(&file_key) { + error!(target: LOG_TARGET, "Failed to delete file {:?}: {:?}", file_key, e); + } } } } diff --git a/pallets/file-system/src/utils.rs b/pallets/file-system/src/utils.rs index 503a50aac..5a7032da9 100644 --- a/pallets/file-system/src/utils.rs +++ b/pallets/file-system/src/utils.rs @@ -146,8 +146,12 @@ where } }; + // Get the current tick number + let current_tick = + ::get_current_tick(); + // Get the threshold for the BSP to be able to volunteer for the storage request. - // The current eligibility value of this storage request for this BSP has to be greater than + // The current eligibility value of this storage request for this BSP has to be greater than or equal to // this for the BSP to be able to volunteer. let bsp_volunteering_threshold = Self::get_volunteer_threshold_of_bsp(&bsp_id, &file_key); @@ -164,26 +168,52 @@ where // Calculate the difference between the BSP's threshold and the current eligibility value. let eligibility_diff = match bsp_volunteering_threshold.checked_sub(&bsp_current_eligibility_value) { - Some(diff) => diff, - None => { - // The BSP's threshold is less than the eligibility current value, which means the BSP is already eligible to volunteer. - let current_tick = - ::get_current_tick(); + Some(diff) if !diff.is_zero() => diff, + _ => { + // The BSP's threshold is less than or equal to the current eligibility value, + // which means the BSP is already eligible to volunteer. return Ok(current_tick); } }; // If the BSP can't volunteer yet, calculate the number of ticks it has to wait for before it can. - let min_ticks_to_wait_to_volunteer = - match eligibility_diff.checked_div(&bsp_eligibility_slope) { - Some(ticks) => max(ticks, T::ThresholdType::one()), - None => { - return Err(QueryFileEarliestVolunteerTickError::ThresholdArithmeticError); + // We use ceiling division to ensure the BSP waits long enough for the threshold to be met. + // Formula: ceil(a / b) = floor(a / b) + (1 if a % b != 0 else 0) + let min_ticks_to_wait_to_volunteer = match eligibility_diff + .checked_div(&bsp_eligibility_slope) + { + Some(quotient) => { + // Check if there's a remainder by verifying if quotient * slope == eligibility_diff + // If not equal, there's a remainder and we need to round up + let has_remainder = match quotient.checked_mul(&bsp_eligibility_slope) { + Some(product) => product != eligibility_diff, + None => { + return Err(QueryFileEarliestVolunteerTickError::ThresholdArithmeticError); + } + }; + + if !has_remainder { + // Exact division, no rounding needed + max(quotient, T::ThresholdType::one()) + } else { + // Round up by adding 1 to the quotient + match quotient.checked_add(&T::ThresholdType::one()) { + Some(result) => max(result, T::ThresholdType::one()), + None => { + return Err( + QueryFileEarliestVolunteerTickError::ThresholdArithmeticError, + ); + } + } } - }; + } + None => { + return Err(QueryFileEarliestVolunteerTickError::ThresholdArithmeticError); + } + }; // Compute the earliest tick number at which the BSP can send the volunteer request. - let earliest_volunteer_tick = storage_request_tick.saturating_add( + let earliest_volunteer_tick = current_tick.saturating_add( T::ThresholdTypeToTickNumber::convert(min_ticks_to_wait_to_volunteer), ); diff --git a/test/suites/integration/bsp/bsp-thresholds.test.ts b/test/suites/integration/bsp/bsp-thresholds.test.ts index ddfb4088a..2933908fa 100644 --- a/test/suites/integration/bsp/bsp-thresholds.test.ts +++ b/test/suites/integration/bsp/bsp-thresholds.test.ts @@ -12,7 +12,11 @@ import { await describeBspNet( "BSPNet: BSP Volunteering Thresholds", - { initialised: false, bspStartingWeight: 100n, networkConfig: "standard" }, + { + initialised: false, + bspStartingWeight: 100n, + networkConfig: "standard" + }, ({ before, it, createUserApi, createBspApi }) => { let userApi: EnrichedBspApi; let bspApi: EnrichedBspApi; @@ -181,297 +185,321 @@ await describeBspNet( }); it("lower reputation can still volunteer and be accepted", async () => { - const basicReplicationTargetRuntimeParameter = { - RuntimeConfig: { - BasicReplicationTarget: [null, 5] - } - }; - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(basicReplicationTargetRuntimeParameter) - ) - ] - }); + let bspDownApi: EnrichedBspApi | undefined; + try { + const basicReplicationTargetRuntimeParameter = { + RuntimeConfig: { + BasicReplicationTarget: [null, 5] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(basicReplicationTargetRuntimeParameter) + ) + ] + }); - const tickToMaximumThresholdRuntimeParameter = { - RuntimeConfig: { - TickRangeToMaximumThreshold: [null, 500] - } - }; - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(tickToMaximumThresholdRuntimeParameter) - ) - ] - }); + const tickToMaximumThresholdRuntimeParameter = { + RuntimeConfig: { + TickRangeToMaximumThreshold: [null, 500] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(tickToMaximumThresholdRuntimeParameter) + ) + ] + }); - const storageRequestTtlRuntimeParameter = { - RuntimeConfig: { - StorageRequestTtl: [null, 550] - } - }; - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(storageRequestTtlRuntimeParameter) + const storageRequestTtlRuntimeParameter = { + RuntimeConfig: { + StorageRequestTtl: [null, 550] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(storageRequestTtlRuntimeParameter) + ) + ] + }); + + // Create a new BSP and onboard with no reputation + const { rpcPort } = await addBsp(userApi, bspDownKey, userApi.accounts.sudo, { + name: "sh-bsp-down", + bspId: ShConsts.BSP_DOWN_ID, + additionalArgs: ["--keystore-path=/keystore/bsp-down"], + bspStartingWeight: 1n + }); + bspDownApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); + + // Wait for it to catch up to the tip of the chain + await userApi.wait.nodeCatchUpToChainTip(bspDownApi); + + const { fileKey } = await userApi.file.createBucketAndSendNewStorageRequest( + "res/whatsup.jpg", + "test/whatsup.jpg", + "bucket-1" + ); + + const lowReputationVolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.BSP_DOWN_ID, + fileKey ) - ] - }); + ).asOk.toNumber(); - // Create a new BSP and onboard with no reputation - const { rpcPort } = await addBsp(userApi, bspDownKey, userApi.accounts.sudo, { - name: "sh-bsp-down", - bspId: ShConsts.BSP_DOWN_ID, - additionalArgs: ["--keystore-path=/keystore/bsp-down"], - bspStartingWeight: 1n - }); - const bspDownApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); + const normalReputationVolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.DUMMY_BSP_ID, + fileKey + ) + ).asOk.toNumber(); - // Wait for it to catch up to the tip of the chain - await userApi.wait.nodeCatchUpToChainTip(bspDownApi); + const currentTick = (await userApi.call.proofsDealerApi.getCurrentTick()).toNumber(); + assert( + currentTick === normalReputationVolunteerTick, + "The BSP with high reputation should be able to volunteer immediately" + ); + assert( + currentTick < lowReputationVolunteerTick, + "The volunteer tick for the low reputation BSP should be in the future" + ); - const { fileKey } = await userApi.file.createBucketAndSendNewStorageRequest( - "res/whatsup.jpg", - "test/whatsup.jpg", - "bucket-1" - ); + // Checking volunteering and confirming for the high reputation BSP + await userApi.wait.bspVolunteer(1); + await bspApi.wait.fileStorageComplete(fileKey); + await userApi.wait.bspStored({ expectedExts: 1 }); - const lowReputationVolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.BSP_DOWN_ID, - fileKey - ) - ).asOk.toNumber(); + // Checking volunteering and confirming for the low reputation BSP + // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X + await userApi.block.skipTo(lowReputationVolunteerTick - 1); - const normalReputationVolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.DUMMY_BSP_ID, - fileKey - ) - ).asOk.toNumber(); + // Wait for the BSP to catch up to the new block height after skipping + await userApi.wait.nodeCatchUpToChainTip(bspDownApi); - const currentBlockNumber = (await userApi.rpc.chain.getHeader()).number.toNumber(); - assert( - currentBlockNumber === normalReputationVolunteerTick, - "The BSP with high reputation should be able to volunteer immediately" - ); - assert( - currentBlockNumber < lowReputationVolunteerTick, - "The volunteer tick for the low reputation BSP should be in the future" - ); + await userApi.wait.bspVolunteer(1); + const matchedEvents = await userApi.assert.eventMany("fileSystem", "AcceptedBspVolunteer"); // T1 - // Checking volunteering and confirming for the high reputation BSP - await userApi.wait.bspVolunteer(1); - await bspApi.wait.fileStorageComplete(fileKey); - await userApi.wait.bspStored({ expectedExts: 1 }); - - // Checking volunteering and confirming for the low reputation BSP - // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X - await userApi.block.skipTo(lowReputationVolunteerTick - 1); - await userApi.wait.bspVolunteer(1); - const matchedEvents = await userApi.assert.eventMany("fileSystem", "AcceptedBspVolunteer"); // T1 - - // Check that it is in fact the BSP with low reputation that just volunteered - const filtered = matchedEvents.filter( - ({ event }) => - (userApi.events.fileSystem.AcceptedBspVolunteer.is(event) && - event.data.bspId.toString()) === ShConsts.BSP_DOWN_ID - ); + // Check that it is in fact the BSP with low reputation that just volunteered + const filtered = matchedEvents.filter( + ({ event }) => + (userApi.events.fileSystem.AcceptedBspVolunteer.is(event) && + event.data.bspId.toString()) === ShConsts.BSP_DOWN_ID + ); - assert( - filtered.length === 1, - "Zero reputation BSP should be able to volunteer and be accepted" - ); - await bspDownApi.disconnect(); - await userApi.docker.stopContainer("sh-bsp-down"); + assert( + filtered.length === 1, + "Zero reputation BSP should be able to volunteer and be accepted" + ); + } finally { + if (bspDownApi) { + await bspDownApi.disconnect(); + } + await userApi.docker.stopContainer("sh-bsp-down"); + } }); it("BSP two eventually volunteers after threshold curve is met", async () => { - const basicReplicationTargetRuntimeParameter = { - RuntimeConfig: { - BasicReplicationTarget: [null, 2] - } - }; - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(basicReplicationTargetRuntimeParameter) - ) - ] - }); + let bspTwoApi: EnrichedBspApi | undefined; + try { + const basicReplicationTargetRuntimeParameter = { + RuntimeConfig: { + BasicReplicationTarget: [null, 2] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(basicReplicationTargetRuntimeParameter) + ) + ] + }); - const tickToMaximumThresholdRuntimeParameter = { - RuntimeConfig: { - TickRangeToMaximumThreshold: [null, 20] - } - }; - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(tickToMaximumThresholdRuntimeParameter) - ) - ] - }); + const tickToMaximumThresholdRuntimeParameter = { + RuntimeConfig: { + TickRangeToMaximumThreshold: [null, 20] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(tickToMaximumThresholdRuntimeParameter) + ) + ] + }); - // Add the second BSP - const { rpcPort } = await addBsp(userApi, bspTwoKey, userApi.accounts.sudo, { - name: "sh-bsp-two", - bspId: ShConsts.BSP_TWO_ID, - additionalArgs: ["--keystore-path=/keystore/bsp-two"] - }); - const bspTwoApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); + // Add the second BSP + const { rpcPort } = await addBsp(userApi, bspTwoKey, userApi.accounts.sudo, { + name: "sh-bsp-two", + bspId: ShConsts.BSP_TWO_ID, + additionalArgs: ["--keystore-path=/keystore/bsp-two"] + }); + bspTwoApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); - // Wait for it to catch up to the tip of the chain - await userApi.wait.nodeCatchUpToChainTip(bspTwoApi); + // Wait for it to catch up to the tip of the chain + await userApi.wait.nodeCatchUpToChainTip(bspTwoApi); - // Create a new storage request - const { fileKey } = await userApi.file.createBucketAndSendNewStorageRequest( - "res/cloud.jpg", - "test/cloud.jpg", - "bucket-2" - ); + // Create a new storage request + const { fileKey } = await userApi.file.createBucketAndSendNewStorageRequest( + "res/cloud.jpg", + "test/cloud.jpg", + "bucket-2" + ); - // Check where the BSPs would be allowed to volunteer for it - const bsp1VolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.DUMMY_BSP_ID, - fileKey - ) - ).asOk.toNumber(); - const bsp2VolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.BSP_TWO_ID, - fileKey - ) - ).asOk.toNumber(); + // Check where the BSPs would be allowed to volunteer for it + const bsp1VolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.DUMMY_BSP_ID, + fileKey + ) + ).asOk.toNumber(); + const bsp2VolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.BSP_TWO_ID, + fileKey + ) + ).asOk.toNumber(); - assert(bsp1VolunteerTick < bsp2VolunteerTick, "BSP one should be able to volunteer first"); - const currentBlockNumber = (await userApi.rpc.chain.getHeader()).number.toNumber(); - assert( - currentBlockNumber === bsp1VolunteerTick, - "BSP one should be able to volunteer immediately" - ); + assert(bsp1VolunteerTick < bsp2VolunteerTick, "BSP one should be able to volunteer first"); + const currentBlockNumber = (await userApi.rpc.chain.getHeader()).number.toNumber(); + assert( + currentBlockNumber === bsp1VolunteerTick, + "BSP one should be able to volunteer immediately" + ); - await userApi.wait.bspVolunteer(1); - await bspApi.wait.fileStorageComplete(fileKey); - await userApi.wait.bspStored({ expectedExts: 1 }); + await userApi.wait.bspVolunteer(1); + await bspApi.wait.fileStorageComplete(fileKey); + await userApi.wait.bspStored({ expectedExts: 1 }); - // Then wait for the second BSP to volunteer and confirm storing the file - // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X - await userApi.block.skipTo(bsp2VolunteerTick - 1); + // Then wait for the second BSP to volunteer and confirm storing the file + // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X + await userApi.block.skipTo(bsp2VolunteerTick - 1); - await userApi.wait.bspVolunteer(1); - await bspTwoApi.wait.fileStorageComplete(fileKey); - await userApi.wait.bspStored({ expectedExts: 1 }); + // Wait for BSP two to catch up to the new block height after skipping + await userApi.wait.nodeCatchUpToChainTip(bspTwoApi); - await bspTwoApi.disconnect(); - await userApi.docker.stopContainer("sh-bsp-two"); + await userApi.wait.bspVolunteer(1); + await bspTwoApi.wait.fileStorageComplete(fileKey); + await userApi.wait.bspStored({ expectedExts: 1 }); + } finally { + if (bspTwoApi) { + await bspTwoApi.disconnect(); + } + await userApi.docker.stopContainer("sh-bsp-two"); + } }); it("BSP with reputation is prioritised", async () => { - // Add a new, high reputation BSP - const { rpcPort } = await addBsp(userApi, bspThreeKey, userApi.accounts.sudo, { - name: "sh-bsp-three", - bspId: ShConsts.BSP_THREE_ID, - additionalArgs: ["--keystore-path=/keystore/bsp-three"], - bspStartingWeight: 800_000_000n - }); - const bspThreeApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); + let bspThreeApi: EnrichedBspApi | undefined; + try { + // Add a new, high reputation BSP + const { rpcPort } = await addBsp(userApi, bspThreeKey, userApi.accounts.sudo, { + name: "sh-bsp-three", + bspId: ShConsts.BSP_THREE_ID, + additionalArgs: ["--keystore-path=/keystore/bsp-three"], + bspStartingWeight: 800_000_000n + }); + bspThreeApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); - // Wait for it to catch up to the top of the chain - await userApi.wait.nodeCatchUpToChainTip(bspThreeApi); + // Wait for it to catch up to the top of the chain + await userApi.wait.nodeCatchUpToChainTip(bspThreeApi); - // Set max replication target and tick to maximum threshold to small numbers - const maxReplicationTargetRuntimeParameter = { - RuntimeConfig: { - MaxReplicationTarget: [null, 5] - } - }; - // In order to test the reputation prioritisation, we need to set the tick to maximum - // threshold to a high enough number such that - // highReputationBspVolunteerTick - initialBspVolunteerTick > 2 (not 1!). - const tickRangeToMaximumThresholdRuntimeParameter = { - RuntimeConfig: { - TickRangeToMaximumThreshold: [null, 9001] - } - }; - const storageRequestTtlRuntimeParameter = { - RuntimeConfig: { - StorageRequestTtl: [null, 110] - } - }; - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(maxReplicationTargetRuntimeParameter) - ) - ] - }); - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(tickRangeToMaximumThresholdRuntimeParameter) + // Set max replication target and tick to maximum threshold to small numbers + const maxReplicationTargetRuntimeParameter = { + RuntimeConfig: { + MaxReplicationTarget: [null, 5] + } + }; + // In order to test the reputation prioritisation, we need to set the tick to maximum + // threshold to a high enough number such that + // highReputationBspVolunteerTick - initialBspVolunteerTick > 2 (not 1!). + const tickRangeToMaximumThresholdRuntimeParameter = { + RuntimeConfig: { + TickRangeToMaximumThreshold: [null, 9001] + } + }; + const storageRequestTtlRuntimeParameter = { + RuntimeConfig: { + StorageRequestTtl: [null, 110] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(maxReplicationTargetRuntimeParameter) + ) + ] + }); + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(tickRangeToMaximumThresholdRuntimeParameter) + ) + ] + }); + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(storageRequestTtlRuntimeParameter) + ) + ] + }); + + // Create a new storage request + const { fileKey } = await userApi.file.createBucketAndSendNewStorageRequest( + "res/adolphus.jpg", + "test/adolphus.jpg", + "bucket-4" + ); // T0 + + // Query the earliest volunteer tick for the dummy BSP and the new BSP + const initialBspVolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.DUMMY_BSP_ID, + fileKey ) - ] - }); - await userApi.block.seal({ - calls: [ - userApi.tx.sudo.sudo( - userApi.tx.parameters.setParameter(storageRequestTtlRuntimeParameter) + ).asOk.toNumber(); + const highReputationBspVolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.BSP_THREE_ID, + fileKey ) - ] - }); - - // Create a new storage request - const { fileKey } = await userApi.file.createBucketAndSendNewStorageRequest( - "res/adolphus.jpg", - "test/adolphus.jpg", - "bucket-4" - ); // T0 - - // Query the earliest volunteer tick for the dummy BSP and the new BSP - const initialBspVolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.DUMMY_BSP_ID, - fileKey - ) - ).asOk.toNumber(); - const highReputationBspVolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.BSP_THREE_ID, - fileKey - ) - ).asOk.toNumber(); + ).asOk.toNumber(); - // Ensure that the new BSP should be able to volunteer first - assert( - highReputationBspVolunteerTick < initialBspVolunteerTick, - "New BSP should be able to volunteer first" - ); + // Ensure that the new BSP should be able to volunteer first + assert( + highReputationBspVolunteerTick < initialBspVolunteerTick, + "New BSP should be able to volunteer first" + ); - // Advance to the tick where the new BSP can volunteer - const currentBlockNumber = (await userApi.rpc.chain.getHeader()).number.toNumber(); - assert( - currentBlockNumber === highReputationBspVolunteerTick, - "BSP with high reputation should be able to volunteer immediately" - ); + // Advance to the tick where the new BSP can volunteer + const currentBlockNumber = (await userApi.rpc.chain.getHeader()).number.toNumber(); + assert( + currentBlockNumber === highReputationBspVolunteerTick, + "BSP with high reputation should be able to volunteer immediately" + ); - // Wait until the new BSP volunteers - await userApi.wait.bspVolunteer(1); - const matchedEvents = await userApi.assert.eventMany("fileSystem", "AcceptedBspVolunteer"); // T1 + // Wait until the new BSP volunteers + await userApi.wait.bspVolunteer(1); + const matchedEvents = await userApi.assert.eventMany("fileSystem", "AcceptedBspVolunteer"); // T1 - const filtered = matchedEvents.filter( - ({ event }) => - (userApi.events.fileSystem.AcceptedBspVolunteer.is(event) && - event.data.bspId.toString()) === ShConsts.BSP_THREE_ID - ); + const filtered = matchedEvents.filter( + ({ event }) => + (userApi.events.fileSystem.AcceptedBspVolunteer.is(event) && + event.data.bspId.toString()) === ShConsts.BSP_THREE_ID + ); - // Verify that the BSP with reputation is prioritised over the lower reputation BSPs - assert(filtered.length === 1, "BSP with reputation should be prioritised"); - await bspThreeApi.disconnect(); - await userApi.docker.stopContainer("sh-bsp-three"); + // Verify that the BSP with reputation is prioritised over the lower reputation BSPs + assert(filtered.length === 1, "BSP with reputation should be prioritised"); + } finally { + if (bspThreeApi) { + await bspThreeApi.disconnect(); + } + await userApi.docker.stopContainer("sh-bsp-three"); + } }); it( diff --git a/test/suites/integration/bsp/maintenance-mode.test.ts b/test/suites/integration/bsp/maintenance-mode.test.ts index ed7c423b4..1b9073e4b 100644 --- a/test/suites/integration/bsp/maintenance-mode.test.ts +++ b/test/suites/integration/bsp/maintenance-mode.test.ts @@ -3,7 +3,7 @@ import { bspTwoKey, describeBspNet, type EnrichedBspApi, ShConsts } from "../../ await describeBspNet( "BSPNet: Maintenance Mode Test", - ({ before, it, createUserApi, createApi }) => { + ({ before, it, createUserApi, createApi, after }) => { let userApi: EnrichedBspApi; let maintenanceBspApi: EnrichedBspApi; @@ -25,6 +25,11 @@ await describeBspNet( }); }); + after(async () => { + await maintenanceBspApi?.disconnect(); + await userApi?.docker.stopContainer("sh-bsp-maintenance"); + }); + it("BSP in maintenance mode does not execute actions after block imports", async () => { // Onboard a BSP in maintenance mode const { rpcPort: maintenanceBspRpcPort } = await userApi.docker.onboardBsp({ @@ -83,10 +88,6 @@ await describeBspNet( // The specific result doesn't matter - what matters is that the call worked and didn't throw strictEqual(result !== undefined, true, "RPC calls should still work in maintenance mode"); - - // Disconnect the maintenance BSP - await userApi.docker.stopContainer("sh-bsp-maintenance"); - await maintenanceBspApi.disconnect(); }); } ); diff --git a/test/suites/integration/bsp/single-volunteer.test.ts b/test/suites/integration/bsp/single-volunteer.test.ts index 00c978e8c..498ae842c 100644 --- a/test/suites/integration/bsp/single-volunteer.test.ts +++ b/test/suites/integration/bsp/single-volunteer.test.ts @@ -2,6 +2,7 @@ import assert, { notEqual, strictEqual } from "node:assert"; import { u8aToHex } from "@polkadot/util"; import { decodeAddress } from "@polkadot/util-crypto"; import { + bspTwoKey, describeBspNet, type EnrichedBspApi, type FileMetadata, @@ -197,6 +198,159 @@ await describeBspNet( } ); +await describeBspNet( + "BSPNet: Initial volunteer tick is different for different BSPs and stays constant over time", + { + initialised: false, + bspStartingWeight: 1n + }, + ({ before, createBspApi, it, createUserApi }) => { + let userApi: EnrichedBspApi; + let bspApi: EnrichedBspApi; + + const source = "res/adolphus.jpg"; + const destination = "test/adolphus.jpg"; + const bucketName = "initial-volunteer-tick-test"; + + let fileMetadata: FileMetadata; + let bspId: string; + let bsp2Id: string; + + before(async () => { + userApi = await createUserApi(); + bspApi = await createBspApi(); + + // Set TickRangeToMaximumThreshold to 1000 to ensure BSPs need to wait + const tickToMaximumThresholdRuntimeParameter = { + RuntimeConfig: { + TickRangeToMaximumThreshold: [null, 1000] + } + }; + await userApi.block.seal({ + calls: [ + userApi.tx.sudo.sudo( + userApi.tx.parameters.setParameter(tickToMaximumThresholdRuntimeParameter) + ) + ] + }); + + // Get the BSP IDs for later use + bspId = userApi.shConsts.DUMMY_BSP_ID; + bsp2Id = userApi.shConsts.BSP_TWO_ID; + + // Add the second BSP with weight=1 + await userApi.docker.onboardBsp({ + bspSigner: bspTwoKey, + name: "sh-bsp-two", + bspId: bsp2Id, + additionalArgs: ["--keystore-path=/keystore/bsp-two"], + waitForIdle: true, + bspStartingWeight: 1n + }); + }); + + it("Network launches and can be queried", async () => { + const userNodePeerId = await userApi.rpc.system.localPeerId(); + strictEqual(userNodePeerId.toString(), userApi.shConsts.NODE_INFOS.user.expectedPeerId); + + const bspNodePeerId = await bspApi.rpc.system.localPeerId(); + strictEqual(bspNodePeerId.toString(), userApi.shConsts.NODE_INFOS.bsp.expectedPeerId); + }); + + it("Earliest volunteer tick is stable and deterministic for both BSPs", async () => { + // Stop the BSPs so they don't volunteer for the file + await userApi.docker.stopContainer("sh-bsp-1"); + await userApi.docker.stopContainer("sh-bsp-two"); + + // Create a new bucket and issue a storage request with replication target = 1 + fileMetadata = await userApi.file.createBucketAndSendNewStorageRequest( + source, + destination, + bucketName, + null, + null, + null, + 1 + ); + + // Get the current tick for reference + const initialTick = (await userApi.call.proofsDealerApi.getCurrentTick()).toNumber(); + + // Query the earliest volunteer tick for both BSPs + const bsp1EarliestTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick(bspId, fileMetadata.fileKey) + ).asOk.toNumber(); + const bsp2EarliestTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + bsp2Id, + fileMetadata.fileKey + ) + ).asOk.toNumber(); + + // The two BSPs should have different thresholds so they should have different earliest volunteer ticks. + // We know that the BSP 2 can't volunteer immediately because the BSP IDs and the file key are + // deterministic in this test. + notEqual( + bsp1EarliestTick, + bsp2EarliestTick, + "Different BSPs should have different earliest volunteer ticks (based on their thresholds)" + ); + + // The BSP 1 should be able to volunteer immediately. Again, we know this is the case because + // the BSP IDs and the file key are deterministic in this test. + strictEqual(bsp1EarliestTick, initialTick, "BSP1 should be able to volunteer immediately"); + + // Seal a few blocks and query again - values should remain stable + await userApi.block.seal(); + await userApi.block.seal(); + await userApi.block.seal(); + + // Get the current tick after advancing time + const tickAfterAdvance = (await userApi.call.proofsDealerApi.getCurrentTick()).toNumber(); + + // Query both BSPs after advancing time + // The earliest volunteer tick should remain stable (except if the BSP was already eligible or + // became eligible in the meantime, in which case it should have increased to the current tick) + const bsp1EarliestTickAfterAdvance = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick(bspId, fileMetadata.fileKey) + ).asOk.toNumber(); + const bsp2EarliestTickAfterAdvance = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + bsp2Id, + fileMetadata.fileKey + ) + ).asOk.toNumber(); + + // Check if BSPs are now eligible (current tick >= their earliest tick) + if (tickAfterAdvance >= bsp1EarliestTick) { + assert( + bsp1EarliestTickAfterAdvance === tickAfterAdvance, + "If the BSP is eligible, its earliest volunteer tick should be the same as the current tick" + ); + } else { + strictEqual( + bsp1EarliestTickAfterAdvance, + bsp1EarliestTick, + "If the BSP is not yet eligible, its earliest volunteer tick should remain stable" + ); + } + + if (tickAfterAdvance >= bsp2EarliestTick) { + assert( + bsp2EarliestTickAfterAdvance === tickAfterAdvance, + "If the BSP is eligible, its earliest volunteer tick should be the same as the current tick" + ); + } else { + strictEqual( + bsp2EarliestTickAfterAdvance, + bsp2EarliestTick, + "If the BSP is not yet eligible, its earliest volunteer tick should remain stable" + ); + } + }); + } +); + await describeBspNet( "BSPNet: Single BSP multi-volunteers", { initialised: false }, diff --git a/test/suites/integration/bsp/storage-capacity.test.ts b/test/suites/integration/bsp/storage-capacity.test.ts index f3953fc1b..49a2a9a1f 100644 --- a/test/suites/integration/bsp/storage-capacity.test.ts +++ b/test/suites/integration/bsp/storage-capacity.test.ts @@ -328,102 +328,106 @@ await describeBspNet("BSPNet: Change capacity tests.", ({ before, it, createUser }); it("BSP does not increase its capacity over its configured maximum (and skips volunteering if that would be needed).", async () => { - // Max storage capacity such that the BSP can store one of the files we will request but no more. - const MAX_STORAGE_CAPACITY = 416600; - - // Add a second BSP with the configured maximum storage capacity limit. - const { rpcPort } = await addBsp(userApi, bspTwoKey, userApi.accounts.sudo, { - name: "sh-bsp-two", - bspId: ShConsts.BSP_TWO_ID, - maxStorageCapacity: MAX_STORAGE_CAPACITY, - initialCapacity: BigInt(MAX_STORAGE_CAPACITY), - additionalArgs: ["--keystore-path=/keystore/bsp-two"] - }); - await userApi.assert.eventPresent("providers", "BspSignUpSuccess"); - - // Wait until the new BSP catches up to the chain tip. - const bspTwoApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); - await userApi.wait.nodeCatchUpToChainTip(bspTwoApi); - - // Stop the other BSP so it doesn't volunteer for the files. - await userApi.docker.pauseContainer("storage-hub-sh-bsp-1"); - - // Issue the first storage request. The new BSP should have enough capacity to volunteer for it. - const source1 = "res/cloud.jpg"; - const location1 = "test/cloud.jpg"; - const bucketName1 = "kek1"; - const fileMetadata = await userApi.file.createBucketAndSendNewStorageRequest( - source1, - location1, - bucketName1 - ); + let bspTwoApi: EnrichedBspApi | undefined; + try { + // Max storage capacity such that the BSP can store one of the files we will request but no more. + const MAX_STORAGE_CAPACITY = 416600; + + // Add a second BSP with the configured maximum storage capacity limit. + const { rpcPort } = await addBsp(userApi, bspTwoKey, userApi.accounts.sudo, { + name: "sh-bsp-two", + bspId: ShConsts.BSP_TWO_ID, + maxStorageCapacity: MAX_STORAGE_CAPACITY, + initialCapacity: BigInt(MAX_STORAGE_CAPACITY), + additionalArgs: ["--keystore-path=/keystore/bsp-two"] + }); + await userApi.assert.eventPresent("providers", "BspSignUpSuccess"); + + // Wait until the new BSP catches up to the chain tip. + bspTwoApi = await BspNetTestApi.create(`ws://127.0.0.1:${rpcPort}`); + await userApi.wait.nodeCatchUpToChainTip(bspTwoApi); + + // Stop the other BSP so it doesn't volunteer for the files. + await userApi.docker.pauseContainer("storage-hub-sh-bsp-1"); + + // Issue the first storage request. The new BSP should have enough capacity to volunteer for it. + const source1 = "res/cloud.jpg"; + const location1 = "test/cloud.jpg"; + const bucketName1 = "kek1"; + const fileMetadata = await userApi.file.createBucketAndSendNewStorageRequest( + source1, + location1, + bucketName1 + ); - // Check at which tick the new BSP can volunteer for the file. - // Note: since we set up the network to have instant acceptance, the new BSP should be able to volunteer immediately - // but we still check to be sure. - const bspVolunteerTick = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.BSP_TWO_ID, - fileMetadata.fileKey - ) - ).asOk.toNumber(); + // Check at which tick the new BSP can volunteer for the file. + // Note: since we set up the network to have instant acceptance, the new BSP should be able to volunteer immediately + // but we still check to be sure. + const bspVolunteerTick = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.BSP_TWO_ID, + fileMetadata.fileKey + ) + ).asOk.toNumber(); - // If the BSP can't volunteer yet, skips blocks until it can. - if ((await userApi.rpc.chain.getHeader()).number.toNumber() < bspVolunteerTick - 1) { - // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X - await userApi.block.skipTo(bspVolunteerTick - 1); - } + // If the BSP can't volunteer yet, skips blocks until it can. + if ((await userApi.rpc.chain.getHeader()).number.toNumber() < bspVolunteerTick - 1) { + // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X + await userApi.block.skipTo(bspVolunteerTick - 1); + } - // Wait until the BSP volunteers for the file. - await userApi.wait.bspVolunteer(1); + // Wait until the BSP volunteers for the file. + await userApi.wait.bspVolunteer(1); - // Wait until the BSP confirms storing the file. - const bspTwpAddress = userApi.createType("Address", bspTwoKey.address); - await userApi.wait.bspStored({ expectedExts: 1, bspAccount: bspTwpAddress }); + // Wait until the BSP confirms storing the file. + const bspTwpAddress = userApi.createType("Address", bspTwoKey.address); + await userApi.wait.bspStored({ expectedExts: 1, bspAccount: bspTwpAddress }); + + // Issue the second storage request. The BSP shouldn't be able to volunteer for this one since + // it would have to increase its capacity over its configured maximum. + const source2 = "res/adolphus.jpg"; + const location2 = "test/adolphus.jpg"; + const bucketName2 = "kek2"; + const fileMetadata2 = await userApi.file.createBucketAndSendNewStorageRequest( + source2, + location2, + bucketName2 + ); - // Issue the second storage request. The BSP shouldn't be able to volunteer for this one since - // it would have to increase its capacity over its configured maximum. - const source2 = "res/adolphus.jpg"; - const location2 = "test/adolphus.jpg"; - const bucketName2 = "kek2"; - const fileMetadata2 = await userApi.file.createBucketAndSendNewStorageRequest( - source2, - location2, - bucketName2 - ); + // Check at which tick the new BSP can volunteer for the file. + // Note: since we set up the network to have instant acceptance, the new BSP should be able to volunteer immediately + // but we still check to be sure. + const bspVolunteerTick2 = ( + await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( + ShConsts.BSP_TWO_ID, + fileMetadata2.fileKey + ) + ).asOk.toNumber(); - // Check at which tick the new BSP can volunteer for the file. - // Note: since we set up the network to have instant acceptance, the new BSP should be able to volunteer immediately - // but we still check to be sure. - const bspVolunteerTick2 = ( - await userApi.call.fileSystemApi.queryEarliestFileVolunteerTick( - ShConsts.BSP_TWO_ID, - fileMetadata2.fileKey - ) - ).asOk.toNumber(); + // If the BSP can't volunteer yet, skips blocks until it can. + if ((await userApi.rpc.chain.getHeader()).number.toNumber() < bspVolunteerTick2) { + // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X + await userApi.block.skipTo(bspVolunteerTick2 - 1); + } - // If the BSP can't volunteer yet, skips blocks until it can. - if ((await userApi.rpc.chain.getHeader()).number.toNumber() < bspVolunteerTick2) { - // If a BSP can volunteer in tick X, it sends the extrinsic once it imports block with tick X - 1, so it gets included directly in tick X - await userApi.block.skipTo(bspVolunteerTick2 - 1); + // The BSP should not volunteer for the second file. To check this we check that the wait for + // the BSP volunteer times out and throws. + assert.rejects(userApi.wait.bspVolunteer(1)); + + // Check that the BSP's capacity used is equal to the first file's size + const bspTwo = ( + await userApi.query.providers.backupStorageProviders(ShConsts.BSP_TWO_ID) + ).unwrap(); + assert.equal( + bspTwo.capacityUsed.toNumber(), + fileMetadata.fileSize, + "Used capacity is still equal to the first file's size" + ); + } finally { + if (bspTwoApi) { + await bspTwoApi.disconnect(); + } + await userApi.docker.stopContainer("sh-bsp-two"); } - - // The BSP should not volunteer for the second file. To check this we check that the wait for - // the BSP volunteer times out and throws. - assert.rejects(userApi.wait.bspVolunteer(1)); - - // Check that the BSP's capacity used is equal to the first file's size - const bspTwo = ( - await userApi.query.providers.backupStorageProviders(ShConsts.BSP_TWO_ID) - ).unwrap(); - assert.equal( - bspTwo.capacityUsed.toNumber(), - fileMetadata.fileSize, - "Used capacity is still equal to the first file's size" - ); - - // Disconnect and stop the new BSP. - await userApi.docker.stopContainer("sh-bsp-two"); - await bspTwoApi.disconnect(); }); }); diff --git a/test/suites/integration/bsp/transaction-manager.test.ts b/test/suites/integration/bsp/transaction-manager.test.ts index 7e580bfd1..29a04a222 100644 --- a/test/suites/integration/bsp/transaction-manager.test.ts +++ b/test/suites/integration/bsp/transaction-manager.test.ts @@ -356,34 +356,41 @@ await describeBspNet( timeout: 10000 }); - // Wait for the BSP to retry submitting the volunteer (automatic retry mechanism) - await userApi.wait.bspVolunteerInTxPool(1); - await bspApi.wait.bspVolunteerInTxPool(1); - - // Get the retry volunteer transaction hash - const retryVolunteerExtrinsics = await userApi.assert.extrinsicPresent({ - module: "fileSystem", - method: "bspVolunteer", - checkTxPool: true, - assertLength: 1 - }); - const txPoolRetry = await userApi.rpc.author.pendingExtrinsics(); - const retryVolunteerHash = txPoolRetry[retryVolunteerExtrinsics[0].extIndex].hash.toString(); - const retryVolunteerNonce = - txPoolRetry[retryVolunteerExtrinsics[0].extIndex].nonce.toNumber(); - - // Verify the retry uses the same nonce (gap filling) - strictEqual( - retryVolunteerNonce, - volunteerNonce, - "Retry volunteer should use the same nonce to fill the gap" - ); - - // Drop the retry volunteer transaction to exhaust the retry mechanism - await bspApi.node.dropTxn(retryVolunteerHash as `0x${string}`); - await userApi.node.dropTxn(retryVolunteerHash as `0x${string}`); + // The BSP will retry submitting the volunteer up to max_try_count times (default: 3) + // We need to drop all retry attempts to exhaust the retry mechanism + const maxTryCount = 3; // Default value from BspUploadFileConfig + + for (let retryAttempt = 0; retryAttempt < maxTryCount; retryAttempt++) { + // Wait for the BSP to retry submitting the volunteer (automatic retry mechanism) + await userApi.wait.bspVolunteerInTxPool(1); + await bspApi.wait.bspVolunteerInTxPool(1); + + // Get the retry volunteer transaction hash + const retryVolunteerExtrinsics = await userApi.assert.extrinsicPresent({ + module: "fileSystem", + method: "bspVolunteer", + checkTxPool: true, + assertLength: 1 + }); + const txPoolRetry = await userApi.rpc.author.pendingExtrinsics(); + const retryVolunteerHash = + txPoolRetry[retryVolunteerExtrinsics[0].extIndex].hash.toString(); + const retryVolunteerNonce = + txPoolRetry[retryVolunteerExtrinsics[0].extIndex].nonce.toNumber(); + + // Verify the retry uses the same nonce (gap filling) + strictEqual( + retryVolunteerNonce, + volunteerNonce, + `Retry volunteer attempt ${retryAttempt + 1} should use the same nonce to fill the gap` + ); + + // Drop the retry volunteer transaction + await bspApi.node.dropTxn(retryVolunteerHash as `0x${string}`); + await userApi.node.dropTxn(retryVolunteerHash as `0x${string}`); + } - // Verify the retry volunteer was dropped + // Verify all retry volunteer extrinsics were dropped await userApi.assert.extrinsicPresent({ module: "fileSystem", method: "bspVolunteer",