|
| 1 | +use backoff::future::retry; |
| 2 | +use backoff::ExponentialBackoff; |
| 3 | +use futures::stream::FuturesUnordered; |
1 | 4 | use futures::StreamExt; |
2 | 5 | use parity_scale_codec::Encode; |
3 | 6 | use std::collections::BTreeSet; |
4 | 7 | use std::error::Error; |
| 8 | +use std::future::Future; |
| 9 | +use std::pin::Pin; |
5 | 10 | use std::sync::atomic::{AtomicBool, Ordering}; |
6 | 11 | use std::sync::Arc; |
7 | 12 | use std::time::Duration; |
8 | 13 | use subspace_core_primitives::{PieceIndex, PieceIndexHash}; |
9 | 14 | use subspace_networking::utils::multihash::MultihashCode; |
10 | 15 | use subspace_networking::{Node, ToMultihash}; |
11 | | -use tokio::time::sleep; |
12 | | -use tracing::{debug, error, trace}; |
| 16 | +use tokio::sync::Semaphore; |
| 17 | +use tokio::time::error::Elapsed; |
| 18 | +use tokio::time::timeout; |
| 19 | +use tracing::{debug, error, info, trace}; |
13 | 20 |
|
14 | | -/// Defines a duration between piece publishing calls. |
15 | | -const PUBLISH_PIECE_BY_SECTOR_WAITING_DURATION_IN_SECS: u64 = 1; |
| 21 | +/// Max time allocated for putting piece from DSN before attempt is considered to fail |
| 22 | +const PUT_PIECE_TIMEOUT: Duration = Duration::from_secs(5); |
| 23 | +/// Defines initial duration between put_piece calls. |
| 24 | +const PUT_PIECE_INITIAL_INTERVAL: Duration = Duration::from_secs(1); |
| 25 | +/// Defines max duration between put_piece calls. |
| 26 | +const PUT_PIECE_MAX_INTERVAL: Duration = Duration::from_secs(5); |
16 | 27 |
|
17 | 28 | // Piece-by-sector DSN publishing helper. |
18 | 29 | #[derive(Clone)] |
@@ -42,43 +53,104 @@ impl PieceSectorPublisher { |
42 | 53 | // Publishes pieces-by-sector to DSN in bulk. Supports cancellation. |
43 | 54 | pub(crate) async fn publish_pieces( |
44 | 55 | &self, |
45 | | - pieces_indexes: Vec<PieceIndex>, |
| 56 | + piece_indexes: Vec<PieceIndex>, |
| 57 | + piece_publisher_batch_size: usize, |
46 | 58 | ) -> Result<(), Box<dyn Error + Send + Sync + 'static>> { |
47 | | - for piece_index in pieces_indexes { |
48 | | - 'attempts: loop { |
49 | | - self.check_cancellation()?; |
50 | | - |
51 | | - let key = PieceIndexHash::from_index(piece_index) |
52 | | - .to_multihash_by_code(MultihashCode::Sector); |
53 | | - |
54 | | - // TODO: rework to piece announcing (pull-model) after fixing |
55 | | - // https://github.com/libp2p/rust-libp2p/issues/3048 |
56 | | - let set = BTreeSet::from_iter(vec![self.dsn_node.id().to_bytes()]); |
57 | | - |
58 | | - let result = self.dsn_node.put_value(key, set.encode()).await; |
59 | | - |
60 | | - match result { |
61 | | - Ok(mut stream) => { |
62 | | - if stream.next().await.is_some() { |
63 | | - trace!(%piece_index, ?key, "Piece publishing for a sector succeeded"); |
64 | | - break 'attempts; |
65 | | - } else { |
66 | | - trace!(%piece_index, ?key, "Piece publishing for a sector failed"); |
67 | | - } |
68 | | - } |
69 | | - Err(error) => { |
70 | | - error!(?error, %piece_index, ?key, "Piece publishing for a sector returned an error"); |
71 | | - |
72 | | - // pause before retrying |
73 | | - sleep(Duration::from_secs( |
74 | | - PUBLISH_PIECE_BY_SECTOR_WAITING_DURATION_IN_SECS, |
75 | | - )) |
76 | | - .await; |
77 | | - } |
78 | | - } |
79 | | - } |
| 59 | + let semaphore = Arc::new(Semaphore::new(piece_publisher_batch_size)); |
| 60 | + |
| 61 | + let mut pieces_receiving_futures = piece_indexes |
| 62 | + .iter() |
| 63 | + .map(|piece_index| { |
| 64 | + Box::pin(async { |
| 65 | + let _permit = semaphore |
| 66 | + .acquire() |
| 67 | + .await |
| 68 | + .expect("Should be valid on non-closed semaphore"); |
| 69 | + |
| 70 | + self.publish_single_piece_with_backoff(*piece_index).await |
| 71 | + }) |
| 72 | + }) |
| 73 | + .collect::<FuturesUnordered<_>>(); |
| 74 | + |
| 75 | + while pieces_receiving_futures.next().await.is_some() { |
| 76 | + self.check_cancellation()?; |
80 | 77 | } |
81 | 78 |
|
| 79 | + info!("Piece publishing was successful."); |
| 80 | + |
82 | 81 | Ok(()) |
83 | 82 | } |
| 83 | + |
| 84 | + async fn publish_single_piece_with_backoff( |
| 85 | + &self, |
| 86 | + piece_index: PieceIndex, |
| 87 | + ) -> Result<(), Box<dyn Error + Send + Sync + 'static>> { |
| 88 | + let backoff = ExponentialBackoff { |
| 89 | + initial_interval: PUT_PIECE_INITIAL_INTERVAL, |
| 90 | + max_interval: PUT_PIECE_MAX_INTERVAL, |
| 91 | + // Try until we get a valid piece |
| 92 | + max_elapsed_time: None, |
| 93 | + ..ExponentialBackoff::default() |
| 94 | + }; |
| 95 | + |
| 96 | + retry(backoff, || async { |
| 97 | + self.check_cancellation() |
| 98 | + .map_err(backoff::Error::Permanent)?; |
| 99 | + |
| 100 | + let publish_timeout_result: Result<Result<(), _>, Elapsed> = timeout( |
| 101 | + PUT_PIECE_TIMEOUT, |
| 102 | + Box::pin(self.publish_single_piece(piece_index)) |
| 103 | + as Pin<Box<dyn Future<Output = _> + Send>>, |
| 104 | + ) |
| 105 | + .await; |
| 106 | + |
| 107 | + if let Ok(publish_result) = publish_timeout_result { |
| 108 | + if publish_result.is_ok() { |
| 109 | + return Ok(()); |
| 110 | + } |
| 111 | + } |
| 112 | + |
| 113 | + error!(%piece_index, "Couldn't publish a piece. Retrying..."); |
| 114 | + |
| 115 | + Err(backoff::Error::transient( |
| 116 | + "Couldn't publish piece to DSN".into(), |
| 117 | + )) |
| 118 | + }) |
| 119 | + .await |
| 120 | + } |
| 121 | + |
| 122 | + async fn publish_single_piece( |
| 123 | + &self, |
| 124 | + piece_index: PieceIndex, |
| 125 | + ) -> Result<(), Box<dyn Error + Send + Sync + 'static>> { |
| 126 | + self.check_cancellation()?; |
| 127 | + |
| 128 | + let key = |
| 129 | + PieceIndexHash::from_index(piece_index).to_multihash_by_code(MultihashCode::Sector); |
| 130 | + |
| 131 | + // TODO: rework to piece announcing (pull-model) after fixing |
| 132 | + // https://github.com/libp2p/rust-libp2p/issues/3048 |
| 133 | + let set = BTreeSet::from_iter(vec![self.dsn_node.id().to_bytes()]); |
| 134 | + |
| 135 | + let result = self.dsn_node.put_value(key, set.encode()).await; |
| 136 | + |
| 137 | + match result { |
| 138 | + Err(error) => { |
| 139 | + debug!(?error, %piece_index, ?key, "Piece publishing for a sector returned an error"); |
| 140 | + |
| 141 | + Err("Piece publishing failed".into()) |
| 142 | + } |
| 143 | + Ok(mut stream) => { |
| 144 | + if stream.next().await.is_some() { |
| 145 | + trace!(%piece_index, ?key, "Piece publishing for a sector succeeded"); |
| 146 | + |
| 147 | + Ok(()) |
| 148 | + } else { |
| 149 | + debug!(%piece_index, ?key, "Piece publishing for a sector failed"); |
| 150 | + |
| 151 | + Err("Piece publishing was unsuccessful".into()) |
| 152 | + } |
| 153 | + } |
| 154 | + } |
| 155 | + } |
84 | 156 | } |
0 commit comments