fix(backend): cleanups part 1

backend/backend_config.toml

@@ -26,9 +26,6 @@ jwt_secret = "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef
 # WebSocket RPC endpoint for the StorageHub node
 # Use ws:// for unencrypted or wss:// for TLS-encrypted connections
 rpc_url = "ws://localhost:9944"
-# URL for node to communicate with the backend (e.g for uploading a file)
-# e.g: if running an msp in a container: http://host.docker.internal:8080
-msp_callback_url = "http://localhost:8080"
 # Request timeout in seconds (optional, default: 30)
 # Increase this value for slower networks or heavy operations
 timeout_secs = 30
@@ -42,6 +39,17 @@ verify_tls = true
 # When true, uses mock RPC connections instead of real network calls
 # mock_mode = false
 
+# MSP-specific configuration
+[msp]
+# URL for node to communicate with the backend (e.g for uploading a file)
+# e.g: if running an msp in a container: http://host.docker.internal:8080
+callback_url = "http://localhost:8080"
+# Number of retry attempts for file upload operations (default: 3)
+# Increase for unreliable network conditions
+upload_retry_attempts = 3
+# Delay in seconds between file upload retry attempts (default: 1)
+upload_retry_delay_secs = 1
+
 # Database configuration
 [database]
 # PostgreSQL connection URL

backend/bin/src/main.rs

@@ -75,7 +75,7 @@ async fn main() -> Result<()> {
     );
     debug!(target: "main", database_url = %config.database.url, "Database configuration");
     debug!(target: "main", rpc_url = %config.storage_hub.rpc_url, "RPC configuration");
-    debug!(target: "main", msp_callback_url = %config.storage_hub.msp_callback_url, "MSP callback configuration");
+    debug!(target: "main", msp_callback_url = %config.msp.callback_url, "MSP callback configuration");
 
     let memory_storage = InMemoryStorage::new();
     let storage = Arc::new(BoxedStorageWrapper::new(memory_storage));
@@ -136,7 +136,7 @@ fn load_config() -> Result<Config> {
         config.storage_hub.rpc_url = rpc_url;
     }
     if let Some(msp_callback_url) = args.msp_callback_url {
-        config.storage_hub.msp_callback_url = msp_callback_url;
+        config.msp.callback_url = msp_callback_url;
     }
 
     Ok(config)

backend/lib/Cargo.toml

@@ -15,7 +15,7 @@ chrono = { workspace = true, features = ["serde"] }
 diesel = { workspace = true }
 diesel-async = { workspace = true }
 futures = { workspace = true }
-alloy-core = "0.8"
+alloy-core = { version = "0.8", features = ["serde"] }
 alloy-signer = "0.8"
 headers = { workspace = true }
 hex = { workspace = true }

backend/lib/src/api/handlers/auth.rs

@@ -93,15 +93,15 @@ mod tests {
 
         // Step 1: Get nonce challenge
         let nonce_request = NonceRequest {
-            address: address.clone(),
+            address,
             chain_id: 1,
         };
 
         let response = server.post(AUTH_NONCE_ENDPOINT).json(&nonce_request).await;
 
         assert_eq!(response.status_code(), StatusCode::OK);
         let nonce_response: NonceResponse = response.json();
-        assert!(nonce_response.message.contains(&address));
+        assert!(nonce_response.message.contains(&address.to_string()));
 
         // Step 2: Sign the message and login
         let signature = sign_message(&signing_key, &nonce_response.message);
@@ -163,17 +163,14 @@ mod tests {
         let app = mock_app().await;
         let server = TestServer::new(app).unwrap();
 
-        let invalid_request = NonceRequest {
-            address: "not_an_eth_address".to_string(),
-            chain_id: 1,
-        };
+        let invalid_json = serde_json::json!({
+            "address": "not_an_eth_address",
+            "chainId": 1
+        });
 
-        let response = server
-            .post(AUTH_NONCE_ENDPOINT)
-            .json(&invalid_request)
-            .await;
+        let response = server.post(AUTH_NONCE_ENDPOINT).json(&invalid_json).await;
 
-        assert_eq!(response.status_code(), StatusCode::BAD_REQUEST);
+        assert_eq!(response.status_code(), StatusCode::UNPROCESSABLE_ENTITY);
     }
 
     #[tokio::test]
@@ -299,7 +296,7 @@ mod tests {
         // Unfortunately we can't easily advance system time
         // so instead we create an "old" token
         let old_claims = JwtClaims {
-            address: MOCK_ADDRESS.to_string(),
+            address: MOCK_ADDRESS,
             iat: Utc::now().timestamp() - 10, // issued 10 seconds ago
             exp: Utc::now().timestamp() + 10, // expires in 10 seconds
         };
@@ -408,7 +405,7 @@ mod tests {
 
         // Login with first wallet
         let nonce_request1 = NonceRequest {
-            address: address1.clone(),
+            address: address1,
             chain_id: 1,
         };
 
@@ -428,7 +425,7 @@ mod tests {
 
         // Login with second wallet
         let nonce_request2 = NonceRequest {
-            address: address2.clone(),
+            address: address2,
             chain_id: 1,
         };
 
@@ -485,7 +482,7 @@ mod tests {
         // until the token is expired
         // so we create a token that's already expired
         let expired_claims = JwtClaims {
-            address: MOCK_ADDRESS.to_string(),
+            address: MOCK_ADDRESS,
             exp: Utc::now().timestamp() - 3600, // 1 hour ago
             iat: Utc::now().timestamp() - 7200, // 2 hours ago
         };

backend/lib/src/api/handlers/buckets.rs

@@ -62,7 +62,7 @@ pub async fn get_files(
 
     let response = FileListResponse {
         bucket_id: bucket_id.clone(),
-        files: vec![file_tree],
+        tree: file_tree,
     };
 
     Ok(Json(response))

backend/lib/src/api/handlers/files.rs

@@ -1,6 +1,4 @@
 //! This module contains the handlers for the file management endpoints
-//!
-//! TODO: move the rest of the endpoints as they are implemented
 
 use axum::{
     body::Bytes,
@@ -28,35 +26,33 @@ use crate::{
 pub async fn get_file_info(
     State(services): State<Services>,
     AuthenticatedUser { address }: AuthenticatedUser,
-    Path((bucket_id, file_key)): Path<(String, String)>,
+    Path((_bucket_id, file_key)): Path<(String, String)>,
 ) -> Result<impl IntoResponse, Error> {
     debug!(
-        bucket_id = %bucket_id,
         file_key = %file_key,
         user = %address,
         "GET file info"
     );
-    let response = services
-        .msp
-        .get_file_info(&bucket_id, &address, &file_key)
-        .await?;
+    let response = services.msp.get_file_info(&address, &file_key).await?;
     Ok(Json(response))
 }
 
-// Internal endpoint used by the MSP RPC to upload a file to the backend
-// The file is only temporary and will be deleted after the stream is closed
+/// Internal endpoint used by the MSP RPC to upload a file to the backend
+/// The file is only temporary and will be deleted after the stream is closed
+// TODO(AUTH): Add MSP Node authentication
+// Currently this internal endpoint doesn't authenticate that
+// the client connecting to it is the MSP Node
 pub async fn internal_upload_by_key(
     State(_services): State<Services>,
     Path(file_key): Path<String>,
     body: Bytes,
 ) -> (StatusCode, impl IntoResponse) {
     debug!(file_key = %file_key, "PUT internal upload");
-    // TODO: re-add auth
-    // FIXME: make this only callable by the rpc itself
-    // let _auth = extract_bearer_token(&auth)?;
+
     if let Err(e) = tokio::fs::create_dir_all("/tmp/uploads").await {
         return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string());
     }
+
     // Validate file_key is a hex string
     let key = file_key.trim_start_matches("0x");
     if hex::decode(key).is_err() {
@@ -82,8 +78,7 @@ pub async fn download_by_key(
         return Err(Error::BadRequest("Invalid file key".to_string()));
     }
 
-    // TODO(AUTH): verify that user has permissions to access this file
-    let download_result = services.msp.get_file_from_key(&file_key).await?;
+    let download_result = services.msp.get_file_from_key(&address, &file_key).await?;
 
     // Extract filename from location or use file_key as fallback
     let filename = download_result
@@ -133,16 +128,14 @@ pub async fn download_by_key(
 pub async fn upload_file(
     State(services): State<Services>,
     AuthenticatedUser { address }: AuthenticatedUser,
-    Path((bucket_id, file_key)): Path<(String, String)>,
+    Path((_bucket_id, file_key)): Path<(String, String)>,
     mut multipart: Multipart,
 ) -> Result<impl IntoResponse, Error> {
     debug!(
-        bucket_id = %bucket_id,
         file_key = %file_key,
         user = %address,
         "PUT upload file"
     );
-    // TODO(AUTH): verify that user has permissions to access this file
 
     // Pre-check with MSP whether this file key is expected before doing heavy processing
     let is_expected = services
@@ -203,25 +196,8 @@ pub async fn upload_file(
     // Process and upload the file using the MSP service
     let response = services
         .msp
-        .process_and_upload_file(&bucket_id, &file_key, file_data_stream, file_metadata)
+        .process_and_upload_file(&address, &file_key, file_data_stream, file_metadata)
         .await?;
 
     Ok((StatusCode::CREATED, Json(response)))
 }
-
-pub async fn distribute_file(
-    State(services): State<Services>,
-    AuthenticatedUser { address }: AuthenticatedUser,
-    Path((bucket_id, file_key)): Path<(String, String)>,
-) -> Result<impl IntoResponse, Error> {
-    debug!(
-        bucket_id = %bucket_id,
-        file_key = %file_key,
-        user = %address,
-        "POST distribute file"
-    );
-    // TODO(AUTH): verify that user has permissions to access this file
-
-    let response = services.msp.distribute_file(&bucket_id, &file_key).await?;
-    Ok(Json(response))
-}

backend/lib/src/api/routes.rs

@@ -52,10 +52,6 @@ pub fn routes(services: Services) -> Router {
         )
         .merge(file_upload)
         .merge(internal_file_upload)
-        .route(
-            "/buckets/{bucket_id}/distribute/{file_key}",
-            post(handlers::files::distribute_file),
-        )
         .route(
             "/download/{file_key}",
             get(handlers::files::download_by_key),
@@ -68,12 +64,23 @@ pub fn routes(services: Services) -> Router {
 
 #[cfg(all(test, feature = "mocks"))]
 mod tests {
-    use crate::{constants::mocks::DOWNLOAD_FILE_CONTENT, services::health::HealthService};
+    use crate::{
+        api::create_app,
+        constants::{
+            mocks::{DOWNLOAD_FILE_CONTENT, MOCK_ADDRESS},
+            rpc::DUMMY_MSP_ID,
+            test::{bucket::DEFAULT_BUCKET_NAME, file::DEFAULT_FINGERPRINT},
+        },
+        services::{health::HealthService, Services},
+        test_utils::random_bytes_32,
+    };
 
     use std::path::Path;
 
     use axum::http::StatusCode;
     use axum_test::TestServer;
+    use shc_indexer_db::OnchainMspId;
+    use shp_types::Hash;
 
     #[tokio::test]
     async fn test_health_route() {
@@ -89,10 +96,54 @@ mod tests {
 
     #[tokio::test]
     async fn test_download_by_key_streams_and_cleans_temp() {
-        let app = crate::api::mock_app().await;
+        // setup file and bucket to satisfy backend checks
+        let services = Services::mocks().await;
+        let file_key = {
+            let client = &services.postgres;
+
+            // Create MSP with the ID that matches the default config
+            let msp = client
+                .create_msp(
+                    &MOCK_ADDRESS.to_string(),
+                    OnchainMspId::new(Hash::from_slice(&DUMMY_MSP_ID)),
+                )
+                .await
+                .expect("should create MSP");
+
+            // Create a test bucket for the mock user
+            let bucket_onchain_id = random_bytes_32();
+            let bucket = client
+                .create_bucket(
+                    &MOCK_ADDRESS.to_string(),
+                    Some(msp.id),
+                    DEFAULT_BUCKET_NAME.as_bytes(),
+                    &bucket_onchain_id,
+                    false,
+                )
+                .await
+                .expect("should create bucket");
+
+            // Create a test file for the mock user in the test bucket
+            let file_key = random_bytes_32();
+            let file = client
+                .create_file(
+                    MOCK_ADDRESS.to_string().as_bytes(),
+                    file_key.as_slice(),
+                    bucket.id,
+                    &bucket_onchain_id,
+                    hex::encode(file_key).as_bytes(), // RPC mock response has location as file key
+                    DEFAULT_FINGERPRINT,
+                    DOWNLOAD_FILE_CONTENT.as_bytes().len() as i64,
+                )
+                .await
+                .expect("should create file");
+
+            hex::encode(file.file_key)
+        };
+
+        let app = create_app(services);
         let server = TestServer::new(app).unwrap();
 
-        let file_key = "0xde4a17999bc1482ba71737367e5d858a133ed1e13327a29c495ab976004a138f";
         let temp_path = format!("/tmp/uploads/{}", file_key);
 
         let response = server.get(&format!("/download/{}", file_key)).await;