Add support for github and hugginface clients; add identity support for the storage blob client

e2e-test-framework/test-data-store/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 azure_core = "0.20.0"
+azure_identity = "0.20.0"
 azure_storage = "0.20.0"
 azure_storage_blobs = "0.20.0"
 base64 = "0.22"

e2e-test-framework/test-data-store/src/test_repo_storage/models.rs

@@ -1299,6 +1299,74 @@ mod tests {
         assert!(reaction.output_handler.is_none()); // handler field is ignored
     }
 
+    #[test]
+    fn test_script_based_config_file() {
+        // Test parsing a config file that use Script as the source type
+        let script_test = r#"
+        {
+            "test_id": "script_test",
+            "version": 1,
+            "description": "A test defined by scripts",
+            "test_folder": "script_test",
+            "sources": [
+                {
+                    "test_source_id": "script-source",
+                    "kind": "Script",
+                    "bootstrap_data_generator": {
+                        "kind": "Script",
+                        "script_file_folder": "bootstrap_scripts",
+                        "time_mode": "live"
+                    },
+                    "source_change_generator": {
+                        "kind": "Script",
+                        "script_file_folder": "change_scripts",
+                        "spacing_mode": "1000",
+                        "time_mode": "recorded"
+                    }
+                }
+            ],
+            "queries": [],
+            "reactions": []
+        }
+        "#;
+
+        let result: Result<LocalTestDefinition, _> = serde_json::from_str(script_test);
+        assert!(
+            result.is_ok(),
+            "Failed to parse LocalTestDefinition: {:?}",
+            result.err()
+        );
+
+        let local_test_def = result.unwrap();
+        assert_eq!(local_test_def.test_id, "script_test");
+        assert_eq!(local_test_def.sources.len(), 1);
+
+        match &local_test_def.sources[0] {
+            TestSourceDefinition::Script(source) => {
+                assert_eq!(source.common.test_source_id, "script-source");
+
+                match source.bootstrap_data_generator.as_ref().unwrap() {
+                    BootstrapDataGeneratorDefinition::Script(definition) => {
+                        assert_eq!(definition.common.time_mode, TimeMode::Live);
+                        assert_eq!(definition.script_file_folder, "bootstrap_scripts");
+                    }
+                }
+
+                match source.source_change_generator.as_ref().unwrap() {
+                    SourceChangeGeneratorDefinition::Script(definition) => {
+                        assert_eq!(
+                            definition.common.spacing_mode,
+                            SpacingMode::Rate(NonZeroU32::new(1000).unwrap())
+                        );
+                        assert_eq!(definition.common.time_mode, TimeMode::Recorded);
+                        assert_eq!(definition.script_file_folder, "change_scripts");
+                    }
+                }
+            }
+            _ => panic!("Expected ScriptTestSourceDefinition"),
+        }
+    }
+
     #[test]
     fn test_spacing_mode_from_str() {
         assert_eq!("none".parse::<SpacingMode>().unwrap(), SpacingMode::None);

e2e-test-framework/test-data-store/src/test_repo_storage/repo_clients/azure_storage_blob_test_repo_client.rs

@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::{collections::HashMap, path::PathBuf};
+use std::{collections::HashMap, path::PathBuf, sync::Arc};
 
 use async_trait::async_trait;
+use azure_identity::{DefaultAzureCredential, TokenCredentialOptions};
 use azure_storage::prelude::*;
 use azure_storage_blobs::container::operations::BlobItem;
 use azure_storage_blobs::prelude::*;
@@ -42,11 +43,25 @@ impl AzureStorageBlobTestRepoClientSettings {
         common_config: CommonTestRepoConfig,
         unique_config: AzureStorageBlobTestRepoConfig,
     ) -> anyhow::Result<Self> {
-        // Create storage credentials from the account name and access key.
-        let storage_credentials = StorageCredentials::access_key(
-            unique_config.account_name.clone(),
-            unique_config.access_key.clone(),
-        );
+        // Create storage credentials based on whether an access key is provided.
+        let storage_credentials = match unique_config.access_key {
+            Some(access_key) => {
+                // Use access key authentication if provided
+                log::info!("Using access key authentication for Azure Storage account: {}", unique_config.account_name);
+                StorageCredentials::access_key(
+                    unique_config.account_name.clone(),
+                    access_key,
+                )
+            }
+            None => {
+                // Use Azure identity (managed identity, Azure CLI, environment variables, etc.)
+                log::info!("Using identity-based authentication (DefaultAzureCredential) for Azure Storage account: {}", unique_config.account_name);
+                let credential = Arc::new(
+                    DefaultAzureCredential::create(TokenCredentialOptions::default())?
+                );
+                StorageCredentials::token_credential(credential)
+            }
+        };
 
         Ok(Self {
             force_cache_refresh: unique_config.force_cache_refresh,
@@ -259,7 +274,13 @@ async fn download_test_repo_folder(
                     // Create the local file path for the blob.
                     let stripped_blob_file_name = blob_name
                         .strip_prefix(&remote_repo_folder)
-                        .unwrap_or(&blob_name);
+                        .ok_or_else(|| {
+                            anyhow::anyhow!(
+                                "Blob name '{}' does not start with expected prefix '{}'",
+                                blob_name,
+                                remote_repo_folder
+                            )
+                        })?;
                     let local_file_path = local_repo_folder.clone().join(stripped_blob_file_name);
 
                     // Process the blob as a directory if it doesn't have an extension.