Do not remove all plan working directories at startup

Instead, only remove the working directories of plans that are no longer
configured. The working directories of configured plans are cleaned up regularly
and shouldn't be removed at startup.

SUP-23960

Author: jherbel

src/bin/scheduler/setup/general.rs

@@ -14,10 +14,6 @@ use std::collections::{HashMap, HashSet};
 use std::fs::{create_dir_all, remove_dir_all, remove_file};
 
 pub fn setup(global_config: &GlobalConfig, plans: Vec<Plan>) -> AnyhowResult<Vec<Plan>> {
-    if global_config.working_directory.exists() {
-        remove_dir_all(&global_config.working_directory)
-            .context("Failed to remove working directory")?;
-    }
     create_dir_all(&global_config.working_directory)
         .context("Failed to create working directory")?;
     setup_results_directories(global_config, &plans)?;
@@ -31,6 +27,8 @@ fn setup_working_directories(
     global_config: &GlobalConfig,
     plans: Vec<Plan>,
 ) -> AnyhowResult<Vec<Plan>> {
+    clean_up_working_directories(global_config, &plans)?;
+
     let (surviving_plans, plan_failures) = setup_plans_working_directory(plans);
     let (surviving_plans, rcc_failures) =
         setup_rcc_working_directories(&global_config.working_directory, surviving_plans);
@@ -49,6 +47,33 @@ fn setup_working_directories(
     Ok(surviving_plans)
 }
 
+fn clean_up_working_directories(global_config: &GlobalConfig, plans: &[Plan]) -> AnyhowResult<()> {
+    for dir_to_be_removed in [
+        environment_building_working_directory(&global_config.working_directory),
+        rcc_setup_working_directory(&global_config.working_directory),
+    ] {
+        if dir_to_be_removed.exists() {
+            remove_dir_all(dir_to_be_removed)?;
+        }
+    }
+
+    let plan_working_directory = &global_config.working_directory.join("plans");
+    if !plan_working_directory.is_dir() {
+        return Ok(());
+    }
+
+    let plan_working_directories_to_keep =
+        HashSet::<&Utf8PathBuf>::from_iter(plans.iter().map(|plan| &plan.working_directory));
+
+    for entry in top_level_directories(plan_working_directory)? {
+        if !plan_working_directories_to_keep.contains(&entry) {
+            remove_dir_all(&entry)?;
+        }
+    }
+
+    Ok(())
+}
+
 fn setup_plans_working_directory(plans: Vec<Plan>) -> (Vec<Plan>, HashMap<String, String>) {
     let mut surviving_plans = Vec::new();
     let mut failures = HashMap::new();
@@ -235,26 +260,34 @@ fn clean_up_plan_results_directory(
     Ok(())
 }
 
-fn top_level_files(directory: &Utf8Path) -> AnyhowResult<Vec<Utf8PathBuf>> {
-    let mut result_files = vec![];
+fn top_level_directories(directory: &Utf8Path) -> anyhow::Result<Vec<Utf8PathBuf>> {
+    Ok(top_level_directory_entries(directory)?
+        .into_iter()
+        .filter(|path| path.is_dir())
+        .collect())
+}
 
-    for dir_entry in directory.read_dir_utf8().context(format!(
-        "Failed to read entries of results directory {directory}",
-    ))? {
-        let dir_entry = dir_entry.context(format!(
-            "Failed to read entries of results directory {directory}",
-        ))?;
-        if dir_entry
-            .file_type()
-            .context(format!(
-                "Failed to determine file type of {}",
-                dir_entry.path()
-            ))?
-            .is_file()
-        {
-            result_files.push(dir_entry.path().to_path_buf())
-        }
+fn top_level_files(directory: &Utf8Path) -> anyhow::Result<Vec<Utf8PathBuf>> {
+    Ok(top_level_directory_entries(directory)?
+        .into_iter()
+        .filter(|path| path.is_file())
+        .collect())
+}
+
+fn top_level_directory_entries(directory: &Utf8Path) -> anyhow::Result<Vec<Utf8PathBuf>> {
+    let mut entries = vec![];
+
+    for dir_entry in directory
+        .read_dir_utf8()
+        .context(format!("Failed to read entries of directory {directory}",))?
+    {
+        entries.push(
+            dir_entry
+                .context(format!("Failed to read entries of directory {directory}",))?
+                .path()
+                .to_path_buf(),
+        )
     }
 
-    Ok(result_files)
+    Ok(entries)
 }

tests/test_scheduler.rs

@@ -22,7 +22,18 @@ use walkdir::WalkDir;
 #[ignore]
 async fn test_scheduler() -> AnyhowResult<()> {
     let test_dir = Utf8PathBuf::from(var("TEST_DIR")?);
+    let unconfigured_plan_working_dir = test_dir
+        .join("working")
+        .join("plans")
+        .join("should_be_removed_during_scheduler_setup");
+    let configured_plan_previous_execution_dir = test_dir
+        .join("working")
+        .join("plans")
+        .join("rcc_headless")
+        .join("should_still_exist_after_scheduler_run");
     create_dir_all(&test_dir)?;
+    create_dir_all(&unconfigured_plan_working_dir)?;
+    create_dir_all(&configured_plan_previous_execution_dir)?;
     let current_user_name = var("UserName")?;
     let config = create_config(
         &test_dir,
@@ -39,6 +50,8 @@ async fn test_scheduler() -> AnyhowResult<()> {
     run_scheduler(&test_dir, &config, var("RUN_FOR")?.parse::<u64>()?).await?;
 
     assert_working_directory(&config.working_directory, &current_user_name).await?;
+    assert!(!unconfigured_plan_working_dir.exists());
+    assert!(configured_plan_previous_execution_dir.is_dir());
     assert_results_directory(&config.results_directory);
     assert_rcc(&config.rcc_config, &current_user_name).await?;
     assert_tasks().await?;