light/superlight to replace initial (#17)

dev_scripts/py_rust.sh

@@ -21,10 +21,8 @@ ensure_venv () {
         source ./py_rust/.venv/bin/activate
     fi
 
-    ./dev_scripts/utils.sh py_install_if_missing typing-extensions
-    ./dev_scripts/utils.sh py_install_if_missing maturin
-    ./dev_scripts/utils.sh py_install_if_missing pyright
-    ./dev_scripts/utils.sh py_install_if_missing pytest
+    # Install any dev requirements that aren't managed by maturin:
+    pip install -r ./py_rust/dev_requirements.txt
 
     ./dev_scripts/utils.sh py_install_if_missing ruff
 }

py_rust/Cargo.toml

@@ -17,7 +17,7 @@ path = "src/lib.rs"
 colored = '2'
 tracing = "0.1"
 error-stack = "0.4"
-bitbazaar = { version = "0.0.38", features = ["cli", "timing", "clap"] }
+bitbazaar = { version = "0.0.38", features = ["cli", "timing"] }
 pyo3 = { version = '0.20.0', features = ['extension-module', 'chrono', 'generate-import-lib'] }
 parking_lot = { version = "0.12", features = ['deadlock_detection', 'serde'] }
 strum = { version = '0.25', features = ['derive'] }

py_rust/dev_requirements.txt

@@ -0,0 +1,6 @@
+maturin==1.4.0
+typing-extensions==4.9.0
+pyright==1.1.351
+pytest==8.0.1
+pytest-xdist==3.5.0
+pytest-profiling==1.7.0

py_rust/src/args.rs

@@ -37,7 +37,7 @@ pub fn get_version_info() -> String {
     }
 }
 
-#[derive(Debug, Parser)]
+#[derive(Clone, Debug, Parser)]
 #[command(
     author,
     name = "zetch",
@@ -49,7 +49,7 @@ pub struct Args {
     #[command(subcommand)]
     pub command: Command,
     #[clap(flatten)]
-    pub log_level_args: bitbazaar::log::ClapLogLevelArgs,
+    pub log_level_args: ClapLogLevelArgs,
     /// The config file to use. Note if render command, relative and not found from working directory, will search entered root directory.
     #[arg(
         short,
@@ -61,7 +61,7 @@ pub struct Args {
     pub config: PathBuf,
 }
 
-#[derive(Debug, clap::Subcommand)]
+#[derive(Clone, Debug, clap::Subcommand)]
 pub enum Command {
     /// Render all templates found whilst traversing the given root (default).
     Render(RenderCommand),
@@ -95,9 +95,19 @@ pub struct RenderCommand {
     /// The target directory to search and render.
     #[clap(default_value = ".")]
     pub root: PathBuf,
+
+    /// No tasks will be run, cli vars will be ignored and treated as empty strings if "light" defaults not specified.
+    #[arg(short, long, default_value = "false")]
+    pub light: bool,
+
+    /// Same as light, but user defined custom extensions are also treated as empty strings, pure rust speeds!
+    #[arg(long, default_value = "false")]
+    pub superlight: bool,
+
     /// Force write all rendered files, ignore existing lockfile.
     #[arg(short, long, default_value = "false")]
     pub force: bool,
+
     /// Comma separated list of env ctx vars to ignore defaults for and raise if not in env. E.g. --ban-defaults FOO,BAR...
     ///
     /// If no vars are provided, all defaults will be ignored.
@@ -218,3 +228,34 @@ pub enum HelpFormat {
     Text,
     Json,
 }
+
+/// A simple clap argument group for controlling the log level for cli usage.
+#[derive(Clone, Debug, clap::Args)]
+pub struct ClapLogLevelArgs {
+    /// Enable verbose logging.
+    #[arg(
+        short,
+        long,
+        global = true,
+        group = "verbosity",
+        help_heading = "Log levels"
+    )]
+    pub verbose: bool,
+    /// Print diagnostics, but nothing else.
+    #[arg(
+        short,
+        long,
+        global = true,
+        group = "verbosity",
+        help_heading = "Log levels"
+    )]
+    /// Disable all logging (but still exit with status code "1" upon detecting diagnostics).
+    #[arg(
+        short,
+        long,
+        global = true,
+        group = "verbosity",
+        help_heading = "Log levels"
+    )]
+    pub silent: bool,
+}

py_rust/src/config/context.rs

@@ -66,7 +66,7 @@ impl CtxEnvVar {
 pub struct CtxCliVar {
     pub commands: Vec<String>,
     pub coerce: Option<Coerce>,
-    pub initial: Option<serde_json::Value>,
+    pub light: Option<serde_json::Value>,
 }
 
 impl CtxCliVar {

py_rust/src/config/schema.json

@@ -148,8 +148,8 @@
                                     },
                                     "minItems": 1
                                 },
-                                "initial": {
-                                    "description": "You might find one of your commands fails on a fresh build if zetch hasn't run yet due to dependencies on other zetched files, i.e. a circular dependency. When no lockfile is found, or --force is used, if any cli variable has initial set zetch will run twice, on the first run cli vars will use their initials where they have them, on the second render the real commands will compute the values."
+                                "light": {
+                                    "description": "The value to use when in rendering in --light or --superlight mode. If not set, the var will be treated as an empty string."
                                 },
                                 "coerce": {
                                     "type": "string",

py_rust/src/custom_exts/py_interface.rs

@@ -50,6 +50,14 @@ pub fn load_custom_exts(exts: &[String], state: &State) -> Result<HashMap<String
     Python::with_gil(|py| {
         // Pythonize a copy of the context and add to the global PY_CONTEXT so its usable from zetch.context():
         let mut py_ctx = PY_CONTEXT.lock();
+
+        if py_ctx.is_some() {
+            return Err(zerr!(
+                Zerr::InternalError,
+                "Custom extensions loaded more than once."
+            ));
+        }
+
         *py_ctx = Some(pythonize(py, &state.ctx).change_context(Zerr::InternalError)?);
 
         let syspath: &PyList = py
@@ -110,10 +118,8 @@ pub fn load_custom_exts(exts: &[String], state: &State) -> Result<HashMap<String
         Ok::<_, error_stack::Report<Zerr>>(())
     })?;
 
-    // Extract a copy of the user funcs to add to minijinja env:
-    // Note copying as env might be created multiple times (e.g. initial)
-    // TODO: instead of this maybe reusing an env? In general need a bit of a refactor!
-    Ok(PY_USER_FUNCS.lock().clone())
+    // Extra the loaded user funcs, this fn is checked to only run once. So no need to clone and maintain global var.
+    Ok(std::mem::take(&mut *PY_USER_FUNCS.lock()))
 }
 
 pub fn mini_values_to_py_params(

py_rust/src/render/debug.rs

@@ -1,8 +1,11 @@
-use crate::state::State;
+use std::collections::HashMap;
+
+use crate::config::conf::Config;
 
 #[derive(Debug, serde::Serialize)]
 pub struct Debug {
-    pub state: State,
+    pub conf: Config,
+    pub ctx: HashMap<String, serde_json::Value>,
     pub written: Vec<String>,
     pub identical: Vec<String>,
     pub matched_templates: Vec<String>,

py_rust/src/render/mini_env.rs

@@ -60,33 +60,42 @@ pub fn new_mini_env<'a>(root: &Path, state: &'a State) -> Result<minijinja::Envi
             ));
         }
 
-        // Add the rust-wrapped python fn to the minijinja environment:
-        env.add_function(
-            name.clone(),
-            move |
-                    values: minijinja::value::Rest<minijinja::Value>|
-                    -> core::result::Result<minijinja::Value, minijinja::Error> {
-                let result =
-                    Python::with_gil(|py| -> Result<serde_json::Value, Zerr> {
-                        let (py_args, py_kwargs) = py_interface::mini_values_to_py_params(py, values)?;
-                        let py_result = py_fn
-                            .call(py, py_args, py_kwargs)
-                            .map_err(|e: PyErr| zerr!(Zerr::CustomPyFunctionError, "{}", e))?;
-                        let rustified: serde_json::Value =
-                            depythonize(py_result.as_ref(py)).change_context(Zerr::CustomPyFunctionError).attach_printable_lazy(|| {
-                                "Failed to convert python result to a rust-like value."
-                            })?;
-                        Ok(rustified)
-                    });
-                match result {
-                    Err(e) => Err(minijinja::Error::new(
-                        minijinja::ErrorKind::InvalidOperation,
-                        format!("Failed to call custom filter '{}'. Err: \n{:?}", name, e),
-                    )),
-                    Ok(result) => Ok(minijinja::Value::from_serializable(&result)),
-                }
-            },
-        )
+        // If superlight, add a pseudo fn that returns an empty string
+        if state.superlight {
+            let empty_str = minijinja::Value::from_safe_string("".to_string());
+            env.add_function(
+                name.clone(),
+                move |_values: minijinja::value::Rest<minijinja::Value>| empty_str.clone(),
+            );
+        } else {
+            // Add the rust-wrapped python fn to the minijinja environment:
+            env.add_function(
+                name.clone(),
+                move |
+                        values: minijinja::value::Rest<minijinja::Value>|
+                        -> core::result::Result<minijinja::Value, minijinja::Error> {
+                    let result =
+                        Python::with_gil(|py| -> Result<serde_json::Value, Zerr> {
+                            let (py_args, py_kwargs) = py_interface::mini_values_to_py_params(py, values)?;
+                            let py_result = py_fn
+                                .call(py, py_args, py_kwargs)
+                                .map_err(|e: PyErr| zerr!(Zerr::CustomPyFunctionError, "{}", e))?;
+                            let rustified: serde_json::Value =
+                                depythonize(py_result.as_ref(py)).change_context(Zerr::CustomPyFunctionError).attach_printable_lazy(|| {
+                                    "Failed to convert python result to a rust-like value."
+                                })?;
+                            Ok(rustified)
+                        });
+                    match result {
+                        Err(e) => Err(minijinja::Error::new(
+                            minijinja::ErrorKind::InvalidOperation,
+                            format!("Failed to call custom filter '{}'. Err: \n{:?}", name, e),
+                        )),
+                        Ok(result) => Ok(minijinja::Value::from_serializable(&result)),
+                    }
+                },
+            )
+        }
     }
 
     Ok(env)

py_rust/src/render/mod.rs

@@ -23,43 +23,26 @@ pub fn render(args: &crate::args::Args, render_args: &RenderCommand) -> Result<b
         self::lockfile::Lockfile::load(render_args.root.clone(), render_args.force)
     });
 
+    // TODO double prints what's that about
+
     let mut state = State::new(args)?;
+    state.load_all_vars()?;
+    debug!("State: {:#?}", state);
 
-    // If newly created, should use cli initials if any have them:
-    let use_cli_initials = lockfile.newly_created
-        && state
-            .conf
-            .context
-            .cli
-            .iter()
-            .any(|(_, v)| v.initial.is_some());
-    state.load_all_vars(Some(render_args), use_cli_initials)?;
-
-    // Need to run twice and rebuild config with real cli vars if initials used in the first state build:
-    let (written, identical) = if use_cli_initials {
-        warn!("Lockfile newly created/force updated and some cli vars have initials, will double render and use initials first time round.");
-        // Conf from second as that has the real cli vars, template info from the first as the second will be inaccurate due to the first having run.
-        let (first_written, first_identical) = render_inner(&state, render_args, &mut lockfile)?;
-
-        // Reload with real cli vars:
-        state.load_all_vars(Some(render_args), false)?;
-
-        // Re-render:
-        render_inner(&state, render_args, &mut lockfile)?;
-        (first_written, first_identical)
-    } else {
-        render_inner(&state, render_args, &mut lockfile)?
-    };
+    let (written, identical) = render_inner(&state, render_args, &mut lockfile)?;
 
-    // Run post-tasks:
-    state.conf.tasks.run_post(&state)?;
+    // Run post-tasks only if not light/superlight:
+    if !state.light {
+        state.conf.tasks.run_post(&state)?;
+    }
 
     timeit!("Syncing lockfile", { lockfile.sync() })?;
 
     // Write only when hidden cli flag --debug is set, to allow testing internals from python without having to setup custom interfaces:
     if render_args.debug {
         let debug = debug::Debug {
-            state: state.clone(),
+            conf: state.conf.clone(),
+            ctx: state.ctx.clone(),
             written: written
                 .iter()
                 .map(|t| t.out_path.display().to_string())
@@ -85,7 +68,11 @@ pub fn render(args: &crate::args::Args, render_args: &RenderCommand) -> Result<b
             .change_context(Zerr::InternalError)?;
     }
 
-    let num_tasks = state.conf.tasks.pre.len() + state.conf.tasks.post.len();
+    let num_tasks = if state.light {
+        0
+    } else {
+        state.conf.tasks.pre.len() + state.conf.tasks.post.len()
+    };
     println!(
         "{} {} template{} written, {} identical.{} Lockfile {}. {} elapsed.",
         "zetch:".bold(),

py_rust/src/run.rs

@@ -68,14 +68,6 @@ pub fn run() -> Result<(), Zerr> {
             .change_context(Zerr::InternalError)?;
     }
 
-    // Stdout if enabled:
-    if let Some(level) = args.log_level_args.level() {
-        builder = builder
-            .stdout(true, true)
-            .level_from(level)
-            .change_context(Zerr::InternalError)?;
-    }
-
     // Build and set as global logger:
     let log = builder.build().change_context(Zerr::InternalError)?;
     log.register_global().change_context(Zerr::InternalError)?;