diff --git a/CHANGELOG.md b/CHANGELOG.md index fece79bd..673e69c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - test: Bump to Vector 0.46.1 ([#620]). - test: Bump OPA to `1.4.2` ([#624]). - Deprecate airflow `2.10.4` ([#625]). +- Move the git-sync implementation to operator-rs ([#623]). The functionality should not have changed. ### Fixed @@ -37,6 +38,7 @@ [#608]: https://github.com/stackabletech/airflow-operator/pull/608 [#613]: https://github.com/stackabletech/airflow-operator/pull/613 [#620]: https://github.com/stackabletech/airflow-operator/pull/620 +[#623]: https://github.com/stackabletech/airflow-operator/pull/623 [#624]: https://github.com/stackabletech/airflow-operator/pull/624 [#625]: https://github.com/stackabletech/airflow-operator/pull/625 diff --git a/Cargo.lock b/Cargo.lock index 4127359d..d8aff1e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,40 +173,13 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ - "axum-core 0.5.2", + "axum-core", "bytes", "form_urlencoded", "futures-util", @@ -216,7 +189,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -234,26 +207,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.2" @@ -602,6 +555,26 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "digest" version = "0.10.7" @@ -1467,22 +1440,21 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c75b990324f09bef15e791606b7b7a296d02fc88a344f6eba9390970a870ad5" +checksum = "aa60a41b57ae1a0a071af77dbcf89fc9819cfe66edaf2beeb204c34459dcf0b2" dependencies = [ "base64 0.22.1", "chrono", "schemars", "serde", - "serde-value", "serde_json", ] [[package]] name = "k8s-version" -version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +version = "0.1.3" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ "darling", "regex", @@ -1491,9 +1463,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4eb20010536b48abe97fec37d23d43069bcbe9686adcf9932202327bc5ca6e" +checksum = "1b49c39074089233c2bb7b1791d1b6c06c84dbab26757491fad9d233db0d432f" dependencies = [ "k8s-openapi", "kube-client", @@ -1504,9 +1476,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc2ed952042df20d15ac2fe9614d0ec14b6118eab89633985d4b36e688dccf1" +checksum = "e199797b1b08865041c9c698f0d11a91de0a8143e808b71e250cd4a1d7ce2b9f" dependencies = [ "base64 0.22.1", "bytes", @@ -1541,11 +1513,12 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff0d0793db58e70ca6d689489183816cb3aa481673e7433dc618cf7e8007c675" +checksum = "1bdefbba89dea2d99ea822a1d7cd6945535efbfb10b790056ee9284bf9e698e7" dependencies = [ "chrono", + "derive_more", "form_urlencoded", "http", "json-patch", @@ -1559,9 +1532,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c562f58dc9f7ca5feac8a6ee5850ca221edd6f04ce0dd2ee873202a88cd494c9" +checksum = "8e609a3633689a50869352a3c16e01d863b6137863c80eeb038383d5ab9f83bf" dependencies = [ "darling", "proc-macro2", @@ -1573,14 +1546,13 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.99.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f34cfab9b4bd8633062e0e85edb81df23cb09f159f2e31c60b069ae826ffdc" +checksum = "1d4bd8a4554786f8f9a87bfa977fb7dbaa1d7f102a30477338b044b65de29d8e" dependencies = [ "ahash", "async-broadcast", "async-stream", - "async-trait", "backon", "educe", "futures 0.3.31", @@ -1666,12 +1638,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -1758,9 +1724,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" -version = "0.28.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" +checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" dependencies = [ "futures-core", "futures-sink", @@ -1772,9 +1738,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.28.1" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c513c7af3bec30113f3d4620134ff923295f1e9c580fda2b8abe0831f925ddc0" +checksum = "e716f864eb23007bdd9dc4aec381e188a1cee28eecf22066772b5fd822b9727d" dependencies = [ "opentelemetry", "tracing", @@ -1784,9 +1750,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253" +checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" dependencies = [ "async-trait", "bytes", @@ -1798,11 +1764,10 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91" +checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" dependencies = [ - "async-trait", "futures-core", "http", "opentelemetry", @@ -1819,9 +1784,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d" +checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1831,18 +1796,17 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" +checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" dependencies = [ - "async-trait", "futures-channel", "futures-executor", "futures-util", "glob", "opentelemetry", "percent-encoding", - "rand", + "rand 0.9.1", "serde_json", "thiserror 2.0.12", "tokio", @@ -2087,8 +2051,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -2098,7 +2072,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -2110,6 +2094,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -2666,8 +2659,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.92.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +version = "0.93.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ "chrono", "clap", @@ -2704,7 +2697,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ "darling", "proc-macro2", @@ -2715,7 +2708,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ "kube", "semver", @@ -2727,9 +2720,9 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ - "axum 0.8.3", + "axum", "clap", "futures-util", "opentelemetry", @@ -2750,7 +2743,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.7.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ "stackable-versioned-macros", ] @@ -2758,7 +2751,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.7.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#d990019c690ed85aea29b095a03661c015676caa" dependencies = [ "convert_case", "darling", @@ -3025,13 +3018,10 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", "flate2", - "h2", "http", "http-body", "http-body-util", @@ -3041,7 +3031,6 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "socket2", "tokio", "tokio-stream", "tower 0.4.13", @@ -3061,7 +3050,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -3175,9 +3164,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721f2d2569dce9f3dfbbddee5906941e953bfcdf736a62da3377f5751650cc36" +checksum = "fd8e764bd6f5813fd8bebc3117875190c5b0415be8f7f8059bffb6ecd979c444" dependencies = [ "js-sys", "once_cell", diff --git a/Cargo.nix b/Cargo.nix index 0b47c582..bb106b6c 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -516,123 +516,7 @@ rec { ]; }; - "axum 0.7.9" = rec { - crateName = "axum"; - version = "0.7.9"; - edition = "2021"; - sha256 = "07z7wqczi9i8xb4460rvn39p4wjqwr32hx907crd1vwb2fy8ijpd"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "axum-core"; - packageId = "axum-core 0.4.5"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "http"; - packageId = "http"; - } - { - name = "http-body"; - packageId = "http-body"; - } - { - name = "http-body-util"; - packageId = "http-body-util"; - } - { - name = "itoa"; - packageId = "itoa"; - } - { - name = "matchit"; - packageId = "matchit 0.7.3"; - } - { - name = "memchr"; - packageId = "memchr"; - } - { - name = "mime"; - packageId = "mime"; - } - { - name = "percent-encoding"; - packageId = "percent-encoding"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "rustversion"; - packageId = "rustversion"; - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "sync_wrapper"; - packageId = "sync_wrapper"; - } - { - name = "tower"; - packageId = "tower 0.5.2"; - usesDefaultFeatures = false; - features = [ "util" ]; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - ]; - devDependencies = [ - { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; - } - { - name = "tower"; - packageId = "tower 0.5.2"; - rename = "tower"; - features = [ "util" "timeout" "limit" "load-shed" "steer" "filter" ]; - } - ]; - features = { - "__private_docs" = [ "axum-core/__private_docs" "tower/full" "dep:tower-http" ]; - "default" = [ "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; - "form" = [ "dep:serde_urlencoded" ]; - "http1" = [ "dep:hyper" "hyper?/http1" "hyper-util?/http1" ]; - "http2" = [ "dep:hyper" "hyper?/http2" "hyper-util?/http2" ]; - "json" = [ "dep:serde_json" "dep:serde_path_to_error" ]; - "macros" = [ "dep:axum-macros" ]; - "multipart" = [ "dep:multer" ]; - "query" = [ "dep:serde_urlencoded" ]; - "tokio" = [ "dep:hyper-util" "dep:tokio" "tokio/net" "tokio/rt" "tower/make" "tokio/macros" ]; - "tower-log" = [ "tower/log" ]; - "tracing" = [ "dep:tracing" "axum-core/tracing" ]; - "ws" = [ "dep:hyper" "tokio" "dep:tokio-tungstenite" "dep:sha1" "dep:base64" ]; - }; - }; - "axum 0.8.3" = rec { + "axum" = rec { crateName = "axum"; version = "0.8.3"; edition = "2021"; @@ -640,7 +524,7 @@ rec { dependencies = [ { name = "axum-core"; - packageId = "axum-core 0.5.2"; + packageId = "axum-core"; } { name = "bytes"; @@ -686,7 +570,7 @@ rec { } { name = "matchit"; - packageId = "matchit 0.8.4"; + packageId = "matchit"; } { name = "memchr"; @@ -811,78 +695,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "form" "http1" "json" "matched-path" "original-uri" "query" "tokio" "tower-log" "tracing" ]; }; - "axum-core 0.4.5" = rec { - crateName = "axum-core"; - version = "0.4.5"; - edition = "2021"; - sha256 = "16b1496c4gm387q20hkv5ic3k5bd6xmnvk50kwsy6ymr8rhvvwh9"; - libName = "axum_core"; - dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } - { - name = "bytes"; - packageId = "bytes"; - } - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - { - name = "http"; - packageId = "http"; - } - { - name = "http-body"; - packageId = "http-body"; - } - { - name = "http-body-util"; - packageId = "http-body-util"; - } - { - name = "mime"; - packageId = "mime"; - } - { - name = "pin-project-lite"; - packageId = "pin-project-lite"; - } - { - name = "rustversion"; - packageId = "rustversion"; - } - { - name = "sync_wrapper"; - packageId = "sync_wrapper"; - } - { - name = "tower-layer"; - packageId = "tower-layer"; - } - { - name = "tower-service"; - packageId = "tower-service"; - } - ]; - devDependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - features = [ "alloc" ]; - } - ]; - features = { - "__private_docs" = [ "dep:tower-http" ]; - "tracing" = [ "dep:tracing" ]; - }; - }; - "axum-core 0.5.2" = rec { + "axum-core" = rec { crateName = "axum-core"; version = "0.5.2"; edition = "2021"; @@ -1901,6 +1714,94 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "powerfmt" "std" ]; }; + "derive_more" = rec { + crateName = "derive_more"; + version = "2.0.1"; + edition = "2021"; + sha256 = "0y3n97cc7rsvgnj211p92y1ppzh6jzvq5kvk6340ghkhfp7l4ch9"; + authors = [ + "Jelte Fennema " + ]; + dependencies = [ + { + name = "derive_more-impl"; + packageId = "derive_more-impl"; + } + ]; + features = { + "add" = [ "derive_more-impl/add" ]; + "add_assign" = [ "derive_more-impl/add_assign" ]; + "as_ref" = [ "derive_more-impl/as_ref" ]; + "constructor" = [ "derive_more-impl/constructor" ]; + "debug" = [ "derive_more-impl/debug" ]; + "default" = [ "std" ]; + "deref" = [ "derive_more-impl/deref" ]; + "deref_mut" = [ "derive_more-impl/deref_mut" ]; + "display" = [ "derive_more-impl/display" ]; + "error" = [ "derive_more-impl/error" ]; + "from" = [ "derive_more-impl/from" ]; + "from_str" = [ "derive_more-impl/from_str" ]; + "full" = [ "add" "add_assign" "as_ref" "constructor" "debug" "deref" "deref_mut" "display" "error" "from" "from_str" "index" "index_mut" "into" "into_iterator" "is_variant" "mul" "mul_assign" "not" "sum" "try_from" "try_into" "try_unwrap" "unwrap" ]; + "index" = [ "derive_more-impl/index" ]; + "index_mut" = [ "derive_more-impl/index_mut" ]; + "into" = [ "derive_more-impl/into" ]; + "into_iterator" = [ "derive_more-impl/into_iterator" ]; + "is_variant" = [ "derive_more-impl/is_variant" ]; + "mul" = [ "derive_more-impl/mul" ]; + "mul_assign" = [ "derive_more-impl/mul_assign" ]; + "not" = [ "derive_more-impl/not" ]; + "sum" = [ "derive_more-impl/sum" ]; + "testing-helpers" = [ "derive_more-impl/testing-helpers" "dep:rustc_version" ]; + "try_from" = [ "derive_more-impl/try_from" ]; + "try_into" = [ "derive_more-impl/try_into" ]; + "try_unwrap" = [ "derive_more-impl/try_unwrap" ]; + "unwrap" = [ "derive_more-impl/unwrap" ]; + }; + resolvedDefaultFeatures = [ "default" "from" "std" ]; + }; + "derive_more-impl" = rec { + crateName = "derive_more-impl"; + version = "2.0.1"; + edition = "2021"; + sha256 = "1wqxcb7d5lzvpplz9szp4rwy1r23f5wmixz0zd2vcjscqknji9mx"; + procMacro = true; + libName = "derive_more_impl"; + authors = [ + "Jelte Fennema " + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.100"; + } + ]; + features = { + "as_ref" = [ "syn/extra-traits" "syn/visit" ]; + "debug" = [ "syn/extra-traits" "dep:unicode-xid" ]; + "display" = [ "syn/extra-traits" "dep:unicode-xid" ]; + "error" = [ "syn/extra-traits" ]; + "from" = [ "syn/extra-traits" ]; + "full" = [ "add" "add_assign" "as_ref" "constructor" "debug" "deref" "deref_mut" "display" "error" "from" "from_str" "index" "index_mut" "into" "into_iterator" "is_variant" "mul" "mul_assign" "not" "sum" "try_from" "try_into" "try_unwrap" "unwrap" ]; + "into" = [ "syn/extra-traits" ]; + "is_variant" = [ "dep:convert_case" ]; + "mul" = [ "syn/extra-traits" ]; + "mul_assign" = [ "syn/extra-traits" ]; + "not" = [ "syn/extra-traits" ]; + "testing-helpers" = [ "dep:rustc_version" ]; + "try_into" = [ "syn/extra-traits" ]; + "try_unwrap" = [ "dep:convert_case" ]; + "unwrap" = [ "dep:convert_case" ]; + }; + resolvedDefaultFeatures = [ "default" "from" ]; + }; "digest" = rec { crateName = "digest"; version = "0.10.7"; @@ -3760,7 +3661,7 @@ rec { "tokio" = [ "dep:tokio" "tokio/net" "tokio/rt" "tokio/time" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "client" "client-legacy" "default" "http1" "http2" "server" "server-auto" "service" "tokio" ]; + resolvedDefaultFeatures = [ "client" "client-legacy" "default" "http1" "server" "service" "tokio" "tracing" ]; }; "iana-time-zone" = rec { crateName = "iana-time-zone"; @@ -4613,10 +4514,10 @@ rec { }; "k8s-openapi" = rec { crateName = "k8s-openapi"; - version = "0.24.0"; + version = "0.25.0"; edition = "2021"; - links = "k8s-openapi-0.24.0"; - sha256 = "1m8ahw59g44kp9p4yd4ar0px15m2nyvhc5krbvqvw2ag6a8bjx9c"; + links = "k8s-openapi-0.25.0"; + sha256 = "1cphvicl9hq4nbp2pbzdcvz9r0f9kzwbqzgp383hl6mfawds8q5a"; libName = "k8s_openapi"; authors = [ "Arnav Singh " @@ -4645,11 +4546,6 @@ rec { packageId = "serde"; usesDefaultFeatures = false; } - { - name = "serde-value"; - packageId = "serde-value"; - usesDefaultFeatures = false; - } { name = "serde_json"; packageId = "serde_json"; @@ -4658,21 +4554,22 @@ rec { } ]; features = { - "earliest" = [ "v1_28" ]; - "latest" = [ "v1_32" ]; + "default" = [ "std" ]; + "earliest" = [ "v1_30" ]; + "latest" = [ "v1_33" ]; "schemars" = [ "dep:schemars" ]; }; - resolvedDefaultFeatures = [ "schemars" "v1_32" ]; + resolvedDefaultFeatures = [ "schemars" "v1_33" ]; }; "k8s-version" = rec { crateName = "k8s-version"; - version = "0.1.2"; + version = "0.1.3"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; libName = "k8s_version"; authors = [ @@ -4695,14 +4592,15 @@ rec { ]; features = { "darling" = [ "dep:darling" ]; + "serde" = [ "dep:serde" ]; }; resolvedDefaultFeatures = [ "darling" ]; }; "kube" = rec { crateName = "kube"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "0vnaqmxk40i2jgwxqsk8x75rn1j37p93gv3zx6mlhssk200b4kls"; + sha256 = "0bs31pdk7lnrza8p8x96mgdq8v60nv8r25vvpg1374h8fj8c6j8v"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4773,9 +4671,9 @@ rec { }; "kube-client" = rec { crateName = "kube-client"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "1wfciml6xcylhlwn72dbiq8vc57cs0a9dzn2bb8j1ps242ayvhkz"; + sha256 = "17rbrvbs3m0c4lgbf2788f0hmpli3b8z1666r50m11h83dxpk6g1"; libName = "kube_client"; authors = [ "clux " @@ -4858,7 +4756,7 @@ rec { name = "hyper-util"; packageId = "hyper-util"; optional = true; - features = [ "client" "client-legacy" "http1" "tokio" ]; + features = [ "client" "client-legacy" "http1" "tokio" "tracing" ]; } { name = "jsonpath-rust"; @@ -5014,9 +4912,9 @@ rec { }; "kube-core" = rec { crateName = "kube-core"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "0xf60y07xkqqqqyl7rvk2r4amcvch61r2j49ssk0rrsqvf9hf3gz"; + sha256 = "1rwqwvwlna79dq2r1dqhzgxmwls5d76xg892m2gdk8nyi6xgpphv"; libName = "kube_core"; authors = [ "clux " @@ -5030,6 +4928,11 @@ rec { usesDefaultFeatures = false; features = [ "now" ]; } + { + name = "derive_more"; + packageId = "derive_more"; + features = [ "from" ]; + } { name = "form_urlencoded"; packageId = "form_urlencoded"; @@ -5091,9 +4994,9 @@ rec { }; "kube-derive" = rec { crateName = "kube-derive"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "1jclsj6ah0ijhzpd43ff0ipxs7i2r985ivm6r3m5zjppr66zaqn5"; + sha256 = "1gw3kymxb0w30gmhxj33g09vcqyq05pc38sjjf3516k86cv9lq4f"; procMacro = true; libName = "kube_derive"; authors = [ @@ -5140,9 +5043,9 @@ rec { }; "kube-runtime" = rec { crateName = "kube-runtime"; - version = "0.99.0"; + version = "1.0.0"; edition = "2021"; - sha256 = "1p7z4vl9l1hbqqqjx7qmkyq3rwhxp3nqa3if0qrqdgdlp7x4rww8"; + sha256 = "13lxw9fvci5h71rlfc1a21zivanvnxzrgykvm3wzi1j7anjdhjqx"; libName = "kube_runtime"; authors = [ "clux " @@ -5162,10 +5065,6 @@ rec { name = "async-stream"; packageId = "async-stream"; } - { - name = "async-trait"; - packageId = "async-trait"; - } { name = "backon"; packageId = "backon"; @@ -5460,19 +5359,7 @@ rec { ]; }; - "matchit 0.7.3" = rec { - crateName = "matchit"; - version = "0.7.3"; - edition = "2021"; - sha256 = "156bgdmmlv4crib31qhgg49nsjk88dxkdqp80ha2pk2rk6n6ax0f"; - authors = [ - "Ibraheem Ahmed " - ]; - features = { - }; - resolvedDefaultFeatures = [ "default" ]; - }; - "matchit 0.8.4" = rec { + "matchit" = rec { crateName = "matchit"; version = "0.8.4"; edition = "2021"; @@ -5715,9 +5602,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.28.0"; + version = "0.29.1"; edition = "2021"; - sha256 = "09k43sgaarw3zx5j434ngq1canpcjibsbxaqqa8dyp0acxxncvi3"; + sha256 = "0v6ijlxs486yip2zbjdpgqc525q8l8k9s8ddz6b4ixvm4xz271wy"; dependencies = [ { name = "futures-core"; @@ -5753,7 +5640,8 @@ rec { } ]; features = { - "default" = [ "trace" "metrics" "logs" "internal-logs" ]; + "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; @@ -5761,16 +5649,16 @@ rec { "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; - "trace" = [ "pin-project-lite" "futures-sink" "futures-core" "thiserror" ]; + "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.28.1"; + version = "0.29.1"; edition = "2021"; - sha256 = "1h6x4pwk225yi8mxl3sqkhg5ya93z57i68267lzi2c7c7fpwf4y5"; + sha256 = "0zbjp4idhprbfxk21wpcivicx8c8w60w7bn4kpfpn013xdjgh5p7"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -5797,11 +5685,17 @@ rec { } ]; devDependencies = [ + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + features = [ "std" ]; + } { name = "tracing-subscriber"; packageId = "tracing-subscriber"; usesDefaultFeatures = false; - features = [ "registry" "std" "env-filter" ]; + features = [ "env-filter" "registry" "std" "fmt" ]; } ]; features = { @@ -5815,9 +5709,9 @@ rec { }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "0lv2sbsdr7b8bxnly92zzhlm1wzjbynib1xlkw9hs0qh56pkz1m8"; + sha256 = "1vf86z9d4dr9msck3k2xan9w5k35rfk9bylhpnav9d97p0rapms6"; libName = "opentelemetry_http"; dependencies = [ { @@ -5866,15 +5760,11 @@ rec { }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "148xq13ar11bvmk7pxbslrhh5pgf40bv83n6dlysigj1dm613vsv"; + sha256 = "0mjnx260qn4x1p9pyip35m7764kkszn087f0f6xcq5k9w07p56fq"; libName = "opentelemetry_otlp"; dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - } { name = "futures-core"; packageId = "futures-core"; @@ -5949,6 +5839,12 @@ rec { usesDefaultFeatures = false; features = [ "macros" "rt-multi-thread" ]; } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + features = [ "server" ]; + } ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; @@ -5985,9 +5881,9 @@ rec { }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "0vbl4si1mny87pmqxxg6wday45pcc8bvpcrf46cpwwi4606qgy2n"; + sha256 = "1cq96c16hxsfvcd26ip1v3sg9952mi89snqdawc5whw14cjdlh4c"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6017,7 +5913,7 @@ rec { "base64" = [ "dep:base64" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/transport" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; "gen-tonic-messages" = [ "tonic" "prost" ]; "hex" = [ "dep:hex" ]; "internal-logs" = [ "tracing" ]; @@ -6038,15 +5934,10 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.28.0"; + version = "0.29.0"; edition = "2021"; - sha256 = "0w4mycm070f4knvi1x5v199apd1fvi0712qiyv0pz70889havpw4"; + sha256 = "02r99lz30zrb8870vivww8cvwhdi78v5fv5sq6mr8wyls4hzppmg"; dependencies = [ - { - name = "async-trait"; - packageId = "async-trait"; - optional = true; - } { name = "futures-channel"; packageId = "futures-channel"; @@ -6077,10 +5968,10 @@ rec { } { name = "rand"; - packageId = "rand"; + packageId = "rand 0.9.1"; optional = true; usesDefaultFeatures = false; - features = [ "std" "std_rng" "small_rng" ]; + features = [ "std" "std_rng" "small_rng" "os_rng" "thread_rng" ]; } { name = "serde_json"; @@ -6112,10 +6003,9 @@ rec { } ]; features = { - "async-std" = [ "dep:async-std" ]; - "async-trait" = [ "dep:async-trait" ]; "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" ]; + "experimental_logs_concurrent_log_processor" = [ "logs" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" ]; "experimental_trace_batch_span_processor_with_async_runtime" = [ "trace" ]; @@ -6124,25 +6014,24 @@ rec { "internal-logs" = [ "tracing" ]; "jaeger_remote_sampler" = [ "trace" "opentelemetry-http" "http" "serde" "serde_json" "url" ]; "logs" = [ "opentelemetry/logs" "serde_json" ]; - "metrics" = [ "opentelemetry/metrics" "glob" "async-trait" ]; + "metrics" = [ "opentelemetry/metrics" "glob" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "rand" = [ "dep:rand" ]; - "rt-async-std" = [ "async-std" "experimental_async_runtime" ]; "rt-tokio" = [ "tokio" "tokio-stream" "experimental_async_runtime" ]; "rt-tokio-current-thread" = [ "tokio" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-async-std" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "tracing" = [ "dep:tracing" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "async-trait" "default" "experimental_async_runtime" "glob" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "serde_json" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "glob" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "serde_json" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" "tracing" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -6741,7 +6630,7 @@ rec { "rustc-dep-of-std" = [ "compiler_builtins/rustc-dep-of-std" "core" ]; }; }; - "rand" = rec { + "rand 0.8.5" = rec { crateName = "rand"; version = "0.8.5"; edition = "2018"; @@ -6760,13 +6649,13 @@ rec { } { name = "rand_chacha"; - packageId = "rand_chacha"; + packageId = "rand_chacha 0.3.1"; optional = true; usesDefaultFeatures = false; } { name = "rand_core"; - packageId = "rand_core"; + packageId = "rand_core 0.6.4"; } ]; features = { @@ -6785,7 +6674,40 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "getrandom" "libc" "rand_chacha" "small_rng" "std" "std_rng" ]; }; - "rand_chacha" = rec { + "rand 0.9.1" = rec { + crateName = "rand"; + version = "0.9.1"; + edition = "2021"; + sha256 = "15yxfcxbgmwba5cv7mjg9bhc1r5c9483dfcdfspg62x4jk8dkgwz"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "rand_chacha"; + packageId = "rand_chacha 0.9.0"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rand_core"; + packageId = "rand_core 0.9.3"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "std" "std_rng" "os_rng" "small_rng" "thread_rng" ]; + "log" = [ "dep:log" ]; + "os_rng" = [ "rand_core/os_rng" ]; + "serde" = [ "dep:serde" "rand_core/serde" ]; + "std" = [ "rand_core/std" "rand_chacha?/std" "alloc" ]; + "std_rng" = [ "dep:rand_chacha" ]; + "thread_rng" = [ "std" "std_rng" "os_rng" ]; + }; + resolvedDefaultFeatures = [ "alloc" "os_rng" "small_rng" "std" "std_rng" "thread_rng" ]; + }; + "rand_chacha 0.3.1" = rec { crateName = "rand_chacha"; version = "0.3.1"; edition = "2018"; @@ -6804,7 +6726,7 @@ rec { } { name = "rand_core"; - packageId = "rand_core"; + packageId = "rand_core 0.6.4"; } ]; features = { @@ -6815,7 +6737,44 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "rand_core" = rec { + "rand_chacha 0.9.0" = rec { + crateName = "rand_chacha"; + version = "0.9.0"; + edition = "2021"; + sha256 = "1jr5ygix7r60pz0s1cv3ms1f6pd1i9pcdmnxzzhjc3zn3mgjn0nk"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + "The CryptoCorrosion Contributors" + ]; + dependencies = [ + { + name = "ppv-lite86"; + packageId = "ppv-lite86"; + usesDefaultFeatures = false; + features = [ "simd" ]; + } + { + name = "rand_core"; + packageId = "rand_core 0.9.3"; + } + ]; + devDependencies = [ + { + name = "rand_core"; + packageId = "rand_core 0.9.3"; + features = [ "os_rng" ]; + } + ]; + features = { + "default" = [ "std" ]; + "os_rng" = [ "rand_core/os_rng" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "ppv-lite86/std" "rand_core/std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; + "rand_core 0.6.4" = rec { crateName = "rand_core"; version = "0.6.4"; edition = "2018"; @@ -6839,6 +6798,29 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "getrandom" "std" ]; }; + "rand_core 0.9.3" = rec { + crateName = "rand_core"; + version = "0.9.3"; + edition = "2021"; + sha256 = "0f3xhf16yks5ic6kmgxcpv1ngdhp48mmfy4ag82i1wnwh8ws3ncr"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "getrandom"; + packageId = "getrandom 0.3.2"; + optional = true; + } + ]; + features = { + "os_rng" = [ "dep:getrandom" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "getrandom?/std" ]; + }; + resolvedDefaultFeatures = [ "os_rng" "std" ]; + }; "redox_syscall" = rec { crateName = "redox_syscall"; version = "0.5.11"; @@ -8684,13 +8666,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.92.0"; + version = "0.93.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; libName = "stackable_operator"; authors = [ @@ -8745,7 +8727,7 @@ rec { name = "k8s-openapi"; packageId = "k8s-openapi"; usesDefaultFeatures = false; - features = [ "schemars" "v1_32" ]; + features = [ "schemars" "v1_33" ]; } { name = "kube"; @@ -8848,8 +8830,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8883,8 +8865,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; libName = "stackable_shared"; authors = [ @@ -8924,8 +8906,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; libName = "stackable_telemetry"; authors = [ @@ -8934,7 +8916,7 @@ rec { dependencies = [ { name = "axum"; - packageId = "axum 0.8.3"; + packageId = "axum"; } { name = "clap"; @@ -9029,8 +9011,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; libName = "stackable_versioned"; authors = [ @@ -9055,8 +9037,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "5fdc47a10de685e4eea49fd0a3f6c3a15a4966c1"; - sha256 = "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw"; + rev = "d990019c690ed85aea29b095a03661c015676caa"; + sha256 = "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -9081,7 +9063,7 @@ rec { packageId = "k8s-openapi"; optional = true; usesDefaultFeatures = false; - features = [ "schemars" "v1_32" ]; + features = [ "schemars" "v1_33" ]; } { name = "k8s-version"; @@ -9113,7 +9095,7 @@ rec { name = "k8s-openapi"; packageId = "k8s-openapi"; usesDefaultFeatures = false; - features = [ "schemars" "v1_32" ]; + features = [ "schemars" "v1_33" ]; } ]; features = { @@ -9787,7 +9769,7 @@ rec { "time" = [ "tokio/time" ]; "tokio-util" = [ "dep:tokio-util" ]; }; - resolvedDefaultFeatures = [ "default" "net" "time" ]; + resolvedDefaultFeatures = [ "default" "time" ]; }; "tokio-util" = rec { crateName = "tokio-util"; @@ -9903,22 +9885,11 @@ rec { "Lucio Franco " ]; dependencies = [ - { - name = "async-stream"; - packageId = "async-stream"; - optional = true; - } { name = "async-trait"; packageId = "async-trait"; optional = true; } - { - name = "axum"; - packageId = "axum 0.7.9"; - optional = true; - usesDefaultFeatures = false; - } { name = "base64"; packageId = "base64 0.22.1"; @@ -9932,11 +9903,6 @@ rec { packageId = "flate2"; optional = true; } - { - name = "h2"; - packageId = "h2"; - optional = true; - } { name = "http"; packageId = "http"; @@ -9981,12 +9947,6 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } - { - name = "socket2"; - packageId = "socket2"; - optional = true; - features = [ "all" ]; - } { name = "tokio"; packageId = "tokio"; @@ -10044,7 +10004,7 @@ rec { "transport" = [ "server" "channel" ]; "zstd" = [ "dep:zstd" ]; }; - resolvedDefaultFeatures = [ "channel" "codegen" "gzip" "prost" "router" "server" "transport" ]; + resolvedDefaultFeatures = [ "channel" "codegen" "gzip" "prost" ]; }; "tower 0.4.13" = rec { crateName = "tower"; @@ -10084,7 +10044,7 @@ rec { } { name = "rand"; - packageId = "rand"; + packageId = "rand 0.8.5"; optional = true; features = [ "small_rng" ]; } @@ -10584,9 +10544,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.29.0"; + version = "0.30.0"; edition = "2021"; - sha256 = "0dnca0b7bxbp6gd64skkvzy3p58yjh35kvnxpggz7sfwd4jjs7vj"; + sha256 = "0i64g7cyrdpzkc2zixz8bd0v1icha63ifcdwpvc3z0gmsr5pd3px"; libName = "tracing_opentelemetry"; dependencies = [ { diff --git a/Cargo.toml b/Cargo.toml index ab2e7d98..810a3117 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.92.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.93.1" } anyhow = "1.0" built = { version = "0.7", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 9dcf61ef..81f487ff 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,10 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#k8s-version@0.1.2": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-operator-derive@0.3.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-operator@0.92.0": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-shared@0.0.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-telemetry@0.6.0": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-versioned-macros@0.7.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-versioned@0.7.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#k8s-version@0.1.3": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-operator-derive@0.3.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-operator@0.93.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-shared@0.0.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-telemetry@0.6.0": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-versioned-macros@0.7.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.93.1#stackable-versioned@0.7.1": "16faz0f3dsv095hk94kmb7mk3pr6lban1v3k0g6yawak6gk5xln6", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 49e32896..b736bc02 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -455,7 +455,7 @@ spec: items: properties: authenticationClass: - description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users. + description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users type: string oidc: description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. @@ -466,7 +466,7 @@ spec: type: string extraScopes: default: [] - description: An optional list of extra scopes which get merged with the scopes defined in the [`AuthenticationClass`]. + description: An optional list of extra scopes which get merged with the scopes defined in the `AuthenticationClass`. items: type: string type: array @@ -539,24 +539,29 @@ spec: branch: default: main description: |- - The branch to clone. Defaults to `main`. + The branch to clone; defaults to `main`. Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string credentialsSecret: - description: 'The name of the Secret used to access the repository if it is not public. This should include two fields: `user` and `password`. The `password` field can either be an actual password (not recommended) or a GitHub token, as described [here](https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual).' + description: |- + The name of the Secret used to access the repository if it is not public. + + The referenced Secret must include two fields: `user` and `password`. The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the git-sync [documentation]. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual nullable: true type: string depth: default: 1 - description: The depth of syncing i.e. the number of commits to clone; defaults to 1. + description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. format: uint32 minimum: 0.0 type: integer gitFolder: default: / description: |- - The location of the DAG folder, relative to the synced repository root. + Location in the Git repository containing the resource; defaults to the root folder. It can optionally start with `/`, however, no trailing slash is recommended. An empty string (``) or slash (`/`) corresponds to the root folder in Git. type: string @@ -564,15 +569,21 @@ spec: additionalProperties: type: string default: {} - description: A map of optional configuration settings that are listed in the [git-sync documentation](https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual). Read the [git sync example](https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example). + description: |- + A map of optional configuration settings that are listed in the git-sync [documentation]. + + Also read the git-sync [example] in our documentation. These settings are not verified. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example type: object repo: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.' + format: uri type: string wait: default: 20s description: |- - The synchronization interval, e.g. `20s` or `5m`, defaults to `20s`. + The synchronization interval, e.g. `20s` or `5m`; defaults to `20s`. Since git-sync v4.x.x this field is mapped to the flag `--period`. type: string diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index e7154e9a..569995cb 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -30,21 +30,19 @@ use stackable_operator::{ }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{ - authentication::{AuthenticationClass, ldap}, - listener::{Listener, ListenerPort, ListenerSpec}, - product_image_selection::ResolvedProductImage, - rbac::build_rbac_resources, - }, + commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, config::fragment::ValidationError, + crd::{ + authentication::{core as auth_core, ldap}, + git_sync, listener, + }, k8s_openapi::{ self, DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, EmptyDirVolumeSource, EnvVar, PersistentVolumeClaim, PodTemplateSpec, - Probe, Service, ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, - VolumeMount, + ConfigMap, PersistentVolumeClaim, PodTemplateSpec, Probe, Service, ServiceAccount, + ServicePort, ServiceSpec, TCPSocketAction, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -55,11 +53,12 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, - kvp::{Label, LabelError, Labels}, + kvp::{Annotation, Label, LabelError, Labels}, logging::controller::ReconcilerError, product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, - transform_all_roles_to_config, validate_all_roles_and_groups_config, + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from, + env_vars_from_rolegroup_config, transform_all_roles_to_config, + validate_all_roles_and_groups_config, }, product_logging::{ self, @@ -92,13 +91,9 @@ use crate::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, authorization::AirflowAuthorizationResolved, - build_recommended_labels, - git_sync::{GIT_SYNC_CONTENT, GIT_SYNC_NAME, GIT_SYNC_ROOT, GitSync}, - v1alpha1, - }, - env_vars::{ - self, build_airflow_template_envs, build_gitsync_statefulset_envs, build_gitsync_template, + build_recommended_labels, v1alpha1, }, + env_vars::{self, build_airflow_template_envs}, operations::{ graceful_shutdown::{ add_airflow_graceful_shutdown_config, add_executor_graceful_shutdown_config, @@ -183,7 +178,7 @@ pub enum Error { #[snafu(display("failed to retrieve AuthenticationClass {authentication_class}"))] AuthenticationClassRetrieval { source: stackable_operator::cluster_resources::Error, - authentication_class: ObjectRef, + authentication_class: ObjectRef, }, #[snafu(display( @@ -192,7 +187,7 @@ pub enum Error { ))] AuthenticationClassProviderNotSupported { authentication_class_provider: String, - authentication_class: ObjectRef, + authentication_class: ObjectRef, }, #[snafu(display("failed to build config file for {rolegroup}"))] @@ -226,6 +221,9 @@ pub enum Error { source: stackable_operator::builder::pod::container::Error, }, + #[snafu(display("invalid git-sync specification"))] + InvalidGitSyncSpec { source: git_sync::v1alpha1::Error }, + #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -290,7 +288,7 @@ pub enum Error { #[snafu(display( "failed to build volume or volume mount spec for the LDAP backend TLS config" ))] - VolumeAndMounts { source: ldap::Error }, + VolumeAndMounts { source: ldap::v1alpha1::Error }, #[snafu(display("failed to construct config"))] ConstructConfig { source: config::Error }, @@ -312,9 +310,7 @@ pub enum Error { }, #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] - AddLdapVolumesAndVolumeMounts { - source: stackable_operator::commons::authentication::ldap::Error, - }, + AddLdapVolumesAndVolumeMounts { source: ldap::v1alpha1::Error }, #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] AddTlsVolumesAndVolumeMounts { @@ -483,6 +479,18 @@ pub async fn reconcile_airflow( .merged_config(&airflow_role, &rolegroup) .context(FailedToResolveConfigSnafu)?; + let git_sync_resources = git_sync::v1alpha1::GitSyncResources::new( + &airflow.spec.cluster_config.dags_git_sync, + &resolved_product_image, + &env_vars_from_rolegroup_config(rolegroup_config), + &airflow.volume_mounts(), + LOG_VOLUME_NAME, + &merged_airflow_config + .logging + .for_container(&Container::GitSync), + ) + .context(InvalidGitSyncSpecSnafu)?; + let rg_service = build_rolegroup_service(airflow, &resolved_product_image, &rolegroup)?; cluster_resources.add(client, rg_service).await.context( ApplyRoleGroupServiceSnafu { @@ -501,6 +509,7 @@ pub async fn reconcile_airflow( &rbac_sa, &merged_airflow_config, airflow_executor, + &git_sync_resources, )?; if let Some(listener_class) = @@ -617,6 +626,19 @@ async fn build_executor_template( .with_context(|_| ApplyRoleGroupConfigSnafu { rolegroup: rolegroup.clone(), })?; + + let git_sync_resources = git_sync::v1alpha1::GitSyncResources::new( + &airflow.spec.cluster_config.dags_git_sync, + resolved_product_image, + &env_vars_from(&common_config.env_overrides), + &airflow.volume_mounts(), + LOG_VOLUME_NAME, + &merged_executor_config + .logging + .for_container(&Container::GitSync), + ) + .context(InvalidGitSyncSpecSnafu)?; + let worker_pod_template_config_map = build_executor_template_config_map( airflow, resolved_product_image, @@ -626,6 +648,7 @@ async fn build_executor_template( &common_config.env_overrides, &common_config.pod_overrides, &rolegroup, + &git_sync_resources, )?; cluster_resources .add(client, worker_pod_template_config_map) @@ -821,8 +844,8 @@ pub fn build_group_listener( rolegroup: &RoleGroupRef, listener_class: String, listener_group_name: String, -) -> Result { - Ok(Listener { +) -> Result { + Ok(listener::v1alpha1::Listener { metadata: ObjectMetaBuilder::new() .name_and_namespace(airflow) .name(listener_group_name) @@ -837,10 +860,10 @@ pub fn build_group_listener( )) .context(ObjectMetaSnafu)? .build(), - spec: ListenerSpec { + spec: listener::v1alpha1::ListenerSpec { class_name: Some(listener_class), ports: Some(listener_ports()), - ..ListenerSpec::default() + ..listener::v1alpha1::ListenerSpec::default() }, status: None, }) @@ -848,8 +871,8 @@ pub fn build_group_listener( /// We only use the http port here and intentionally omit /// the metrics one. -fn listener_ports() -> Vec { - vec![ListenerPort { +fn listener_ports() -> Vec { + vec![listener::v1alpha1::ListenerPort { name: HTTP_PORT_NAME.to_string(), port: HTTP_PORT.into(), protocol: Some("TCP".to_string()), @@ -871,6 +894,7 @@ fn build_server_rolegroup_statefulset( service_account: &ServiceAccount, merged_airflow_config: &AirflowConfig, executor: &AirflowExecutor, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, ) -> Result { let binding = airflow.get_role(airflow_role); let role = binding.as_ref().context(NoAirflowRoleSnafu)?; @@ -899,6 +923,13 @@ fn build_server_rolegroup_statefulset( let pb_metadata = ObjectMetaBuilder::new() .with_recommended_labels(recommended_object_labels) .context(ObjectMetaSnafu)? + .with_annotation( + Annotation::try_from(( + "kubectl.kubernetes.io/default-container", + format!("{}", Container::Airflow), + )) + .unwrap(), + ) .build(); pb.metadata(pb_metadata) @@ -948,6 +979,7 @@ fn build_server_rolegroup_statefulset( executor, authentication_config, authorization_config, + git_sync_resources, ) .context(BuildStatefulsetEnvVarsSnafu)?, ); @@ -1010,6 +1042,20 @@ fn build_server_rolegroup_statefulset( .context(AddVolumeMountSnafu)?; } + // If the DAG is modularized we may encounter a timing issue whereby the celery worker + // has started *before* all modules referenced by the DAG have been fetched by gitsync + // and registered. This will result in ModuleNotFoundError errors. This can be avoided + // by running a one-off git-sync process in an init-container so that all DAG + // dependencies are fully loaded. The sidecar git-sync is then used for regular updates. + let use_git_sync_init_containers = matches!(executor, AirflowExecutor::CeleryExecutor { .. }); + add_git_sync_resources( + &mut pb, + &mut airflow_container, + git_sync_resources, + true, + use_git_sync_init_containers, + )?; + pb.add_container(airflow_container.build()); let metrics_container = ContainerBuilder::new("metrics") @@ -1063,43 +1109,6 @@ fn build_server_rolegroup_statefulset( .context(AddVolumeSnafu)?; } - if let Some(gitsync) = airflow.git_sync() { - let gitsync_container = build_gitsync_container( - resolved_product_image, - &gitsync, - false, - &format!("{}-{}", GIT_SYNC_NAME, 1), - build_gitsync_statefulset_envs(rolegroup_config, &gitsync.credentials_secret), - airflow.volume_mounts(), - )?; - - pb.add_volume( - VolumeBuilder::new(GIT_SYNC_CONTENT) - .empty_dir(EmptyDirVolumeSource::default()) - .build(), - ) - .context(AddVolumeSnafu)?; - - pb.add_container(gitsync_container); - - if let AirflowExecutor::CeleryExecutor { .. } = executor { - let gitsync_init_container = build_gitsync_container( - resolved_product_image, - &gitsync, - true, - &format!("{}-{}", GIT_SYNC_NAME, 0), - build_gitsync_statefulset_envs(rolegroup_config, &gitsync.credentials_secret), - airflow.volume_mounts(), - )?; - // If the DAG is modularized we may encounter a timing issue whereby the celery worker has started - // *before* all modules referenced by the DAG have been fetched by gitsync and registered. This - // will result in ModuleNotFoundError errors. This can be avoided by running a one-off git-sync - // process in an init-container so that all DAG dependencies are fully loaded. The sidecar - // git-sync is then used for regular updates. - pb.add_init_container(gitsync_init_container); - } - } - if merged_airflow_config.logging.enable_vector_agent { match &airflow .spec @@ -1161,7 +1170,7 @@ fn build_server_rolegroup_statefulset( match_labels: Some(statefulset_match_labels.into()), ..LabelSelector::default() }, - service_name: rolegroup_ref.object_name(), + service_name: Some(rolegroup_ref.object_name()), template: pod_template, volume_claim_templates: pvcs, ..StatefulSetSpec::default() @@ -1205,6 +1214,7 @@ fn build_executor_template_config_map( env_overrides: &HashMap, pod_overrides: &PodTemplateSpec, rolegroup_ref: &RoleGroupRef, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, ) -> Result { let mut pb = PodBuilder::new(); let pb_metadata = ObjectMetaBuilder::new() @@ -1248,10 +1258,12 @@ fn build_executor_template_config_map( airflow_container .image_from_product_image(resolved_product_image) .resources(merged_executor_config.resources.clone().into()) - .add_env_vars( - build_airflow_template_envs(airflow, env_overrides, merged_executor_config) - .context(BuildStatefulsetEnvVarsSnafu)?, - ) + .add_env_vars(build_airflow_template_envs( + airflow, + env_overrides, + merged_executor_config, + git_sync_resources, + )) .add_volume_mounts(airflow.volume_mounts()) .context(AddVolumeMountSnafu)? .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) @@ -1261,6 +1273,14 @@ fn build_executor_template_config_map( .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)?; + add_git_sync_resources( + &mut pb, + &mut airflow_container, + git_sync_resources, + false, + true, + )?; + pb.add_container(airflow_container.build()); pb.add_volumes(airflow.volumes().clone()) .context(AddVolumeSnafu)?; @@ -1273,24 +1293,6 @@ fn build_executor_template_config_map( )) .context(AddVolumeSnafu)?; - if let Some(gitsync) = airflow.git_sync() { - let gitsync_container = build_gitsync_container( - resolved_product_image, - &gitsync, - true, - &format!("{}-{}", GIT_SYNC_NAME, 0), - build_gitsync_template(env_overrides, &gitsync.credentials_secret), - airflow.volume_mounts(), - )?; - pb.add_volume( - VolumeBuilder::new(GIT_SYNC_CONTENT) - .empty_dir(EmptyDirVolumeSource::default()) - .build(), - ) - .context(AddVolumeSnafu)?; - pb.add_init_container(gitsync_container); - } - if merged_executor_config.logging.enable_vector_agent { match &airflow .spec @@ -1347,42 +1349,6 @@ fn build_executor_template_config_map( cm_builder.build().context(PodTemplateConfigMapSnafu) } -fn build_gitsync_container( - resolved_product_image: &ResolvedProductImage, - gitsync: &&GitSync, - one_time: bool, - name: &str, - env_vars: Vec, - volume_mounts: Vec, -) -> Result { - let gitsync_container = ContainerBuilder::new(name) - .context(InvalidContainerNameSnafu)? - .add_env_vars(env_vars) - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![gitsync.get_args(one_time).join("\n")]) - .add_volume_mount(GIT_SYNC_CONTENT, GIT_SYNC_ROOT) - .context(AddVolumeMountSnafu)? - .add_volume_mounts(volume_mounts) - .context(AddVolumeMountSnafu)? - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("64Mi") - .with_memory_limit("64Mi") - .build(), - ) - .build(); - Ok(gitsync_container) -} - pub fn error_policy( _obj: Arc>, error: &Error, @@ -1432,3 +1398,28 @@ fn add_authentication_volumes_and_volume_mounts( } Ok(()) } + +fn add_git_sync_resources( + pb: &mut PodBuilder, + cb: &mut ContainerBuilder, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + add_sidecar_containers: bool, + add_init_containers: bool, +) -> Result<()> { + if add_sidecar_containers { + for container in git_sync_resources.git_sync_containers.iter().cloned() { + pb.add_container(container); + } + } + if add_init_containers { + for container in git_sync_resources.git_sync_init_containers.iter().cloned() { + pb.add_init_container(container); + } + } + pb.add_volumes(git_sync_resources.git_content_volumes.to_owned()) + .context(AddVolumeSnafu)?; + cb.add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) + .context(AddVolumeMountSnafu)?; + + Ok(()) +} diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 3952b140..8bc5ca28 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -2,9 +2,9 @@ use std::collections::BTreeMap; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_operator::commons::{ - authentication::{ldap::AuthenticationProvider, oidc}, - tls_verification::TlsVerification, +use stackable_operator::{ + commons::tls_verification::TlsVerification, + crd::authentication::{ldap, oidc}, }; use crate::crd::{ @@ -28,19 +28,13 @@ type Result = std::result::Result; #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("Failed to create LDAP endpoint url."))] - FailedToCreateLdapEndpointUrl { - source: stackable_operator::commons::authentication::ldap::Error, - }, + FailedToCreateLdapEndpointUrl { source: ldap::v1alpha1::Error }, #[snafu(display("invalid OIDC endpoint"))] - InvalidOidcEndpoint { - source: stackable_operator::commons::authentication::oidc::Error, - }, + InvalidOidcEndpoint { source: oidc::v1alpha1::Error }, #[snafu(display("invalid well-known OIDC configuration URL"))] - InvalidWellKnownConfigUrl { - source: stackable_operator::commons::authentication::oidc::Error, - }, + InvalidWellKnownConfigUrl { source: oidc::v1alpha1::Error }, } pub fn add_airflow_config( @@ -116,7 +110,7 @@ fn append_authentication_config( fn append_ldap_config( config: &mut BTreeMap, - ldap: &AuthenticationProvider, + ldap: &ldap::v1alpha1::AuthenticationProvider, ) -> Result<()> { config.insert( AirflowConfigOptions::AuthType.to_string(), @@ -206,8 +200,8 @@ fn append_ldap_config( fn append_oidc_config( config: &mut BTreeMap, providers: &[( - &oidc::AuthenticationProvider, - &oidc::ClientAuthenticationOptions<()>, + &oidc::v1alpha1::AuthenticationProvider, + &oidc::v1alpha1::ClientAuthenticationOptions<()>, )], ) -> Result<(), Error> { // Debatable: AUTH_OAUTH or AUTH_OID @@ -221,7 +215,7 @@ fn append_oidc_config( for (oidc, client_options) in providers { let (env_client_id, env_client_secret) = - oidc::AuthenticationProvider::client_credentials_env_names( + oidc::v1alpha1::AuthenticationProvider::client_credentials_env_names( &client_options.client_credentials_secret_ref, ); let mut scopes = oidc.scopes.clone(); @@ -233,7 +227,7 @@ fn append_oidc_config( .unwrap_or(&DEFAULT_OIDC_PROVIDER); let oauth_providers_config_entry = match oidc_provider { - oidc::IdentityProviderHint::Keycloak => { + oidc::v1alpha1::IdentityProviderHint::Keycloak => { let endpoint_url = oidc.endpoint_url().context(InvalidOidcEndpointSnafu)?; let mut api_base_url = endpoint_url.as_str().trim_end_matches('/').to_owned(); api_base_url.push_str("/protocol/"); @@ -316,7 +310,7 @@ mod tests { use indoc::formatdoc; use rstest::rstest; use stackable_operator::{ - commons::authentication::{ldap, oidc}, + crd::authentication::{ldap, oidc}, time::Duration, }; @@ -373,7 +367,7 @@ mod tests { secretClass: openldap-tls "#; let deserializer = serde_yaml::Deserializer::from_str(ldap_provider_yaml); - let ldap_provider: ldap::AuthenticationProvider = + let ldap_provider: ldap::v1alpha1::AuthenticationProvider = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let authentication_config = AirflowClientAuthenticationDetailsResolved { @@ -433,7 +427,7 @@ mod tests { " ); let deserializer = serde_yaml::Deserializer::from_str(&oidc_provider_yaml1); - let oidc_provider1: oidc::AuthenticationProvider = + let oidc_provider1: oidc::v1alpha1::AuthenticationProvider = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let oidc_provider_yaml2 = r#" @@ -444,14 +438,14 @@ mod tests { provider_hint: Keycloak "#; let deserializer = serde_yaml::Deserializer::from_str(oidc_provider_yaml2); - let oidc_provider2: oidc::AuthenticationProvider = + let oidc_provider2: oidc::v1alpha1::AuthenticationProvider = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let authentication_config = AirflowClientAuthenticationDetailsResolved { authentication_classes_resolved: vec![ AirflowAuthenticationClassResolved::Oidc { provider: oidc_provider1, - oidc: oidc::ClientAuthenticationOptions { + oidc: oidc::v1alpha1::ClientAuthenticationOptions { client_credentials_secret_ref: "test-client-secret1".to_string(), extra_scopes: vec!["roles".to_string()], product_specific_fields: (), @@ -459,7 +453,7 @@ mod tests { }, AirflowAuthenticationClassResolved::Oidc { provider: oidc_provider2, - oidc: oidc::ClientAuthenticationOptions { + oidc: oidc::v1alpha1::ClientAuthenticationOptions { client_credentials_secret_ref: "test-client-secret2".to_string(), extra_scopes: vec![], product_specific_fields: (), diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 46f19fae..acc2e632 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -4,19 +4,17 @@ use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu, ensure}; use stackable_operator::{ client::Client, - commons::authentication::{ - AuthenticationClass, AuthenticationClassProvider, ClientAuthenticationDetails, ldap, - oidc::{self, IdentityProviderHint}, - }, + crd::authentication::{core as auth_core, ldap, oidc}, schemars::{self, JsonSchema}, }; use tracing::info; const SUPPORTED_AUTHENTICATION_CLASS_PROVIDERS: [&str; 2] = ["LDAP", "OIDC"]; -const SUPPORTED_OIDC_PROVIDERS: &[oidc::IdentityProviderHint] = - &[oidc::IdentityProviderHint::Keycloak]; +const SUPPORTED_OIDC_PROVIDERS: &[oidc::v1alpha1::IdentityProviderHint] = + &[oidc::v1alpha1::IdentityProviderHint::Keycloak]; // The assumed OIDC provider if no hint is given in the AuthClass -pub const DEFAULT_OIDC_PROVIDER: oidc::IdentityProviderHint = oidc::IdentityProviderHint::Keycloak; +pub const DEFAULT_OIDC_PROVIDER: oidc::v1alpha1::IdentityProviderHint = + oidc::v1alpha1::IdentityProviderHint::Keycloak; #[derive(Snafu, Debug)] pub enum Error { @@ -69,9 +67,7 @@ pub enum Error { ))] DifferentSyncRolesAtSettingsNotAllowed, #[snafu(display("Invalid OIDC configuration"))] - OidcConfigurationInvalid { - source: stackable_operator::commons::authentication::Error, - }, + OidcConfigurationInvalid { source: auth_core::v1alpha1::Error }, #[snafu(display( "{configured:?} is not a supported principalClaim in Airflow for the Keycloak OIDC provider. Please use {supported:?} in the AuthenticationClass {auth_class_name:?}" ))] @@ -87,7 +83,7 @@ type Result = std::result::Result; #[serde(rename_all = "camelCase")] pub struct AirflowClientAuthenticationDetails { #[serde(flatten)] - pub common: ClientAuthenticationDetails<()>, + pub common: auth_core::v1alpha1::ClientAuthenticationDetails<()>, /// Allow users who are not already in the FAB DB. /// Gets mapped to `AUTH_USER_REGISTRATION` @@ -132,11 +128,11 @@ pub struct AirflowClientAuthenticationDetailsResolved { #[derive(Clone, Debug, Eq, PartialEq)] pub enum AirflowAuthenticationClassResolved { Ldap { - provider: ldap::AuthenticationProvider, + provider: ldap::v1alpha1::AuthenticationProvider, }, Oidc { - provider: oidc::AuthenticationProvider, - oidc: oidc::ClientAuthenticationOptions<()>, + provider: oidc::v1alpha1::AuthenticationProvider, + oidc: oidc::v1alpha1::ClientAuthenticationOptions<()>, }, } @@ -145,18 +141,24 @@ impl AirflowClientAuthenticationDetailsResolved { auth_details: &[AirflowClientAuthenticationDetails], client: &Client, ) -> Result { - let resolve_auth_class = |auth_details: ClientAuthenticationDetails| async move { - auth_details.resolve_class(client).await - }; + let resolve_auth_class = + |auth_details: auth_core::v1alpha1::ClientAuthenticationDetails| async move { + auth_details.resolve_class(client).await + }; AirflowClientAuthenticationDetailsResolved::resolve(auth_details, resolve_auth_class).await } pub async fn resolve( auth_details: &[AirflowClientAuthenticationDetails], - resolve_auth_class: impl Fn(ClientAuthenticationDetails) -> R, + resolve_auth_class: impl Fn(auth_core::v1alpha1::ClientAuthenticationDetails) -> R, ) -> Result where - R: Future>, + R: Future< + Output = Result< + auth_core::v1alpha1::AuthenticationClass, + stackable_operator::client::Error, + >, + >, { let mut resolved_auth_classes: Vec = Vec::new(); let mut user_registration = None; @@ -179,7 +181,7 @@ impl AirflowClientAuthenticationDetailsResolved { .context(AuthenticationClassRetrievalFailedSnafu)?; match &auth_class.spec.provider { - AuthenticationClassProvider::Ldap(provider) => { + auth_core::v1alpha1::AuthenticationClassProvider::Ldap(provider) => { let resolved_auth_class = AirflowAuthenticationClassResolved::Ldap { provider: provider.to_owned(), }; @@ -197,7 +199,7 @@ impl AirflowClientAuthenticationDetailsResolved { resolved_auth_classes.push(resolved_auth_class); } - AuthenticationClassProvider::Oidc(provider) => { + auth_core::v1alpha1::AuthenticationClassProvider::Oidc(provider) => { let resolved_auth_class = AirflowClientAuthenticationDetailsResolved::from_oidc( auth_class_name, @@ -214,9 +216,9 @@ impl AirflowClientAuthenticationDetailsResolved { resolved_auth_classes.push(resolved_auth_class); //`&Static(_)`, `&Tls(_)` and `&Kerberos(_)` not covered } - AuthenticationClassProvider::Kerberos(_) - | AuthenticationClassProvider::Static(_) - | AuthenticationClassProvider::Tls(_) => { + auth_core::v1alpha1::AuthenticationClassProvider::Kerberos(_) + | auth_core::v1alpha1::AuthenticationClassProvider::Static(_) + | auth_core::v1alpha1::AuthenticationClassProvider::Tls(_) => { return Err(Error::AuthenticationProviderNotSupported { auth_class_name: auth_class_name.to_owned(), provider: auth_class.spec.provider.to_string(), @@ -263,7 +265,7 @@ impl AirflowClientAuthenticationDetailsResolved { fn from_oidc( auth_class_name: &str, - provider: &oidc::AuthenticationProvider, + provider: &oidc::v1alpha1::AuthenticationProvider, auth_details: &AirflowClientAuthenticationDetails, ) -> Result { let oidc_provider = match &provider.provider_hint { @@ -289,7 +291,7 @@ impl AirflowClientAuthenticationDetailsResolved { // We have to enforce preferred_username here due to the flask implementation // https://github.com/dpgaspar/Flask-AppBuilder/blob/6d44e6d581433dcea475764c4bb1270c24bbd6de/flask_appbuilder/security/manager.py#L719 match oidc_provider { - IdentityProviderHint::Keycloak => { + oidc::v1alpha1::IdentityProviderHint::Keycloak => { ensure!( &provider.principal_claim == "preferred_username", OidcPrincipalClaimNotSupportedSnafu { @@ -324,7 +326,6 @@ mod tests { use indoc::indoc; use stackable_operator::{ commons::{ - authentication::oidc, networking::HostName, tls_verification::{ CaCert, Tls, TlsClientDetails, TlsServerVerification, TlsVerification, @@ -453,7 +454,7 @@ mod tests { AirflowClientAuthenticationDetailsResolved { authentication_classes_resolved: vec![ AirflowAuthenticationClassResolved::Oidc { - provider: oidc::AuthenticationProvider::new( + provider: oidc::v1alpha1::AuthenticationProvider::new( HostName::try_from("first.oidc.server".to_string()).unwrap(), Some(443), "/realms/main".into(), @@ -466,16 +467,16 @@ mod tests { }, "preferred_username".into(), vec!["openid".into(), "email".into(), "profile".into()], - Some(IdentityProviderHint::Keycloak) + Some(oidc::v1alpha1::IdentityProviderHint::Keycloak) ), - oidc: oidc::ClientAuthenticationOptions { + oidc: oidc::v1alpha1::ClientAuthenticationOptions { client_credentials_secret_ref: "airflow-oidc-client1".into(), extra_scopes: vec!["groups".into()], product_specific_fields: () } }, AirflowAuthenticationClassResolved::Oidc { - provider: oidc::AuthenticationProvider::new( + provider: oidc::v1alpha1::AuthenticationProvider::new( HostName::try_from("second.oidc.server".to_string()).unwrap(), None, "/realms/test".into(), @@ -484,7 +485,7 @@ mod tests { vec!["openid".into(), "email".into(), "profile".into()], None ), - oidc: oidc::ClientAuthenticationOptions { + oidc: oidc::v1alpha1::ClientAuthenticationOptions { client_credentials_secret_ref: "airflow-oidc-client2".into(), extra_scopes: Vec::new(), product_specific_fields: () @@ -904,7 +905,7 @@ mod tests { /// Deserialize the given `AuthenticationClass` YAML documents. /// /// Fail if the given string cannot be deserialized. - fn deserialize_auth_classes(input: &str) -> Vec { + fn deserialize_auth_classes(input: &str) -> Vec { if input.is_empty() { Vec::new() } else { @@ -924,13 +925,20 @@ mod tests { /// `stackable_operator::commons::authentication::ClientAuthenticationDetails` /// which requires a Kubernetes client. fn create_auth_class_resolver( - auth_classes: Vec, + auth_classes: Vec, ) -> impl Fn( - ClientAuthenticationDetails, + auth_core::v1alpha1::ClientAuthenticationDetails, ) -> Pin< - Box>>, + Box< + dyn Future< + Output = Result< + auth_core::v1alpha1::AuthenticationClass, + stackable_operator::client::Error, + >, + >, + >, > { - move |auth_details: ClientAuthenticationDetails| { + move |auth_details: auth_core::v1alpha1::ClientAuthenticationDetails| { let auth_classes = auth_classes.clone(); Box::pin(async move { auth_classes diff --git a/rust/operator-binary/src/crd/git_sync.rs b/rust/operator-binary/src/crd/git_sync.rs deleted file mode 100644 index 87ce0d8d..00000000 --- a/rust/operator-binary/src/crd/git_sync.rs +++ /dev/null @@ -1,329 +0,0 @@ -use std::{collections::BTreeMap, path::PathBuf}; - -use serde::{Deserialize, Serialize}; -use stackable_operator::{ - schemars::{self, JsonSchema}, - time::Duration, - utils::COMMON_BASH_TRAP_FUNCTIONS, -}; - -pub const GIT_SYNC_CONTENT: &str = "content-from-git"; -pub const GIT_SYNC_SAFE_DIR: &str = "safe.directory"; -pub const GIT_SYNC_DIR: &str = "/stackable/app/git"; -pub const GIT_SYNC_ROOT: &str = "/tmp/git"; -pub const GIT_SYNC_LINK: &str = "current"; -pub const GIT_SYNC_NAME: &str = "gitsync"; - -#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct GitSync { - /// The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`. - pub repo: String, - - /// The branch to clone. Defaults to `main`. - /// - /// Since git-sync v4.x.x this field is mapped to the flag `--ref`. - #[serde(default = "GitSync::default_branch")] - pub branch: String, - - /// The location of the DAG folder, relative to the synced repository root. - /// - /// It can optionally start with `/`, however, no trailing slash is recommended. - /// An empty string (``) or slash (`/`) corresponds to the root folder in Git. - #[serde(default = "GitSync::default_git_folder")] - pub git_folder: PathBuf, - - /// The depth of syncing i.e. the number of commits to clone; defaults to 1. - #[serde(default = "GitSync::default_depth")] - pub depth: u32, - - /// The synchronization interval, e.g. `20s` or `5m`, defaults to `20s`. - /// - /// Since git-sync v4.x.x this field is mapped to the flag `--period`. - #[serde(default = "GitSync::default_wait")] - pub wait: Duration, - - /// The name of the Secret used to access the repository if it is not public. - /// This should include two fields: `user` and `password`. - /// The `password` field can either be an actual password (not recommended) or a GitHub token, - /// as described [here](https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual). - pub credentials_secret: Option, - - /// A map of optional configuration settings that are listed in the [git-sync documentation](https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual). - /// Read the [git sync example](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_example). - #[serde(default)] - pub git_sync_conf: BTreeMap, -} - -impl GitSync { - fn default_branch() -> String { - "main".to_string() - } - - fn default_git_folder() -> PathBuf { - PathBuf::from("/") - } - - fn default_depth() -> u32 { - 1 - } - - fn default_wait() -> Duration { - Duration::from_secs(20) - } - - /// Returns the command arguments for calling git-sync. If git-sync is called when using the - /// KubernetesExecutor it should only be called once, and from an initContainer; otherwise, the container - /// is not terminated and the job can not be cleaned up properly (the job/task is created by airflow from - /// a pod template and is terminated by airflow itself). The `one_time` parameter is used - /// to indicate this. - pub fn get_args(&self, one_time: bool) -> Vec { - let mut git_config = format!("{GIT_SYNC_SAFE_DIR}:{GIT_SYNC_ROOT}"); - let mut git_sync_command = vec![ - "/stackable/git-sync".to_string(), - format!("--repo={}", self.repo.clone()), - format!("--ref={}", self.branch), - format!("--depth={}", self.depth), - format!("--period={}s", self.wait.as_secs()), - format!("--link={GIT_SYNC_LINK}"), - format!("--root={GIT_SYNC_ROOT}"), - ]; - if !self.git_sync_conf.is_empty() { - for (key, value) in &self.git_sync_conf { - // config options that are internal details have - // constant values and will be ignored here - if key.eq_ignore_ascii_case("--dest") || key.eq_ignore_ascii_case("--root") { - tracing::warn!("Config option {:?} will be ignored...", key); - } else { - // both "-git-config" and "--gitconfig" are recognized by gitsync - if key.to_lowercase().ends_with("-git-config") { - if value.to_lowercase().contains(GIT_SYNC_SAFE_DIR) { - tracing::warn!("Config option {value:?} contains a value for {GIT_SYNC_SAFE_DIR} that overrides - the value of this operator. Git-sync functionality will probably not work as expected!"); - } - git_config = format!("{git_config},{value}"); - } else { - git_sync_command.push(format!("{key}={value}")); - } - } - } - git_sync_command.push(format!("--git-config='{git_config}'")); - } - - let mut args: Vec = vec![]; - - if one_time { - // for one-time git-sync calls (which is the case when git-sync runs as an init - // container in a job created by the KubernetesExecutor), specify this with the relevant - // parameter and do not push the process into the background - git_sync_command.push("--one-time=true".to_string()); - args.push(git_sync_command.join(" ")); - } else { - // otherwise, we need the signal termination code and the process pushed to the background - git_sync_command.push("&".to_string()); - args.append(&mut vec![ - COMMON_BASH_TRAP_FUNCTIONS.to_string(), - "prepare_signal_handlers".to_string(), - ]); - args.push(git_sync_command.join(" ")); - args.push("wait_for_termination $!".to_string()); - } - args - } -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::*; - use crate::v1alpha1::AirflowCluster; - - #[test] - fn test_git_sync_defaults() { - let cluster = " - apiVersion: airflow.stackable.tech/v1alpha1 - kind: AirflowCluster - metadata: - name: airflow - spec: - image: - productVersion: 2.10.5 - clusterConfig: - loadExamples: false - exposeConfig: false - credentialsSecret: simple-airflow-credentials - dagsGitSync: - - name: git-sync - repo: https://github.com/stackabletech/airflow-operator - webservers: - roleGroups: - default: - config: {} - celeryExecutors: - roleGroups: - default: - config: {} - schedulers: - roleGroups: - default: - config: {} - "; - - let deserializer = serde_yaml::Deserializer::from_str(cluster); - let airflow: AirflowCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let git_sync = airflow.git_sync().expect("git_sync needs to be present"); - - // Check values (including defaults) - assert_eq!( - git_sync.repo, - "https://github.com/stackabletech/airflow-operator" - ); - assert_eq!(git_sync.branch, "main"); - assert_eq!(git_sync.git_folder, PathBuf::from("/")); - assert_eq!(git_sync.depth, 1); - assert_eq!(git_sync.wait, Duration::from_secs(20)); - assert_eq!(git_sync.git_sync_conf, BTreeMap::new()); - - // Check resulting command - assert!(git_sync.get_args(false).contains( - &"/stackable/git-sync --repo=https://github.com/stackabletech/airflow-operator --ref=main --depth=1 --period=20s --link=current --root=/tmp/git &".to_string() - )); - assert!(git_sync.get_args(true).contains( - &"/stackable/git-sync --repo=https://github.com/stackabletech/airflow-operator --ref=main --depth=1 --period=20s --link=current --root=/tmp/git --one-time=true".to_string() - )); - assert_eq!( - crate::env_vars::get_dags_folder(&airflow).unwrap(), - "/stackable/app/git/current/" - ); - } - - #[test] - fn test_git_sync_config() { - let cluster = " - apiVersion: airflow.stackable.tech/v1alpha1 - kind: AirflowCluster - metadata: - name: airflow - spec: - image: - productVersion: 2.10.5 - clusterConfig: - loadExamples: false - exposeConfig: false - credentialsSecret: simple-airflow-credentials - dagsGitSync: - - name: git-sync - repo: https://github.com/stackabletech/airflow-operator - branch: feat/git-sync - wait: 42h - gitSyncConf: - --ref: c63921857618a8c392ad757dda13090fff3d879a - # Intentionally using trailing slashes here - gitFolder: ////////tests/templates/kuttl/mount-dags-gitsync/dags - webservers: - roleGroups: - default: - config: {} - celeryExecutors: - roleGroups: - default: - config: {} - schedulers: - roleGroups: - default: - config: {} - "; - - let deserializer = serde_yaml::Deserializer::from_str(cluster); - let airflow: AirflowCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let git_sync = airflow.git_sync().expect("git_sync needs to be present"); - - // Check resulting command - assert!(git_sync.get_args(false).contains( - &"/stackable/git-sync --repo=https://github.com/stackabletech/airflow-operator --ref=feat/git-sync --depth=1 --period=151200s --link=current --root=/tmp/git --ref=c63921857618a8c392ad757dda13090fff3d879a --git-config='safe.directory:/tmp/git' &".to_string() - )); - assert!(git_sync.get_args(true).contains( - &"/stackable/git-sync --repo=https://github.com/stackabletech/airflow-operator --ref=feat/git-sync --depth=1 --period=151200s --link=current --root=/tmp/git --ref=c63921857618a8c392ad757dda13090fff3d879a --git-config='safe.directory:/tmp/git' --one-time=true".to_string() - )); - assert_eq!( - crate::env_vars::get_dags_folder(&airflow).unwrap(), - "/stackable/app/git/current/tests/templates/kuttl/mount-dags-gitsync/dags" - ); - } - - #[rstest] - #[case( - "\"--git-config\": \"http.sslCAInfo:/tmp/ca-cert/ca.crt\"", - "--git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt'" - )] - #[case( - "\"-git-config\": \"http.sslCAInfo:/tmp/ca-cert/ca.crt\"", - "--git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt'" - )] - #[case( - "\"--git-config\": http.sslCAInfo:/tmp/ca-cert/ca.crt", - "--git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt'" - )] - #[case( - "--git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt", - "--git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt'" - )] - #[case( - "'--git-config': 'http.sslCAInfo:/tmp/ca-cert/ca.crt'", - "--git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt'" - )] - #[case( - "--git-config: 'http.sslCAInfo:/tmp/ca-cert/ca.crt,safe.directory:/tmp/git2'", - "--git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt,safe.directory:/tmp/git2'" - )] - fn test_git_sync_git_config(#[case] input: &str, #[case] output: &str) { - let cluster = format!( - " - apiVersion: airflow.stackable.tech/v1alpha1 - kind: AirflowCluster - metadata: - name: airflow - spec: - image: - productVersion: 2.10.5 - clusterConfig: - loadExamples: false - exposeConfig: false - credentialsSecret: simple-airflow-credentials - dagsGitSync: - - name: git-sync - repo: https://github.com/stackabletech/airflow-operator - gitSyncConf: - {input} - webservers: - roleGroups: - default: - replicas: 1 - celeryExecutors: - roleGroups: - default: - replicas: 1 - schedulers: - roleGroups: - default: - replicas: 1 - " - ); - - let deserializer = serde_yaml::Deserializer::from_str(cluster.as_str()); - let cluster: AirflowCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - - assert!( - cluster - .git_sync() - .unwrap() - .get_args(false) - .iter() - .any(|c| c.contains(output)) - ); - } -} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 785540a8..4b74eb04 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -19,6 +19,7 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, + crd::git_sync, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, @@ -50,13 +51,11 @@ use crate::crd::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetails, AirflowClientAuthenticationDetailsResolved, }, - git_sync::{GIT_SYNC_CONTENT, GIT_SYNC_DIR, GitSync}, }; pub mod affinity; pub mod authentication; pub mod authorization; -pub mod git_sync; pub const AIRFLOW_UID: i64 = 1000; pub const APP_NAME: &str = "airflow"; @@ -232,7 +231,7 @@ pub mod versioned { /// Learn more in the /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). #[serde(default)] - pub dags_git_sync: Vec, + pub dags_git_sync: Vec, /// for internal use only - not for production use. #[serde(default)] @@ -322,28 +321,7 @@ impl v1alpha1::AirflowCluster { } pub fn volume_mounts(&self) -> Vec { - let mut mounts = self.spec.cluster_config.volume_mounts.clone(); - if self.git_sync().is_some() { - mounts.push(VolumeMount { - name: GIT_SYNC_CONTENT.into(), - mount_path: GIT_SYNC_DIR.into(), - ..VolumeMount::default() - }); - } - mounts - } - - pub fn git_sync(&self) -> Option<&GitSync> { - let dags_git_sync = &self.spec.cluster_config.dags_git_sync; - // dags_git_sync is a list but only the first element is considered - // (this avoids a later breaking change when all list elements are processed) - if dags_git_sync.len() > 1 { - tracing::warn!( - "{:?} git-sync elements: only first will be considered...", - dags_git_sync.len() - ); - } - dags_git_sync.first() + self.spec.cluster_config.volume_mounts.clone() } /// The name of the role-level load-balanced Kubernetes `Service` @@ -733,6 +711,7 @@ pub enum Container { Airflow, Vector, Base, + GitSync, } #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index 1da5bd3d..bc3bbd69 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -4,9 +4,11 @@ use std::{ }; use product_config::types::PropertyNameKind; -use snafu::{OptionExt, Snafu}; +use snafu::Snafu; use stackable_operator::{ - commons::authentication::oidc, k8s_openapi::api::core::v1::EnvVar, kube::ResourceExt, + crd::{authentication::oidc, git_sync}, + k8s_openapi::api::core::v1::EnvVar, + kube::ResourceExt, product_logging::framework::create_vector_shutdown_file_command, }; @@ -18,7 +20,6 @@ use crate::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, authorization::AirflowAuthorizationResolved, - git_sync::{GIT_SYNC_DIR, GIT_SYNC_LINK, GitSync}, v1alpha1, }, util::env_var_from_secret, @@ -48,9 +49,6 @@ const ADMIN_LASTNAME: &str = "ADMIN_LASTNAME"; const ADMIN_PASSWORD: &str = "ADMIN_PASSWORD"; const ADMIN_EMAIL: &str = "ADMIN_EMAIL"; -const GITSYNC_USERNAME: &str = "GITSYNC_USERNAME"; -const GITSYNC_PASSWORD: &str = "GITSYNC_PASSWORD"; - const PYTHONPATH: &str = "PYTHONPATH"; #[derive(Snafu, Debug)] @@ -71,10 +69,11 @@ pub fn build_airflow_statefulset_envs( executor: &AirflowExecutor, auth_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, ) -> Result, Error> { let mut env: BTreeMap = BTreeMap::new(); - env.extend(static_envs(airflow)?); + env.extend(static_envs(git_sync_resources)); // environment variables let env_vars = rolegroup_config.get(&PropertyNameKind::Env); @@ -124,7 +123,7 @@ pub fn build_airflow_statefulset_envs( } } - let dags_folder = get_dags_folder(airflow)?; + let dags_folder = get_dags_folder(git_sync_resources); env.insert(AIRFLOW_CORE_DAGS_FOLDER.into(), EnvVar { name: AIRFLOW_CORE_DAGS_FOLDER.into(), value: Some(dags_folder), @@ -230,37 +229,33 @@ pub fn build_airflow_statefulset_envs( Ok(transform_map_to_vec(env)) } -pub fn get_dags_folder(airflow: &v1alpha1::AirflowCluster) -> Result { - match airflow.git_sync() { - Some(GitSync { git_folder, .. }) => { - let mut dag_folder = PathBuf::from(GIT_SYNC_DIR); - dag_folder.push(GIT_SYNC_LINK); - // Remove trailing slash - let sanitized_git_folder = git_folder.strip_prefix("/").unwrap_or(git_folder); - dag_folder.push(sanitized_git_folder); - Ok(dag_folder - .to_str() - .with_context(|| ConstructGitDagFolderSnafu { - dag_folder: dag_folder.clone(), - })? - .to_string()) - } - None => { - // if this has not been set for dag-provisioning via gitsync (above), set the default value - // so that PYTHONPATH can refer to this. N.B. nested variables need to be resolved, so that - // /stackable/airflow is used instead of $AIRFLOW_HOME. - // See https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#dags-folder - Ok("/stackable/airflow/dags".to_string()) - } +pub fn get_dags_folder(git_sync_resources: &git_sync::v1alpha1::GitSyncResources) -> String { + let git_sync_count = git_sync_resources.git_content_folders.len(); + if git_sync_count > 1 { + tracing::warn!( + "There are {git_sync_count} git-sync entries: Only the first one will be considered.", + ); } + + // If DAG provisioning via git-sync is not configured, set a default value + // so that PYTHONPATH can refer to it. N.B. nested variables need to be + // resolved, so that /stackable/airflow is used instead of $AIRFLOW_HOME. + // see https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#dags-folder + git_sync_resources + .git_content_folders_as_string() + .first() + .cloned() + .unwrap_or("/stackable/airflow/dags".to_string()) } // This set of environment variables is a standard set that is not dependent on any // conditional logic and should be applied to the statefulset or the executor template config map. -fn static_envs(airflow: &v1alpha1::AirflowCluster) -> Result, Error> { +fn static_envs( + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, +) -> BTreeMap { let mut env: BTreeMap = BTreeMap::new(); - let dags_folder = get_dags_folder(airflow)?; + let dags_folder = get_dags_folder(git_sync_resources); env.insert(PYTHONPATH.into(), EnvVar { // PYTHONPATH must be extended to include the dags folder so that dag @@ -305,45 +300,8 @@ fn static_envs(airflow: &v1alpha1::AirflowCluster) -> Result` that is not linked to a role. -pub fn build_gitsync_statefulset_envs( - rolegroup_config: &HashMap>, - git_credentials_secret: &Option, -) -> Vec { - let mut env: BTreeMap = BTreeMap::new(); - add_gitsync_credentials(git_credentials_secret, &mut env); - - if let Some(git_env) = rolegroup_config.get(&PropertyNameKind::Env) { - for (k, v) in git_env.iter() { - gitsync_vars_map(k, &mut env, v); - } - } - - tracing::debug!("Env-var set [{:?}]", env); - transform_map_to_vec(env) -} - -fn add_gitsync_credentials( - git_credentials_secret: &Option, - env: &mut BTreeMap, -) { - if let Some(git_credentials_secret) = &git_credentials_secret { - env.insert( - GITSYNC_USERNAME.to_string(), - env_var_from_secret(GITSYNC_USERNAME, git_credentials_secret, "user"), - ); - env.insert( - GITSYNC_PASSWORD.to_string(), - env_var_from_secret(GITSYNC_PASSWORD, git_credentials_secret, "password"), - ); - } + env } /// Return environment variables to be applied to the configuration map used in conjunction with @@ -352,7 +310,8 @@ pub fn build_airflow_template_envs( airflow: &v1alpha1::AirflowCluster, env_overrides: &HashMap, config: &ExecutorConfig, -) -> Result, Error> { + git_sync_resources: &git_sync::v1alpha1::GitSyncResources, +) -> Vec { let mut env: BTreeMap = BTreeMap::new(); let secret = airflow.spec.cluster_config.credentials_secret.as_str(); @@ -379,14 +338,14 @@ pub fn build_airflow_template_envs( // the config map also requires the dag-folder location as this will be passed on // to the pods started by airflow. - let dags_folder = get_dags_folder(airflow)?; + let dags_folder = get_dags_folder(git_sync_resources); env.insert(AIRFLOW_CORE_DAGS_FOLDER.into(), EnvVar { name: AIRFLOW_CORE_DAGS_FOLDER.into(), value: Some(dags_folder), ..Default::default() }); - env.extend(static_envs(airflow)?); + env.extend(static_envs(git_sync_resources)); // _STACKABLE_POST_HOOK will contain a command to create a shutdown hook that will be // evaluated in the wrapper for each stackable spark container: this is necessary for pods @@ -415,36 +374,10 @@ pub fn build_airflow_template_envs( }); } - tracing::debug!("Env-var set [{:?}]", env); - Ok(transform_map_to_vec(env)) -} - -/// Return environment variables to be applied to the configuration map used in conjunction with -/// the `kubernetesExecutor` worker: applied to the gitsync `initContainer`. -pub fn build_gitsync_template( - env_overrides: &HashMap, - git_credentials_secret: &Option, -) -> Vec { - let mut env: BTreeMap = BTreeMap::new(); - - add_gitsync_credentials(git_credentials_secret, &mut env); - - for (k, v) in env_overrides.iter().collect::>() { - gitsync_vars_map(k, &mut env, v); - } - tracing::debug!("Env-var set [{:?}]", env); transform_map_to_vec(env) } -fn gitsync_vars_map(k: &String, env: &mut BTreeMap, v: &String) { - env.insert(k.to_string(), EnvVar { - name: k.to_string(), - value: Some(v.to_string()), - ..Default::default() - }); -} - // Internally the environment variable collection uses a map so that overrides can actually // override existing keys. The returned collection will be a vector. fn transform_map_to_vec(env_map: BTreeMap) -> Vec { @@ -473,7 +406,7 @@ fn authentication_env_vars( oidc_client_credentials_secrets .iter() .cloned() - .flat_map(oidc::AuthenticationProvider::client_credentials_env_var_mounts) + .flat_map(oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts) .collect() } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index cbb42e79..d3410673 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -5,7 +5,7 @@ use futures::StreamExt; use stackable_operator::{ YamlSchema, cli::{Command, ProductOperatorRun}, - commons::authentication::AuthenticationClass, + crd::authentication::core as auth_core, k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service}, @@ -115,7 +115,8 @@ async fn main() -> anyhow::Result<()> { ) .shutdown_on_signal() .watches( - client.get_api::>(&()), + client + .get_api::>(&()), watcher::Config::default(), move |authentication_class| { authentication_class_store @@ -174,7 +175,7 @@ async fn main() -> anyhow::Result<()> { fn references_authentication_class( airflow: &DeserializeGuard, - authentication_class: &DeserializeGuard, + authentication_class: &DeserializeGuard, ) -> bool { let Ok(airflow) = &airflow.0 else { return false; diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index a4594b4e..12289fa0 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -70,6 +70,9 @@ spec: vectorAggregatorConfigMapName: airflow-vector-aggregator-discovery loadExamples: true credentialsSecret: test-airflow-credentials + dagsGitSync: + - repo: https://github.com/stackabletech/example-dags + gitFolder: dags webservers: config: resources: @@ -93,6 +96,14 @@ spec: loggers: ROOT: level: INFO + git-sync: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO vector: console: level: INFO @@ -128,6 +139,8 @@ spec: cpu: min: 50m max: 250m + memory: + limit: 3Gi roleGroups: automatic-log-config: replicas: 1 @@ -143,6 +156,14 @@ spec: loggers: ROOT: level: DEBUG + git-sync: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO vector: console: level: INFO @@ -181,6 +202,14 @@ spec: loggers: ROOT: level: DEBUG + git-sync: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO vector: console: level: INFO @@ -198,6 +227,8 @@ spec: cpu: min: 100m max: 500m + memory: + limit: 1Gi roleGroups: automatic-log-config: replicas: 1 @@ -213,6 +244,14 @@ spec: loggers: ROOT: level: INFO + git-sync: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO vector: console: level: INFO diff --git a/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 index ff16ca6b..714e3476 100644 --- a/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 @@ -30,6 +30,12 @@ customConfig: condition: >- .pod == "airflow-webserver-automatic-log-config-0" && .container == "airflow" + filteredAutomaticLogConfigWebserverGitSync: + type: filter + inputs: [validEvents] + condition: >- + .pod == "airflow-webserver-automatic-log-config-0" && + .container == "git-sync-0" filteredAutomaticLogConfigWebserverVector: type: filter inputs: [validEvents] @@ -54,6 +60,12 @@ customConfig: condition: >- .pod == "airflow-scheduler-automatic-log-config-0" && .container == "airflow" + filteredAutomaticLogConfigSchedulerGitSync: + type: filter + inputs: [validEvents] + condition: >- + .pod == "airflow-scheduler-automatic-log-config-0" && + .container == "git-sync-0" filteredAutomaticLogConfigSchedulerVector: type: filter inputs: [validEvents] @@ -79,6 +91,18 @@ customConfig: condition: >- .pod == "airflow-worker-automatic-log-config-0" && .container == "airflow" + filteredAutomaticLogConfigWorkerGitSync: + type: filter + inputs: [validEvents] + condition: >- + .pod == "airflow-worker-automatic-log-config-0" && + .container == "git-sync-0" + filteredAutomaticLogConfigWorkerGitSyncInit: + type: filter + inputs: [validEvents] + condition: >- + .pod == "airflow-worker-automatic-log-config-0" && + .container == "git-sync-0-init" filteredAutomaticLogConfigWorkerVector: type: filter inputs: [validEvents] @@ -104,6 +128,12 @@ customConfig: condition: >- starts_with(string!(.pod), "example-trigger-target-dag-bash-task-") && .container == "base" + filteredExampleTriggerTargetDagBashGitSyncInit: + type: filter + inputs: [validEvents] + condition: >- + starts_with(string!(.pod), "example-trigger-target-dag-bash-task-") && + .container == "git-sync-0-init" filteredExampleTriggerTargetDagBashTaskVector: type: filter inputs: [validEvents] @@ -116,6 +146,12 @@ customConfig: condition: >- starts_with(string!(.pod), "example-trigger-target-dag-run-this-") && .container == "base" + filteredExampleTriggerTargetDagRunThisGitSyncInit: + type: filter + inputs: [validEvents] + condition: >- + starts_with(string!(.pod), "example-trigger-target-dag-run-this-") && + .container == "git-sync-0-init" filteredExampleTriggerTargetDagRunThisVector: type: filter inputs: [validEvents]