From 4f5d268e7d5ad46b873cda100c77da9436f9843e Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Thu, 7 Sep 2023 21:03:21 +0200 Subject: [PATCH] Implement metastore shard API (#3773) --- quickwit/Cargo.lock | 469 +++++++++-------- .../quickwit-config/src/index_config/mod.rs | 3 +- .../src/actors/index_serializer.rs | 7 +- .../quickwit-indexing/src/actors/indexer.rs | 16 +- .../src/actors/indexing_pipeline.rs | 27 +- .../src/actors/merge_executor.rs | 8 +- .../quickwit-indexing/src/actors/packager.rs | 12 +- .../quickwit-indexing/src/actors/publisher.rs | 33 +- .../quickwit-indexing/src/actors/uploader.rs | 24 +- .../src/models/indexed_split.rs | 17 +- .../src/models/packaged_split.rs | 38 +- .../src/models/publisher_message.rs | 3 +- .../src/source/kafka_source.rs | 10 +- .../src/source/pulsar_source.rs | 10 +- quickwit/quickwit-metastore/src/checkpoint.rs | 96 ++-- .../src/metastore/control_plane_metastore.rs | 4 +- .../file_backed_index/mod.rs | 188 +++++-- .../file_backed_index/serialize.rs | 47 +- .../file_backed_index/shards.rs | 483 ++++++++++++++++++ .../metastore/file_backed_metastore/mod.rs | 100 +++- .../metastore/grpc_metastore/grpc_adapter.rs | 34 +- .../src/metastore/grpc_metastore/mod.rs | 42 +- .../src/metastore/index_metadata/mod.rs | 3 +- .../src/metastore/instrumented_metastore.rs | 4 +- .../metastore/metastore_event_publisher.rs | 4 +- .../quickwit-metastore/src/metastore/mod.rs | 3 +- .../src/metastore/postgresql_metastore.rs | 3 +- .../src/metastore/retrying_metastore/mod.rs | 8 +- .../src/metastore/retrying_metastore/test.rs | 5 +- .../quickwit-metastore/src/split_metadata.rs | 4 +- quickwit/quickwit-metastore/src/tests.rs | 103 +++- .../file-backed-index/v0.4.expected.json | 3 +- .../file-backed-index/v0.5.expected.json | 3 +- .../file-backed-index/v0.6.expected.json | 23 +- .../test-data/file-backed-index/v0.6.json | 23 +- .../index-metadata/v0.6.expected.json | 4 +- .../test-data/index-metadata/v0.6.json | 4 +- .../split-metadata/v0.6.expected.json | 2 +- .../test-data/split-metadata/v0.6.json | 2 +- quickwit/quickwit-proto/Cargo.toml | 2 +- quickwit/quickwit-proto/build.rs | 17 + .../protos/quickwit/control_plane.proto | 27 +- .../protos/quickwit/ingest.proto | 22 +- .../protos/quickwit/metastore.proto | 8 +- .../quickwit/quickwit.control_plane.rs | 353 ++++++------- .../src/codegen/quickwit/quickwit.ingest.rs | 27 +- .../codegen/quickwit/quickwit.metastore.rs | 5 +- quickwit/quickwit-proto/src/types.rs | 3 + 48 files changed, 1648 insertions(+), 688 deletions(-) create mode 100644 quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/shards.rs diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 1e76fb854a9..29a6652a4cd 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -10,9 +10,9 @@ checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -69,9 +69,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anstyle-parse" @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" +checksum = "d495b6dc0184693324491a5ac05f559acc97bf937ab31d7a1c33dd0016be6d2b" dependencies = [ "flate2", "futures-core", @@ -342,7 +342,7 @@ dependencies = [ "http", "hyper", "ring", - "time 0.3.25", + "time 0.3.26", "tokio", "tower", "tracing", @@ -537,7 +537,7 @@ dependencies = [ "percent-encoding", "regex", "sha2", - "time 0.3.25", + "time 0.3.26", "tracing", ] @@ -593,7 +593,7 @@ dependencies = [ "hyper-rustls 0.23.2", "lazy_static", "pin-project-lite", - "rustls 0.20.8", + "rustls 0.20.9", "serde", "serde_json", "tokio", @@ -695,7 +695,7 @@ dependencies = [ "itoa", "num-integer", "ryu", - "time 0.3.25", + "time 0.3.26", ] [[package]] @@ -775,7 +775,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b0f0eea648347e40f5f7f7e6bfea4553bcefad0fbf52044ea339e5ce3aba61" dependencies = [ "async-trait", - "base64 0.21.2", + "base64 0.21.3", "bytes", "dyn-clone", "futures", @@ -790,7 +790,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "time 0.3.25", + "time 0.3.26", "url", "uuid", ] @@ -813,7 +813,7 @@ dependencies = [ "serde_derive", "serde_json", "sha2", - "time 0.3.25", + "time 0.3.26", "url", "uuid", ] @@ -833,7 +833,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "time 0.3.25", + "time 0.3.26", "url", "uuid", ] @@ -854,9 +854,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -881,9 +881,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64-simd" @@ -1131,9 +1131,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", @@ -1188,9 +1188,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "f56b4c72906975ca04becb8a30e102dfecddd0c06181e3e95ddc444be28881f8" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1199,7 +1199,7 @@ dependencies = [ "serde", "time 0.1.45", "wasm-bindgen", - "winapi 0.3.9", + "windows-targets 0.48.5", ] [[package]] @@ -1297,9 +1297,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "cmake" @@ -1697,9 +1697,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.0" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", "hashbrown 0.14.0", @@ -1758,9 +1758,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" dependencies = [ "serde", ] @@ -1917,9 +1917,9 @@ dependencies = [ [[package]] name = "elasticsearch-dsl" -version = "0.4.15" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0289994b78ad51661d9195cd2fb071ebdc68f3b03de0e0e7420ede8fb320dc" +checksum = "78e17a6bd2ee53e64da09f261bed7215235b7adfe3accbd20f20d66b91f2e7fb" dependencies = [ "chrono", "num-traits", @@ -2065,18 +2065,18 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc978899517288e3ebbd1a3bfc1d9537dbb87eeab149e53ea490e63bcdff561a" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" dependencies = [ "serde", ] [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -2246,7 +2246,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" dependencies = [ - "rustix 0.38.8", + "rustix 0.38.10", "windows-sys 0.48.0", ] @@ -2426,9 +2426,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "glob" @@ -2443,7 +2443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb6624c70caf330298b84a9ad1537ee7a5de788a5b9a06a3bbe206260943011" dependencies = [ "async-trait", - "base64 0.21.2", + "base64 0.21.3", "google-cloud-metadata", "google-cloud-token", "home", @@ -2452,7 +2452,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tracing", "urlencoding", @@ -2465,7 +2465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "931bedb2264cb00f914b0a6a5c304e34865c34306632d3932e0951a073e4a67d" dependencies = [ "async-trait", - "base64 0.21.2", + "base64 0.21.3", "google-cloud-metadata", "google-cloud-token", "home", @@ -2474,7 +2474,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tracing", "urlencoding", @@ -2605,9 +2605,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -2658,9 +2658,9 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ "hashbrown 0.14.0", ] @@ -2798,9 +2798,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "http-serde" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" +checksum = "6f560b665ad9f1572cfcaf034f7fb84338a7ce945216d64a90fd81f046a3caee" dependencies = [ "http", "serde", @@ -2878,7 +2878,7 @@ dependencies = [ "http", "hyper", "log", - "rustls 0.20.8", + "rustls 0.20.9", "rustls-native-certs", "tokio", "tokio-rustls 0.23.4", @@ -2894,7 +2894,7 @@ dependencies = [ "http", "hyper", "log", - "rustls 0.21.6", + "rustls 0.21.7", "rustls-native-certs", "tokio", "tokio-rustls 0.24.1", @@ -3035,9 +3035,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a53088c87cf71c9d4f3372a2cb9eea1e7b8a0b1bf8b7f7d23fe5b76dbb07e63b" +checksum = "e1be380c410bf0595e94992a648ea89db4dd3f3354ba54af206fd2a68cf5ac8e" [[package]] name = "io-lifetimes" @@ -3072,7 +3072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.8", + "rustix 0.38.10", "windows-sys 0.48.0", ] @@ -3130,7 +3130,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "pem", "ring", "serde", @@ -3236,9 +3236,9 @@ dependencies = [ [[package]] name = "lindera-cc-cedict" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69c983c7c5068266e882449172a9583b04745045180d5118a52ea0e69743476b" +checksum = "53a46035785bfdf38a81713ac93d9df9adc527b7189f7264f2ce290459318935" dependencies = [ "bincode", "byteorder", @@ -3252,9 +3252,9 @@ dependencies = [ [[package]] name = "lindera-cc-cedict-builder" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2e8f2ca97ddf952fe340642511b9c14b373cb2eef711d526bb8ef2ca0969b8" +checksum = "6f567a47e47b5420908424de2c6c5e424e3cafe588d0146bd128c0f3755758a3" dependencies = [ "anyhow", "bincode", @@ -3272,9 +3272,9 @@ dependencies = [ [[package]] name = "lindera-compress" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f72b460559bcbe8a9cee85ea4a5056133ed3abf373031191589236e656d65b59" +checksum = "49f3e553d55ebe9881fa5e5de588b0a153456e93564d17dfbef498912caf63a2" dependencies = [ "anyhow", "flate2", @@ -3283,9 +3283,9 @@ dependencies = [ [[package]] name = "lindera-core" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f586eb8a9393c32d5525e0e9336a3727bd1329674740097126f3b0bff8a1a1ea" +checksum = "a9a2440cc156a4a911a174ec68203543d1efb10df3a700a59b6bf581e453c726" dependencies = [ "anyhow", "bincode", @@ -3300,9 +3300,9 @@ dependencies = [ [[package]] name = "lindera-decompress" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb1facd8da698072fcc7338bd757730db53d59f313f44dd583fa03681dcc0e1" +checksum = "e077a410e61c962cb526f71b7effd62ffc607488a8f61869c937582d2ccb529b" dependencies = [ "anyhow", "flate2", @@ -3311,9 +3311,9 @@ dependencies = [ [[package]] name = "lindera-dictionary" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7be7410b1da7017a8948986b87af67082f605e9a716f0989790d795d677f0c" +checksum = "d9f57491adf7b311a3ee87f5e4a36454df16a2ec73de4ef28b2106fac80bd782" dependencies = [ "anyhow", "bincode", @@ -3332,9 +3332,9 @@ dependencies = [ [[package]] name = "lindera-ipadic" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db2c39d5b69cb7b69df6edb44863d38991e0eb0037d38396604c1e65106a5db" +checksum = "639d95328fc40c8bc7041f7c669d38319cc6e069077764914aa07c9c16f8fe1e" dependencies = [ "bincode", "byteorder", @@ -3349,9 +3349,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-builder" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705d07f8a45d04fd95149f7ad41a26d1f9e56c9c00402be6f9dd05e3d88b99c6" +checksum = "a3476ec7748aebd2eb23d496ddfce5e7e0a5c031cffcd214451043e02d029f11" dependencies = [ "anyhow", "bincode", @@ -3371,9 +3371,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-neologd-builder" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633a93983ba13fba42328311a501091bd4a7aff0c94ae9eaa9d4733dd2b0468a" +checksum = "7b1c7576a02d5e4af2bf62de51790a01bc4b8bc0d0b6a6b86a46b157f5cb306d" dependencies = [ "anyhow", "bincode", @@ -3392,9 +3392,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a428e0d316b6c86f51bd919479692bc41ad840dba266ebc044663970f431ea18" +checksum = "b713ecd5b827d7d448c3c5eb3c6d5899ecaf22cd17087599996349a02c76828d" dependencies = [ "bincode", "byteorder", @@ -3409,9 +3409,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic-builder" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5288704c6b8a069c0a1705c38758e836497698b50453373ab3d56c6f9a7ef8" +checksum = "3e545752f6487be87b572529ad594cb3b48d2ef20821516f598b2d152d23277b" dependencies = [ "anyhow", "bincode", @@ -3429,9 +3429,9 @@ dependencies = [ [[package]] name = "lindera-tokenizer" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106ba439b2e87529d9bbedbb88d69f635baba1195c26502b308f55a85885fc81" +checksum = "24a2d4606a5a4da62ac4a3680ee884a75da7f0c892dc967fc9cb983ceba39a8f" dependencies = [ "bincode", "byteorder", @@ -3444,9 +3444,9 @@ dependencies = [ [[package]] name = "lindera-unidic-builder" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b698227fdaeac32289173ab389b990d4eb00a40cbc9912020f69a0c491dabf55" +checksum = "cdfa3e29a22c047da57fadd960ff674b720de15a1e2fb35b5ed67f3408afb469" dependencies = [ "anyhow", "bincode", @@ -3598,9 +3598,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "5486aed0026218e61b8a01d5fbd5a0a134649abb71a0e53b7bc088529dced86e" [[package]] name = "memmap2" @@ -3812,9 +3812,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -3937,9 +3937,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] @@ -4003,11 +4003,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.56" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -4044,9 +4044,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.91" +version = "0.9.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" +checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" dependencies = [ "cc", "libc", @@ -4145,9 +4145,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.8.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7417b1484e3641a8791af3c3123cdc083ac60a0d262a2f281b6125d58917caf4" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ "num-traits", ] @@ -4246,7 +4246,7 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec", - "windows-targets 0.48.3", + "windows-targets 0.48.5", ] [[package]] @@ -4324,12 +4324,12 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap 2.0.0", ] [[package]] @@ -4401,9 +4401,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -4559,9 +4559,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] name = "postcard" @@ -4984,7 +4984,7 @@ dependencies = [ "thousands", "tikv-jemalloc-ctl", "tikv-jemallocator", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-util", "toml 0.7.6", @@ -5011,7 +5011,7 @@ dependencies = [ "serde_json", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tonic 0.9.2", @@ -5156,7 +5156,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tonic 0.9.2", @@ -5173,7 +5173,7 @@ dependencies = [ "serde", "serde_json", "tantivy", - "time 0.3.25", + "time 0.3.26", "time-fmt", ] @@ -5193,7 +5193,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tracing", ] @@ -5203,7 +5203,7 @@ name = "quickwit-doc-mapper" version = "0.6.3" dependencies = [ "anyhow", - "base64 0.21.2", + "base64 0.21.3", "criterion", "dyn-clone", "fnv", @@ -5227,7 +5227,7 @@ dependencies = [ "siphasher", "tantivy", "thiserror", - "time 0.3.25", + "time 0.3.26", "tracing", "typetag", "utoipa", @@ -5259,7 +5259,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tokio-util", @@ -5318,7 +5318,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tracing", @@ -5402,7 +5402,7 @@ version = "0.6.3" dependencies = [ "anyhow", "async-trait", - "base64 0.21.2", + "base64 0.21.3", "itertools 0.11.0", "once_cell", "prost", @@ -5423,7 +5423,7 @@ dependencies = [ "serde_json", "tantivy", "tempfile", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tonic 0.9.2", @@ -5459,7 +5459,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tracing", @@ -5515,7 +5515,7 @@ dependencies = [ "sqlx", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tower", @@ -5531,7 +5531,7 @@ version = "0.6.3" dependencies = [ "anyhow", "async-trait", - "base64 0.21.2", + "base64 0.21.3", "once_cell", "prost", "quickwit-actors", @@ -5586,7 +5586,7 @@ name = "quickwit-query" version = "0.6.3" dependencies = [ "anyhow", - "base64 0.21.2", + "base64 0.21.3", "criterion", "fnv", "hex", @@ -5602,7 +5602,7 @@ dependencies = [ "serde_with 3.3.0", "tantivy", "thiserror", - "time 0.3.25", + "time 0.3.26", "tracing", "whichlang", ] @@ -5638,7 +5638,7 @@ dependencies = [ "anyhow", "assert-json-diff 2.0.2", "async-trait", - "base64 0.21.2", + "base64 0.21.3", "bytes", "chitchat", "fnv", @@ -5733,7 +5733,7 @@ dependencies = [ "tempfile", "termcolor", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tower", @@ -5759,7 +5759,7 @@ dependencies = [ "azure_core", "azure_storage", "azure_storage_blobs", - "base64 0.21.2", + "base64 0.21.3", "bytes", "fnv", "futures", @@ -5957,9 +5957,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.5.0+1.9.2" +version = "4.6.0+2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bb0676c2112342ac7165decdedbc4e7086c0af384479ccce534546b10687a5d" +checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" dependencies = [ "cmake", "libc", @@ -6002,14 +6002,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] @@ -6023,13 +6023,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -6040,9 +6040,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "rend" @@ -6055,11 +6055,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes", "encoding_rs", "futures-core", @@ -6078,7 +6078,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.6", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_json", @@ -6093,7 +6093,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", "winreg", ] @@ -6232,13 +6232,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.31.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea" +checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" dependencies = [ "arrayvec 0.7.4", "borsh", - "byteorder", "bytes", "num-traits", "rand 0.8.5", @@ -6284,9 +6283,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" dependencies = [ "bitflags 2.4.0", "errno", @@ -6297,9 +6296,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", "ring", @@ -6309,13 +6308,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", - "rustls-webpki 0.101.3", + "rustls-webpki 0.101.4", "sct", ] @@ -6337,7 +6336,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", ] [[package]] @@ -6352,9 +6351,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.3" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261e9e0888cba427c3316e6322805653c9425240b6fd96cee7cb671ab70ab8d0" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -6524,9 +6523,9 @@ dependencies = [ [[package]] name = "serde_plain" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6018081315db179d0ce57b1fe4b62a12a0028c9cf9bbef868c9cf477b3c34ae" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" dependencies = [ "serde", ] @@ -6593,7 +6592,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "chrono", "hex", "indexmap 1.9.3", @@ -6601,7 +6600,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros 3.3.0", - "time 0.3.25", + "time 0.3.26", ] [[package]] @@ -6755,14 +6754,14 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.25", + "time 0.3.26", ] [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "sketches-ddsketch" @@ -6775,9 +6774,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -6913,7 +6912,7 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls 0.21.6", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_json", @@ -6921,7 +6920,7 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", - "time 0.3.25", + "time 0.3.26", "tokio", "tokio-stream", "tracing", @@ -6975,7 +6974,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ "atoi", - "base64 0.21.2", + "base64 0.21.3", "bitflags 2.4.0", "byteorder", "bytes", @@ -7006,7 +7005,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time 0.3.25", + "time 0.3.26", "tracing", "whoami", ] @@ -7018,7 +7017,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ "atoi", - "base64 0.21.2", + "base64 0.21.3", "bitflags 2.4.0", "byteorder", "crc", @@ -7046,7 +7045,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time 0.3.25", + "time 0.3.26", "tracing", "whoami", ] @@ -7069,7 +7068,7 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", - "time 0.3.25", + "time 0.3.26", "tracing", "url", ] @@ -7201,7 +7200,7 @@ dependencies = [ "aho-corasick", "arc-swap", "async-trait", - "base64 0.21.2", + "base64 0.21.3", "bitpacking", "byteorder", "census", @@ -7241,7 +7240,7 @@ dependencies = [ "tantivy-tokenizer-api", "tempfile", "thiserror", - "time 0.3.25", + "time 0.3.26", "uuid", "winapi 0.3.9", "zstd 0.12.4", @@ -7279,7 +7278,7 @@ dependencies = [ "byteorder", "ownedbytes", "serde", - "time 0.3.25", + "time 0.3.26", ] [[package]] @@ -7354,7 +7353,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.8", + "rustix 0.38.10", "windows-sys 0.48.0", ] @@ -7464,9 +7463,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" dependencies = [ "deranged", "itoa", @@ -7490,14 +7489,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bfd61bca99323ce96911bd2c443259115460615e44f1d449cee8cb3831a1dd" dependencies = [ "thiserror", - "time 0.3.25", + "time 0.3.26", ] [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" dependencies = [ "time-core", ] @@ -7604,7 +7603,7 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls 0.20.8", + "rustls 0.20.9", "tokio", "webpki", ] @@ -7615,7 +7614,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.6", + "rustls 0.21.7", "tokio", ] @@ -7745,7 +7744,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.2", + "base64 0.21.3", "bytes", "flate2", "futures-core", @@ -7917,7 +7916,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", - "time 0.3.25", + "time 0.3.26", "tracing", "tracing-core", "tracing-log", @@ -8003,9 +8002,9 @@ dependencies = [ [[package]] name = "ulid" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a3aaa69b04e5b66cc27309710a569ea23593612387d67daaf102e73aa974fd" +checksum = "0f9d3475df4ff8a8f7804c0fc3394b44fdcfc4fb635717bf05fbb7c41c83a376" dependencies = [ "rand 0.8.5", "serde", @@ -8019,9 +8018,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -8085,9 +8084,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -8184,7 +8183,7 @@ dependencies = [ "bytes", "chrono", "once_cell", - "ordered-float 3.8.0", + "ordered-float 3.9.1", "path", "regex", "serde", @@ -8228,7 +8227,7 @@ dependencies = [ "getrandom 0.2.10", "indoc", "lalrpop-util", - "ordered-float 3.8.0", + "ordered-float 3.9.1", "paste", "path", "regex", @@ -8251,7 +8250,7 @@ dependencies = [ "chrono-tz", "derivative", "nom", - "ordered-float 3.8.0", + "ordered-float 3.9.1", "path", "serde", "serde_json", @@ -8276,7 +8275,7 @@ source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0 dependencies = [ "lalrpop", "lalrpop-util", - "ordered-float 3.8.0", + "ordered-float 3.9.1", "paste", "path", "thiserror", @@ -8290,7 +8289,7 @@ source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0 dependencies = [ "aes", "base16", - "base64 0.21.2", + "base64 0.21.3", "bytes", "cbc", "cfb-mode", @@ -8311,7 +8310,7 @@ dependencies = [ "nom", "ofb", "once_cell", - "ordered-float 3.8.0", + "ordered-float 3.9.1", "path", "percent-encoding", "quoted_printable", @@ -8518,9 +8517,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -8549,15 +8548,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki", -] - [[package]] name = "webpki-roots" version = "0.23.1" @@ -8573,9 +8563,15 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki 0.101.3", + "rustls-webpki 0.101.4", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "which" version = "4.4.0" @@ -8647,7 +8643,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.3", + "windows-targets 0.48.5", ] [[package]] @@ -8665,7 +8661,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.3", + "windows-targets 0.48.5", ] [[package]] @@ -8685,17 +8681,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f51fb4c64f8b770a823c043c7fad036323e1c48f55287b7bbb7987b2fcdf3b" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.3", - "windows_aarch64_msvc 0.48.3", - "windows_i686_gnu 0.48.3", - "windows_i686_msvc 0.48.3", - "windows_x86_64_gnu 0.48.3", - "windows_x86_64_gnullvm 0.48.3", - "windows_x86_64_msvc 0.48.3", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -8706,9 +8702,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1bb55ae4ce76a597a8566d82c57432bc69c039449d61572a7a353da28f68c" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -8718,9 +8714,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1513e8d48365a78adad7322fd6b5e4c4e99d92a69db8df2d435b25b1f1f286d4" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -8730,9 +8726,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60587c0265d2b842298f5858e1a5d79d146f9ee0c37be5782e92a6eb5e1d7a83" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -8742,9 +8738,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224fe0e0ffff5d2ea6a29f82026c8f43870038a0ffc247aa95a52b47df381ac4" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -8754,9 +8750,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc52a0f50a088de499712cbc012df7ebd94e2d6eb948435449d76a6287e7ad" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" @@ -8766,9 +8762,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2093925509d91ea3d69bcd20238f4c2ecdb1a29d3c281d026a09705d0dd35f3d" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -8778,26 +8774,27 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ade45bc8bf02ae2aa34a9d54ba660a1a58204da34ba793c00d83ca3730b5f1" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi 0.3.9", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] @@ -8808,7 +8805,7 @@ checksum = "c6f71803d3a1c80377a06221e0530be02035d5b3e854af56c6ece7ac20ac441d" dependencies = [ "assert-json-diff 2.0.2", "async-trait", - "base64 0.21.2", + "base64 0.21.3", "deadpool", "futures", "futures-timer", @@ -8899,7 +8896,7 @@ dependencies = [ "hmac", "pbkdf2", "sha1", - "time 0.3.25", + "time 0.3.26", "zstd 0.11.2+zstd.1.5.2", ] diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs index e589079e327..80ce657a968 100644 --- a/quickwit/quickwit-config/src/index_config/mod.rs +++ b/quickwit/quickwit-config/src/index_config/mod.rs @@ -35,6 +35,7 @@ use quickwit_doc_mapper::{ DefaultDocMapper, DefaultDocMapperBuilder, DocMapper, FieldMappingEntry, Mode, ModeType, QuickwitJsonOptions, TokenizerEntry, }; +use quickwit_proto::IndexId; use serde::{Deserialize, Serialize}; pub use serialize::load_index_config_from_user_config; @@ -296,7 +297,7 @@ fn prepend_at_char(schedule: &str) -> String { #[serde(into = "VersionedIndexConfig")] #[serde(try_from = "VersionedIndexConfig")] pub struct IndexConfig { - pub index_id: String, + pub index_id: IndexId, pub index_uri: Uri, pub doc_mapping: DocMapping, pub indexing_settings: IndexingSettings, diff --git a/quickwit/quickwit-indexing/src/actors/index_serializer.rs b/quickwit/quickwit-indexing/src/actors/index_serializer.rs index c2750ea8f4f..97c6ba32843 100644 --- a/quickwit/quickwit-indexing/src/actors/index_serializer.rs +++ b/quickwit/quickwit-indexing/src/actors/index_serializer.rs @@ -94,11 +94,12 @@ impl Handler for IndexSerializer { splits.push(split); } let indexed_split_batch = IndexedSplitBatch { - batch_parent_span: batch_builder.batch_parent_span, splits, - checkpoint_delta: batch_builder.checkpoint_delta, + checkpoint_delta_opt: batch_builder.checkpoint_delta_opt, publish_lock: batch_builder.publish_lock, - merge_operation: None, + publish_token_opt: batch_builder.publish_token_opt, + merge_operation_opt: None, + batch_parent_span: batch_builder.batch_parent_span, }; ctx.send_message(&self.packager_mailbox, indexed_split_batch) .await?; diff --git a/quickwit/quickwit-indexing/src/actors/indexer.rs b/quickwit/quickwit-indexing/src/actors/indexer.rs index 1b02250c119..6ebdbc1649e 100644 --- a/quickwit/quickwit-indexing/src/actors/indexer.rs +++ b/quickwit/quickwit-indexing/src/actors/indexer.rs @@ -40,6 +40,7 @@ use quickwit_doc_mapper::DocMapper; use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta}; use quickwit_metastore::Metastore; use quickwit_proto::indexing::IndexingPipelineId; +use quickwit_proto::PublishToken; use quickwit_query::get_quickwit_fastfield_normalizer_manager; use serde::Serialize; use tantivy::schema::Schema; @@ -205,6 +206,7 @@ impl IndexerState { }, indexing_permit, publish_lock: self.publish_lock.clone(), + publish_token_opt: None, // TODO: Get publish token from source (next PR) last_delete_opstamp, memory_usage: Byte::from_bytes(0), }; @@ -318,6 +320,7 @@ struct IndexingWorkbench { checkpoint_delta: IndexCheckpointDelta, indexing_permit: Option, publish_lock: PublishLock, + publish_token_opt: Option, // On workbench creation, we fetch from the metastore the last delete task opstamp. // We use this value to set the `delete_opstamp` of the workbench splits. last_delete_opstamp: u64, @@ -550,6 +553,7 @@ impl Indexer { other_indexed_split_opt, checkpoint_delta, publish_lock, + publish_token_opt, batch_parent_span, indexing_permit, .. @@ -574,9 +578,10 @@ impl Indexer { &self.index_serializer_mailbox, EmptySplit { index_uid: self.indexer_state.pipeline_id.index_uid.clone(), - batch_parent_span, checkpoint_delta, publish_lock, + publish_token_opt, + batch_parent_span, }, ) .await?; @@ -589,11 +594,12 @@ impl Indexer { ctx.send_message( &self.index_serializer_mailbox, IndexedSplitBatchBuilder { - batch_parent_span, splits, - checkpoint_delta: Some(checkpoint_delta), + checkpoint_delta_opt: Some(checkpoint_delta), publish_lock, + publish_token_opt, commit_trigger, + batch_parent_span, }, ) .await?; @@ -771,7 +777,7 @@ mod tests { for split in batch.splits.iter() { assert_eq!(split.split_attrs.delete_opstamp, last_delete_opstamp); } - let index_checkpoint = batch.checkpoint_delta.unwrap(); + let index_checkpoint = batch.checkpoint_delta_opt.unwrap(); assert_eq!(index_checkpoint.source_id, "test-source"); assert_eq!( index_checkpoint.source_delta, @@ -1497,7 +1503,7 @@ mod tests { let mut metastore = MockMetastore::default(); metastore .expect_publish_splits() - .returning(move |_, splits, _, _| { + .returning(move |_, splits, _, _, _| { assert!(splits.is_empty()); Ok(()) }); diff --git a/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs b/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs index 9b48297125c..44dcda5780e 100644 --- a/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs +++ b/quickwit/quickwit-indexing/src/actors/indexing_pipeline.rs @@ -604,7 +604,12 @@ mod tests { metastore .expect_publish_splits() .withf( - |index_uid, splits, replaced_splits, checkpoint_delta_opt| -> bool { + |index_uid, + splits, + replaced_splits, + checkpoint_delta_opt, + _publish_token_opt| + -> bool { let checkpoint_delta = checkpoint_delta_opt.as_ref().unwrap(); index_uid.to_string() == "test-index:11111111111111111111111111" && checkpoint_delta.source_id == "test-source" @@ -614,7 +619,7 @@ mod tests { .ends_with(":(00000000000000000000..00000000000000001030])") }, ) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let node_id = "test-node"; let metastore = Arc::new(metastore); let pipeline_id = IndexingPipelineId { @@ -698,7 +703,12 @@ mod tests { metastore .expect_publish_splits() .withf( - |index_uid, splits, replaced_split_ids, checkpoint_delta_opt| -> bool { + |index_uid, + splits, + replaced_split_ids, + checkpoint_delta_opt, + _publish_token_opt| + -> bool { let checkpoint_delta = checkpoint_delta_opt.as_ref().unwrap(); index_uid.to_string() == "test-index:11111111111111111111111111" && splits.len() == 1 @@ -708,7 +718,7 @@ mod tests { .ends_with(":(00000000000000000000..00000000000000001030])") }, ) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let universe = Universe::new(); let node_id = "test-node"; let metastore = Arc::new(metastore); @@ -872,7 +882,12 @@ mod tests { metastore .expect_publish_splits() .withf( - |index_uid, splits, replaced_split_ids, checkpoint_delta_opt| -> bool { + |index_uid, + splits, + replaced_split_ids, + checkpoint_delta_opt, + _publish_token_opt| + -> bool { let checkpoint_delta = checkpoint_delta_opt.as_ref().unwrap(); index_uid.to_string() == "test-index:11111111111111111111111111" && splits.is_empty() @@ -882,7 +897,7 @@ mod tests { .ends_with(":(00000000000000000000..00000000000000001030])") }, ) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let universe = Universe::new(); let node_id = "test-node"; let metastore = Arc::new(metastore); diff --git a/quickwit/quickwit-indexing/src/actors/merge_executor.rs b/quickwit/quickwit-indexing/src/actors/merge_executor.rs index 8845286a8a0..97918c50f89 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_executor.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_executor.rs @@ -128,11 +128,12 @@ impl Handler for MergeExecutor { ctx.send_message( &self.merge_packager_mailbox, IndexedSplitBatch { - batch_parent_span: merge_op.merge_parent_span.clone(), splits: vec![indexed_split], - checkpoint_delta: Default::default(), + checkpoint_delta_opt: Default::default(), publish_lock: PublishLock::default(), - merge_operation: Some(merge_op), + publish_token_opt: None, + batch_parent_span: merge_op.merge_parent_span.clone(), + merge_operation_opt: Some(merge_op), }, ) .await?; @@ -716,6 +717,7 @@ mod tests { &[new_split_metadata.split_id()], &[split_metadata.split_id()], None, + None, ) .await .unwrap(); diff --git a/quickwit/quickwit-indexing/src/actors/packager.rs b/quickwit/quickwit-indexing/src/actors/packager.rs index 07a5c3e756d..1190a139dfc 100644 --- a/quickwit/quickwit-indexing/src/actors/packager.rs +++ b/quickwit/quickwit-indexing/src/actors/packager.rs @@ -136,7 +136,7 @@ impl Handler for Packager { "start-packaging-splits" ); fail_point!("packager:before"); - let mut packaged_splits = Vec::new(); + let mut packaged_splits = Vec::with_capacity(batch.splits.len()); for split in batch.splits { if batch.publish_lock.is_dead() { // TODO: Remove the junk right away? @@ -153,9 +153,10 @@ impl Handler for Packager { &self.uploader_mailbox, PackagedSplitBatch::new( packaged_splits, - batch.checkpoint_delta, + batch.checkpoint_delta_opt, batch.publish_lock, - batch.merge_operation, + batch.publish_token_opt, + batch.merge_operation_opt, batch.batch_parent_span, ), ) @@ -465,10 +466,11 @@ mod tests { packager_mailbox .send_message(IndexedSplitBatch { splits: vec![indexed_split], - checkpoint_delta: IndexCheckpointDelta::for_test("source_id", 10..20).into(), + checkpoint_delta_opt: IndexCheckpointDelta::for_test("source_id", 10..20).into(), publish_lock: PublishLock::default(), + publish_token_opt: None, + merge_operation_opt: None, batch_parent_span: Span::none(), - merge_operation: None, }) .await?; assert_eq!( diff --git a/quickwit/quickwit-indexing/src/actors/publisher.rs b/quickwit/quickwit-indexing/src/actors/publisher.rs index aaaad7e1d8e..b78bd09de93 100644 --- a/quickwit/quickwit-indexing/src/actors/publisher.rs +++ b/quickwit/quickwit-indexing/src/actors/publisher.rs @@ -117,8 +117,8 @@ impl Handler for Publisher { replaced_split_ids, checkpoint_delta_opt, publish_lock, - merge_operation: _, - parent_span: _, + publish_token_opt, + .. } = split_update; let split_ids: Vec<&str> = new_splits.iter().map(|split| split.split_id()).collect(); @@ -132,6 +132,7 @@ impl Handler for Publisher { &split_ids[..], &replaced_split_ids_ref_vec, checkpoint_delta_opt.clone(), + publish_token_opt, )) .await .context("Failed to publish splits.")?; @@ -205,7 +206,11 @@ mod tests { mock_metastore .expect_publish_splits() .withf( - |index_uid, split_ids, replaced_split_ids, checkpoint_delta_opt| { + |index_uid, + split_ids, + replaced_split_ids, + checkpoint_delta_opt, + _publish_token_opt| { let checkpoint_delta = checkpoint_delta_opt.as_ref().unwrap(); index_uid.to_string() == "index:11111111111111111111111111" && checkpoint_delta.source_id == "source" @@ -215,7 +220,7 @@ mod tests { }, ) .times(1) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let (merge_planner_mailbox, merge_planner_inbox) = universe.create_test_mailbox(); let (source_mailbox, source_inbox) = universe.create_test_mailbox(); @@ -241,6 +246,7 @@ mod tests { source_delta: SourceCheckpointDelta::from_range(1..3), }), publish_lock: PublishLock::default(), + publish_token_opt: None, merge_operation: None, parent_span: tracing::Span::none(), }) @@ -277,7 +283,11 @@ mod tests { mock_metastore .expect_publish_splits() .withf( - |index_uid, split_ids, replaced_split_ids, checkpoint_delta_opt| { + |index_uid, + split_ids, + replaced_split_ids, + checkpoint_delta_opt, + _publish_token_opt| { let checkpoint_delta = checkpoint_delta_opt.as_ref().unwrap(); index_uid.to_string() == "index:11111111111111111111111111" && checkpoint_delta.source_id == "source" @@ -287,7 +297,7 @@ mod tests { }, ) .times(1) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let (merge_planner_mailbox, merge_planner_inbox) = universe.create_test_mailbox(); let (source_mailbox, source_inbox) = universe.create_test_mailbox(); @@ -310,6 +320,7 @@ mod tests { source_delta: SourceCheckpointDelta::from_range(1..3), }), publish_lock: PublishLock::default(), + publish_token_opt: None, merge_operation: None, parent_span: tracing::Span::none(), }) @@ -347,7 +358,11 @@ mod tests { mock_metastore .expect_publish_splits() .withf( - |index_uid, new_split_ids, replaced_split_ids, checkpoint_delta_opt| { + |index_uid, + new_split_ids, + replaced_split_ids, + checkpoint_delta_opt, + _publish_token_opt| { index_uid.to_string() == "index:11111111111111111111111111" && new_split_ids[..] == ["split3"] && replaced_split_ids[..] == ["split1", "split2"] @@ -355,7 +370,7 @@ mod tests { }, ) .times(1) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let (merge_planner_mailbox, merge_planner_inbox) = universe.create_test_mailbox(); let publisher = Publisher::new( PublisherType::MainPublisher, @@ -373,6 +388,7 @@ mod tests { replaced_split_ids: vec!["split1".to_string(), "split2".to_string()], checkpoint_delta_opt: None, publish_lock: PublishLock::default(), + publish_token_opt: None, merge_operation: None, parent_span: Span::none(), }; @@ -414,6 +430,7 @@ mod tests { replaced_split_ids: Vec::new(), checkpoint_delta_opt: None, publish_lock, + publish_token_opt: None, merge_operation: None, parent_span: Span::none(), }) diff --git a/quickwit/quickwit-indexing/src/actors/uploader.rs b/quickwit/quickwit-indexing/src/actors/uploader.rs index bde305b28d6..4b3a76b8918 100644 --- a/quickwit/quickwit-indexing/src/actors/uploader.rs +++ b/quickwit/quickwit-indexing/src/actors/uploader.rs @@ -31,7 +31,7 @@ use once_cell::sync::OnceCell; use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity}; use quickwit_metastore::checkpoint::IndexCheckpointDelta; use quickwit_metastore::{Metastore, SplitMetadata}; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use quickwit_storage::SplitPayloadBuilder; use serde::Serialize; use tantivy::TrackedObject; @@ -261,7 +261,7 @@ impl Handler for Uploader { type Reply = (); #[instrument(name = "uploader", - parent=batch.parent_span.id(), + parent=batch.batch_parent_span.id(), skip_all)] async fn handle( &mut self, @@ -349,11 +349,12 @@ impl Handler for Uploader { let splits_update = make_publish_operation( index_uid, - batch.publish_lock, packaged_splits_and_metadata, batch.checkpoint_delta_opt, - batch.merge_operation, - batch.parent_span, + batch.publish_lock, + batch.publish_token_opt, + batch.merge_operation_opt, + batch.batch_parent_span, ); split_update_sender.send(splits_update, &ctx_clone).await?; @@ -393,6 +394,7 @@ impl Handler for Uploader { replaced_split_ids: Vec::new(), checkpoint_delta_opt: Some(empty_split.checkpoint_delta), publish_lock: empty_split.publish_lock, + publish_token_opt: empty_split.publish_token_opt, merge_operation: None, parent_span: empty_split.batch_parent_span, }; @@ -404,9 +406,10 @@ impl Handler for Uploader { fn make_publish_operation( index_uid: IndexUid, - publish_lock: PublishLock, packaged_splits_and_metadatas: Vec<(PackagedSplit, SplitMetadata)>, checkpoint_delta_opt: Option, + publish_lock: PublishLock, + publish_token_opt: Option, merge_operation: Option>, parent_span: Span, ) -> SplitsUpdate { @@ -417,13 +420,14 @@ fn make_publish_operation( .collect::>(); SplitsUpdate { index_uid, - publish_lock, new_splits: packaged_splits_and_metadatas .into_iter() .map(|split_and_meta| split_and_meta.1) .collect_vec(), replaced_split_ids: Vec::from_iter(replaced_split_ids), checkpoint_delta_opt, + publish_lock, + publish_token_opt, merge_operation, parent_span, } @@ -540,6 +544,7 @@ mod tests { checkpoint_delta_opt, PublishLock::default(), None, + None, Span::none(), )) .await?; @@ -674,6 +679,7 @@ mod tests { None, PublishLock::default(), None, + None, Span::none(), )) .await?; @@ -785,6 +791,7 @@ mod tests { checkpoint_delta_opt, PublishLock::default(), None, + None, Span::none(), )) .await?; @@ -832,9 +839,10 @@ mod tests { uploader_mailbox .send_message(EmptySplit { index_uid: IndexUid::new("test-index"), - batch_parent_span: Span::none(), checkpoint_delta, publish_lock: PublishLock::default(), + publish_token_opt: None, + batch_parent_span: Span::none(), }) .await?; assert_eq!( diff --git a/quickwit/quickwit-indexing/src/models/indexed_split.rs b/quickwit/quickwit-indexing/src/models/indexed_split.rs index 7fda336b32d..ce285e2fcd9 100644 --- a/quickwit/quickwit-indexing/src/models/indexed_split.rs +++ b/quickwit/quickwit-indexing/src/models/indexed_split.rs @@ -24,7 +24,7 @@ use quickwit_common::io::IoControls; use quickwit_common::temp_dir::TempDirectory; use quickwit_metastore::checkpoint::IndexCheckpointDelta; use quickwit_proto::indexing::IndexingPipelineId; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use tantivy::directory::MmapDirectory; use tantivy::{IndexBuilder, TrackedObject}; use tracing::{instrument, Span}; @@ -153,15 +153,16 @@ impl IndexedSplitBuilder { #[derive(Debug)] pub struct IndexedSplitBatch { - pub batch_parent_span: Span, pub splits: Vec, - pub checkpoint_delta: Option, + pub checkpoint_delta_opt: Option, pub publish_lock: PublishLock, + pub publish_token_opt: Option, /// A [`MergeOperation`] tracked by either the `MergePlanner` or the `DeleteTaskPlanner` /// in the `MergePipeline` or `DeleteTaskPipeline`. /// See planners docs to understand the usage. /// If `None`, the split batch was built in the `IndexingPipeline`. - pub merge_operation: Option>, + pub merge_operation_opt: Option>, + pub batch_parent_span: Span, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -176,18 +177,20 @@ pub enum CommitTrigger { #[derive(Debug)] pub struct IndexedSplitBatchBuilder { - pub batch_parent_span: Span, pub splits: Vec, - pub checkpoint_delta: Option, + pub checkpoint_delta_opt: Option, pub publish_lock: PublishLock, + pub publish_token_opt: Option, pub commit_trigger: CommitTrigger, + pub batch_parent_span: Span, } /// Sends notifications to the Publisher that the last batch of splits was emtpy. #[derive(Debug)] pub struct EmptySplit { pub index_uid: IndexUid, - pub batch_parent_span: Span, pub checkpoint_delta: IndexCheckpointDelta, pub publish_lock: PublishLock, + pub publish_token_opt: Option, + pub batch_parent_span: Span, } diff --git a/quickwit/quickwit-indexing/src/models/packaged_split.rs b/quickwit/quickwit-indexing/src/models/packaged_split.rs index c1dd252bb0c..a8055c31214 100644 --- a/quickwit/quickwit-indexing/src/models/packaged_split.rs +++ b/quickwit/quickwit-indexing/src/models/packaged_split.rs @@ -17,12 +17,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::collections::{BTreeSet, HashSet}; +use std::collections::BTreeSet; use std::fmt; +use itertools::Itertools; use quickwit_common::temp_dir::TempDirectory; use quickwit_metastore::checkpoint::IndexCheckpointDelta; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use tantivy::TrackedObject; use tracing::Span; @@ -38,6 +39,10 @@ pub struct PackagedSplit { } impl PackagedSplit { + pub fn index_uid(&self) -> &IndexUid { + &self.split_attrs.pipeline_id.index_uid + } + pub fn split_id(&self) -> &str { &self.split_attrs.split_id } @@ -56,45 +61,46 @@ impl fmt::Debug for PackagedSplit { #[derive(Debug)] pub struct PackagedSplitBatch { - pub parent_span: Span, pub splits: Vec, pub checkpoint_delta_opt: Option, + pub publish_lock: PublishLock, + pub publish_token_opt: Option, /// A [`MergeOperation`] tracked by either the `MergePlanner` or the `DeleteTaskPlanner` /// in the `MergePipeline` or `DeleteTaskPipeline`. /// See planners docs to understand the usage. /// If `None`, the split batch was built in the `IndexingPipeline`. - pub merge_operation: Option>, - pub publish_lock: PublishLock, + pub merge_operation_opt: Option>, + pub batch_parent_span: Span, } impl PackagedSplitBatch { /// Instantiate a consistent [`PackagedSplitBatch`] that /// satisfies two constraints: /// - a batch must have at least one split - /// - all splits must be on the same `index_id`. + /// - all splits must belong to the same `index_uid`. pub fn new( splits: Vec, checkpoint_delta_opt: Option, publish_lock: PublishLock, - merge_operation: Option>, - span: Span, + publish_token_opt: Option, + merge_operation_opt: Option>, + batch_parent_span: Span, ) -> Self { assert!(!splits.is_empty()); - assert_eq!( + assert!( splits .iter() - .map(|split| split.split_attrs.pipeline_id.index_uid.clone()) - .collect::>() - .len(), - 1, - "All splits must be on the same `index_id`." + .tuple_windows() + .all(|(left_split, right_split)| left_split.index_uid() == right_split.index_uid()), + "All splits must belong to the same `index_uid`." ); Self { - parent_span: span, splits, checkpoint_delta_opt, publish_lock, - merge_operation, + publish_token_opt, + merge_operation_opt, + batch_parent_span, } } diff --git a/quickwit/quickwit-indexing/src/models/publisher_message.rs b/quickwit/quickwit-indexing/src/models/publisher_message.rs index e5874e0d630..e6c58b82282 100644 --- a/quickwit/quickwit-indexing/src/models/publisher_message.rs +++ b/quickwit/quickwit-indexing/src/models/publisher_message.rs @@ -22,7 +22,7 @@ use std::fmt; use itertools::Itertools; use quickwit_metastore::checkpoint::IndexCheckpointDelta; use quickwit_metastore::SplitMetadata; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use tantivy::TrackedObject; use tracing::Span; @@ -35,6 +35,7 @@ pub struct SplitsUpdate { pub replaced_split_ids: Vec, pub checkpoint_delta_opt: Option, pub publish_lock: PublishLock, + pub publish_token_opt: Option, /// A [`MergeOperation`] tracked by either the `MergePlanner` or the `DeleteTaskPlanner` /// in the `MergePipeline` or `DeleteTaskPipeline`. /// See planners docs to understand the usage. diff --git a/quickwit/quickwit-indexing/src/source/kafka_source.rs b/quickwit/quickwit-indexing/src/source/kafka_source.rs index 2e801b433bd..ef10b823f2e 100644 --- a/quickwit/quickwit-indexing/src/source/kafka_source.rs +++ b/quickwit/quickwit-indexing/src/source/kafka_source.rs @@ -911,12 +911,18 @@ mod kafka_broker_tests { ) .unwrap(); } - let index_delta = IndexCheckpointDelta { + let checkpoint_delta = IndexCheckpointDelta { source_id: source_id.to_string(), source_delta, }; metastore - .publish_splits(index_uid.clone(), &[&split_id], &[], Some(index_delta)) + .publish_splits( + index_uid.clone(), + &[&split_id], + &[], + Some(checkpoint_delta), + None, + ) .await .unwrap(); index_uid diff --git a/quickwit/quickwit-indexing/src/source/pulsar_source.rs b/quickwit/quickwit-indexing/src/source/pulsar_source.rs index c8fb0d2aaea..114b35113f5 100644 --- a/quickwit/quickwit-indexing/src/source/pulsar_source.rs +++ b/quickwit/quickwit-indexing/src/source/pulsar_source.rs @@ -511,12 +511,18 @@ mod pulsar_broker_tests { ) .unwrap(); } - let index_delta = IndexCheckpointDelta { + let checkpoint_delta = IndexCheckpointDelta { source_id: source_id.to_string(), source_delta, }; metastore - .publish_splits(index_uid.clone(), &[&split_id], &[], Some(index_delta)) + .publish_splits( + index_uid.clone(), + &[&split_id], + &[], + Some(checkpoint_delta), + None, + ) .await .unwrap(); index_uid diff --git a/quickwit/quickwit-metastore/src/checkpoint.rs b/quickwit/quickwit-metastore/src/checkpoint.rs index 0baabcb9d7c..3526c925dc8 100644 --- a/quickwit/quickwit-metastore/src/checkpoint.rs +++ b/quickwit/quickwit-metastore/src/checkpoint.rs @@ -25,15 +25,23 @@ use std::iter::FromIterator; use std::ops::Range; use std::sync::Arc; +use quickwit_proto::SourceId; use serde::ser::SerializeMap; use serde::{Deserialize, Serialize}; use thiserror::Error; use tracing::{info, warn}; -/// PartitionId identifies a partition for a given source. +/// A `PartitionId` uniquely identifies a partition for a given source. #[derive(Clone, Debug, Default, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct PartitionId(pub Arc); +impl PartitionId { + /// Returns the partition ID as a `u64`. + pub fn as_u64(&self) -> Option { + self.0.parse().ok() + } +} + impl fmt::Display for PartitionId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", &self.0) @@ -69,12 +77,12 @@ impl From for PartitionId { /// Marks a position within a specific partition of a source. /// /// The nature of the position may very depending on the source. -/// Each source needs to encode it as a String in such a way that +/// Each source needs to encode it as a `String` in such a way that /// the lexicographical order matches the natural order of the /// position. /// -/// For instance, for u64 a 0-left-padded decimal representation -/// can be used. Alternatively a base64 representation of their +/// For instance, for u64, a 20-left-padded decimal representation +/// can be used. Alternatively, a base64 representation of their /// Big Endian representation can be used. /// /// The empty string can be used to represent the beginning of the source, @@ -120,8 +128,8 @@ impl From for Position { } } -impl<'a> From<&'a str> for Position { - fn from(position_str: &'a str) -> Self { +impl From<&str> for Position { + fn from(position_str: &str) -> Self { match position_str { "" => Position::Beginning, _ => Position::Offset(Arc::new(position_str.to_string())), @@ -129,10 +137,17 @@ impl<'a> From<&'a str> for Position { } } +/// A partition delta represents an interval (from, to] over a partition of a source. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct PartitionDelta { + pub from: Position, + pub to: Position, +} + #[derive(Default, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct IndexCheckpoint { #[serde(flatten)] - per_source: BTreeMap, + per_source: BTreeMap, } impl fmt::Debug for IndexCheckpoint { @@ -143,14 +158,14 @@ impl fmt::Debug for IndexCheckpoint { } } -impl From> for IndexCheckpoint { - fn from(per_source: BTreeMap) -> Self { - IndexCheckpoint { per_source } +impl From> for IndexCheckpoint { + fn from(per_source: BTreeMap) -> Self { + Self { per_source } } } impl IndexCheckpoint { - /// Updates a given source checkpoint. Returns whether the checkpoint was modified. + /// Updates a checkpoint in place. Returns whether the checkpoint was modified. /// /// If the checkpoint delta is not compatible with the /// current checkpoint, an error is returned, and the @@ -177,7 +192,7 @@ impl IndexCheckpoint { self.per_source.remove(source_id).is_some() } - /// Returns the checkpoint associated to a given source. + /// Returns the checkpoint associated with a given source. /// /// All registered source have an associated checkpoint (that is possibly empty). /// @@ -215,6 +230,11 @@ pub struct SourceCheckpoint { } impl SourceCheckpoint { + /// Adds a partition to the checkpoint. + pub fn add_partition(&mut self, partition_id: PartitionId, position: Position) { + self.per_partition.insert(partition_id, position); + } + /// Returns the number of partitions covered by the checkpoint. pub fn num_partitions(&self) -> usize { self.per_partition.len() @@ -275,16 +295,16 @@ impl<'de> Deserialize<'de> for SourceCheckpoint { /// the checkpoint. #[derive(Clone, Debug, Error, Eq, PartialEq, Serialize, Deserialize)] #[error( - "Incompatible checkpoint delta at partition `{partition_id}`: cur_pos:{current_position:?} \ - delta_pos:{delta_position_from:?}" + "Incompatible checkpoint delta at partition `{partition_id}`: cur_pos:{partition_position:?} \ + delta_pos:{delta_from_position:?}" )] pub struct IncompatibleCheckpointDelta { /// The partition ID for which the incompatibility has been detected. pub partition_id: PartitionId, - /// The current position within this partition. - pub current_position: Position, - /// The origin position for the delta. - pub delta_position_from: Position, + /// The current position (inclusive) within this partition. + pub partition_position: Position, + /// The start position (exclusive) for the delta. + pub delta_from_position: Position, } #[derive(Clone, Debug, Error, Serialize, Deserialize, PartialEq, Eq)] @@ -318,7 +338,7 @@ impl SourceCheckpoint { .map(|(partition_id, position)| (partition_id.clone(), position.clone())) } - fn check_compatibility( + pub fn check_compatibility( &self, delta: &SourceCheckpointDelta, ) -> Result<(), IncompatibleCheckpointDelta> { @@ -335,8 +355,8 @@ impl SourceCheckpoint { Ordering::Greater => { return Err(IncompatibleCheckpointDelta { partition_id: delta_partition.clone(), - current_position: position.clone(), - delta_position_from: delta_position.from.clone(), + partition_position: position.clone(), + delta_from_position: delta_position.from.clone(), }); } } @@ -390,13 +410,6 @@ impl fmt::Debug for SourceCheckpoint { } } -/// A partition delta represents an interval (from, to] over a partition of a source. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -struct PartitionDelta { - pub from: Position, - pub to: Position, -} - /// A checkpoint delta represents a checkpoint update. /// /// It is shipped as part of a split to convey the update @@ -407,11 +420,6 @@ struct PartitionDelta { /// partition not only a new position, but also an expected /// `from` position. This makes it possible to defensively check that /// we are not trying to add documents to the index that were already indexed. -#[derive(Default, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub struct SourceCheckpointDelta { - per_partition: BTreeMap, -} - #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct IndexCheckpointDelta { pub source_id: String, @@ -425,7 +433,7 @@ impl IndexCheckpointDelta { #[cfg(any(test, feature = "testsuite"))] pub fn for_test(source_id: &str, pos_range: Range) -> Self { - IndexCheckpointDelta { + Self { source_id: source_id.to_string(), source_delta: SourceCheckpointDelta::from_range(pos_range), } @@ -439,6 +447,11 @@ impl fmt::Debug for IndexCheckpointDelta { } } +#[derive(Default, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct SourceCheckpointDelta { + per_partition: BTreeMap, +} + impl fmt::Debug for SourceCheckpointDelta { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str("∆(")?; @@ -509,6 +522,13 @@ impl SourceCheckpointDelta { source_checkpoint } + /// Returns an iterator of partition IDs and associated deltas. + pub fn iter(&self) -> impl Iterator + '_ { + self.per_partition + .iter() + .map(|(partition_id, partition_delta)| (partition_id.clone(), partition_delta.clone())) + } + /// Records a `(from, to]` partition delta for a given partition. pub fn record_partition_delta( &mut self, @@ -532,8 +552,8 @@ impl SourceCheckpointDelta { } else { return Err(PartitionDeltaError::from(IncompatibleCheckpointDelta { partition_id: occupied_entry.key().clone(), - current_position: occupied_entry.get().to.clone(), - delta_position_from: from_position, + partition_position: occupied_entry.get().to.clone(), + delta_from_position: from_position, })); } } @@ -775,8 +795,8 @@ mod tests { result, Err(PartitionDeltaError::from(IncompatibleCheckpointDelta { partition_id: PartitionId::from("a"), - current_position: Position::from("00128"), - delta_position_from: Position::from("00130") + partition_position: Position::from("00128"), + delta_from_position: Position::from("00130") })) ); } diff --git a/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs b/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs index 77e45470721..ba2c5b07e83 100644 --- a/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/control_plane_metastore.rs @@ -30,7 +30,7 @@ use quickwit_proto::metastore::{ DeleteShardsResponse, DeleteSourceRequest, DeleteTask, ListShardsRequest, ListShardsResponse, MetastoreResult, OpenShardsRequest, OpenShardsResponse, ToggleSourceRequest, }; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use crate::checkpoint::IndexCheckpointDelta; use crate::{IndexMetadata, ListIndexesQuery, ListSplitsQuery, Metastore, Split, SplitMetadata}; @@ -149,6 +149,7 @@ impl Metastore for ControlPlaneMetastore { staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { self.metastore .publish_splits( @@ -156,6 +157,7 @@ impl Metastore for ControlPlaneMetastore { staged_split_ids, replaced_split_ids, checkpoint_delta_opt, + publish_token_opt, ) .await } diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/mod.rs b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/mod.rs index 2a2c7307254..1148c928029 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/mod.rs @@ -22,22 +22,28 @@ //! anything from here directly. mod serialize; +mod shards; use std::collections::HashMap; use std::fmt::Debug; use std::ops::Bound; +use itertools::Either; use quickwit_common::PrettySample; -use quickwit_config::SourceConfig; +use quickwit_config::{SourceConfig, INGEST_SOURCE_ID}; use quickwit_proto::metastore::{ - DeleteQuery, DeleteTask, EntityKind, MetastoreError, MetastoreResult, + CloseShardsFailure, CloseShardsSubrequest, CloseShardsSuccess, DeleteQuery, + DeleteShardsSubrequest, DeleteTask, EntityKind, ListShardsSubrequest, ListShardsSubresponse, + MetastoreError, MetastoreResult, OpenShardsSubrequest, OpenShardsSubresponse, }; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken, SourceId, SplitId}; use serde::{Deserialize, Serialize}; use serialize::VersionedFileBackedIndex; +use shards::Shards; use time::OffsetDateTime; use tracing::{info, warn}; +use super::MutationOccurred; use crate::checkpoint::IndexCheckpointDelta; use crate::{split_tag_filter, IndexMetadata, ListSplitsQuery, Split, SplitMetadata, SplitState}; @@ -46,11 +52,13 @@ use crate::{split_tag_filter, IndexMetadata, ListSplitsQuery, Split, SplitMetada #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(into = "VersionedFileBackedIndex")] #[serde(from = "VersionedFileBackedIndex")] -pub struct FileBackedIndex { +pub(crate) struct FileBackedIndex { /// Metadata specific to the index. metadata: IndexMetadata, /// List of splits belonging to the index. - splits: HashMap, + splits: HashMap, + /// Shards of each source. + per_source_shards: HashMap, /// Delete tasks. delete_tasks: Vec, /// Stamper. @@ -71,7 +79,14 @@ pub struct FileBackedIndex { #[cfg(any(test, feature = "testsuite"))] impl quickwit_config::TestableForRegression for FileBackedIndex { fn sample_for_regression() -> Self { + use quickwit_proto::ingest::Shard; + + use self::shards::SerdeShards; + let index_metadata = IndexMetadata::sample_for_regression(); + let index_uid = index_metadata.index_uid.clone(); + let source_id = INGEST_SOURCE_ID.to_string(); + let split_metadata = SplitMetadata::sample_for_regression(); let split = Split { split_state: SplitState::Published, @@ -80,22 +95,41 @@ impl quickwit_config::TestableForRegression for FileBackedIndex { publish_timestamp: Some(1789), }; let splits = vec![split]; + + let shard = Shard { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + shard_id: 1, + leader_id: "leader-ingester".to_string(), + follower_id: Some("follower-ingester".to_string()), + ..Default::default() + }; + + let serde_shards = SerdeShards { + next_shard_id: 2, + shards: vec![shard], + }; + let shards = Shards::from_serde_shards(index_uid.clone(), source_id.clone(), serde_shards); + let per_source_shards = HashMap::from_iter([(source_id, shards)]); + let delete_task = DeleteTask { create_timestamp: 0, opstamp: 10, delete_query: Some(DeleteQuery { - index_uid: "index:11111111111111111111111111".to_string(), + index_uid: index_uid.into(), start_timestamp: None, end_timestamp: None, query_ast: quickwit_query::query_ast::qast_json_helper("Harry Potter", &["body"]), }), }; - FileBackedIndex::new(index_metadata, splits, vec![delete_task]) + let delete_tasks = vec![delete_task]; + FileBackedIndex::new(index_metadata, splits, per_source_shards, delete_tasks) } fn test_equality(&self, other: &Self) { self.metadata().test_equality(other.metadata()); - assert_eq!(self.splits(), other.splits()); + assert_eq!(self.splits, other.splits); + assert_eq!(self.per_source_shards, other.per_source_shards); assert_eq!(self.delete_tasks, other.delete_tasks); } } @@ -105,6 +139,7 @@ impl From for FileBackedIndex { Self { metadata: index_metadata, splits: Default::default(), + per_source_shards: Default::default(), delete_tasks: Default::default(), stamper: Default::default(), recently_modified: false, @@ -122,18 +157,25 @@ enum DeleteSplitOutcome { impl FileBackedIndex { /// Constructor. - pub fn new(metadata: IndexMetadata, splits: Vec, delete_tasks: Vec) -> Self { + pub fn new( + metadata: IndexMetadata, + splits: Vec, + per_source_shards: HashMap, + delete_tasks: Vec, + ) -> Self { let last_opstamp = delete_tasks .iter() .map(|delete_task| delete_task.opstamp) .max() .unwrap_or(0) as usize; + let splits = splits + .into_iter() + .map(|split| (split.split_id().to_string(), split)) + .collect(); Self { metadata, - splits: splits - .into_iter() - .map(|split| (split.split_id().to_string(), split)) - .collect(), + splits, + per_source_shards, delete_tasks, stamper: Stamper::new(last_opstamp), recently_modified: false, @@ -166,11 +208,6 @@ impl FileBackedIndex { &self.metadata } - /// Splits accessor. - pub fn splits(&self) -> &HashMap { - &self.splits - } - /// Stages a single split. /// /// If a split already exists and is in the [SplitState::Staged] state, @@ -305,25 +342,37 @@ impl FileBackedIndex { /// Publishes splits. pub(crate) fn publish_splits<'a>( &mut self, - split_ids: &[&'a str], + staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { if let Some(checkpoint_delta) = checkpoint_delta_opt { let source_id = checkpoint_delta.source_id.clone(); - self.metadata - .checkpoint - .try_apply_delta(checkpoint_delta) - .map_err(|error| { - let entity = EntityKind::CheckpointDelta { - index_id: self.index_id().to_string(), - source_id, - }; - let message = error.to_string(); - MetastoreError::FailedPrecondition { entity, message } + + if source_id == INGEST_SOURCE_ID { + let publish_token = publish_token_opt.ok_or_else(|| { + let message = format!( + "publish token is required for publishing splits for source `{source_id}`" + ); + MetastoreError::InvalidArgument { message } })?; + self.try_apply_delta_v2(checkpoint_delta, publish_token)?; + } else { + self.metadata + .checkpoint + .try_apply_delta(checkpoint_delta) + .map_err(|error| { + let entity = EntityKind::CheckpointDelta { + index_id: self.index_id().to_string(), + source_id, + }; + let message = error.to_string(); + MetastoreError::FailedPrecondition { entity, message } + })?; + } } - self.mark_splits_as_published_helper(split_ids)?; + self.mark_splits_as_published_helper(staged_split_ids)?; self.mark_splits_for_deletion(replaced_split_ids, &[SplitState::Published], true)?; Ok(()) } @@ -394,10 +443,17 @@ impl FileBackedIndex { } /// Adds a source. - pub(crate) fn add_source(&mut self, source: SourceConfig) -> MetastoreResult<()> { - self.metadata.add_source(source) + pub(crate) fn add_source(&mut self, source_config: SourceConfig) -> MetastoreResult<()> { + let source_id = source_config.source_id.clone(); + self.metadata.add_source(source_config)?; + self.per_source_shards.insert( + source_id.clone(), + Shards::empty(self.index_uid(), source_id), + ); + Ok(()) } + /// Enables or disables a source. Returns whether a mutation occurred. pub(crate) fn toggle_source(&mut self, source_id: &str, enable: bool) -> MetastoreResult { self.metadata.toggle_source(source_id, enable) } @@ -463,6 +519,68 @@ impl FileBackedIndex { .collect(); Ok(delete_tasks) } + + // Shard API + + fn get_shards_for_source(&self, source_id: &str) -> MetastoreResult<&Shards> { + self.per_source_shards.get(source_id).ok_or_else(|| { + MetastoreError::NotFound(EntityKind::Source { + index_id: self.index_id().to_string(), + source_id: source_id.to_string(), + }) + }) + } + + fn get_shards_for_source_mut(&mut self, source_id: &str) -> MetastoreResult<&mut Shards> { + self.per_source_shards.get_mut(source_id).ok_or_else(|| { + MetastoreError::NotFound(EntityKind::Source { + index_id: self.metadata.index_id().to_string(), + source_id: source_id.to_string(), + }) + }) + } + + pub(crate) fn open_shards( + &mut self, + subrequest: OpenShardsSubrequest, + ) -> MetastoreResult> { + self.get_shards_for_source_mut(&subrequest.source_id)? + .open_shards(subrequest) + } + + pub(crate) fn close_shards( + &mut self, + subrequest: CloseShardsSubrequest, + ) -> MetastoreResult>> { + self.get_shards_for_source_mut(&subrequest.source_id)? + .close_shards(subrequest) + } + + pub(crate) fn delete_shards( + &mut self, + subrequest: DeleteShardsSubrequest, + force: bool, + ) -> MetastoreResult> { + self.get_shards_for_source_mut(&subrequest.source_id)? + .delete_shards(subrequest, force) + } + + pub(crate) fn list_shards( + &mut self, + subrequest: ListShardsSubrequest, + ) -> MetastoreResult> { + self.get_shards_for_source(&subrequest.source_id)? + .list_shards(subrequest) + } + + pub(crate) fn try_apply_delta_v2( + &mut self, + checkpoint_delta: IndexCheckpointDelta, + publish_token: PublishToken, + ) -> MetastoreResult> { + self.get_shards_for_source_mut(&checkpoint_delta.source_id)? + .try_apply_delta(checkpoint_delta.source_delta, publish_token) + } } /// Stamper provides Opstamps, which is just an auto-increment id to label @@ -471,9 +589,9 @@ impl FileBackedIndex { struct Stamper(usize); impl Stamper { - /// Creates a [`Stamper`]. - pub fn new(first_opstamp: usize) -> Self { - Self(first_opstamp) + /// Creates a new [`Stamper`]. + pub fn new(initial_opstamp: usize) -> Self { + Self(initial_opstamp) } /// Increments the stamper by 1 and returns the incremented value. diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/serialize.rs b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/serialize.rs index db8c0246e26..5e266ba7090 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/serialize.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/serialize.rs @@ -17,9 +17,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::collections::HashMap; + use itertools::Itertools; +use quickwit_proto::SourceId; use serde::{Deserialize, Serialize}; +use super::shards::{SerdeShards, Shards}; use crate::file_backed_metastore::file_backed_index::FileBackedIndex; use crate::metastore::DeleteTask; use crate::{IndexMetadata, Split}; @@ -54,23 +58,33 @@ pub(crate) struct FileBackedIndexV0_6 { metadata: IndexMetadata, splits: Vec, #[serde(default)] + shards: HashMap, + #[serde(default)] delete_tasks: Vec, } impl From for FileBackedIndexV0_6 { fn from(index: FileBackedIndex) -> Self { + let splits = index + .splits + .into_values() + .sorted_by_key(|split| split.update_timestamp) + .collect(); + let shards = index + .per_source_shards + .into_iter() + .map(|(source_id, shards)| (source_id, SerdeShards::from(shards))) + .collect(); + let delete_tasks = index + .delete_tasks + .into_iter() + .sorted_by_key(|delete_task| delete_task.opstamp) + .collect(); Self { metadata: index.metadata, - splits: index - .splits - .into_values() - .sorted_by_key(|split| split.update_timestamp) - .collect(), - delete_tasks: index - .delete_tasks - .into_iter() - .sorted_by_key(|delete_task| delete_task.opstamp) - .collect(), + splits, + shards, + delete_tasks, } } } @@ -83,6 +97,17 @@ impl From for FileBackedIndex { split.split_metadata.index_uid = index.metadata.index_uid.clone(); } } - Self::new(index.metadata, index.splits, index.delete_tasks) + let shards = index + .shards + .into_iter() + .map(|(source_id, serde_shards)| { + let index_uid = index.metadata.index_uid.clone(); + ( + source_id.clone(), + Shards::from_serde_shards(index_uid, source_id, serde_shards), + ) + }) + .collect(); + Self::new(index.metadata, index.splits, shards, index.delete_tasks) } } diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/shards.rs b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/shards.rs new file mode 100644 index 00000000000..a2a21a511e5 --- /dev/null +++ b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/file_backed_index/shards.rs @@ -0,0 +1,483 @@ +// Copyright (C) 2023 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::fmt; + +use itertools::Either; +use quickwit_proto::ingest::{Shard, ShardState}; +use quickwit_proto::metastore::{ + CloseShardsFailure, CloseShardsSubrequest, CloseShardsSuccess, DeleteShardsSubrequest, + EntityKind, ListShardsSubrequest, ListShardsSubresponse, MetastoreError, MetastoreResult, + OpenShardsSubrequest, OpenShardsSubresponse, +}; +use quickwit_proto::tonic::Code; +use quickwit_proto::types::ShardId; +use quickwit_proto::{queue_id, IndexUid, SourceId}; +use serde::{Deserialize, Serialize}; +use tracing::{info, warn}; + +use crate::checkpoint::{PartitionId, Position, SourceCheckpoint, SourceCheckpointDelta}; +use crate::file_backed_metastore::MutationOccurred; + +/// Manages the shards of a source. +#[derive(Clone, Eq, PartialEq)] +pub(crate) struct Shards { + index_uid: IndexUid, + source_id: SourceId, + checkpoint: SourceCheckpoint, + pub next_shard_id: ShardId, + pub shards: HashMap, +} + +impl fmt::Debug for Shards { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Shards") + .field("index_uid", &self.index_uid) + .field("source_id", &self.source_id) + .field("num_shards", &self.shards.len()) + .finish() + } +} + +impl Shards { + pub(super) fn empty(index_uid: IndexUid, source_id: SourceId) -> Self { + Self { + index_uid, + source_id, + checkpoint: SourceCheckpoint::default(), + next_shard_id: 1, // `1` matches the PostgreSQL sequence min value. + shards: HashMap::new(), + } + } + + pub(super) fn from_serde_shards( + index_uid: IndexUid, + source_id: SourceId, + serde_shards: SerdeShards, + ) -> Self { + let mut checkpoint = SourceCheckpoint::default(); + let mut shards = HashMap::with_capacity(serde_shards.shards.len()); + + for shard in serde_shards.shards { + checkpoint.add_partition( + PartitionId::from(shard.shard_id), + Position::from(shard.publish_position_inclusive.clone()), + ); + shards.insert(shard.shard_id, shard); + } + + Self { + index_uid, + source_id, + checkpoint, + next_shard_id: serde_shards.next_shard_id, + shards, + } + } + + fn get_shard(&self, shard_id: ShardId) -> MetastoreResult<&Shard> { + self.shards.get(&shard_id).ok_or_else(|| { + let queue_id = queue_id(self.index_uid.as_str(), &self.source_id, shard_id); + MetastoreError::NotFound(EntityKind::Shard { queue_id }) + }) + } + + fn get_shard_mut(&mut self, shard_id: ShardId) -> MetastoreResult<&mut Shard> { + self.shards.get_mut(&shard_id).ok_or_else(|| { + let queue_id = queue_id(self.index_uid.as_str(), &self.source_id, shard_id); + MetastoreError::NotFound(EntityKind::Shard { queue_id }) + }) + } + + pub(super) fn open_shards( + &mut self, + subrequest: OpenShardsSubrequest, + ) -> MetastoreResult> { + let mut mutation_occurred = false; + + if subrequest.next_shard_id + 1 < self.next_shard_id + || subrequest.next_shard_id > self.next_shard_id + { + warn!( + "control plane and metastore next shard IDs do not match, expected `{}`, got `{}`", + self.next_shard_id, subrequest.next_shard_id + ) + // TODO: Return an error and crash the control plane. + } + + let entry = self.shards.entry(subrequest.next_shard_id); + let shard = match entry { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let shard = Shard { + index_uid: self.index_uid.clone().into(), + source_id: self.source_id.clone(), + shard_id: self.next_shard_id, + leader_id: subrequest.leader_id.clone(), + follower_id: subrequest.follower_id.clone(), + ..Default::default() + }; + mutation_occurred = true; + entry.insert(shard.clone()); + self.next_shard_id += 1; + + info!( + index_id=%self.index_uid.index_id(), + source_id=%self.source_id, + shard_id=%shard.shard_id, + leader_id=%shard.leader_id, + follower_id=?shard.follower_id, + "Opened shard." + ); + shard + } + }; + let open_shards = vec![shard]; + let next_shard_id = self.next_shard_id; + + let response = OpenShardsSubresponse { + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + open_shards, + next_shard_id, + }; + if mutation_occurred { + Ok(MutationOccurred::Yes(response)) + } else { + Ok(MutationOccurred::No(response)) + } + } + + pub(super) fn close_shards( + &mut self, + subrequest: CloseShardsSubrequest, + ) -> MetastoreResult>> { + let Some(shard) = self.shards.get_mut(&subrequest.shard_id) else { + let failure = CloseShardsFailure { + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + error_code: Code::NotFound as u32, + error_message: "Shard not found.".to_string(), + }; + return Ok(MutationOccurred::No(Either::Right(failure))); + }; + match subrequest.shard_state() { + ShardState::Closing => { + shard.shard_state = ShardState::Closing as i32; + info!( + index_id=%self.index_uid.index_id(), + source_id=%shard.source_id, + shard_id=%shard.shard_id, + "Closing shard.", + ); + } + ShardState::Closed => { + shard.shard_state = ShardState::Closed as i32; + shard.replication_position_inclusive = shard + .replication_position_inclusive + .min(subrequest.replication_position_inclusive); + info!( + index_id=%self.index_uid.index_id(), + source_id=%shard.source_id, + shard_id=%shard.shard_id, + "Closed shard.", + ); + } + other => { + let error_message = format!( + "Invalid `shard_state` argument: expected `Closing` or `Closed` state, got \ + `{other:?}`.", + ); + let failure = CloseShardsFailure { + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + error_code: Code::InvalidArgument as u32, + error_message, + }; + return Ok(MutationOccurred::No(Either::Right(failure))); + } + } + let success = CloseShardsSuccess { + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + }; + Ok(MutationOccurred::Yes(Either::Left(success))) + } + + pub(super) fn delete_shards( + &mut self, + subrequest: DeleteShardsSubrequest, + force: bool, + ) -> MetastoreResult> { + let mut mutation_occurred = false; + for shard_id in subrequest.shard_ids { + if let Entry::Occupied(entry) = self.shards.entry(shard_id) { + let shard = entry.get(); + if force || shard.is_deletable() { + mutation_occurred = true; + info!( + index_id=%self.index_uid.index_id(), + source_id=%self.source_id, + shard_id=%shard.shard_id, + "Deleted shard.", + ); + entry.remove(); + continue; + } + let message = format!("shard `{shard_id}` is not deletable"); + return Err(MetastoreError::InvalidArgument { message }); + } + } + if mutation_occurred { + Ok(MutationOccurred::Yes(())) + } else { + Ok(MutationOccurred::No(())) + } + } + + pub(super) fn list_shards( + &self, + subrequest: ListShardsSubrequest, + ) -> MetastoreResult> { + let shards = self.list_shards_inner(subrequest.shard_state); + let response = ListShardsSubresponse { + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shards, + next_shard_id: self.next_shard_id, + }; + Ok(MutationOccurred::No(response)) + } + + pub(super) fn try_apply_delta( + &mut self, + checkpoint_delta: SourceCheckpointDelta, + publish_token: String, + ) -> MetastoreResult> { + if checkpoint_delta.is_empty() { + return Ok(MutationOccurred::No(())); + } + self.checkpoint + .check_compatibility(&checkpoint_delta) + .map_err(|_error| { + let message = "incompatible partition delta: FIXME".to_string(); + MetastoreError::InvalidArgument { message } + })?; + + let mut shard_ids = Vec::with_capacity(checkpoint_delta.num_partitions()); + + for (partition_id, partition_delta) in checkpoint_delta.iter() { + let shard_id = partition_id.as_u64().ok_or_else(|| { + let message = format!( + "invalid partition ID: expected a 64-bit unsigned integer, got \ + `{partition_id}`" + ); + MetastoreError::InvalidArgument { message } + })?; + let shard = self.get_shard(shard_id)?; + + if shard.publish_token() != publish_token { + let message = "failed to apply checkpoint delta: invalid publish token".to_string(); + return Err(MetastoreError::InvalidArgument { message }); + } + let publish_position_inclusive = partition_delta.to.as_str().to_string(); + shard_ids.push((shard_id, publish_position_inclusive)) + } + self.checkpoint + .try_apply_delta(checkpoint_delta) + .expect("delta compatibility should have been checked"); + + for (shard_id, publish_position_inclusive) in shard_ids { + let shard = self.get_shard_mut(shard_id).expect("shard should exist"); + shard.publish_position_inclusive = publish_position_inclusive + } + Ok(MutationOccurred::Yes(())) + } + + fn list_shards_inner(&self, shard_state: Option) -> Vec { + if let Some(shard_state) = shard_state { + self.shards + .values() + .filter(|shard| shard.shard_state == shard_state) + .cloned() + .collect() + } else { + self.shards.values().cloned().collect() + } + } +} + +/// The serialized representation of [`SourceShards`]. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct SerdeShards { + pub next_shard_id: ShardId, + pub shards: Vec, +} + +impl From for SerdeShards { + fn from(shards: Shards) -> Self { + Self { + next_shard_id: shards.next_shard_id, + shards: shards.shards.into_values().collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_open_shards() { + let index_uid: IndexUid = "test-index:0".into(); + let source_id = "test-source".to_string(); + let mut shards = Shards::empty(index_uid.clone(), source_id.clone()); + + let subrequest = OpenShardsSubrequest { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + leader_id: "leader_id".to_string(), + follower_id: None, + next_shard_id: 1, + }; + let MutationOccurred::Yes(subresponse) = shards.open_shards(subrequest.clone()).unwrap() + else { + panic!("Expected `MutationOccured::Yes`"); + }; + assert_eq!(subresponse.index_uid, index_uid.as_str()); + assert_eq!(subresponse.source_id, source_id); + assert_eq!(subresponse.open_shards.len(), 1); + + let shard = &subresponse.open_shards[0]; + assert_eq!(shard.index_uid, index_uid.as_str()); + assert_eq!(shard.source_id, source_id); + assert_eq!(shard.shard_id, 1); + assert_eq!(shard.shard_state, 0); + assert_eq!(shard.leader_id, "leader_id"); + assert_eq!(shard.follower_id, None); + assert_eq!(shard.replication_position_inclusive, None); + assert_eq!(shard.publish_position_inclusive, ""); + + assert_eq!(shards.shards.get(&1).unwrap(), shard); + + let MutationOccurred::No(subresponse) = shards.open_shards(subrequest).unwrap() else { + panic!("Expected `MutationOccured::No`"); + }; + assert_eq!(subresponse.open_shards.len(), 1); + + let shard = &subresponse.open_shards[0]; + assert_eq!(shards.shards.get(&1).unwrap(), shard); + + let subrequest = OpenShardsSubrequest { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + leader_id: "leader_id".to_string(), + follower_id: Some("follower_id".to_string()), + next_shard_id: 2, + }; + let MutationOccurred::Yes(subresponse) = shards.open_shards(subrequest).unwrap() else { + panic!("Expected `MutationOccured::No`"); + }; + assert_eq!(subresponse.index_uid, index_uid.as_str()); + assert_eq!(subresponse.source_id, source_id); + assert_eq!(subresponse.open_shards.len(), 1); + + let shard = &subresponse.open_shards[0]; + assert_eq!(shard.index_uid, index_uid.as_str()); + assert_eq!(shard.source_id, source_id); + assert_eq!(shard.shard_id, 2); + assert_eq!(shard.shard_state, 0); + assert_eq!(shard.leader_id, "leader_id"); + assert_eq!(shard.follower_id.as_ref().unwrap(), "follower_id"); + assert_eq!(shard.replication_position_inclusive, None); + assert_eq!(shard.publish_position_inclusive, ""); + + assert_eq!(shards.shards.get(&2).unwrap(), shard); + } + + #[test] + fn test_close_shard() {} + + #[test] + fn test_list_shards() { + let index_uid: IndexUid = "test-index:0".into(); + let source_id = "test-source".to_string(); + let mut shards = Shards::empty(index_uid.clone(), source_id.clone()); + + let subrequest = ListShardsSubrequest { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + shard_state: None, + }; + let MutationOccurred::No(subresponse) = shards.list_shards(subrequest).unwrap() else { + panic!("Expected `MutationOccured::No`"); + }; + assert_eq!(subresponse.index_uid, index_uid.as_str()); + assert_eq!(subresponse.source_id, source_id); + assert_eq!(subresponse.shards.len(), 0); + + let shard_0 = Shard { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + shard_id: 0, + shard_state: ShardState::Open as i32, + ..Default::default() + }; + let shard_1 = Shard { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + shard_id: 1, + shard_state: ShardState::Closed as i32, + ..Default::default() + }; + shards.shards.insert(0, shard_0); + shards.shards.insert(1, shard_1); + + let subrequest = ListShardsSubrequest { + index_uid: index_uid.clone().into(), + source_id: source_id.clone(), + shard_state: None, + }; + let MutationOccurred::No(mut subresponse) = shards.list_shards(subrequest).unwrap() else { + panic!("Expected `MutationOccured::No`"); + }; + subresponse + .shards + .sort_unstable_by_key(|shard| shard.shard_id); + assert_eq!(subresponse.shards.len(), 2); + assert_eq!(subresponse.shards[0].shard_id, 0); + assert_eq!(subresponse.shards[1].shard_id, 1); + + let subrequest = ListShardsSubrequest { + index_uid: index_uid.into(), + source_id, + shard_state: Some(ShardState::Closed as i32), + }; + let MutationOccurred::No(subresponse) = shards.list_shards(subrequest).unwrap() else { + panic!("Expected `MutationOccured::No`"); + }; + assert_eq!(subresponse.shards.len(), 1); + assert_eq!(subresponse.shards[0].shard_id, 1); + } +} diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/mod.rs b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/mod.rs index dcf97acfd74..417990fb9ac 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed_metastore/mod.rs @@ -33,13 +33,15 @@ use std::time::Duration; use async_trait::async_trait; use futures::future::try_join_all; -use itertools::Itertools; +use itertools::{Either, Itertools}; use quickwit_common::uri::Uri; use quickwit_config::{validate_index_id_pattern, IndexConfig, SourceConfig}; use quickwit_proto::metastore::{ - DeleteQuery, DeleteTask, EntityKind, MetastoreError, MetastoreResult, + CloseShardsRequest, CloseShardsResponse, DeleteQuery, DeleteShardsRequest, + DeleteShardsResponse, DeleteTask, EntityKind, ListShardsRequest, ListShardsResponse, + MetastoreError, MetastoreResult, OpenShardsRequest, OpenShardsResponse, }; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use quickwit_storage::Storage; use regex::RegexSet; use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; @@ -467,12 +469,18 @@ impl Metastore for FileBackedMetastore { async fn publish_splits<'a>( &self, index_uid: IndexUid, - split_ids: &[&'a str], + staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { self.mutate(index_uid, |index| { - index.publish_splits(split_ids, replaced_split_ids, checkpoint_delta_opt)?; + index.publish_splits( + staged_split_ids, + replaced_split_ids, + checkpoint_delta_opt, + publish_token_opt, + )?; Ok(MutationOccurred::Yes(())) }) .await?; @@ -642,6 +650,84 @@ impl Metastore for FileBackedMetastore { check_indexes_states_exist(self.storage.clone()).await } + // Shard API + + async fn open_shards(&self, request: OpenShardsRequest) -> MetastoreResult { + let mut subresponses = Vec::with_capacity(request.subrequests.len()); + + for subrequest in request.subrequests { + let index_uid: IndexUid = subrequest.index_uid.clone().into(); + let subresponse = self + .mutate(index_uid, |index| index.open_shards(subrequest)) + .await?; + subresponses.push(subresponse); + } + let response = OpenShardsResponse { subresponses }; + Ok(response) + } + + async fn close_shards( + &self, + request: CloseShardsRequest, + ) -> MetastoreResult { + let mut successes = Vec::with_capacity(request.subrequests.len()); + let mut failures = Vec::new(); + + for subrequest in request.subrequests { + let index_uid: IndexUid = subrequest.index_uid.clone().into(); + match self + .mutate(index_uid, |index| index.close_shards(subrequest)) + .await + { + Ok(Either::Left(success)) => { + successes.push(success); + } + Ok(Either::Right(failure)) => { + failures.push(failure); + } + Err(error) => return Err(error), + } + } + let response = CloseShardsResponse { + successes, + failures, + }; + Ok(response) + } + + async fn delete_shards( + &self, + request: DeleteShardsRequest, + ) -> MetastoreResult { + let mut subresponses = Vec::with_capacity(request.subrequests.len()); + + for subrequest in request.subrequests { + let index_uid: IndexUid = subrequest.index_uid.clone().into(); + let subresponse = self + .mutate(index_uid, |index| { + index.delete_shards(subrequest, request.force) + }) + .await?; + subresponses.push(subresponse); + } + let response = DeleteShardsResponse {}; + Ok(response) + } + + async fn list_shards(&self, request: ListShardsRequest) -> MetastoreResult { + let mut subresponses = Vec::with_capacity(request.subrequests.len()); + + for subrequest in request.subrequests { + let index_uid: IndexUid = subrequest.index_uid.clone().into(); + let subresponse = self + .mutate(index_uid, |index| index.list_shards(subrequest)) + .await?; + subresponses.push(subresponse); + } + let response = ListShardsResponse { subresponses }; + Ok(response) + } + /// ------------------------------------------------------------------------------- /// Delete tasks @@ -909,7 +995,7 @@ mod tests { // publish split fails let err = metastore - .publish_splits(index_uid.clone(), &[split_id], &[], None) + .publish_splits(index_uid.clone(), &[split_id], &[], None, None) .await; assert!(err.is_err()); @@ -1087,7 +1173,7 @@ mod tests { // publish split let split_id = format!("split-{i}"); metastore - .publish_splits(index_uid.clone(), &[&split_id], &[], None) + .publish_splits(index_uid.clone(), &[&split_id], &[], None, None) .await .unwrap(); } diff --git a/quickwit/quickwit-metastore/src/metastore/grpc_metastore/grpc_adapter.rs b/quickwit/quickwit-metastore/src/metastore/grpc_metastore/grpc_adapter.rs index fb3a538916f..c13830c9f54 100644 --- a/quickwit/quickwit-metastore/src/metastore/grpc_metastore/grpc_adapter.rs +++ b/quickwit/quickwit-metastore/src/metastore/grpc_metastore/grpc_adapter.rs @@ -23,17 +23,17 @@ use async_trait::async_trait; use itertools::Itertools; use quickwit_config::IndexConfig; use quickwit_proto::metastore::{ - AddSourceRequest, CloseShardsRequest, CloseShardsResponse, CreateIndexRequest, - CreateIndexResponse, DeleteIndexRequest, DeleteQuery, DeleteShardsRequest, - DeleteShardsResponse, DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, EmptyResponse, - IndexMetadataRequest, IndexMetadataResponse, LastDeleteOpstampRequest, - LastDeleteOpstampResponse, ListAllSplitsRequest, ListDeleteTasksRequest, - ListDeleteTasksResponse, ListIndexesMetadatasRequest, ListIndexesMetadatasResponse, - ListShardsRequest, ListShardsResponse, ListSplitsRequest, ListSplitsResponse, - ListStaleSplitsRequest, MarkSplitsForDeletionRequest, MetastoreError, MetastoreService, - OpenShardsRequest, OpenShardsResponse, PublishSplitsRequest, ResetSourceCheckpointRequest, - StageSplitsRequest, ToggleSourceRequest, UpdateSplitsDeleteOpstampRequest, - UpdateSplitsDeleteOpstampResponse, + serde_utils as metastore_serde_utils, AddSourceRequest, CloseShardsRequest, + CloseShardsResponse, CreateIndexRequest, CreateIndexResponse, DeleteIndexRequest, DeleteQuery, + DeleteShardsRequest, DeleteShardsResponse, DeleteSourceRequest, DeleteSplitsRequest, + DeleteTask, EmptyResponse, IndexMetadataRequest, IndexMetadataResponse, + LastDeleteOpstampRequest, LastDeleteOpstampResponse, ListAllSplitsRequest, + ListDeleteTasksRequest, ListDeleteTasksResponse, ListIndexesMetadatasRequest, + ListIndexesMetadatasResponse, ListShardsRequest, ListShardsResponse, ListSplitsRequest, + ListSplitsResponse, ListStaleSplitsRequest, MarkSplitsForDeletionRequest, MetastoreError, + MetastoreService, OpenShardsRequest, OpenShardsResponse, PublishSplitsRequest, + ResetSourceCheckpointRequest, StageSplitsRequest, ToggleSourceRequest, + UpdateSplitsDeleteOpstampRequest, UpdateSplitsDeleteOpstampResponse, }; use quickwit_proto::tonic::{Request, Response, Status}; use quickwit_proto::{set_parent_span_from_request_metadata, tonic}; @@ -224,13 +224,10 @@ impl MetastoreService for GrpcMetastoreAdapter { .map(|split_id| split_id.as_str()) .collect_vec(); let checkpoint_delta_opt = publish_request - .index_checkpoint_delta_serialized_json - .map(|json| serde_json::from_str(&json)) - .transpose() - .map_err(|error| MetastoreError::JsonDeserializeError { - struct_name: "IndexCheckpointDelta".to_string(), - message: error.to_string(), - })?; + .index_checkpoint_delta_json_opt + .as_deref() + .map(metastore_serde_utils::from_json_str) + .transpose()?; let publish_splits_reply = self .0 .publish_splits( @@ -238,6 +235,7 @@ impl MetastoreService for GrpcMetastoreAdapter { &split_ids, &replaced_split_ids, checkpoint_delta_opt, + publish_request.publish_token_opt, ) .await .map(|_| EmptyResponse {})?; diff --git a/quickwit/quickwit-metastore/src/metastore/grpc_metastore/mod.rs b/quickwit/quickwit-metastore/src/metastore/grpc_metastore/mod.rs index 13c97b5f42c..45554f5cecf 100644 --- a/quickwit/quickwit-metastore/src/metastore/grpc_metastore/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/grpc_metastore/mod.rs @@ -31,16 +31,17 @@ use quickwit_common::tower::BalanceChannel; use quickwit_common::uri::Uri as QuickwitUri; use quickwit_config::{IndexConfig, SourceConfig}; use quickwit_proto::metastore::{ - AddSourceRequest, CreateIndexRequest, DeleteIndexRequest, DeleteQuery, DeleteSourceRequest, - DeleteSplitsRequest, DeleteTask, IndexMetadataRequest, LastDeleteOpstampRequest, - ListAllSplitsRequest, ListDeleteTasksRequest, ListIndexesMetadatasRequest, ListSplitsRequest, - ListStaleSplitsRequest, MarkSplitsForDeletionRequest, MetastoreError, MetastoreResult, - MetastoreServiceClient, PublishSplitsRequest, ResetSourceCheckpointRequest, StageSplitsRequest, - ToggleSourceRequest, UpdateSplitsDeleteOpstampRequest, + serde_utils as metastore_serde_utils, AddSourceRequest, CreateIndexRequest, DeleteIndexRequest, + DeleteQuery, DeleteSourceRequest, DeleteSplitsRequest, DeleteTask, IndexMetadataRequest, + LastDeleteOpstampRequest, ListAllSplitsRequest, ListDeleteTasksRequest, + ListIndexesMetadatasRequest, ListSplitsRequest, ListStaleSplitsRequest, + MarkSplitsForDeletionRequest, MetastoreError, MetastoreResult, MetastoreServiceClient, + PublishSplitsRequest, ResetSourceCheckpointRequest, StageSplitsRequest, ToggleSourceRequest, + UpdateSplitsDeleteOpstampRequest, }; use quickwit_proto::tonic::codegen::InterceptedService; use quickwit_proto::tonic::Status; -use quickwit_proto::{IndexUid, SpanContextInterceptor}; +use quickwit_proto::{IndexUid, PublishToken, SpanContextInterceptor}; use tower::timeout::error::Elapsed; use crate::checkpoint::IndexCheckpointDelta; @@ -247,28 +248,29 @@ impl Metastore for MetastoreGrpcClient { async fn publish_splits<'a>( &self, index_uid: IndexUid, - split_ids: &[&'a str], + staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { - let staged_split_ids: Vec = - split_ids.iter().map(|split| split.to_string()).collect(); - let replaced_split_ids_vec: Vec = replaced_split_ids + let staged_split_ids: Vec = staged_split_ids + .iter() + .map(|split| split.to_string()) + .collect(); + let replaced_split_ids: Vec = replaced_split_ids .iter() .map(|split_id| split_id.to_string()) .collect(); - let index_checkpoint_delta_serialized_json = checkpoint_delta_opt - .map(|checkpoint_delta| serde_json::to_string(&checkpoint_delta)) - .transpose() - .map_err(|error| MetastoreError::JsonSerializeError { - struct_name: "IndexCheckpointDelta".to_string(), - message: error.to_string(), - })?; + let index_checkpoint_delta_json_opt = checkpoint_delta_opt + .as_ref() + .map(metastore_serde_utils::to_json_str) + .transpose()?; let request = PublishSplitsRequest { index_uid: index_uid.into(), staged_split_ids, - replaced_split_ids: replaced_split_ids_vec, - index_checkpoint_delta_serialized_json, + replaced_split_ids, + index_checkpoint_delta_json_opt, + publish_token_opt, }; self.underlying .clone() diff --git a/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs b/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs index 36fda2afb56..909b2be6b3c 100644 --- a/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/index_metadata/mod.rs @@ -29,6 +29,7 @@ use quickwit_proto::{IndexUid, SourceId}; use serde::{Deserialize, Serialize}; use serialize::VersionedIndexMetadata; use time::OffsetDateTime; +use ulid::Ulid; use crate::checkpoint::{ IndexCheckpoint, PartitionId, Position, SourceCheckpoint, SourceCheckpointDelta, @@ -147,7 +148,7 @@ impl TestableForRegression for IndexMetadata { let checkpoint = IndexCheckpoint::from(per_source_checkpoint); let index_config = IndexConfig::sample_for_regression(); let mut index_metadata = IndexMetadata { - index_uid: IndexUid::from_parts("test", "11111111111111111111111111"), + index_uid: IndexUid::from_parts(index_config.index_id.clone(), Ulid::nil()), index_config, checkpoint, create_timestamp: 1789, diff --git a/quickwit/quickwit-metastore/src/metastore/instrumented_metastore.rs b/quickwit/quickwit-metastore/src/metastore/instrumented_metastore.rs index 4a22f572b4a..8253d0eb050 100644 --- a/quickwit/quickwit-metastore/src/metastore/instrumented_metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/instrumented_metastore.rs @@ -24,7 +24,7 @@ use itertools::Itertools; use quickwit_common::uri::Uri; use quickwit_config::{IndexConfig, SourceConfig}; use quickwit_proto::metastore::{DeleteQuery, DeleteTask, MetastoreResult}; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use crate::checkpoint::IndexCheckpointDelta; use crate::{IndexMetadata, ListIndexesQuery, ListSplitsQuery, Metastore, Split, SplitMetadata}; @@ -153,6 +153,7 @@ impl Metastore for InstrumentedMetastore { split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { instrument!( self.underlying @@ -161,6 +162,7 @@ impl Metastore for InstrumentedMetastore { split_ids, replaced_split_ids, checkpoint_delta_opt, + publish_token_opt, ) .await, [publish_splits, index_uid.index_id()] diff --git a/quickwit/quickwit-metastore/src/metastore/metastore_event_publisher.rs b/quickwit/quickwit-metastore/src/metastore/metastore_event_publisher.rs index 9a730efbebd..065a50b745f 100644 --- a/quickwit/quickwit-metastore/src/metastore/metastore_event_publisher.rs +++ b/quickwit/quickwit-metastore/src/metastore/metastore_event_publisher.rs @@ -28,7 +28,7 @@ use quickwit_proto::metastore::events::{ AddSourceEvent, DeleteIndexEvent, DeleteSourceEvent, ToggleSourceEvent, }; use quickwit_proto::metastore::{DeleteQuery, DeleteTask, MetastoreResult}; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use tracing::info; use crate::checkpoint::IndexCheckpointDelta; @@ -115,6 +115,7 @@ impl Metastore for MetastoreEventPublisher { split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { self.underlying .publish_splits( @@ -122,6 +123,7 @@ impl Metastore for MetastoreEventPublisher { split_ids, replaced_split_ids, checkpoint_delta_opt, + publish_token_opt, ) .await } diff --git a/quickwit/quickwit-metastore/src/metastore/mod.rs b/quickwit/quickwit-metastore/src/metastore/mod.rs index 27a276cef4b..84f6807c4df 100644 --- a/quickwit/quickwit-metastore/src/metastore/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/mod.rs @@ -42,7 +42,7 @@ use quickwit_proto::metastore::{ DeleteShardsResponse, DeleteTask, EntityKind, ListShardsRequest, ListShardsResponse, MetastoreError, MetastoreResult, OpenShardsRequest, OpenShardsResponse, }; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use time::OffsetDateTime; use crate::checkpoint::IndexCheckpointDelta; @@ -190,6 +190,7 @@ pub trait Metastore: fmt::Debug + Send + Sync + 'static { staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()>; /// Lists the splits. diff --git a/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs b/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs index 322a0afc3ff..dcdff57b947 100644 --- a/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs @@ -36,7 +36,7 @@ use quickwit_doc_mapper::tag_pruning::TagFilterAst; use quickwit_proto::metastore::{ DeleteQuery, DeleteTask, EntityKind, MetastoreError, MetastoreResult, }; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use sqlx::migrate::Migrator; use sqlx::postgres::{PgConnectOptions, PgDatabaseError, PgPoolOptions}; use sqlx::{ConnectOptions, Pool, Postgres, Transaction}; @@ -638,6 +638,7 @@ impl Metastore for PostgresqlMetastore { staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + _publish_token_opt: Option, ) -> MetastoreResult<()> { run_with_tx!(self.connection_pool, tx, { let mut index_metadata = index_metadata(tx, index_uid.index_id()).await?; diff --git a/quickwit/quickwit-metastore/src/metastore/retrying_metastore/mod.rs b/quickwit/quickwit-metastore/src/metastore/retrying_metastore/mod.rs index 5c629f2edfc..44e525ae9e8 100644 --- a/quickwit/quickwit-metastore/src/metastore/retrying_metastore/mod.rs +++ b/quickwit/quickwit-metastore/src/metastore/retrying_metastore/mod.rs @@ -28,7 +28,7 @@ use quickwit_common::retry::RetryParams; use quickwit_common::uri::Uri; use quickwit_config::{IndexConfig, SourceConfig}; use quickwit_proto::metastore::{DeleteQuery, DeleteTask, MetastoreResult}; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use self::retry::retry; use crate::checkpoint::IndexCheckpointDelta; @@ -126,17 +126,19 @@ impl Metastore for RetryingMetastore { async fn publish_splits<'a>( &self, index_uid: IndexUid, - split_ids: &[&'a str], + staged_split_ids: &[&'a str], replaced_split_ids: &[&'a str], checkpoint_delta_opt: Option, + publish_token_opt: Option, ) -> MetastoreResult<()> { retry(&self.retry_params, || async { self.inner .publish_splits( index_uid.clone(), - split_ids, + staged_split_ids, replaced_split_ids, checkpoint_delta_opt.clone(), + publish_token_opt.clone(), ) .await }) diff --git a/quickwit/quickwit-metastore/src/metastore/retrying_metastore/test.rs b/quickwit/quickwit-metastore/src/metastore/retrying_metastore/test.rs index 7f654ebfaae..183e396adf0 100644 --- a/quickwit/quickwit-metastore/src/metastore/retrying_metastore/test.rs +++ b/quickwit/quickwit-metastore/src/metastore/retrying_metastore/test.rs @@ -27,7 +27,7 @@ use quickwit_config::{IndexConfig, SourceConfig}; use quickwit_proto::metastore::{ DeleteQuery, DeleteTask, EntityKind, MetastoreError, MetastoreResult, }; -use quickwit_proto::IndexUid; +use quickwit_proto::{IndexUid, PublishToken}; use crate::checkpoint::IndexCheckpointDelta; use crate::{ @@ -132,9 +132,10 @@ impl Metastore for RetryTestMetastore { async fn publish_splits<'a>( &self, _index_uid: IndexUid, - _split_ids: &[&'a str], + _staged_split_ids: &[&'a str], _replaced_split_ids: &[&'a str], _checkpoint_delta_opt: Option, + _publish_token_opt: Option, ) -> MetastoreResult<()> { self.try_success() } diff --git a/quickwit/quickwit-metastore/src/split_metadata.rs b/quickwit/quickwit-metastore/src/split_metadata.rs index 99b91f2fccd..d471324b644 100644 --- a/quickwit/quickwit-metastore/src/split_metadata.rs +++ b/quickwit/quickwit-metastore/src/split_metadata.rs @@ -216,9 +216,11 @@ pub struct SplitInfo { #[cfg(any(test, feature = "testsuite"))] impl quickwit_config::TestableForRegression for SplitMetadata { fn sample_for_regression() -> Self { + use ulid::Ulid; + SplitMetadata { split_id: "split".to_string(), - index_uid: IndexUid::from_parts("my-index", "11111111111111111111111111"), + index_uid: IndexUid::from_parts("my-index", Ulid::nil()), source_id: "source".to_string(), node_id: "node".to_string(), delete_opstamp: 10, diff --git a/quickwit/quickwit-metastore/src/tests.rs b/quickwit/quickwit-metastore/src/tests.rs index a8653f76b1e..8bf86dfefa3 100644 --- a/quickwit/quickwit-metastore/src/tests.rs +++ b/quickwit/quickwit-metastore/src/tests.rs @@ -551,7 +551,7 @@ pub mod test_suite { .await .unwrap(); metastore - .publish_splits(index_uid.clone(), &[split_id], &[], None) + .publish_splits(index_uid.clone(), &[split_id], &[], None, None) .await .unwrap(); } @@ -631,6 +631,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -656,6 +657,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -718,6 +720,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -739,6 +742,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -753,7 +757,7 @@ pub mod test_suite { let index_uid = metastore.create_index(index_config.clone()).await.unwrap(); let error = metastore - .publish_splits(index_uid.clone(), &["split-not-found"], &[], None) + .publish_splits(index_uid.clone(), &["split-not-found"], &[], None, None) .await .unwrap_err(); assert!(matches!( @@ -774,7 +778,7 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_1], &[], None, None) .await .unwrap(); @@ -800,6 +804,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -814,6 +819,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -847,6 +853,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -866,6 +873,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -899,6 +907,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -929,6 +938,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -943,6 +953,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -973,6 +984,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -992,6 +1004,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -1027,6 +1040,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -1056,6 +1070,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -1070,6 +1085,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -1106,6 +1122,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -1120,6 +1137,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap_err(); @@ -1184,6 +1202,7 @@ pub mod test_suite { &[&split_id], &[], Some(checkpoint_delta), + None, ) .await .unwrap(); @@ -1249,6 +1268,7 @@ pub mod test_suite { &["split-not-found-1"], &["split-not-found-2"], None, + None, ) .await .unwrap_err(); @@ -1269,6 +1289,7 @@ pub mod test_suite { &["split-not-found-1"], &["split-not-found-2"], None, + None, ) .await .unwrap_err(); @@ -1290,13 +1311,19 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_1], &[], None, None) .await .unwrap(); // TODO Source id let error = metastore - .publish_splits(index_uid.clone(), &[&split_id_2], &[&split_id_1], None) + .publish_splits( + index_uid.clone(), + &[&split_id_2], + &[&split_id_1], + None, + None, + ) .await .unwrap_err(); assert!(matches!( @@ -1320,7 +1347,13 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1, &split_id_2], &[], None) + .publish_splits( + index_uid.clone(), + &[&split_id_1, &split_id_2], + &[], + None, + None, + ) .await .unwrap(); @@ -1331,7 +1364,13 @@ pub mod test_suite { // TODO source_id let error = metastore - .publish_splits(index_uid.clone(), &[&split_id_2], &[&split_id_1], None) + .publish_splits( + index_uid.clone(), + &[&split_id_2], + &[&split_id_1], + None, + None, + ) .await .unwrap_err(); assert!(matches!( @@ -1355,7 +1394,7 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_1], &[], None, None) .await .unwrap(); @@ -1370,6 +1409,7 @@ pub mod test_suite { &[&split_id_2, &split_id_3], &[&split_id_1], None, + None, ) // TODO source id .await .unwrap_err(); @@ -1391,7 +1431,7 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_1], &[], None, None) .await .unwrap(); @@ -1406,7 +1446,13 @@ pub mod test_suite { .unwrap(); let error = metastore - .publish_splits(index_uid.clone(), &[&split_id_2], &[&split_id_1], None) + .publish_splits( + index_uid.clone(), + &[&split_id_2], + &[&split_id_1], + None, + None, + ) .await .unwrap_err(); assert!( @@ -1426,7 +1472,7 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_1], &[], None, None) .await .unwrap(); @@ -1445,6 +1491,7 @@ pub mod test_suite { &[&split_id_2, &split_id_3], &[&split_id_1], None, + None, ) .await .unwrap(); @@ -1504,7 +1551,7 @@ pub mod test_suite { .await .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_2], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_2], &[], None, None) .await .unwrap(); @@ -1520,7 +1567,7 @@ pub mod test_suite { .await .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_3], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_3], &[], None, None) .await .unwrap(); @@ -1626,7 +1673,7 @@ pub mod test_suite { .await .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_1], &[], None, None) .await .unwrap(); @@ -1761,7 +1808,13 @@ pub mod test_suite { .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1, &split_id_2], &[], None) + .publish_splits( + index_uid.clone(), + &[&split_id_1, &split_id_2], + &[], + None, + None, + ) .await .unwrap(); @@ -2297,6 +2350,7 @@ pub mod test_suite { IndexCheckpointDelta::for_test(&source_id, offsets) } .into(), + None, ) .await .unwrap(); @@ -2618,7 +2672,7 @@ pub mod test_suite { .await .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_4], &[], None) + .publish_splits(index_uid.clone(), &[&split_id_4], &[], None, None) .await .unwrap(); // Sleep for 1 second to have different publish timestamps. @@ -2629,6 +2683,7 @@ pub mod test_suite { &[&split_id_1, &split_id_2, &split_id_5], &[], None, + None, ) .await .unwrap(); @@ -2738,7 +2793,13 @@ pub mod test_suite { .await .unwrap(); metastore - .publish_splits(index_uid.clone(), &[&split_id_1, &split_id_2], &[], None) + .publish_splits( + index_uid.clone(), + &[&split_id_1, &split_id_2], + &[], + None, + None, + ) .await .unwrap(); @@ -2833,7 +2894,13 @@ pub mod test_suite { .expect("Pre-existing staged splits should be updated."); metastore - .publish_splits(index_uid.clone(), &[&split_id_1, &split_id_2], &[], None) + .publish_splits( + index_uid.clone(), + &[&split_id_1, &split_id_2], + &[], + None, + None, + ) .await .unwrap(); let error = metastore diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json index 50d02a2d7fc..da0d915432b 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json @@ -28,7 +28,6 @@ "stored": true, "tokenizer": "raw" }, - "index_field_presence": false, "field_mappings": [ { "coerce": true, @@ -73,6 +72,7 @@ "type": "text" } ], + "index_field_presence": false, "max_num_partitions": 100, "mode": "dynamic", "partition_key": "tenant", @@ -131,6 +131,7 @@ ], "version": "0.6" }, + "shards": {}, "splits": [ { "create_timestamp": 3, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json index ee73527c51a..475102b82e5 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json @@ -28,7 +28,6 @@ "stored": true, "tokenizer": "raw" }, - "index_field_presence": false, "field_mappings": [ { "coerce": true, @@ -73,6 +72,7 @@ "type": "text" } ], + "index_field_presence": false, "max_num_partitions": 100, "mode": "dynamic", "partition_key": "tenant", @@ -135,6 +135,7 @@ ], "version": "0.6" }, + "shards": {}, "splits": [ { "create_timestamp": 3, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json index 5a475a379a9..00dec078ed7 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json @@ -3,7 +3,7 @@ { "create_timestamp": 0, "delete_query": { - "index_uid": "index:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}}]}" }, "opstamp": 10 @@ -28,7 +28,6 @@ "stored": true, "tokenizer": "raw" }, - "index_field_presence": true, "field_mappings": [ { "coerce": true, @@ -73,6 +72,7 @@ "type": "text" } ], + "index_field_presence": true, "max_num_partitions": 100, "mode": "dynamic", "partition_key": "tenant_id", @@ -120,7 +120,7 @@ }, "version": "0.6" }, - "index_uid": "test:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "sources": [ { "desired_num_pipelines": 2, @@ -142,6 +142,21 @@ ], "version": "0.6" }, + "shards": { + "_ingest-source": { + "next_shard_id": 2, + "shards": [ + { + "follower_id": "follower-ingester", + "index_uid": "my-index:00000000000000000000000000", + "leader_id": "leader-ingester", + "shard_id": 1, + "shard_state": 0, + "source_id": "_ingest-source" + } + ] + } + }, "splits": [ { "create_timestamp": 3, @@ -150,7 +165,7 @@ "end": 2000, "start": 1000 }, - "index_uid": "my-index:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "maturity": { "maturation_period_millis": 4000, "type": "immature" diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json index 5a475a379a9..00dec078ed7 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json @@ -3,7 +3,7 @@ { "create_timestamp": 0, "delete_query": { - "index_uid": "index:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}}]}" }, "opstamp": 10 @@ -28,7 +28,6 @@ "stored": true, "tokenizer": "raw" }, - "index_field_presence": true, "field_mappings": [ { "coerce": true, @@ -73,6 +72,7 @@ "type": "text" } ], + "index_field_presence": true, "max_num_partitions": 100, "mode": "dynamic", "partition_key": "tenant_id", @@ -120,7 +120,7 @@ }, "version": "0.6" }, - "index_uid": "test:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "sources": [ { "desired_num_pipelines": 2, @@ -142,6 +142,21 @@ ], "version": "0.6" }, + "shards": { + "_ingest-source": { + "next_shard_id": 2, + "shards": [ + { + "follower_id": "follower-ingester", + "index_uid": "my-index:00000000000000000000000000", + "leader_id": "leader-ingester", + "shard_id": 1, + "shard_state": 0, + "source_id": "_ingest-source" + } + ] + } + }, "splits": [ { "create_timestamp": 3, @@ -150,7 +165,7 @@ "end": 2000, "start": 1000 }, - "index_uid": "my-index:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "maturity": { "maturation_period_millis": 4000, "type": "immature" diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json index 46bd1d830ea..7842dbefd13 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json @@ -17,7 +17,6 @@ "stored": true, "tokenizer": "raw" }, - "index_field_presence": true, "field_mappings": [ { "coerce": true, @@ -62,6 +61,7 @@ "type": "text" } ], + "index_field_presence": true, "max_num_partitions": 100, "mode": "dynamic", "partition_key": "tenant_id", @@ -109,7 +109,7 @@ }, "version": "0.6" }, - "index_uid": "test:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "sources": [ { "desired_num_pipelines": 2, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json index 46bd1d830ea..7842dbefd13 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json @@ -17,7 +17,6 @@ "stored": true, "tokenizer": "raw" }, - "index_field_presence": true, "field_mappings": [ { "coerce": true, @@ -62,6 +61,7 @@ "type": "text" } ], + "index_field_presence": true, "max_num_partitions": 100, "mode": "dynamic", "partition_key": "tenant_id", @@ -109,7 +109,7 @@ }, "version": "0.6" }, - "index_uid": "test:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "sources": [ { "desired_num_pipelines": 2, diff --git a/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json index 42ffd916b0e..9c1f6311c86 100644 --- a/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json @@ -5,7 +5,7 @@ "end": 2000, "start": 1000 }, - "index_uid": "my-index:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "maturity": { "maturation_period_millis": 4000, "type": "immature" diff --git a/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.json b/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.json index 42ffd916b0e..9c1f6311c86 100644 --- a/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.json +++ b/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.json @@ -5,7 +5,7 @@ "end": 2000, "start": 1000 }, - "index_uid": "my-index:11111111111111111111111111", + "index_uid": "my-index:00000000000000000000000000", "maturity": { "maturation_period_millis": 4000, "type": "immature" diff --git a/quickwit/quickwit-proto/Cargo.toml b/quickwit/quickwit-proto/Cargo.toml index c1b0e8bb3a7..9efa4db9fc9 100644 --- a/quickwit/quickwit-proto/Cargo.toml +++ b/quickwit/quickwit-proto/Cargo.toml @@ -24,8 +24,8 @@ serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true, optional = true } thiserror = { workspace = true } -tonic = { workspace = true } tokio = { workspace = true } +tonic = { workspace = true } tower = { workspace = true } tracing = { workspace = true } tracing-opentelemetry = { workspace = true } diff --git a/quickwit/quickwit-proto/build.rs b/quickwit/quickwit-proto/build.rs index efb87666651..8fe75a035c5 100644 --- a/quickwit/quickwit-proto/build.rs +++ b/quickwit/quickwit-proto/build.rs @@ -84,9 +84,26 @@ fn main() -> Result<(), Box> { "DeleteQuery.end_timestamp", "#[serde(skip_serializing_if = \"Option::is_none\")]", ) + .field_attribute( + "Shard.follower_id", + "#[serde(default, skip_serializing_if = \"Option::is_none\")]", + ) + .field_attribute( + "Shard.publish_position_inclusive", + "#[serde(default, skip_serializing_if = \"String::is_empty\")]", + ) + .field_attribute( + "Shard.publish_token", + "#[serde(default, skip_serializing_if = \"Option::is_none\")]", + ) + .field_attribute( + "Shard.replication_position_inclusive", + "#[serde(default, skip_serializing_if = \"Option::is_none\")]", + ) .type_attribute(".", "#[derive(Serialize, Deserialize, utoipa::ToSchema)]") .type_attribute("PartialHit.sort_value", "#[derive(Copy)]") .type_attribute("SearchRequest", "#[derive(Eq, Hash)]") + .type_attribute("Shard", "#[derive(Eq)]") .type_attribute("SortByValue", "#[derive(Ord, PartialOrd)]") .type_attribute("SortField", "#[derive(Eq, Hash)]") .out_dir("src/codegen/quickwit") diff --git a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto index 21392349f57..fbdd824a594 100644 --- a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto +++ b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto @@ -25,8 +25,11 @@ import "quickwit/ingest.proto"; import "quickwit/metastore.proto"; service ControlPlaneService { - // The control plane acts as a proxy for the metastore so it can track its state accurately and react to events in real-time. - // Index API + // The control plane acts as a proxy for the metastore for a subset of the API so it can track the state of the metastore accurately and react to events in real-time. + + // Metastore index API proxy: + // - `create_index` + // - `delete_index` // Creates a new index. rpc CreateIndex(quickwit.metastore.CreateIndexRequest) returns (quickwit.metastore.CreateIndexResponse); @@ -34,7 +37,10 @@ service ControlPlaneService { // Deletes an index. rpc DeleteIndex(quickwit.metastore.DeleteIndexRequest) returns (quickwit.metastore.EmptyResponse); - // Source API + // Metastore source API proxy: + // - `add_source` + // - `toggle_source` + // - `delete_source` // Adds a source to an index. rpc AddSource(quickwit.metastore.AddSourceRequest) returns (quickwit.metastore.EmptyResponse); @@ -45,6 +51,14 @@ service ControlPlaneService { // Removes a source from an index. rpc DeleteSource(quickwit.metastore.DeleteSourceRequest) returns (quickwit.metastore.EmptyResponse); + // Shard API + + /// Returns the list of open shards for one or several sources. If the control plane is not able to find any + /// for a source, it will pick a pair of leader-follower ingesters and will open a new shard. + rpc GetOpenShards(GetOpenShardsRequest) returns (GetOpenShardsResponse); + + rpc CloseShards(CloseShardsRequest) returns (CloseShardsResponse); + /// Notify the Control Plane that a change on an index occurred. The change /// can be an index creation, deletion, or update that includes a source creation/deletion/num pipeline update. // Note(fmassot): it's not very clear for a user to know which change triggers a control plane notification. @@ -53,13 +67,6 @@ service ControlPlaneService { // However, these attributes will not be used by the Control Plane, at least at short term. rpc NotifyIndexChange(NotifyIndexChangeRequest) returns (NotifyIndexChangeResponse); - // Shard API - - /// Returns the list of open shards for one or several sources. If the control plane is not able to find any - /// for a source, it will pick a pair of leader-follower ingesters and will open a new shard. - rpc GetOpenShards(GetOpenShardsRequest) returns (GetOpenShardsResponse); - - rpc CloseShards(CloseShardsRequest) returns (CloseShardsResponse); } message NotifyIndexChangeRequest {} diff --git a/quickwit/quickwit-proto/protos/quickwit/ingest.proto b/quickwit/quickwit-proto/protos/quickwit/ingest.proto index 48e121cb437..653370633bb 100644 --- a/quickwit/quickwit-proto/protos/quickwit/ingest.proto +++ b/quickwit/quickwit-proto/protos/quickwit/ingest.proto @@ -34,13 +34,13 @@ message DocBatchV2 { } enum ShardState { - /// The shard is open and accepts write requests. + // The shard is open and accepts write requests. OPEN = 0; - /// The shard is open and still accepts write requests, but should no longer be advertised to ingest routers. - /// It is waiting for the its leader or follower to close it with its final replication position, after which write requests will be rejected. + // The shard is open and still accepts write requests, but should no longer be advertised to ingest routers. + // It is waiting for its leader or follower to close it with its final replication position, after which write requests will be rejected. CLOSING = 1; - /// The shard is closed and cannot be written to. - /// It can be safely deleted if the publish position is superior or equal to the replication position. + // The shard is closed and cannot be written to. + // It can be safely deleted if the publish position is superior or equal to the replication position. CLOSED = 2; } @@ -49,17 +49,19 @@ message Shard { string index_uid = 1; string source_id = 2; uint64 shard_id = 3; - /// The node ID of the ingester to which all the write requests for this shard should be sent to. + // The node ID of the ingester to which all the write requests for this shard should be sent to. string leader_id = 4; - /// The node ID of the ingester holding a copy of the data. + // The node ID of the ingester holding a copy of the data. optional string follower_id = 5; // Mutable fields ShardState shard_state = 8; - /// Position up to which the follower has acknowledged replication of the records written in its log. + // Position up to which the follower has acknowledged replication of the records written in its log. optional uint64 replication_position_inclusive = 9; - /// Position up to which indexers have indexed and published the records stored in the shard. - /// It is updated asynchronously in a best effort manner by the indexers and indicates the position up to which the log can be safely truncated. + // Position up to which indexers have indexed and published the records stored in the shard. + // It is updated asynchronously in a best effort manner by the indexers and indicates the position up to which the log can be safely truncated. string publish_position_inclusive = 10; + // A publish token that ensures only one indexer works on a given shard at a time. + // For instance, if an indexer goes rogue, eventually the control plane will detect it and assign the shard to another indexer, which will override the publish token. optional string publish_token = 11; } diff --git a/quickwit/quickwit-proto/protos/quickwit/metastore.proto b/quickwit/quickwit-proto/protos/quickwit/metastore.proto index cb30688c404..751c2293449 100644 --- a/quickwit/quickwit-proto/protos/quickwit/metastore.proto +++ b/quickwit/quickwit-proto/protos/quickwit/metastore.proto @@ -100,6 +100,10 @@ service MetastoreService { /// /// Shard API /// + /// Note that for the file-backed metastore implementation, the requests are not processed atomically. + /// Indeed, each request comprises one or more subrequests that target different indexes and sources processed + /// independently. Responses list the requests that succeeded or failed in the fields `successes` and + /// `failures`. rpc OpenShards(OpenShardsRequest) returns (OpenShardsResponse); @@ -162,8 +166,8 @@ message PublishSplitsRequest { string index_uid = 1; repeated string staged_split_ids = 2; repeated string replaced_split_ids = 3; - optional string index_checkpoint_delta_serialized_json = 4; - // optional string publish_token = 5; + optional string index_checkpoint_delta_json_opt = 4; + optional string publish_token_opt = 5; } message MarkSplitsForDeletionRequest { diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs index 4996efed153..0ef1ffdddcd 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs @@ -94,10 +94,6 @@ pub trait ControlPlaneService: std::fmt::Debug + dyn_clone::DynClone + Send + Sy &mut self, request: super::metastore::DeleteSourceRequest, ) -> crate::control_plane::ControlPlaneResult; - async fn notify_index_change( - &mut self, - request: NotifyIndexChangeRequest, - ) -> crate::control_plane::ControlPlaneResult; async fn get_open_shards( &mut self, request: GetOpenShardsRequest, @@ -106,6 +102,10 @@ pub trait ControlPlaneService: std::fmt::Debug + dyn_clone::DynClone + Send + Sy &mut self, request: CloseShardsRequest, ) -> crate::control_plane::ControlPlaneResult; + async fn notify_index_change( + &mut self, + request: NotifyIndexChangeRequest, + ) -> crate::control_plane::ControlPlaneResult; } dyn_clone::clone_trait_object!(ControlPlaneService); #[cfg(any(test, feature = "testsuite"))] @@ -206,12 +206,6 @@ impl ControlPlaneService for ControlPlaneServiceClient { ) -> crate::control_plane::ControlPlaneResult { self.inner.delete_source(request).await } - async fn notify_index_change( - &mut self, - request: NotifyIndexChangeRequest, - ) -> crate::control_plane::ControlPlaneResult { - self.inner.notify_index_change(request).await - } async fn get_open_shards( &mut self, request: GetOpenShardsRequest, @@ -224,6 +218,12 @@ impl ControlPlaneService for ControlPlaneServiceClient { ) -> crate::control_plane::ControlPlaneResult { self.inner.close_shards(request).await } + async fn notify_index_change( + &mut self, + request: NotifyIndexChangeRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.inner.notify_index_change(request).await + } } #[cfg(any(test, feature = "testsuite"))] pub mod control_plane_service_mock { @@ -274,12 +274,6 @@ pub mod control_plane_service_mock { > { self.inner.lock().await.delete_source(request).await } - async fn notify_index_change( - &mut self, - request: super::NotifyIndexChangeRequest, - ) -> crate::control_plane::ControlPlaneResult { - self.inner.lock().await.notify_index_change(request).await - } async fn get_open_shards( &mut self, request: super::GetOpenShardsRequest, @@ -292,6 +286,12 @@ pub mod control_plane_service_mock { ) -> crate::control_plane::ControlPlaneResult { self.inner.lock().await.close_shards(request).await } + async fn notify_index_change( + &mut self, + request: super::NotifyIndexChangeRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.inner.lock().await.notify_index_change(request).await + } } impl From for ControlPlaneServiceClient { fn from(mock: MockControlPlaneService) -> Self { @@ -390,8 +390,8 @@ for Box { Box::pin(fut) } } -impl tower::Service for Box { - type Response = NotifyIndexChangeResponse; +impl tower::Service for Box { + type Response = GetOpenShardsResponse; type Error = crate::control_plane::ControlPlaneError; type Future = BoxFuture; fn poll_ready( @@ -400,14 +400,14 @@ impl tower::Service for Box { ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } - fn call(&mut self, request: NotifyIndexChangeRequest) -> Self::Future { + fn call(&mut self, request: GetOpenShardsRequest) -> Self::Future { let mut svc = self.clone(); - let fut = async move { svc.notify_index_change(request).await }; + let fut = async move { svc.get_open_shards(request).await }; Box::pin(fut) } } -impl tower::Service for Box { - type Response = GetOpenShardsResponse; +impl tower::Service for Box { + type Response = CloseShardsResponse; type Error = crate::control_plane::ControlPlaneError; type Future = BoxFuture; fn poll_ready( @@ -416,14 +416,14 @@ impl tower::Service for Box { ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } - fn call(&mut self, request: GetOpenShardsRequest) -> Self::Future { + fn call(&mut self, request: CloseShardsRequest) -> Self::Future { let mut svc = self.clone(); - let fut = async move { svc.get_open_shards(request).await }; + let fut = async move { svc.close_shards(request).await }; Box::pin(fut) } } -impl tower::Service for Box { - type Response = CloseShardsResponse; +impl tower::Service for Box { + type Response = NotifyIndexChangeResponse; type Error = crate::control_plane::ControlPlaneError; type Future = BoxFuture; fn poll_ready( @@ -432,9 +432,9 @@ impl tower::Service for Box { ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } - fn call(&mut self, request: CloseShardsRequest) -> Self::Future { + fn call(&mut self, request: NotifyIndexChangeRequest) -> Self::Future { let mut svc = self.clone(); - let fut = async move { svc.close_shards(request).await }; + let fut = async move { svc.notify_index_change(request).await }; Box::pin(fut) } } @@ -466,11 +466,6 @@ struct ControlPlaneServiceTowerBlock { super::metastore::EmptyResponse, crate::control_plane::ControlPlaneError, >, - notify_index_change_svc: quickwit_common::tower::BoxService< - NotifyIndexChangeRequest, - NotifyIndexChangeResponse, - crate::control_plane::ControlPlaneError, - >, get_open_shards_svc: quickwit_common::tower::BoxService< GetOpenShardsRequest, GetOpenShardsResponse, @@ -481,6 +476,11 @@ struct ControlPlaneServiceTowerBlock { CloseShardsResponse, crate::control_plane::ControlPlaneError, >, + notify_index_change_svc: quickwit_common::tower::BoxService< + NotifyIndexChangeRequest, + NotifyIndexChangeResponse, + crate::control_plane::ControlPlaneError, + >, } impl Clone for ControlPlaneServiceTowerBlock { fn clone(&self) -> Self { @@ -490,9 +490,9 @@ impl Clone for ControlPlaneServiceTowerBlock { add_source_svc: self.add_source_svc.clone(), toggle_source_svc: self.toggle_source_svc.clone(), delete_source_svc: self.delete_source_svc.clone(), - notify_index_change_svc: self.notify_index_change_svc.clone(), get_open_shards_svc: self.get_open_shards_svc.clone(), close_shards_svc: self.close_shards_svc.clone(), + notify_index_change_svc: self.notify_index_change_svc.clone(), } } } @@ -530,12 +530,6 @@ impl ControlPlaneService for ControlPlaneServiceTowerBlock { ) -> crate::control_plane::ControlPlaneResult { self.delete_source_svc.ready().await?.call(request).await } - async fn notify_index_change( - &mut self, - request: NotifyIndexChangeRequest, - ) -> crate::control_plane::ControlPlaneResult { - self.notify_index_change_svc.ready().await?.call(request).await - } async fn get_open_shards( &mut self, request: GetOpenShardsRequest, @@ -548,6 +542,12 @@ impl ControlPlaneService for ControlPlaneServiceTowerBlock { ) -> crate::control_plane::ControlPlaneResult { self.close_shards_svc.ready().await?.call(request).await } + async fn notify_index_change( + &mut self, + request: NotifyIndexChangeRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.notify_index_change_svc.ready().await?.call(request).await + } } #[derive(Debug, Default)] pub struct ControlPlaneServiceTowerBlockBuilder { @@ -597,15 +597,6 @@ pub struct ControlPlaneServiceTowerBlockBuilder { >, >, #[allow(clippy::type_complexity)] - notify_index_change_layer: Option< - quickwit_common::tower::BoxLayer< - Box, - NotifyIndexChangeRequest, - NotifyIndexChangeResponse, - crate::control_plane::ControlPlaneError, - >, - >, - #[allow(clippy::type_complexity)] get_open_shards_layer: Option< quickwit_common::tower::BoxLayer< Box, @@ -623,6 +614,15 @@ pub struct ControlPlaneServiceTowerBlockBuilder { crate::control_plane::ControlPlaneError, >, >, + #[allow(clippy::type_complexity)] + notify_index_change_layer: Option< + quickwit_common::tower::BoxLayer< + Box, + NotifyIndexChangeRequest, + NotifyIndexChangeResponse, + crate::control_plane::ControlPlaneError, + >, + >, } impl ControlPlaneServiceTowerBlockBuilder { pub fn shared_layer(mut self, layer: L) -> Self @@ -668,12 +668,6 @@ impl ControlPlaneServiceTowerBlockBuilder { >::Future: Send + 'static, - L::Service: tower::Service< - NotifyIndexChangeRequest, - Response = NotifyIndexChangeResponse, - Error = crate::control_plane::ControlPlaneError, - > + Clone + Send + Sync + 'static, - >::Future: Send + 'static, L::Service: tower::Service< GetOpenShardsRequest, Response = GetOpenShardsResponse, @@ -686,6 +680,12 @@ impl ControlPlaneServiceTowerBlockBuilder { Error = crate::control_plane::ControlPlaneError, > + Clone + Send + Sync + 'static, >::Future: Send + 'static, + L::Service: tower::Service< + NotifyIndexChangeRequest, + Response = NotifyIndexChangeResponse, + Error = crate::control_plane::ControlPlaneError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, { self .create_index_layer = Some( @@ -708,14 +708,17 @@ impl ControlPlaneServiceTowerBlockBuilder { quickwit_common::tower::BoxLayer::new(layer.clone()), ); self - .notify_index_change_layer = Some( + .get_open_shards_layer = Some( quickwit_common::tower::BoxLayer::new(layer.clone()), ); self - .get_open_shards_layer = Some( + .close_shards_layer = Some( quickwit_common::tower::BoxLayer::new(layer.clone()), ); - self.close_shards_layer = Some(quickwit_common::tower::BoxLayer::new(layer)); + self + .notify_index_change_layer = Some( + quickwit_common::tower::BoxLayer::new(layer), + ); self } pub fn create_index_layer(mut self, layer: L) -> Self @@ -793,22 +796,6 @@ impl ControlPlaneServiceTowerBlockBuilder { self.delete_source_layer = Some(quickwit_common::tower::BoxLayer::new(layer)); self } - pub fn notify_index_change_layer(mut self, layer: L) -> Self - where - L: tower::Layer> + Send + Sync + 'static, - L::Service: tower::Service< - NotifyIndexChangeRequest, - Response = NotifyIndexChangeResponse, - Error = crate::control_plane::ControlPlaneError, - > + Clone + Send + Sync + 'static, - >::Future: Send + 'static, - { - self - .notify_index_change_layer = Some( - quickwit_common::tower::BoxLayer::new(layer), - ); - self - } pub fn get_open_shards_layer(mut self, layer: L) -> Self where L: tower::Layer> + Send + Sync + 'static, @@ -835,6 +822,22 @@ impl ControlPlaneServiceTowerBlockBuilder { self.close_shards_layer = Some(quickwit_common::tower::BoxLayer::new(layer)); self } + pub fn notify_index_change_layer(mut self, layer: L) -> Self + where + L: tower::Layer> + Send + Sync + 'static, + L::Service: tower::Service< + NotifyIndexChangeRequest, + Response = NotifyIndexChangeResponse, + Error = crate::control_plane::ControlPlaneError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self + .notify_index_change_layer = Some( + quickwit_common::tower::BoxLayer::new(layer), + ); + self + } pub fn build(self, instance: T) -> ControlPlaneServiceClient where T: ControlPlaneService, @@ -898,18 +901,18 @@ impl ControlPlaneServiceTowerBlockBuilder { } else { quickwit_common::tower::BoxService::new(boxed_instance.clone()) }; - let notify_index_change_svc = if let Some(layer) = self.notify_index_change_layer - { + let get_open_shards_svc = if let Some(layer) = self.get_open_shards_layer { layer.layer(boxed_instance.clone()) } else { quickwit_common::tower::BoxService::new(boxed_instance.clone()) }; - let get_open_shards_svc = if let Some(layer) = self.get_open_shards_layer { + let close_shards_svc = if let Some(layer) = self.close_shards_layer { layer.layer(boxed_instance.clone()) } else { quickwit_common::tower::BoxService::new(boxed_instance.clone()) }; - let close_shards_svc = if let Some(layer) = self.close_shards_layer { + let notify_index_change_svc = if let Some(layer) = self.notify_index_change_layer + { layer.layer(boxed_instance.clone()) } else { quickwit_common::tower::BoxService::new(boxed_instance.clone()) @@ -920,9 +923,9 @@ impl ControlPlaneServiceTowerBlockBuilder { add_source_svc, toggle_source_svc, delete_source_svc, - notify_index_change_svc, get_open_shards_svc, close_shards_svc, + notify_index_change_svc, }; ControlPlaneServiceClient::new(tower_block) } @@ -1044,15 +1047,6 @@ where crate::control_plane::ControlPlaneError, >, > - + tower::Service< - NotifyIndexChangeRequest, - Response = NotifyIndexChangeResponse, - Error = crate::control_plane::ControlPlaneError, - Future = BoxFuture< - NotifyIndexChangeResponse, - crate::control_plane::ControlPlaneError, - >, - > + tower::Service< GetOpenShardsRequest, Response = GetOpenShardsResponse, @@ -1070,6 +1064,15 @@ where CloseShardsResponse, crate::control_plane::ControlPlaneError, >, + > + + tower::Service< + NotifyIndexChangeRequest, + Response = NotifyIndexChangeResponse, + Error = crate::control_plane::ControlPlaneError, + Future = BoxFuture< + NotifyIndexChangeResponse, + crate::control_plane::ControlPlaneError, + >, >, { async fn create_index( @@ -1104,12 +1107,6 @@ where ) -> crate::control_plane::ControlPlaneResult { self.call(request).await } - async fn notify_index_change( - &mut self, - request: NotifyIndexChangeRequest, - ) -> crate::control_plane::ControlPlaneResult { - self.call(request).await - } async fn get_open_shards( &mut self, request: GetOpenShardsRequest, @@ -1122,6 +1119,12 @@ where ) -> crate::control_plane::ControlPlaneResult { self.call(request).await } + async fn notify_index_change( + &mut self, + request: NotifyIndexChangeRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.call(request).await + } } #[derive(Debug, Clone)] pub struct ControlPlaneServiceGrpcClientAdapter { @@ -1197,16 +1200,6 @@ where .map(|response| response.into_inner()) .map_err(|error| error.into()) } - async fn notify_index_change( - &mut self, - request: NotifyIndexChangeRequest, - ) -> crate::control_plane::ControlPlaneResult { - self.inner - .notify_index_change(request) - .await - .map(|response| response.into_inner()) - .map_err(|error| error.into()) - } async fn get_open_shards( &mut self, request: GetOpenShardsRequest, @@ -1227,6 +1220,16 @@ where .map(|response| response.into_inner()) .map_err(|error| error.into()) } + async fn notify_index_change( + &mut self, + request: NotifyIndexChangeRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.inner + .notify_index_change(request) + .await + .map(|response| response.into_inner()) + .map_err(|error| error.into()) + } } #[derive(Debug)] pub struct ControlPlaneServiceGrpcServerAdapter { @@ -1298,17 +1301,6 @@ for ControlPlaneServiceGrpcServerAdapter { .map(tonic::Response::new) .map_err(|error| error.into()) } - async fn notify_index_change( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - self.inner - .clone() - .notify_index_change(request.into_inner()) - .await - .map(tonic::Response::new) - .map_err(|error| error.into()) - } async fn get_open_shards( &self, request: tonic::Request, @@ -1331,6 +1323,17 @@ for ControlPlaneServiceGrpcServerAdapter { .map(tonic::Response::new) .map_err(|error| error.into()) } + async fn notify_index_change( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .clone() + .notify_index_change(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(|error| error.into()) + } } /// Generated client implementations. pub mod control_plane_service_grpc_client { @@ -1578,17 +1581,13 @@ pub mod control_plane_service_grpc_client { ); self.inner.unary(req, path, codec).await } - /// / Notify the Control Plane that a change on an index occurred. The change - /// / can be an index creation, deletion, or update that includes a source creation/deletion/num pipeline update. - /// Note(fmassot): it's not very clear for a user to know which change triggers a control plane notification. - /// This can be explicited in the attributes of `NotifyIndexChangeRequest` with an enum that describes the - /// type of change. The index ID and/or source ID could also be added. - /// However, these attributes will not be used by the Control Plane, at least at short term. - pub async fn notify_index_change( + /// / Returns the list of open shards for one or several sources. If the control plane is not able to find any + /// / for a source, it will pick a pair of leader-follower ingesters and will open a new shard. + pub async fn get_open_shards( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> std::result::Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -1602,25 +1601,23 @@ pub mod control_plane_service_grpc_client { })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( - "/quickwit.control_plane.ControlPlaneService/NotifyIndexChange", + "/quickwit.control_plane.ControlPlaneService/GetOpenShards", ); let mut req = request.into_request(); req.extensions_mut() .insert( GrpcMethod::new( "quickwit.control_plane.ControlPlaneService", - "NotifyIndexChange", + "GetOpenShards", ), ); self.inner.unary(req, path, codec).await } - /// / Returns the list of open shards for one or several sources. If the control plane is not able to find any - /// / for a source, it will pick a pair of leader-follower ingesters and will open a new shard. - pub async fn get_open_shards( + pub async fn close_shards( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> std::result::Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -1634,23 +1631,29 @@ pub mod control_plane_service_grpc_client { })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( - "/quickwit.control_plane.ControlPlaneService/GetOpenShards", + "/quickwit.control_plane.ControlPlaneService/CloseShards", ); let mut req = request.into_request(); req.extensions_mut() .insert( GrpcMethod::new( "quickwit.control_plane.ControlPlaneService", - "GetOpenShards", + "CloseShards", ), ); self.inner.unary(req, path, codec).await } - pub async fn close_shards( + /// / Notify the Control Plane that a change on an index occurred. The change + /// / can be an index creation, deletion, or update that includes a source creation/deletion/num pipeline update. + /// Note(fmassot): it's not very clear for a user to know which change triggers a control plane notification. + /// This can be explicited in the attributes of `NotifyIndexChangeRequest` with an enum that describes the + /// type of change. The index ID and/or source ID could also be added. + /// However, these attributes will not be used by the Control Plane, at least at short term. + pub async fn notify_index_change( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> std::result::Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -1664,14 +1667,14 @@ pub mod control_plane_service_grpc_client { })?; let codec = tonic::codec::ProstCodec::default(); let path = http::uri::PathAndQuery::from_static( - "/quickwit.control_plane.ControlPlaneService/CloseShards", + "/quickwit.control_plane.ControlPlaneService/NotifyIndexChange", ); let mut req = request.into_request(); req.extensions_mut() .insert( GrpcMethod::new( "quickwit.control_plane.ControlPlaneService", - "CloseShards", + "NotifyIndexChange", ), ); self.inner.unary(req, path, codec).await @@ -1725,19 +1728,6 @@ pub mod control_plane_service_grpc_server { tonic::Response, tonic::Status, >; - /// / Notify the Control Plane that a change on an index occurred. The change - /// / can be an index creation, deletion, or update that includes a source creation/deletion/num pipeline update. - /// Note(fmassot): it's not very clear for a user to know which change triggers a control plane notification. - /// This can be explicited in the attributes of `NotifyIndexChangeRequest` with an enum that describes the - /// type of change. The index ID and/or source ID could also be added. - /// However, these attributes will not be used by the Control Plane, at least at short term. - async fn notify_index_change( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; /// / Returns the list of open shards for one or several sources. If the control plane is not able to find any /// / for a source, it will pick a pair of leader-follower ingesters and will open a new shard. async fn get_open_shards( @@ -1754,6 +1744,19 @@ pub mod control_plane_service_grpc_server { tonic::Response, tonic::Status, >; + /// / Notify the Control Plane that a change on an index occurred. The change + /// / can be an index creation, deletion, or update that includes a source creation/deletion/num pipeline update. + /// Note(fmassot): it's not very clear for a user to know which change triggers a control plane notification. + /// This can be explicited in the attributes of `NotifyIndexChangeRequest` with an enum that describes the + /// type of change. The index ID and/or source ID could also be added. + /// However, these attributes will not be used by the Control Plane, at least at short term. + async fn notify_index_change( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct ControlPlaneServiceGrpcServer { @@ -2078,25 +2081,25 @@ pub mod control_plane_service_grpc_server { }; Box::pin(fut) } - "/quickwit.control_plane.ControlPlaneService/NotifyIndexChange" => { + "/quickwit.control_plane.ControlPlaneService/GetOpenShards" => { #[allow(non_camel_case_types)] - struct NotifyIndexChangeSvc(pub Arc); + struct GetOpenShardsSvc(pub Arc); impl< T: ControlPlaneServiceGrpc, - > tonic::server::UnaryService - for NotifyIndexChangeSvc { - type Response = super::NotifyIndexChangeResponse; + > tonic::server::UnaryService + for GetOpenShardsSvc { + type Response = super::GetOpenShardsResponse; type Future = BoxFuture< tonic::Response, tonic::Status, >; fn call( &mut self, - request: tonic::Request, + request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - (*inner).notify_index_change(request).await + (*inner).get_open_shards(request).await }; Box::pin(fut) } @@ -2108,7 +2111,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let inner = inner.0; - let method = NotifyIndexChangeSvc(inner); + let method = GetOpenShardsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( @@ -2124,25 +2127,25 @@ pub mod control_plane_service_grpc_server { }; Box::pin(fut) } - "/quickwit.control_plane.ControlPlaneService/GetOpenShards" => { + "/quickwit.control_plane.ControlPlaneService/CloseShards" => { #[allow(non_camel_case_types)] - struct GetOpenShardsSvc(pub Arc); + struct CloseShardsSvc(pub Arc); impl< T: ControlPlaneServiceGrpc, - > tonic::server::UnaryService - for GetOpenShardsSvc { - type Response = super::GetOpenShardsResponse; + > tonic::server::UnaryService + for CloseShardsSvc { + type Response = super::CloseShardsResponse; type Future = BoxFuture< tonic::Response, tonic::Status, >; fn call( &mut self, - request: tonic::Request, + request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - (*inner).get_open_shards(request).await + (*inner).close_shards(request).await }; Box::pin(fut) } @@ -2154,7 +2157,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let inner = inner.0; - let method = GetOpenShardsSvc(inner); + let method = CloseShardsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( @@ -2170,25 +2173,25 @@ pub mod control_plane_service_grpc_server { }; Box::pin(fut) } - "/quickwit.control_plane.ControlPlaneService/CloseShards" => { + "/quickwit.control_plane.ControlPlaneService/NotifyIndexChange" => { #[allow(non_camel_case_types)] - struct CloseShardsSvc(pub Arc); + struct NotifyIndexChangeSvc(pub Arc); impl< T: ControlPlaneServiceGrpc, - > tonic::server::UnaryService - for CloseShardsSvc { - type Response = super::CloseShardsResponse; + > tonic::server::UnaryService + for NotifyIndexChangeSvc { + type Response = super::NotifyIndexChangeResponse; type Future = BoxFuture< tonic::Response, tonic::Status, >; fn call( &mut self, - request: tonic::Request, + request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { - (*inner).close_shards(request).await + (*inner).notify_index_change(request).await }; Box::pin(fut) } @@ -2200,7 +2203,7 @@ pub mod control_plane_service_grpc_server { let inner = self.inner.clone(); let fut = async move { let inner = inner.0; - let method = CloseShardsSvc(inner); + let method = NotifyIndexChangeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs index 12d22ff81d5..2dcefc893a9 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs @@ -8,6 +8,7 @@ pub struct DocBatchV2 { pub doc_lengths: ::prost::alloc::vec::Vec, } #[derive(Serialize, Deserialize, utoipa::ToSchema)] +#[derive(Eq)] #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Shard { @@ -18,23 +19,29 @@ pub struct Shard { pub source_id: ::prost::alloc::string::String, #[prost(uint64, tag = "3")] pub shard_id: u64, - /// / The node ID of the ingester to which all the write requests for this shard should be sent to. + /// The node ID of the ingester to which all the write requests for this shard should be sent to. #[prost(string, tag = "4")] pub leader_id: ::prost::alloc::string::String, - /// / The node ID of the ingester holding a copy of the data. + /// The node ID of the ingester holding a copy of the data. #[prost(string, optional, tag = "5")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub follower_id: ::core::option::Option<::prost::alloc::string::String>, /// Mutable fields #[prost(enumeration = "ShardState", tag = "8")] pub shard_state: i32, - /// / Position up to which the follower has acknowledged replication of the records written in its log. + /// Position up to which the follower has acknowledged replication of the records written in its log. #[prost(uint64, optional, tag = "9")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub replication_position_inclusive: ::core::option::Option, - /// / Position up to which indexers have indexed and published the records stored in the shard. - /// / It is updated asynchronously in a best effort manner by the indexers and indicates the position up to which the log can be safely truncated. + /// Position up to which indexers have indexed and published the records stored in the shard. + /// It is updated asynchronously in a best effort manner by the indexers and indicates the position up to which the log can be safely truncated. #[prost(string, tag = "10")] + #[serde(default, skip_serializing_if = "String::is_empty")] pub publish_position_inclusive: ::prost::alloc::string::String, + /// A publish token that ensures only one indexer works on a given shard at a time. + /// For instance, if an indexer goes rogue, eventually the control plane will detect it and assign the shard to another indexer, which will override the publish token. #[prost(string, optional, tag = "11")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub publish_token: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Serialize, Deserialize, utoipa::ToSchema)] @@ -73,13 +80,13 @@ impl CommitTypeV2 { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ShardState { - /// / The shard is open and accepts write requests. + /// The shard is open and accepts write requests. Open = 0, - /// / The shard is open and still accepts write requests, but should no longer be advertised to ingest routers. - /// / It is waiting for the its leader or follower to close it with its final replication position, after which write requests will be rejected. + /// The shard is open and still accepts write requests, but should no longer be advertised to ingest routers. + /// It is waiting for its leader or follower to close it with its final replication position, after which write requests will be rejected. Closing = 1, - /// / The shard is closed and cannot be written to. - /// / It can be safely deleted if the publish position is superior or equal to the replication position. + /// The shard is closed and cannot be written to. + /// It can be safely deleted if the publish position is superior or equal to the replication position. Closed = 2, } impl ShardState { diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs index 494f7da7e5c..892bb588de1 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.metastore.rs @@ -91,11 +91,12 @@ pub struct PublishSplitsRequest { pub staged_split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(string, repeated, tag = "3")] pub replaced_split_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// optional string publish_token = 5; #[prost(string, optional, tag = "4")] - pub index_checkpoint_delta_serialized_json: ::core::option::Option< + pub index_checkpoint_delta_json_opt: ::core::option::Option< ::prost::alloc::string::String, >, + #[prost(string, optional, tag = "5")] + pub publish_token_opt: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Serialize, Deserialize, utoipa::ToSchema)] #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/quickwit/quickwit-proto/src/types.rs b/quickwit/quickwit-proto/src/types.rs index 22fc25d4541..dc943611f46 100644 --- a/quickwit/quickwit-proto/src/types.rs +++ b/quickwit/quickwit-proto/src/types.rs @@ -35,6 +35,9 @@ pub type SplitId = String; pub type ShardId = u64; +/// See the file `ingest.proto` for more details. +pub type PublishToken = String; + /// Uniquely identifies a shard and its underlying mrecordlog queue. pub type QueueId = String; // //