Add topsql_meta_deltalake

Signed-off-by: yibin87 <[email protected]>

Author: yibin87

Cargo.lock

@@ -65,6 +65,7 @@ vector-lib = { git = "https://github.com/vectordotdev/vector", tag = "v0.49.0",
 xz2 = { version = "0.1.7" }
 hashlru = "0.11.1"
 lazy_static = "1.4.0"
+lru = "0.12.5"
 
 [dev-dependencies]
 lazy_static = "1.4.0"

Cargo.toml

@@ -3,4 +3,5 @@ pub mod azure_blob_upload_file;
 pub mod deltalake;
 pub mod gcp_cloud_storage_upload_file;
 pub mod vm_import;
-pub mod topsql_deltalake;
+pub mod topsql_data_deltalake;
+pub mod topsql_meta_deltalake;

src/sinks/mod.rs

@@ -18,7 +18,7 @@ use vector_lib::{
     tls::TlsConfig,
 };
 
-use crate::sinks::topsql_deltalake::processor::TopSQLDeltaLakeSink;
+use crate::sinks::topsql_data_deltalake::processor::TopSQLDeltaLakeSink;
 
 use reqwest::Client;
 use serde_json::Value;
@@ -37,7 +37,7 @@ pub const fn default_max_delay_secs() -> u64 {
 pub use crate::common::deltalake_writer::{DeltaTableConfig, WriteConfig};
 
 /// Configuration for the deltalake sink
-#[configurable_component(sink("topsql_deltalake"))]
+#[configurable_component(sink("topsql_data_deltalake"))]
 #[derive(Debug, Clone)]
 #[serde(deny_unknown_fields)]
 pub struct DeltaLakeConfig {
@@ -228,7 +228,7 @@ impl GenerateConfig for DeltaLakeConfig {
 }
 
 #[async_trait::async_trait]
-#[typetag::serde(name = "topsql_deltalake")]
+#[typetag::serde(name = "topsql_data_deltalake")]
 impl SinkConfig for DeltaLakeConfig {
     async fn build(&self, cx: SinkContext) -> vector::Result<(VectorSink, Healthcheck)> {
         error!(