encoding/: Add Protobuf support (#83)

Add support for the OpenMetrics Protobuf encoding.

Co-authored-by: Max Inden <[email protected]>
Co-authored-by: Diva M <[email protected]>

Author: 3 people

.github/workflows/rust.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --benches
+          args: --benches --all-features
 
   test:
     name: Test Suite
@@ -73,7 +73,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --all
+          args: --all --all-features
 
   fmt:
     name: Rustfmt
@@ -135,7 +135,7 @@ jobs:
           RUSTDOCFLAGS: "--deny broken_intra_doc_links"
         with:
           command: doc
-          args: --verbose --workspace --no-deps --document-private-items
+          args: --verbose --workspace --no-deps --document-private-items --all-features
 
   cross-compile:
     name: Cross compile
@@ -159,4 +159,4 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: --release --target=${{ matrix.target }}
+          args: --release --target=${{ matrix.target }} --all-features

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.19.0] - unreleased
+
+### Added
+- Added support for the OpenMetrics protobuf format. See [PR 83].
+
+### Changed
+
+- Move`Encode` trait from `prometheus_client::encoding::text` to `prometheus_client::encoding`. See [PR 83].
+
+[PR 83]: https://github.com/prometheus/client_rust/pull/83
+
 ## [0.18.0]
 
 ### Changed

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "prometheus-client"
-version = "0.18.0"
+version = "0.19.0"
 authors = ["Max Inden <[email protected]>"]
 edition = "2021"
 description = "Open Metrics client library allowing users to natively instrument applications."
@@ -10,14 +10,20 @@ repository = "https://github.com/prometheus/client_rust"
 homepage = "https://github.com/prometheus/client_rust"
 documentation = "https://docs.rs/prometheus-client"
 
+[features]
+protobuf = ["dep:prost", "dep:prost-types", "dep:prost-build", "dep:void", "prometheus-client-derive-encode/protobuf"]
+
 [workspace]
-members = ["derive-text-encode"]
+members = ["derive-encode"]
 
 [dependencies]
 dtoa = "1.0"
 itoa = "1.0"
 parking_lot = "0.12"
-prometheus-client-derive-text-encode = { version = "0.3.0", path = "derive-text-encode" }
+prometheus-client-derive-encode = { version = "0.3.0", path = "derive-encode" }
+prost = { version = "0.9.0", optional = true }
+prost-types = { version = "0.9.0", optional = true }
+void = { version = "1.0", optional = true }
 
 [dev-dependencies]
 async-std = { version = "1", features = ["attributes"] }
@@ -29,6 +35,9 @@ rand = "0.8.4"
 tide = "0.16"
 actix-web = "4"
 
+[build-dependencies]
+prost-build = { version = "0.9.0", optional = true }
+
 [[bench]]
 name = "family"
 harness = false
@@ -37,3 +46,10 @@ harness = false
 name = "text"
 path = "benches/encoding/text.rs"
 harness = false
+required-features = []
+
+[[bench]]
+name = "proto"
+path = "benches/encoding/proto.rs"
+harness = false
+required-features = ["protobuf"] 

benches/encoding/proto.rs

@@ -0,0 +1,87 @@
+// Benchmark inspired by
+// https://github.com/tikv/rust-prometheus/blob/ab1ca7285d3463504381a5025ae1951e020d6796/benches/text_encoder.rs:write
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use prometheus_client::encoding::proto::{encode, EncodeMetric};
+use prometheus_client::encoding::Encode;
+use prometheus_client::metrics::counter::Counter;
+use prometheus_client::metrics::family::Family;
+use prometheus_client::metrics::histogram::{exponential_buckets, Histogram};
+use prometheus_client::registry::Registry;
+use std::fmt::{Display, Formatter};
+
+pub fn proto(c: &mut Criterion) {
+    c.bench_function("encode", |b| {
+        #[derive(Clone, Hash, PartialEq, Eq, Encode)]
+        struct Labels {
+            path: String,
+            method: Method,
+            some_number: u64,
+        }
+
+        #[derive(Clone, Hash, PartialEq, Eq, Encode)]
+        enum Method {
+            Get,
+            #[allow(dead_code)]
+            Put,
+        }
+
+        impl Display for Method {
+            fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+                match self {
+                    Method::Get => write!(f, "Get"),
+                    Method::Put => write!(f, "Put"),
+                }
+            }
+        }
+
+        #[derive(Clone, Hash, PartialEq, Eq, Encode)]
+        enum Region {
+            Africa,
+            #[allow(dead_code)]
+            Asia,
+        }
+
+        let mut registry = Registry::<Box<dyn EncodeMetric>>::default();
+
+        for i in 0..100 {
+            let counter_family = Family::<Labels, Counter>::default();
+            let histogram_family = Family::<Region, Histogram>::new_with_constructor(|| {
+                Histogram::new(exponential_buckets(1.0, 2.0, 10))
+            });
+
+            registry.register(
+                format!("my_counter{}", i),
+                "My counter",
+                Box::new(counter_family.clone()),
+            );
+            registry.register(
+                format!("my_histogram{}", i),
+                "My histogram",
+                Box::new(histogram_family.clone()),
+            );
+
+            for j in 0_u32..100 {
+                counter_family
+                    .get_or_create(&Labels {
+                        path: format!("/path/{}", i),
+                        method: Method::Get,
+                        some_number: j.into(),
+                    })
+                    .inc();
+
+                histogram_family
+                    .get_or_create(&Region::Africa)
+                    .observe(j.into());
+            }
+        }
+
+        b.iter(|| {
+            let metric_set = encode(&registry);
+            black_box(metric_set);
+        })
+    });
+}
+
+criterion_group!(benches, proto);
+criterion_main!(benches);

benches/encoding/text.rs

@@ -1,7 +1,8 @@
 // Benchmark inspired by https://github.com/tikv/rust-prometheus/blob/ab1ca7285d3463504381a5025ae1951e020d6796/benches/text_encoder.rs
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use prometheus_client::encoding::text::{encode, Encode, EncodeMetric};
+use prometheus_client::encoding::text::{encode, EncodeMetric};
+use prometheus_client::encoding::Encode;
 use prometheus_client::metrics::counter::Counter;
 use prometheus_client::metrics::family::Family;
 use prometheus_client::metrics::histogram::{exponential_buckets, Histogram};
@@ -33,7 +34,7 @@ pub fn text(c: &mut Criterion) {
             Five,
         }
 
-        impl Encode for Status {
+        impl prometheus_client::encoding::text::Encode for Status {
             fn encode(&self, writer: &mut dyn Write) -> Result<(), std::io::Error> {
                 let status = match self {
                     Status::Two => b"200",
@@ -45,6 +46,21 @@ pub fn text(c: &mut Criterion) {
             }
         }
 
+        #[cfg(feature = "protobuf")]
+        impl prometheus_client::encoding::proto::EncodeLabels for Status {
+            fn encode(&self, labels: &mut Vec<prometheus_client::encoding::proto::Label>) {
+                let value = match self {
+                    Status::Two => "200".to_string(),
+                    Status::Four => "400".to_string(),
+                    Status::Five => "500".to_string(),
+                };
+                labels.push(prometheus_client::encoding::proto::Label {
+                    name: "status".to_string(),
+                    value,
+                });
+            }
+        }
+
         let mut registry = Registry::<Box<dyn EncodeMetric>>::default();
 
         for i in 0..100 {

build.rs

@@ -0,0 +1,11 @@
+use std::io::Result;
+
+fn main() -> Result<()> {
+    #[cfg(feature = "protobuf")]
+    prost_build::compile_protos(
+        &["src/encoding/proto/openmetrics_data_model.proto"],
+        &["src/encoding/proto/"],
+    )?;
+
+    Ok(())
+}

derive-text-encode/Cargo.toml derive-encode/Cargo.tomlderive-text-encode/Cargo.toml renamed to derive-encode/Cargo.toml

@@ -1,14 +1,17 @@
 [package]
-name = "prometheus-client-derive-text-encode"
-version = "0.3.0"
+name = "prometheus-client-derive-encode"
+version = "0.3.1"
 authors = ["Max Inden <[email protected]>"]
 edition = "2021"
-description = "Auxiliary crate to derive text Encode trait from prometheus-client."
+description = "Auxiliary crate to derive Encode trait from prometheus-client."
 license = "Apache-2.0 OR MIT"
 repository = "https://github.com/prometheus/client_rust"
 homepage = "https://github.com/prometheus/client_rust"
 documentation = "https://docs.rs/prometheus-client-derive-text-encode"
 
+[features]
+protobuf = []
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -17,7 +20,7 @@ quote = "1"
 syn = "1"
 
 [dev-dependencies]
-prometheus-client = { path = "../" }
+prometheus-client = { path = "../", features = ["protobuf"] }
 
 [lib]
 proc-macro = true

derive-text-encode/src/lib.rs derive-encode/src/lib.rsderive-text-encode/src/lib.rs renamed to derive-encode/src/lib.rs

@@ -10,7 +10,7 @@ pub fn derive_encode(input: TokenStream) -> TokenStream {
     let ast: DeriveInput = syn::parse(input).unwrap();
     let name = &ast.ident;
 
-    let body = match ast.data {
+    let body = match ast.clone().data {
         syn::Data::Struct(s) => match s.fields {
             syn::Fields::Named(syn::FieldsNamed { named, .. }) => named
                 .into_iter()
@@ -70,9 +70,94 @@ pub fn derive_encode(input: TokenStream) -> TokenStream {
             }
         }
     };
+
+    #[cfg(feature = "protobuf")]
+    let gen = {
+        let protobuf = derive_protobuf_encode(ast);
+        quote! {
+            #gen
+
+            #protobuf
+        }
+    };
+
     gen.into()
 }
 
+#[cfg(feature = "protobuf")]
+fn derive_protobuf_encode(ast: DeriveInput) -> TokenStream2 {
+    let name = &ast.ident;
+
+    match ast.data {
+        syn::Data::Struct(s) => match s.fields {
+            syn::Fields::Named(syn::FieldsNamed { named, .. }) => {
+                let push_labels: TokenStream2 = named
+                    .into_iter()
+                    .map(|f| {
+                        let ident = f.ident.unwrap();
+                        let ident_string = KEYWORD_IDENTIFIERS
+                            .iter()
+                            .find(|pair| ident == pair.1)
+                            .map(|pair| pair.0.to_string())
+                            .unwrap_or_else(|| ident.to_string());
+
+                        quote! {
+                            let mut label = {
+                                let mut labels = vec![];
+                                self.#ident.encode(&mut labels);
+                                debug_assert_eq!(1, labels.len(), "Labels encoded from {} should have only one label.", #ident_string);
+                                labels.pop().expect("should have an element")
+                            };
+                            // Override the label name with the field name of this struct.
+                            label.name = #ident_string.to_string();
+                            labels.push(label);
+                        }
+                    })
+                    .collect();
+
+                quote! {
+                    impl prometheus_client::encoding::proto::EncodeLabels for #name {
+                        fn encode(&self, labels: &mut Vec<prometheus_client::encoding::proto::Label>) {
+                            #push_labels
+                        }
+                    }
+                }
+            }
+            syn::Fields::Unnamed(_) => {
+                panic!("Can not derive Encode for struct with unnamed fields.")
+            }
+            syn::Fields::Unit => panic!("Can not derive Encode for struct with unit field."),
+        },
+        syn::Data::Enum(syn::DataEnum { variants, .. }) => {
+            let match_arms: TokenStream2 = variants
+                .into_iter()
+                .map(|v| {
+                    let ident = v.ident;
+                    quote! {
+                        #name::#ident => {
+                            let mut label = prometheus_client::encoding::proto::Label::default();
+                            label.name = stringify!(#name).to_string();
+                            label.value = stringify!(#ident).to_string();
+                            labels.push(label);
+                        }
+                    }
+                })
+                .collect();
+
+            quote! {
+                impl prometheus_client::encoding::proto::EncodeLabels for #name {
+                    fn encode(&self, labels: &mut Vec<prometheus_client::encoding::proto::Label>) {
+                        match self {
+                            #match_arms
+                        };
+                    }
+                }
+            }
+        }
+        syn::Data::Union(_) => panic!("Can not derive Encode for union."),
+    }
+}
+
 // Copied from https://github.com/djc/askama (MIT and APACHE licensed) and
 // modified.
 static KEYWORD_IDENTIFIERS: [(&str, &str); 48] = [