From 86f6222d48d86f08d219560ced7d52c49a7d4752 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Tue, 19 Sep 2023 15:29:09 +0200 Subject: [PATCH 01/14] wip: grpc server --- .../krakend/extended/extra_config/grpc.json | 16 +++++++++++++++ docker-compose.yml | 20 +++++++++---------- 2 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 config/krakend/extended/extra_config/grpc.json diff --git a/config/krakend/extended/extra_config/grpc.json b/config/krakend/extended/extra_config/grpc.json new file mode 100644 index 0000000..f18c374 --- /dev/null +++ b/config/krakend/extended/extra_config/grpc.json @@ -0,0 +1,16 @@ +{ + "catalog": [ + "{{.grpc_catalog}}" + ], + "server" : { + "port": 8910, + "services": [ + { + "name": "Flights", + "methods": [ + {"name": "FindFlight"} + ] + } + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 0c13a85..2ebf7b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,16 +61,16 @@ services: ports: - "16686:16686" - "14268:14268" - elasticsearch: - image: elasticsearch:8.4.1 - environment: - - "discovery.type=single-node" - - "xpack.security.enabled=false" - - "xpack.security.transport.ssl.enabled=false" - - "xpack.security.http.ssl.enabled=false" - ports: - - "19200:9200" - - "9300:9300" +# elasticsearch: +# image: elasticsearch:8.4.1 +# environment: +# - "discovery.type=single-node" +# - "xpack.security.enabled=false" +# - "xpack.security.transport.ssl.enabled=false" +# - "xpack.security.http.ssl.enabled=false" +# ports: +# - "19200:9200" +# - "9300:9300" kibana: image: kibana:8.4.1 ports: From f0315f3aef5c929a051948cbbe784ca97d6a35f6 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Mon, 16 Oct 2023 16:05:36 +0200 Subject: [PATCH 02/14] wip: some changes to test serving gRPC --- .../krakend/extended/extra_config/grpc.json | 24 +++++++++++++++---- .../extended/service_extra_config.json | 9 +++---- config/krakend/fc_config.local.json | 2 +- images/grpc/flights/cmd/client/main.go | 6 +++-- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/config/krakend/extended/extra_config/grpc.json b/config/krakend/extended/extra_config/grpc.json index f18c374..17156bd 100644 --- a/config/krakend/extended/extra_config/grpc.json +++ b/config/krakend/extended/extra_config/grpc.json @@ -2,15 +2,29 @@ "catalog": [ "{{.grpc_catalog}}" ], - "server" : { - "port": 8910, + "server": { "services": [ { - "name": "Flights", + "name": "flight_finder.Flights", "methods": [ - {"name": "FindFlight"} + { + "name": "FindFlight", + "backend": [ + { + "host": [ + "grpc://localhost:4242" + ], + "url_pattern": "/flight_finder.Flights/FindFlight", + "extra_config": { + "backend/grpc": { + "use_request_body": true + } + } + } + ] + } ] } - } + ] } } diff --git a/config/krakend/extended/service_extra_config.json b/config/krakend/extended/service_extra_config.json index 95a8e09..83b674a 100644 --- a/config/krakend/extended/service_extra_config.json +++ b/config/krakend/extended/service_extra_config.json @@ -1,11 +1,8 @@ { - "grpc": { - "catalog": [ - "{{.grpc_catalog}}" - ] - }, + "grpc": { "$ref": "extra_config/grpc.json" }, "router": { "return_error_msg": true, + "use_h2c": true, "disable_gzip": true }, "auth/basic": { @@ -130,4 +127,4 @@ "revoke_server_api_key": "639ee23f-f4c5-40c4-855c-912bf01fae87", "revoke_server_max_workers": 5 } -} \ No newline at end of file +} diff --git a/config/krakend/fc_config.local.json b/config/krakend/fc_config.local.json index 14dcbc9..d3f64b1 100644 --- a/config/krakend/fc_config.local.json +++ b/config/krakend/fc_config.local.json @@ -1,5 +1,5 @@ { - "out": "./extended/composed_conf.json", + "out": "./extended/priv_local_composed_conf.json", "partials": { "paths": ["./extended/partials"] }, diff --git a/images/grpc/flights/cmd/client/main.go b/images/grpc/flights/cmd/client/main.go index 1cec600..4fecd20 100644 --- a/images/grpc/flights/cmd/client/main.go +++ b/images/grpc/flights/cmd/client/main.go @@ -15,9 +15,11 @@ import ( func main() { fmt.Printf("this is a client...\n") - c := NewFlightsClient("localhost:4242") + // c := NewFlightsClient("localhost:4242") + // c := NewFlightsClient("localhost:41979") + c := NewFlightsClient("localhost:7979") c.FindFlight() - c.BookFlight() + // c.BookFlight() } type FlightsClient struct { From fa2b4422746295ba99e1a2bbfd07457076819018 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Tue, 17 Oct 2023 10:45:58 +0200 Subject: [PATCH 03/14] grpc fake flights service: display received metadata --- images/grpc/flights/cmd/client/main.go | 15 ++++++++++++--- images/grpc/flights/cmd/server/main.go | 18 ++++++++++++++---- images/grpc/flights/cmd/server/server.go | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/images/grpc/flights/cmd/client/main.go b/images/grpc/flights/cmd/client/main.go index 4fecd20..02a6c03 100644 --- a/images/grpc/flights/cmd/client/main.go +++ b/images/grpc/flights/cmd/client/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "os" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -13,11 +14,19 @@ import ( // timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) +const ( + ENV_HOST string = "FLIGHTSCLIENT_HOST" +) + func main() { fmt.Printf("this is a client...\n") - // c := NewFlightsClient("localhost:4242") - // c := NewFlightsClient("localhost:41979") - c := NewFlightsClient("localhost:7979") + + host := os.Getenv(ENV_HOST) + if host == "" { + host = "localhost:4242" + fmt.Printf("env var %s not set, using HOST: %s", ENV_HOST, host) + } + c := NewFlightsClient(host) c.FindFlight() // c.BookFlight() } diff --git a/images/grpc/flights/cmd/server/main.go b/images/grpc/flights/cmd/server/main.go index dc88fcb..f30024f 100644 --- a/images/grpc/flights/cmd/server/main.go +++ b/images/grpc/flights/cmd/server/main.go @@ -3,23 +3,33 @@ package main import ( "fmt" "net" + "os" "google.golang.org/grpc" // timestamppb "google.golang.org/protobuf/types/known/timestamppb" flightspb "github.com/krakendio/playground-enterprise/images/grpc/genlib/flights" ) +const ( + ENV_PORT string = "FLIGHTSSERVER_PORT" +) + func main() { fes := NewFlightsEchoServer() s := grpc.NewServer() flightspb.RegisterFlightsServer(s, fes) - // TODO: select the listen port - fmt.Printf("binding to :4242") - ls, err := net.Listen("tcp", ":4242") + port := os.Getenv(ENV_PORT) + if port == "" { + port = "4242" + } + bindAddr := fmt.Sprintf(":%s", port) + + fmt.Printf("binding to %s\n", bindAddr) + ls, err := net.Listen("tcp", bindAddr) if err != nil { - fmt.Printf("cannot bind to port: %s\n", err.Error()) + fmt.Printf("cannot bind to %s: %s\n", bindAddr, err.Error()) return } diff --git a/images/grpc/flights/cmd/server/server.go b/images/grpc/flights/cmd/server/server.go index 3976ca3..5f2e13a 100644 --- a/images/grpc/flights/cmd/server/server.go +++ b/images/grpc/flights/cmd/server/server.go @@ -11,6 +11,8 @@ import ( flightspb "github.com/krakendio/playground-enterprise/images/grpc/genlib/flights" libpb "github.com/krakendio/playground-enterprise/images/grpc/genlib/lib" timestamppb "google.golang.org/protobuf/types/known/timestamppb" + + "google.golang.org/grpc/metadata" ) type FlightsEchoServer struct { @@ -35,11 +37,27 @@ func prettyPrint(title string, i interface{}) { fmt.Printf("* -> [ %s ]:\n%s\n\n", title, string(bytesOut)) } +func printMetadata(ctx context.Context) { + md, exists := metadata.FromIncomingContext(ctx) + if !exists { + fmt.Printf("NO metadata available\n") + return + } + fmt.Printf("* -> [ METADATA ]:\n") + for k, vs := range md { + fmt.Printf(" -> %s\n", k) + for _, v := range vs { + fmt.Printf(" %s\n", v) + } + } +} + func (s *FlightsEchoServer) FindFlight(ctx context.Context, req *flightspb.FindFlightRequest) (*flightspb.FindFlightResponse, error) { tm := time.Now() fmt.Printf("\n-[FindFlight @ %v]-----\n", tm) + printMetadata(ctx) prettyPrint("received", req) resp := s.generateResponse(req) prettyPrint("sending", resp) @@ -52,6 +70,7 @@ func (s *FlightsEchoServer) BookFlight(ctx context.Context, tm := time.Now() fmt.Printf("\n-[BookFlight @ %v]-----\n", tm) + printMetadata(ctx) prettyPrint("received", req) resp := &flightspb.BookFlightResponse{ ConfirmationId: fmt.Sprintf("%v", tm), From d513d481e9af2cf462fae5bfeb18d1680416a459 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Tue, 17 Oct 2023 14:43:55 +0200 Subject: [PATCH 04/14] add example of passing grpc metadata as headers --- config/krakend/extended/extra_config/grpc.json | 3 +++ images/grpc/flights/cmd/client/main.go | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/krakend/extended/extra_config/grpc.json b/config/krakend/extended/extra_config/grpc.json index 17156bd..79d2664 100644 --- a/config/krakend/extended/extra_config/grpc.json +++ b/config/krakend/extended/extra_config/grpc.json @@ -9,6 +9,9 @@ "methods": [ { "name": "FindFlight", + "input_headers": [ + "x-my-custom-header" + ], "backend": [ { "host": [ diff --git a/images/grpc/flights/cmd/client/main.go b/images/grpc/flights/cmd/client/main.go index 02a6c03..aee0873 100644 --- a/images/grpc/flights/cmd/client/main.go +++ b/images/grpc/flights/cmd/client/main.go @@ -8,6 +8,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" flightspb "github.com/krakendio/playground-enterprise/images/grpc/genlib/flights" libpb "github.com/krakendio/playground-enterprise/images/grpc/genlib/lib" @@ -47,7 +48,8 @@ func NewFlightsClient(addr string) *FlightsClient { } func (c *FlightsClient) FindFlight() { - ctx := context.Background() + ctx := metadata.AppendToOutgoingContext(context.Background(), + "X-My-Custom-Header", "ThisIsMyCustomValue") resp, err := c.conn.FindFlight(ctx, &flightspb.FindFlightRequest{ Page: &libpb.Page{ Size: 20, @@ -63,7 +65,8 @@ func (c *FlightsClient) FindFlight() { } func (c *FlightsClient) BookFlight() { - ctx := context.Background() + ctx := metadata.AppendToOutgoingContext(context.Background(), + "X-My-Custom-Header", "ThisIsMyCustomValue") resp, err := c.conn.BookFlight(ctx, &flightspb.BookFlightRequest{ FlightId: "foobar", Passengers: []*flightspb.Passenger{ From f50baf84a5e6325fc445fefd9e797522dbff448c Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Fri, 20 Oct 2023 20:17:53 +0200 Subject: [PATCH 05/14] grpc server add example of get backend with params --- config/krakend/extended/extra_config/grpc.json | 12 ++++++++++++ images/grpc/flights/cmd/client/main.go | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/config/krakend/extended/extra_config/grpc.json b/config/krakend/extended/extra_config/grpc.json index 79d2664..b6634f6 100644 --- a/config/krakend/extended/extra_config/grpc.json +++ b/config/krakend/extended/extra_config/grpc.json @@ -12,7 +12,14 @@ "input_headers": [ "x-my-custom-header" ], + "payload_params": { + "page.cursor": "cursor" + }, "backend": [ + { + "host": [ "http://localhost:8080" ], + "url_pattern": "/__debug/foo/" + }, { "host": [ "grpc://localhost:4242" @@ -23,6 +30,11 @@ "use_request_body": true } } + }, + { + "method": "GET", + "host": [ "http://localhost:7811" ], + "url_pattern": "/fake.json?q={cursor}" } ] } diff --git a/images/grpc/flights/cmd/client/main.go b/images/grpc/flights/cmd/client/main.go index aee0873..054228f 100644 --- a/images/grpc/flights/cmd/client/main.go +++ b/images/grpc/flights/cmd/client/main.go @@ -12,7 +12,7 @@ import ( flightspb "github.com/krakendio/playground-enterprise/images/grpc/genlib/flights" libpb "github.com/krakendio/playground-enterprise/images/grpc/genlib/lib" - // timestamppb "google.golang.org/protobuf/types/known/timestamppb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) const ( @@ -55,6 +55,9 @@ func (c *FlightsClient) FindFlight() { Size: 20, Cursor: "foo", }, + Departure: &libpb.TimeRange{ + Start: timestamppb.Now(), + }, }) if err != nil { From 505f53564aa91930d4a7f97a1113be820ca236d5 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Sun, 21 Jan 2024 15:33:26 +0100 Subject: [PATCH 06/14] fix too much memory in elastic search and use recent lwan version --- docker-compose.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2ebf7b3..73e6222 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,16 +61,17 @@ services: ports: - "16686:16686" - "14268:14268" -# elasticsearch: -# image: elasticsearch:8.4.1 -# environment: -# - "discovery.type=single-node" -# - "xpack.security.enabled=false" -# - "xpack.security.transport.ssl.enabled=false" -# - "xpack.security.http.ssl.enabled=false" -# ports: -# - "19200:9200" -# - "9300:9300" + elasticsearch: + image: elasticsearch:8.4.1 + environment: + - "discovery.type=single-node" + - "xpack.security.enabled=false" + - "xpack.security.transport.ssl.enabled=false" + - "xpack.security.http.ssl.enabled=false" + - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" + ports: + - "19200:9200" + - "9300:9300" kibana: image: kibana:8.4.1 ports: @@ -92,7 +93,7 @@ services: - "15672:15672" - "5672:5672" fake_api: - image: ghcr.io/lpereira/lwan:latest + image: ghcr.io/lpereira/lwan:master volumes: - ./data:/wwwroot ports: From a6b3495cf58d264d942eaf32e46e3ae759ddda68 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Mon, 22 Jan 2024 13:29:18 +0100 Subject: [PATCH 07/14] add tempo and prometheus to test krakend-otel --- config/prometheus/prometheus.env_tmpl.yml | 22 +++++++++ config/prometheus/prometheus.yml | 22 +++++++++ config/prometheus/promtail.yml | 31 ++++++++++++ config/tempo/tempo.yaml | 60 +++++++++++++++++++++++ docker-compose.yml | 22 +++++++++ 5 files changed, 157 insertions(+) create mode 100644 config/prometheus/prometheus.env_tmpl.yml create mode 100644 config/prometheus/prometheus.yml create mode 100644 config/prometheus/promtail.yml create mode 100644 config/tempo/tempo.yaml diff --git a/config/prometheus/prometheus.env_tmpl.yml b/config/prometheus/prometheus.env_tmpl.yml new file mode 100644 index 0000000..2b89d5a --- /dev/null +++ b/config/prometheus/prometheus.env_tmpl.yml @@ -0,0 +1,22 @@ +global: + scrape_interval: 15s + external_labels: + monitor: kotel_monitor + +scrape_configs: + - job_name: krakend_ee_local + scrape_interval: 5s + metrics_path: '/metrics' + static_configs: + - targets: + - '${KRAKEND_LOCAL_IP}:59091' + labels: + app: krakend_ee_local_app + - job_name: krakend_ee_docker + scrape_interval: 5s + metrics_path: '/metrics' + static_configs: + - targets: + - 'krakend_ee:9090' + labels: + app: krakend_ee_docker_app diff --git a/config/prometheus/prometheus.yml b/config/prometheus/prometheus.yml new file mode 100644 index 0000000..5527073 --- /dev/null +++ b/config/prometheus/prometheus.yml @@ -0,0 +1,22 @@ +global: + scrape_interval: 15s + external_labels: + monitor: kotel_monitor + +scrape_configs: + - job_name: krakend_ee_local + scrape_interval: 5s + metrics_path: '/metrics' + static_configs: + - targets: + - '192.168.1.12:59091' + labels: + app: krakend_ee_local_app + - job_name: krakend_ee_docker + scrape_interval: 5s + metrics_path: '/metrics' + static_configs: + - targets: + - 'krakend_ee:9090' + labels: + app: krakend_ee_docker_app diff --git a/config/prometheus/promtail.yml b/config/prometheus/promtail.yml new file mode 100644 index 0000000..c1d96e9 --- /dev/null +++ b/config/prometheus/promtail.yml @@ -0,0 +1,31 @@ +server: + http_listen_port: 3200 + grpc_listen_port: 0 + +clients: + - url: http://kotel_loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: "kotel_example" + pipeline_stages: + - match: + selector: '{app="kotel_app"}' + stages: + - json: + expressions: + level: level + method: method + path: path + file: file + msg: msg + timestamp: time + - labels: + level: + path: + method: + static_configs: + - targets: + - localhost + labels: + app: "kotel_ap" + __path__: "/var/log/tmp/*.log" diff --git a/config/tempo/tempo.yaml b/config/tempo/tempo.yaml new file mode 100644 index 0000000..418cc3a --- /dev/null +++ b/config/tempo/tempo.yaml @@ -0,0 +1,60 @@ +server: + http_listen_port: 3200 + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +# this configuration will listen on all ports and protocols that tempo is capable of. +# the receives all come from the OpenTelemetry collector. more configuration information can +# be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver +# for a production deployment you should only enable the receivers you need! +distributor: + receivers: + jaeger: + protocols: + thrift_http: + grpc: + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + +# cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally +ingester: + max_block_duration: 5m + +# overall Tempo trace retention. set for demo purposes +compactor: + compaction: + block_retention: 1h + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /tmp/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /tmp/tempo/wal # where to store the the wal locally + local: + path: /tmp/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] # enables metrics generator diff --git a/docker-compose.yml b/docker-compose.yml index 73e6222..6d2ecc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,7 @@ version: "3" +volumes: + vol_tempo: + vol_grafana: services: krakend_ee: # The :watch image restarts the service automatically when the configuration files change. @@ -27,6 +30,7 @@ services: ports: - "4000:3000" volumes: + - "vol_grafana:/var/lib/grafana" - "./config/telemetry/grafana/datasources/all.yml:/etc/grafana/provisioning/datasources/all.yml" - "./config/telemetry/grafana/dashboards/all.yml:/etc/grafana/provisioning/dashboards/all.yml" - "./config/telemetry/grafana/krakend:/var/lib/grafana/dashboards/krakend" @@ -56,11 +60,29 @@ services: - "8086:8086" volumes: - "./config/telemetry/influx:/docker-entrypoint-initdb.d" + prometheus: + image: prom/prometheus:latest + ports: + - "59090:9090" + volumes: + - "./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml" jaeger: image: jaegertracing/all-in-one:1 ports: - "16686:16686" - "14268:14268" + tempo: + image: grafana/tempo:latest + domainname: tempo + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - "./config/tempo/tempo.yaml:/etc/tempo.yaml" + - "vol_tempo:/tmp/tempo" + ports: + - "3200:3200" # tempo + - "9095:9095" # tempo grpc + - "54317:4317" # otlp grpc + - "54318:4318" # otlp http elasticsearch: image: elasticsearch:8.4.1 environment: From f5a3f734175c64f667be615d6058fb407835e3c1 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Wed, 31 Jan 2024 18:28:47 +0100 Subject: [PATCH 08/14] add services for opentelemetry testing --- .../krakend/extended/extra_config/otel.json | 60 ++++++ config/krakend/extended/root.local.vars.json | 6 + .../extended/service_extra_config.json | 1 + .../krakend/extended/settings/root.vars.json | 8 +- .../endpoints_telemetry_customization.json | 185 ++++++++++++++++++ config/prometheus/prometheus.yml | 2 +- 6 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 config/krakend/extended/extra_config/otel.json create mode 100644 config/krakend/extended/templates/endpoints_telemetry_customization.json diff --git a/config/krakend/extended/extra_config/otel.json b/config/krakend/extended/extra_config/otel.json new file mode 100644 index 0000000..e245071 --- /dev/null +++ b/config/krakend/extended/extra_config/otel.json @@ -0,0 +1,60 @@ +{ + "service_name": "playground_enterprise_otel", + "metric_reporting_period": 1, + "trace_sample_rate": 0.5, + "exporters": { + "prometheus": [ + { + "name": "local_prometheus", + "port": 59091, + "process_metrics": true, + "go_metrics": true + } + ], + "otlp": [ + { + "name": "local_tempo", + "host": "{{ .opentelemetry.tempogrpc.host }}", + "port": {{ .opentelemetry.tempogrpc.port }}, + "use_http": false + } + ] + }, + "layers": { + "global": { + "disable_metrics": false, + "disable_traces": false, + "disable_propagation": false + }, + "pipe": { + "disable_metrics": false, + "disable_traces": false + }, + "backend": { + "metrics": { + "disable_stage": false, + "round_trip": true, + "read_payload": true, + "detailed_connection": true, + "static_attributes": [ + { + "key": "my_metric_attr", + "value": "my_metric_val" + } + ] + }, + "traces": { + "disable_stage": false, + "round_trip": true, + "read_payload": true, + "detailed_connection": true, + "static_attributes": [ + { + "key": "my_trace_attr", + "value": "my_trace_val" + } + ] + } + } + } +} diff --git a/config/krakend/extended/root.local.vars.json b/config/krakend/extended/root.local.vars.json index 77931c7..b21c82e 100644 --- a/config/krakend/extended/root.local.vars.json +++ b/config/krakend/extended/root.local.vars.json @@ -11,6 +11,12 @@ "grpc_trains": "localhost:4243", "fake_api": "localhost:8000" }, + "opentelemetry": { + "tempogrpc": { + "port": 54317, + "host": "127.0.0.1" + } + }, "endpoints": { "ee_websockets": { "def": { diff --git a/config/krakend/extended/service_extra_config.json b/config/krakend/extended/service_extra_config.json index 83b674a..3bb3360 100644 --- a/config/krakend/extended/service_extra_config.json +++ b/config/krakend/extended/service_extra_config.json @@ -1,5 +1,6 @@ { "grpc": { "$ref": "extra_config/grpc.json" }, + "telemetry/opentelemetry": { "$ref": "extra_config/otel.json" }, "router": { "return_error_msg": true, "use_h2c": true, diff --git a/config/krakend/extended/settings/root.vars.json b/config/krakend/extended/settings/root.vars.json index 8cac895..34bf5bf 100644 --- a/config/krakend/extended/settings/root.vars.json +++ b/config/krakend/extended/settings/root.vars.json @@ -1,4 +1,10 @@ { "conf_dir": "/etc/krakend", - "grpc_catalog": "/etc/krakend/grpc_catalog" + "grpc_catalog": "/etc/krakend/grpc_catalog", + "opentelemetry": { + "tempogrpc": { + "port": 4317, + "host": "tempo" + } + } } diff --git a/config/krakend/extended/templates/endpoints_telemetry_customization.json b/config/krakend/extended/templates/endpoints_telemetry_customization.json new file mode 100644 index 0000000..88d1db4 --- /dev/null +++ b/config/krakend/extended/templates/endpoints_telemetry_customization.json @@ -0,0 +1,185 @@ +{ + "@comment": "Feature: Aggregation + Basic transformation (filtering & mapping) + grouping", + "endpoint": "otel/git/{user}", + "backend": [ + { + "host": [ + "https://api.github.com" + ], + "url_pattern": "/users/{user}", + "allow": [ + "avatar_url", + "name", + "company", + "blog", + "location", + "mail", + "hireable", + "followers", + "public_repos", + "public_gists" + ], + "mapping": { + "blog": "website" + }, + "group": "user" + }, + { + "host": [ + "https://api.github.com" + ], + "url_pattern": "/users/{user}/repos", + "mapping": { + "collection": "repos" + }, + "is_collection": true + } + ], + "extra_config": { + "telemetry/opentelemetry": { + "pipe": { + "disable_metrics": true, + "disable_traces": true + }, + "backend": { + "metrics": { + "disable_stage": true + }, + "traces": { + "disable_stage": true + } + } + }, + "documentation/openapi": { + "summary": "Merges two calls from github and serves under a single endpoint", + "description": "This endpoint fetches in parallel the user information and the repository information from Githuib and returns it together. Non interesting fields are removed.", + "tags": [ + "Aggregation" + ] + } + } +}, +{ + "@comment": "Feature: One-liner advanced data manipulation (filter, sorting & remapping) with JMESPath Query Language", + "endpoint": "otel/cryptos/{currency}", + "output_encoding": "json-collection", + "backend": [ + { + "host": [ + "https://api.coingecko.com" + ], + "url_pattern": "/api/v3/coins/markets?vs_currency={currency}&order=market_cap_desc&per_page=100&page=1&sparkline=false", + "encoding": "safejson" + } + ], + "extra_config": { + "modifier/jmespath": { + "@comment": "Get first 5 crypto by market capitalization, then reverse sort them by name, finally filter and rename some fields", + "expr": "reverse(sort_by(collection[:5], &name))[*].{name: name, token: symbol,price: current_price, price_change: price_change_percentage_24h, market_cap: market_cap}" + }, + "documentation/openapi": { + "summary": "Get TOP 5 crypto capitalization & price data for a given currency", + "description": "This endpoint fetches some crypto capitalization data from Coingecko for a given currency.", + "tags": [ + "Aggregation" + ] + } + } +}, +{ + "@comment": "Feature: Regular expression manipulation", + "endpoint": "otel/user/creditcard", + "backend": [ + { + "url_pattern": "/user/1.json", + "allow": [ + "credit_card" + ], + "extra_config": { + "plugin/req-resp-modifier": { + "name": [ + "content-replacer" + ], + "content-replacer": { + "credit_card.number": { + "@comment": "Ridiculous card masking. Show last 4 digits and remove the rest. Credit card number is nested.", + "find": "^.*(\\d{4})", + "replace": "XXXX-${1}", + "regexp": true + } + } + } + } + } + ] +}, +{ + "@comment": "Feature: Backend cache", + "endpoint": "/otel/market/cached", + "backend": [ + { + "host": [ + "https://api.coingecko.com" + ], + "url_pattern": "/api/v3/coins/markets?vs_currency=eur&ids=bitcoin,ethereum&order=market_cap_desc&per_page=100&page=1&sparkline=false", + "encoding": "safejson", + "mapping": { + "collection": "coins" + }, + "extra_config": { + "qos/http-cache": { + "shared": true + } + } + } + ] +}, +{ + "@comment": "Feature: Mocked response", + "endpoint": "/otel/mocked-response", + "method": "GET", + "output_encoding": "json", + "backend": [ + { + "host": [ + "http://unexistent_backend" + ], + "url_pattern": "/" + } + ], + "extra_config": { + "proxy": { + "static": { + "data": { + "an_integer": 123, + "an_array": [ + "arr1", + "arr2" + ], + "an_object": { + "obj": "obj1" + } + }, + "strategy": "always" + } + } + } +}, +{ + "@comment": "Feature: Sequential calls, using values from 1st call response into 2nd call request", + "endpoint": "/otel/sequential", + "backend": [ + { + "url_pattern": "/hotels/1.json", + "allow": [ "destination_id" ] + }, + { + "url_pattern": "/destinations/{resp0_destination_id}.json" + } + ], + "extra_config": { + "proxy": { + "sequential": true + } + } +} diff --git a/config/prometheus/prometheus.yml b/config/prometheus/prometheus.yml index 5527073..7a27cb1 100644 --- a/config/prometheus/prometheus.yml +++ b/config/prometheus/prometheus.yml @@ -17,6 +17,6 @@ scrape_configs: metrics_path: '/metrics' static_configs: - targets: - - 'krakend_ee:9090' + - 'krakend_ee:59091' labels: app: krakend_ee_docker_app From ffd22d1d692576b1e5c09224893d097273799450 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Thu, 1 Feb 2024 13:40:12 +0100 Subject: [PATCH 09/14] enable OTLP in jaeger docker instance --- config/krakend/extended/extra_config/otel.json | 7 +++++++ config/krakend/extended/root.local.vars.json | 4 ++++ config/krakend/extended/settings/root.vars.json | 4 ++++ .../templates/endpoints_telemetry_customization.json | 12 ++++++++++++ docker-compose.yml | 4 ++++ 5 files changed, 31 insertions(+) diff --git a/config/krakend/extended/extra_config/otel.json b/config/krakend/extended/extra_config/otel.json index e245071..d22cdc5 100644 --- a/config/krakend/extended/extra_config/otel.json +++ b/config/krakend/extended/extra_config/otel.json @@ -17,6 +17,13 @@ "host": "{{ .opentelemetry.tempogrpc.host }}", "port": {{ .opentelemetry.tempogrpc.port }}, "use_http": false + }, + { + "name": "debug_jaeger", + "host": "{{ .opentelemetry.jaegergrpc.host }}", + "port": {{ .opentelemetry.jaegergrpc.port }}, + "use_http": false, + "disable_traces": true } ] }, diff --git a/config/krakend/extended/root.local.vars.json b/config/krakend/extended/root.local.vars.json index b21c82e..957df4d 100644 --- a/config/krakend/extended/root.local.vars.json +++ b/config/krakend/extended/root.local.vars.json @@ -15,6 +15,10 @@ "tempogrpc": { "port": 54317, "host": "127.0.0.1" + }, + "jaegergrpc": { + "port": 64317, + "host": "127.0.0.1" } }, "endpoints": { diff --git a/config/krakend/extended/settings/root.vars.json b/config/krakend/extended/settings/root.vars.json index 34bf5bf..98d98e6 100644 --- a/config/krakend/extended/settings/root.vars.json +++ b/config/krakend/extended/settings/root.vars.json @@ -5,6 +5,10 @@ "tempogrpc": { "port": 4317, "host": "tempo" + }, + "jaegergrpc": { + "port": 4317, + "host": "jaeger" } } } diff --git a/config/krakend/extended/templates/endpoints_telemetry_customization.json b/config/krakend/extended/templates/endpoints_telemetry_customization.json index 88d1db4..1386092 100644 --- a/config/krakend/extended/templates/endpoints_telemetry_customization.json +++ b/config/krakend/extended/templates/endpoints_telemetry_customization.json @@ -73,6 +73,18 @@ } ], "extra_config": { + "telemetry/opentelemetry": { + "pipe": { + "disable_metrics": false, + "disable_traces": false + }, + "exporters_override": { + "metric_reporting_period": 10, + "trace_sample_rate": 1, + "metric_exporters": ["local_prometheus"], + "trace_exporters": ["debug_jaeger"] + } + }, "modifier/jmespath": { "@comment": "Get first 5 crypto by market capitalization, then reverse sort them by name, finally filter and rename some fields", "expr": "reverse(sort_by(collection[:5], &name))[*].{name: name, token: symbol,price: current_price, price_change: price_change_percentage_24h, market_cap: market_cap}" diff --git a/docker-compose.yml b/docker-compose.yml index 6d2ecc1..a824fd5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,9 +68,13 @@ services: - "./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml" jaeger: image: jaegertracing/all-in-one:1 + environment: + - "COLLECTOR_OTLP_ENABLED=true" ports: - "16686:16686" - "14268:14268" + - "64317:4317" # otlp grpc + - "64318:4318" # otlp grpc tempo: image: grafana/tempo:latest domainname: tempo From 533652157be646d2f05dfad78bf93430ffabcadb Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Thu, 8 Feb 2024 16:58:30 +0100 Subject: [PATCH 10/14] clean up config --- .../krakend/extended/extra_config/grpc.json | 2 +- .../krakend/extended/extra_config/otel.json | 19 ++++++- config/krakend/extended/krakend.tmpl | 2 +- config/krakend/extended/root.local.vars.json | 57 ------------------- config/krakend/extended/root.vars.json | 20 ------- .../extended/service_extra_config.json | 2 +- .../settings/{ => environments}/dev/hosts.yml | 4 +- .../DO NOT VERSION endpoints_aggregation.yml | 0 .../local}/hosts.yml | 6 +- .../extended/settings/root.local.vars.json | 14 +++++ .../endpoints_services_connectivity.json | 6 +- config/krakend/fc_config.json | 3 +- config/krakend/fc_config.local.json | 5 +- docker-compose.yml | 11 ++++ 14 files changed, 57 insertions(+), 94 deletions(-) delete mode 100644 config/krakend/extended/root.local.vars.json rename config/krakend/extended/settings/{ => environments}/dev/hosts.yml (71%) rename config/krakend/extended/settings/{all_environments => environments/local}/DO NOT VERSION endpoints_aggregation.yml (100%) rename config/krakend/extended/settings/{all_environments => environments/local}/hosts.yml (63%) create mode 100644 config/krakend/extended/settings/root.local.vars.json diff --git a/config/krakend/extended/extra_config/grpc.json b/config/krakend/extended/extra_config/grpc.json index b6634f6..7436f21 100644 --- a/config/krakend/extended/extra_config/grpc.json +++ b/config/krakend/extended/extra_config/grpc.json @@ -22,7 +22,7 @@ }, { "host": [ - "grpc://localhost:4242" + "localhost:4242" ], "url_pattern": "/flight_finder.Flights/FindFlight", "extra_config": { diff --git a/config/krakend/extended/extra_config/otel.json b/config/krakend/extended/extra_config/otel.json index d22cdc5..83113fc 100644 --- a/config/krakend/extended/extra_config/otel.json +++ b/config/krakend/extended/extra_config/otel.json @@ -16,14 +16,31 @@ "name": "local_tempo", "host": "{{ .opentelemetry.tempogrpc.host }}", "port": {{ .opentelemetry.tempogrpc.port }}, + "use_http": false, + "disable_metrics": true + }, + { + "name": "datadog", + "host": "127.0.0.1", + "port": 14317, "use_http": false }, + { + "name": "newrelic", + "host": "otlp.eu01.nr-data.net", + "port": 4318, + "use_http": true, + "http_headers": [ + {"key": "api-key", "value": "{{ env "NEWRELIC_API_KEY" }}" } + ] + }, { "name": "debug_jaeger", "host": "{{ .opentelemetry.jaegergrpc.host }}", "port": {{ .opentelemetry.jaegergrpc.port }}, "use_http": false, - "disable_traces": true + "disable_traces": true, + "disable_metrics": true } ] }, diff --git a/config/krakend/extended/krakend.tmpl b/config/krakend/extended/krakend.tmpl index 43a2933..2043eb9 100644 --- a/config/krakend/extended/krakend.tmpl +++ b/config/krakend/extended/krakend.tmpl @@ -27,4 +27,4 @@ "extra_config": { "$ref": "./service_extra_config.json" } -} \ No newline at end of file +} diff --git a/config/krakend/extended/root.local.vars.json b/config/krakend/extended/root.local.vars.json deleted file mode 100644 index 957df4d..0000000 --- a/config/krakend/extended/root.local.vars.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "conf_dir": "./", - "grpc_catalog": "../../images/grpc/defs", - "host": ["http://localhost:8666"], - "servers": { - "influxdb": "localhost:8086", - "logstash": "localhost:12201", - "chat": "localhost:8888", - "rabbitmq": "localhost:5672", - "grpc_flights": "localhost:4242", - "grpc_trains": "localhost:4243", - "fake_api": "localhost:8000" - }, - "opentelemetry": { - "tempogrpc": { - "port": 54317, - "host": "127.0.0.1" - }, - "jaegergrpc": { - "port": 64317, - "host": "127.0.0.1" - } - }, - "endpoints": { - "ee_websockets": { - "def": { - "backend": [ - { - "url_pattern": "/ws/{room}", - "disable_host_sanitize": true, - "host": ["ws://localhost:4245"] - } - ] - } - }, - "ee_websockets_ee": { - "def": { - "backend": [ - { - "url_pattern": "/", - "encoding": "no-op", - "host": ["http://localhost:4245"] - } - ] - } - }, - "feat_jwt_signing": { - "extra_config": { - "signer": { - "val": { - "jwk_url": "http://localhost:8000/jwk/symmetric.json" - } - } - } - } - } -} diff --git a/config/krakend/extended/root.vars.json b/config/krakend/extended/root.vars.json index c62fe17..e69de29 100644 --- a/config/krakend/extended/root.vars.json +++ b/config/krakend/extended/root.vars.json @@ -1,20 +0,0 @@ -{ - "version": 3, - "name": "KrakenD Enterprise API Gateway", - "port": 8080, - "host": ["http://fake_api:8080"], - "timeout": "3000ms", - "cache_ttl": "300s", - "debug_endpoint": true, - "servers": { - "influxdb": "influxdb:8086", - "logstash": "logstash:12201", - "chat": "chat:8888", - "rabbitmq": "rabbitmq:5672", - "grpc_flights": "grpc_flights:4242", - "grpc_trains": "grpc_trains:4243", - "fake_api": "fake_api:8080" - }, - "conf_dir": "/etc/krakend", - "grpc_catalog": "/etc/krakend/grpc_catalog" -} diff --git a/config/krakend/extended/service_extra_config.json b/config/krakend/extended/service_extra_config.json index 3bb3360..bce8709 100644 --- a/config/krakend/extended/service_extra_config.json +++ b/config/krakend/extended/service_extra_config.json @@ -123,7 +123,7 @@ "jti" ], "TTL": 1500, - "revoke_server_ping_url": "http://revoker:8081/instances", + "revoke_server_ping_url": "http://{{.hosts.revoker}}/instances", "revoke_server_ping_interval": "30s", "revoke_server_api_key": "639ee23f-f4c5-40c4-855c-912bf01fae87", "revoke_server_max_workers": 5 diff --git a/config/krakend/extended/settings/dev/hosts.yml b/config/krakend/extended/settings/environments/dev/hosts.yml similarity index 71% rename from config/krakend/extended/settings/dev/hosts.yml rename to config/krakend/extended/settings/environments/dev/hosts.yml index 8c6f6d9..07226c3 100644 --- a/config/krakend/extended/settings/dev/hosts.yml +++ b/config/krakend/extended/settings/environments/dev/hosts.yml @@ -1,8 +1,8 @@ -github: "https://api.github.com" influxdb: "influxdb:8086" logstash: "logstash:12201" chat: "chat:8888" rabbitmq: "rabbitmq:5672" grpc_flights: "grpc_flights:4242" grpc_trains: "grpc_trains:4243" -fake_api: "http://fake_api:8080" \ No newline at end of file +revoker: "revoker:8081" +fake_api: "fake_api:8080" diff --git a/config/krakend/extended/settings/all_environments/DO NOT VERSION endpoints_aggregation.yml b/config/krakend/extended/settings/environments/local/DO NOT VERSION endpoints_aggregation.yml similarity index 100% rename from config/krakend/extended/settings/all_environments/DO NOT VERSION endpoints_aggregation.yml rename to config/krakend/extended/settings/environments/local/DO NOT VERSION endpoints_aggregation.yml diff --git a/config/krakend/extended/settings/all_environments/hosts.yml b/config/krakend/extended/settings/environments/local/hosts.yml similarity index 63% rename from config/krakend/extended/settings/all_environments/hosts.yml rename to config/krakend/extended/settings/environments/local/hosts.yml index dd972bd..c0c90ae 100644 --- a/config/krakend/extended/settings/all_environments/hosts.yml +++ b/config/krakend/extended/settings/environments/local/hosts.yml @@ -1,8 +1,8 @@ -github: "https://api.github.com" influxdb: "localhost:8086" logstash: "localhost:12201" -chat: "localhost:8888" +chat: "localhost:4245" rabbitmq: "localhost:5672" grpc_flights: "localhost:4242" grpc_trains: "localhost:4243" -fake_api: "localhost:8000" \ No newline at end of file +revoker: "localhost:8081" +fake_api: "localhost:8000" diff --git a/config/krakend/extended/settings/root.local.vars.json b/config/krakend/extended/settings/root.local.vars.json new file mode 100644 index 0000000..964bc77 --- /dev/null +++ b/config/krakend/extended/settings/root.local.vars.json @@ -0,0 +1,14 @@ +{ + "conf_dir": "./", + "grpc_catalog": "../../images/grpc/defs", + "opentelemetry": { + "tempogrpc": { + "port": 54317, + "host": "127.0.0.1" + }, + "jaegergrpc": { + "port": 64317, + "host": "127.0.0.1" + } + } +} diff --git a/config/krakend/extended/templates/endpoints_services_connectivity.json b/config/krakend/extended/templates/endpoints_services_connectivity.json index f2752d3..3e0be66 100644 --- a/config/krakend/extended/templates/endpoints_services_connectivity.json +++ b/config/krakend/extended/templates/endpoints_services_connectivity.json @@ -170,7 +170,7 @@ "url_pattern": "/ws/{room}", "disable_host_sanitize": true, "host": [ - "ws://chat:8888" + "ws://{{ .hosts.chat }}" ] } ], @@ -202,7 +202,7 @@ { "url_pattern": "/", "host": [ - "http://chat:8888" + "http://{{ .hosts.chat }}" ], "encoding": "no-op" } @@ -276,4 +276,4 @@ "url_pattern": "/__debug/{input_query_strings.route}" } ] - } \ No newline at end of file + } diff --git a/config/krakend/fc_config.json b/config/krakend/fc_config.json index cea5c90..5a9e6f4 100644 --- a/config/krakend/fc_config.json +++ b/config/krakend/fc_config.json @@ -7,8 +7,7 @@ "allow_overwrite": true, "paths": [ "/etc/krakend/extended/settings/root.vars.json", - "/etc/krakend/extended/settings/all_environments", - "/etc/krakend/extended/settings/dev" + "/etc/krakend/extended/settings/environments/dev" ] }, "templates": { diff --git a/config/krakend/fc_config.local.json b/config/krakend/fc_config.local.json index d3f64b1..30ef3e7 100644 --- a/config/krakend/fc_config.local.json +++ b/config/krakend/fc_config.local.json @@ -6,9 +6,8 @@ "settings": { "allow_overwrite": true, "paths": [ - "./extended/root.vars.json", - "./extended/vars", - "./extended/root.local.vars.json" + "./extended/settings/root.vars.json", + "./extended/settings/environments/local" ] }, "templates": { diff --git a/docker-compose.yml b/docker-compose.yml index a824fd5..d8e14f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3" volumes: vol_tempo: vol_grafana: + vol_grafana10: services: krakend_ee: # The :watch image restarts the service automatically when the configuration files change. @@ -39,6 +40,16 @@ services: - GF_SECURITY_ADMIN_PASSWORD=admin # The token below must match the one in influxdb - INFLUXDB_TOKEN=my-super-secret-auth-token + grafana10: + image: grafana/grafana:latest + domainname: grafana10 + ports: + - "4010:3000" + volumes: + - "vol_grafana10:/var/lib/grafana" + environment: + - GF_SECURITY_ADMIN_USER=krakend + - GF_SECURITY_ADMIN_PASSWORD=krakend influxdb: # image: influxdb:1.8.10 # environment: From 15c15567b0c2b4b00cc8ac5d52866fdac86031e2 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Mon, 26 Feb 2024 22:16:46 +0100 Subject: [PATCH 11/14] some changes to test telemetry features --- .../krakend/extended/extra_config/grpc.json | 4 +- .../krakend/extended/extra_config/otel.json | 21 ++--- .../extended/service_extra_config.json | 12 ++- .../settings/environments/dev/hosts.yml | 1 + .../settings/environments/local/hosts.yml | 1 + .../extended/settings/root.local.vars.json | 4 + .../endpoints_telemetry_customization.json | 4 +- config/krakend/fc_config.local.json | 3 +- docker-compose.yml | 14 ++-- .../google/protobuf/bridge/message_set.proto | 76 ------------------- .../google/protobuf/bridge/message_set.pb | 5 -- 11 files changed, 44 insertions(+), 101 deletions(-) delete mode 100644 images/grpc/contracts/google/protobuf/bridge/message_set.proto delete mode 100644 images/grpc/defs/google/protobuf/bridge/message_set.pb diff --git a/config/krakend/extended/extra_config/grpc.json b/config/krakend/extended/extra_config/grpc.json index 7436f21..cec1dd7 100644 --- a/config/krakend/extended/extra_config/grpc.json +++ b/config/krakend/extended/extra_config/grpc.json @@ -33,8 +33,8 @@ }, { "method": "GET", - "host": [ "http://localhost:7811" ], - "url_pattern": "/fake.json?q={cursor}" + "host": [ "http://localhost:8000" ], + "url_pattern": "/articles.json?q={cursor}" } ] } diff --git a/config/krakend/extended/extra_config/otel.json b/config/krakend/extended/extra_config/otel.json index 83113fc..97c261b 100644 --- a/config/krakend/extended/extra_config/otel.json +++ b/config/krakend/extended/extra_config/otel.json @@ -23,16 +23,15 @@ "name": "datadog", "host": "127.0.0.1", "port": 14317, + "disable_metrics": {{ .opentelemetry.datadog.disable_metrics }}, + "disable_traces": {{ .opentelemetry.datadog.disable_traces }}, "use_http": false }, { "name": "newrelic", "host": "otlp.eu01.nr-data.net", "port": 4318, - "use_http": true, - "http_headers": [ - {"key": "api-key", "value": "{{ env "NEWRELIC_API_KEY" }}" } - ] + "use_http": true }, { "name": "debug_jaeger", @@ -48,11 +47,13 @@ "global": { "disable_metrics": false, "disable_traces": false, - "disable_propagation": false + "disable_propagation": false, + "report_headers": true }, - "pipe": { + "proxy": { "disable_metrics": false, - "disable_traces": false + "disable_traces": false, + "report_headers": true }, "backend": { "metrics": { @@ -65,7 +66,8 @@ "key": "my_metric_attr", "value": "my_metric_val" } - ] + ], + "report_headers": true }, "traces": { "disable_stage": false, @@ -77,7 +79,8 @@ "key": "my_trace_attr", "value": "my_trace_val" } - ] + ], + "report_headers": true } } } diff --git a/config/krakend/extended/service_extra_config.json b/config/krakend/extended/service_extra_config.json index bce8709..5da2de4 100644 --- a/config/krakend/extended/service_extra_config.json +++ b/config/krakend/extended/service_extra_config.json @@ -1,6 +1,16 @@ { "grpc": { "$ref": "extra_config/grpc.json" }, "telemetry/opentelemetry": { "$ref": "extra_config/otel.json" }, + "telemetry/opentelemetry-security": { + "otlp": [ + { + "name": "newrelic", + "headers": { + "api-key": "{{ env "NEWRELIC_API_KEY" }}" + } + } + ] + }, "router": { "return_error_msg": true, "use_h2c": true, @@ -91,7 +101,7 @@ }, "exporters": { "jaeger": { - "endpoint": "http://jaeger:14268/api/traces", + "endpoint": "http://{{.hosts.jaeger}}/api/traces", "service_name": "krakend" } } diff --git a/config/krakend/extended/settings/environments/dev/hosts.yml b/config/krakend/extended/settings/environments/dev/hosts.yml index 07226c3..d51c417 100644 --- a/config/krakend/extended/settings/environments/dev/hosts.yml +++ b/config/krakend/extended/settings/environments/dev/hosts.yml @@ -6,3 +6,4 @@ grpc_flights: "grpc_flights:4242" grpc_trains: "grpc_trains:4243" revoker: "revoker:8081" fake_api: "fake_api:8080" +jaeger: "jaeger:14268" diff --git a/config/krakend/extended/settings/environments/local/hosts.yml b/config/krakend/extended/settings/environments/local/hosts.yml index c0c90ae..57b2471 100644 --- a/config/krakend/extended/settings/environments/local/hosts.yml +++ b/config/krakend/extended/settings/environments/local/hosts.yml @@ -6,3 +6,4 @@ grpc_flights: "localhost:4242" grpc_trains: "localhost:4243" revoker: "localhost:8081" fake_api: "localhost:8000" +jaeger: "localhost:14268" diff --git a/config/krakend/extended/settings/root.local.vars.json b/config/krakend/extended/settings/root.local.vars.json index 964bc77..c3d3f7e 100644 --- a/config/krakend/extended/settings/root.local.vars.json +++ b/config/krakend/extended/settings/root.local.vars.json @@ -9,6 +9,10 @@ "jaegergrpc": { "port": 64317, "host": "127.0.0.1" + }, + "datadog": { + "disable_metrics": true, + "disable_traces": true } } } diff --git a/config/krakend/extended/templates/endpoints_telemetry_customization.json b/config/krakend/extended/templates/endpoints_telemetry_customization.json index 1386092..7793b92 100644 --- a/config/krakend/extended/templates/endpoints_telemetry_customization.json +++ b/config/krakend/extended/templates/endpoints_telemetry_customization.json @@ -37,7 +37,7 @@ ], "extra_config": { "telemetry/opentelemetry": { - "pipe": { + "proxy": { "disable_metrics": true, "disable_traces": true }, @@ -74,7 +74,7 @@ ], "extra_config": { "telemetry/opentelemetry": { - "pipe": { + "proxy": { "disable_metrics": false, "disable_traces": false }, diff --git a/config/krakend/fc_config.local.json b/config/krakend/fc_config.local.json index 30ef3e7..fda4b69 100644 --- a/config/krakend/fc_config.local.json +++ b/config/krakend/fc_config.local.json @@ -7,7 +7,8 @@ "allow_overwrite": true, "paths": [ "./extended/settings/root.vars.json", - "./extended/settings/environments/local" + "./extended/settings/environments/local", + "./extended/settings/root.local.vars.json" ] }, "templates": { diff --git a/docker-compose.yml b/docker-compose.yml index d8e14f9..e06e4f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,9 +47,13 @@ services: - "4010:3000" volumes: - "vol_grafana10:/var/lib/grafana" + - "./config/telemetry/grafana10/provisioning/datasources/all.yml:/etc/grafana/provisioning/datasources/all.yml" + - "./config/telemetry/grafana10/provisioning/dashboards/all.yml:/etc/grafana/provisioning/dashboards/all.yml" + - "./config/telemetry/grafana10/data/dashboards:/var/lib/grafana/dashboards" environment: - GF_SECURITY_ADMIN_USER=krakend - GF_SECURITY_ADMIN_PASSWORD=krakend + - GF_AUT_ANONYMOUS_ENABLED=true influxdb: # image: influxdb:1.8.10 # environment: @@ -96,8 +100,8 @@ services: ports: - "3200:3200" # tempo - "9095:9095" # tempo grpc - - "54317:4317" # otlp grpc - - "54318:4318" # otlp http + - "54317:4317" # otlp grpc + - "54318:4318" # otlp http elasticsearch: image: elasticsearch:8.4.1 environment: @@ -130,11 +134,11 @@ services: - "15672:15672" - "5672:5672" fake_api: - image: ghcr.io/lpereira/lwan:master + image: jdkelley/simple-http-server:latest volumes: - - ./data:/wwwroot + - ./data:/serve ports: - - "8000:8080" + - "8000:8000" web: build: context: images/spa-auth-web diff --git a/images/grpc/contracts/google/protobuf/bridge/message_set.proto b/images/grpc/contracts/google/protobuf/bridge/message_set.proto deleted file mode 100644 index 2f23f9b..0000000 --- a/images/grpc/contracts/google/protobuf/bridge/message_set.proto +++ /dev/null @@ -1,76 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2007 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// -// This is proto2's version of MessageSet. See go/messageset to learn what -// MessageSets are and how they are used. -// -// In proto2, we implement MessageSet in terms of extensions, except with a -// special wire format for backwards-compatibility. To define a message that -// goes in a MessageSet in proto2, you must declare within that message's -// scope an extension of MessageSet named "message_set_extension" and with -// the field number matching the type ID. So, for example, this proto1 code: -// message Foo { -// enum TypeId { MESSAGE_TYPE_ID = 1234; } -// } -// becomes this proto2 code: -// message Foo { -// extend google.protobuf.bridge.MessageSet { -// optional Foo message_set_extension = 1234; -// } -// } -// -// Now you can use the usual proto2 extensions accessors to access this -// message. For example, the proto1 code: -// MessageSet mset; -// Foo* foo = mset.get_mutable(); -// becomes this proto2 code: -// google::protobuf::bridge::MessageSet mset; -// Foo* foo = mset.MutableExtension(Foo::message_set_extension); -// -// Of course, new code that doesn't have backwards-compatibility requirements -// should just use extensions themselves and not worry about MessageSet. - -syntax = "proto2"; - -package google.protobuf.bridge; - -option java_outer_classname = "MessageSetProtos"; -option java_multiple_files = true; -option cc_enable_arenas = true; -option objc_class_prefix = "GPB"; - -// This is proto2's version of MessageSet. -message MessageSet { - option message_set_wire_format = true; - - extensions 4 to max; -} diff --git a/images/grpc/defs/google/protobuf/bridge/message_set.pb b/images/grpc/defs/google/protobuf/bridge/message_set.pb deleted file mode 100644 index b48f4a2..0000000 --- a/images/grpc/defs/google/protobuf/bridge/message_set.pb +++ /dev/null @@ -1,5 +0,0 @@ - -} -(google/protobuf/bridge/message_set.protogoogle.protobuf.bridge" - -MessageSet*ÿÿÿÿ:BBMessageSetProtosPø¢GPB \ No newline at end of file From 7b51fa6d513b319da072529f25b8922ffa1086ef Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Wed, 17 Apr 2024 15:24:34 +0200 Subject: [PATCH 12/14] add missing grafana 10 config files --- config/krakend/extended/extra_config/otel.json | 2 +- config/krakend/extended/settings/root.local.vars.json | 4 ++-- .../extended/templates/endpoints_telemetry_customization.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/krakend/extended/extra_config/otel.json b/config/krakend/extended/extra_config/otel.json index 97c261b..c3665e8 100644 --- a/config/krakend/extended/extra_config/otel.json +++ b/config/krakend/extended/extra_config/otel.json @@ -1,7 +1,7 @@ { "service_name": "playground_enterprise_otel", "metric_reporting_period": 1, - "trace_sample_rate": 0.5, + "trace_sample_rate": 0.1, "exporters": { "prometheus": [ { diff --git a/config/krakend/extended/settings/root.local.vars.json b/config/krakend/extended/settings/root.local.vars.json index c3d3f7e..2237443 100644 --- a/config/krakend/extended/settings/root.local.vars.json +++ b/config/krakend/extended/settings/root.local.vars.json @@ -11,8 +11,8 @@ "host": "127.0.0.1" }, "datadog": { - "disable_metrics": true, - "disable_traces": true + "disable_metrics": false, + "disable_traces": false } } } diff --git a/config/krakend/extended/templates/endpoints_telemetry_customization.json b/config/krakend/extended/templates/endpoints_telemetry_customization.json index 7793b92..3e4d33b 100644 --- a/config/krakend/extended/templates/endpoints_telemetry_customization.json +++ b/config/krakend/extended/templates/endpoints_telemetry_customization.json @@ -82,7 +82,7 @@ "metric_reporting_period": 10, "trace_sample_rate": 1, "metric_exporters": ["local_prometheus"], - "trace_exporters": ["debug_jaeger"] + "trace_exporters": ["debug_jaeger", "newrelic", "local_tempo" ] } }, "modifier/jmespath": { From 3d1e8df814c680dfd00cda5eaa615deccf80a4c5 Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Wed, 17 Apr 2024 16:10:19 +0200 Subject: [PATCH 13/14] add a config file for running a local instance --- config/krakend/extended/krakend.local.json | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 config/krakend/extended/krakend.local.json diff --git a/config/krakend/extended/krakend.local.json b/config/krakend/extended/krakend.local.json new file mode 100644 index 0000000..33d9197 --- /dev/null +++ b/config/krakend/extended/krakend.local.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://www.krakend.io/schema/krakend.json", + "version": 3, + "name": "KrakenD Enterprise API Gateway Local", + "port": 8080, + "host": [ + "{{ .hosts.fake_api }}" + ], + "timeout": "3000ms", + "cache_ttl": "300s", + "debug_endpoint": true, + "plugin": { + "folder": "/opt/krakend/plugins/", + "pattern": ".so" + }, + "async_agent": [ + { + "$ref": "./async_agents/async_agent_demo.json" + } + ], + "endpoints": [ + {{ template "endpoints_telemetry_customization.json" . }}, + {{ template "endpoints_req_resp_transformations.json" . }}, + {{ template "endpoints_authentication.json" . }}, + {{ template "endpoints_traffic_management.json" . }}, + {{ template "endpoints_services_connectivity.json" . }} + ], + "extra_config": { + "$ref": "./service_extra_config.json" + } +} From e89071599a3c7a062e4fcbee50e858add65b61cf Mon Sep 17 00:00:00 2001 From: David Hontecillas Date: Wed, 24 Apr 2024 15:39:44 +0200 Subject: [PATCH 14/14] update some local config --- config/krakend/extended/extra_config/otel.json | 6 ++++-- config/krakend/extended/settings/root.local.vars.json | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config/krakend/extended/extra_config/otel.json b/config/krakend/extended/extra_config/otel.json index c3665e8..3845b39 100644 --- a/config/krakend/extended/extra_config/otel.json +++ b/config/krakend/extended/extra_config/otel.json @@ -1,7 +1,7 @@ { "service_name": "playground_enterprise_otel", "metric_reporting_period": 1, - "trace_sample_rate": 0.1, + "trace_sample_rate": 1, "exporters": { "prometheus": [ { @@ -31,7 +31,9 @@ "name": "newrelic", "host": "otlp.eu01.nr-data.net", "port": 4318, - "use_http": true + "use_http": true, + "disable_traces": true, + "disable_metrics": true }, { "name": "debug_jaeger", diff --git a/config/krakend/extended/settings/root.local.vars.json b/config/krakend/extended/settings/root.local.vars.json index 2237443..c3d3f7e 100644 --- a/config/krakend/extended/settings/root.local.vars.json +++ b/config/krakend/extended/settings/root.local.vars.json @@ -11,8 +11,8 @@ "host": "127.0.0.1" }, "datadog": { - "disable_metrics": false, - "disable_traces": false + "disable_metrics": true, + "disable_traces": true } } }