diff --git a/apis/proto/v1/benchmark/benchmark.proto b/apis/proto/v1/benchmark/benchmark.proto index cc1bbc95963..a12fef10e2c 100644 --- a/apis/proto/v1/benchmark/benchmark.proto +++ b/apis/proto/v1/benchmark/benchmark.proto @@ -20,13 +20,13 @@ package benchmark.v1; option go_package = "github.com/vdaas/vald/apis/grpc/v1/benchmark"; option java_multiple_files = true; -option java_package = "org.vdaas.vald.api.v1.benchmark"; option java_outer_classname = "Benchmark"; +option java_package = "org.vdaas.vald.api.v1.benchmark"; service Controller { - // TODO define API spec here + // TODO define API spec here } service Job { - // TODO define API spec here + // TODO define API spec here } diff --git a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml index d85ea0c12b7..f5669958ee6 100644 --- a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml +++ b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml @@ -280,6 +280,34 @@ spec: dimension: type: integer minimum: 1 + global_config: + type: object + properties: + logging: + type: object + properties: + format: + type: string + enum: + - raw + - json + level: + type: string + enum: + - debug + - info + - warn + - error + - fatal + logger: + type: string + enum: + - glg + - zap + time_zone: + type: string + version: + type: string insert_config: type: object properties: @@ -349,6 +377,416 @@ spec: type: number timeout: type: string + server_config: + type: object + properties: + healths: + type: object + properties: + liveness: + type: object + properties: + enabled: + type: boolean + host: + type: string + livenessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + readiness: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + readinessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + startup: + type: object + properties: + enabled: + type: boolean + port: + type: integer + maximum: 65535 + minimum: 0 + startupProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + servers: + type: object + properties: + grpc: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + grpc: + type: object + properties: + bidirectional_stream_concurrency: + type: integer + connection_timeout: + type: string + enable_reflection: + type: boolean + header_table_size: + type: integer + initial_conn_window_size: + type: integer + initial_window_size: + type: integer + interceptors: + type: array + items: + type: string + enum: + - RecoverInterceptor + - AccessLogInterceptor + - TraceInterceptor + - MetricInterceptor + keepalive: + type: object + properties: + max_conn_age: + type: string + max_conn_age_grace: + type: string + max_conn_idle: + type: string + min_time: + type: string + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_header_list_size: + type: integer + max_receive_message_size: + type: integer + max_send_message_size: + type: integer + read_buffer_size: + type: integer + write_buffer_size: + type: integer + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + restart: + type: boolean + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + rest: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 target: type: object properties: diff --git a/charts/vald-benchmark-operator/job-values.schema.json b/charts/vald-benchmark-operator/job-values.schema.json index 40759dd01bc..582302dfdfc 100644 --- a/charts/vald-benchmark-operator/job-values.schema.json +++ b/charts/vald-benchmark-operator/job-values.schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Values", "type": "object", "properties": { @@ -342,6 +342,24 @@ "description": "vector dimension", "minimum": 1 }, + "global_config": { + "type": "object", + "properties": { + "logging": { + "type": "object", + "properties": { + "format": { "type": "string", "enum": ["raw", "json"] }, + "level": { + "type": "string", + "enum": ["debug", "info", "warn", "error", "fatal"] + }, + "logger": { "type": "string", "enum": ["glg", "zap"] } + } + }, + "time_zone": { "type": "string" }, + "version": { "type": "string" } + } + }, "insert_config": { "type": "object", "description": "insert config", @@ -443,6 +461,734 @@ } } }, + "server_config": { + "type": "object", + "properties": { + "healths": { + "type": "object", + "properties": { + "liveness": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "liveness server enabled" + }, + "host": { + "type": "string", + "description": "liveness server host" + }, + "livenessProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer", + "description": "liveness probe failure threshold" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "liveness probe path" + }, + "port": { + "type": "string", + "description": "liveness probe port" + }, + "scheme": { + "type": "string", + "description": "liveness probe scheme" + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "description": "liveness probe initial delay seconds" + }, + "periodSeconds": { + "type": "integer", + "description": "liveness probe period seconds" + }, + "successThreshold": { + "type": "integer", + "description": "liveness probe success threshold" + }, + "timeoutSeconds": { + "type": "integer", + "description": "liveness probe timeout seconds" + } + } + }, + "port": { + "type": "integer", + "description": "liveness server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "liveness server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "liveness server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "liveness server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "liveness server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "liveness server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "liveness server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "liveness server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "liveness server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "liveness server service port", + "maximum": 65535, + "minimum": 0 + } + } + }, + "readiness": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "readiness server enabled" + }, + "host": { + "type": "string", + "description": "readiness server host" + }, + "port": { + "type": "integer", + "description": "readiness server port", + "maximum": 65535, + "minimum": 0 + }, + "readinessProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer", + "description": "readiness probe failure threshold" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "readiness probe path" + }, + "port": { + "type": "string", + "description": "readiness probe port" + }, + "scheme": { + "type": "string", + "description": "readiness probe scheme" + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "description": "readiness probe initial delay seconds" + }, + "periodSeconds": { + "type": "integer", + "description": "readiness probe period seconds" + }, + "successThreshold": { + "type": "integer", + "description": "readiness probe success threshold" + }, + "timeoutSeconds": { + "type": "integer", + "description": "readiness probe timeout seconds" + } + } + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "readiness server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "readiness server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "readiness server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "readiness server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "readiness server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "readiness server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "readiness server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "readiness server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "readiness server service port", + "maximum": 65535, + "minimum": 0 + } + } + }, + "startup": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "startup server enabled" + }, + "port": { + "type": "integer", + "description": "startup server port", + "maximum": 65535, + "minimum": 0 + }, + "startupProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer", + "description": "startup probe failure threshold" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "startup probe path" + }, + "port": { + "type": "string", + "description": "startup probe port" + }, + "scheme": { + "type": "string", + "description": "startup probe scheme" + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "description": "startup probe initial delay seconds" + }, + "periodSeconds": { + "type": "integer", + "description": "startup probe period seconds" + }, + "successThreshold": { + "type": "integer", + "description": "startup probe success threshold" + }, + "timeoutSeconds": { + "type": "integer", + "description": "startup probe timeout seconds" + } + } + } + } + } + } + }, + "servers": { + "type": "object", + "properties": { + "grpc": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "gRPC server enabled" + }, + "host": { "type": "string", "description": "gRPC server host" }, + "port": { + "type": "integer", + "description": "gRPC server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "grpc": { + "type": "object", + "properties": { + "bidirectional_stream_concurrency": { + "type": "integer", + "description": "gRPC server bidirectional stream concurrency" + }, + "connection_timeout": { + "type": "string", + "description": "gRPC server connection timeout" + }, + "enable_reflection": { + "type": "boolean", + "description": "gRPC server reflection option" + }, + "header_table_size": { + "type": "integer", + "description": "gRPC server header table size" + }, + "initial_conn_window_size": { + "type": "integer", + "description": "gRPC server initial connection window size" + }, + "initial_window_size": { + "type": "integer", + "description": "gRPC server initial window size" + }, + "interceptors": { + "type": "array", + "description": "gRPC server interceptors", + "items": { + "type": "string", + "enum": [ + "RecoverInterceptor", + "AccessLogInterceptor", + "TraceInterceptor", + "MetricInterceptor" + ] + } + }, + "keepalive": { + "type": "object", + "properties": { + "max_conn_age": { + "type": "string", + "description": "gRPC server keep alive max connection age" + }, + "max_conn_age_grace": { + "type": "string", + "description": "gRPC server keep alive max connection age grace" + }, + "max_conn_idle": { + "type": "string", + "description": "gRPC server keep alive max connection idle" + }, + "min_time": { + "type": "string", + "description": "gRPC server keep alive min_time" + }, + "permit_without_stream": { + "type": "boolean", + "description": "gRPC server keep alive permit_without_stream" + }, + "time": { + "type": "string", + "description": "gRPC server keep alive time" + }, + "timeout": { + "type": "string", + "description": "gRPC server keep alive timeout" + } + } + }, + "max_header_list_size": { + "type": "integer", + "description": "gRPC server max header list size" + }, + "max_receive_message_size": { + "type": "integer", + "description": "gRPC server max receive message size" + }, + "max_send_message_size": { + "type": "integer", + "description": "gRPC server max send message size" + }, + "read_buffer_size": { + "type": "integer", + "description": "gRPC server read buffer size" + }, + "write_buffer_size": { + "type": "integer", + "description": "gRPC server write buffer size" + } + } + }, + "mode": { + "type": "string", + "description": "gRPC server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "gRPC server probe wait time" + }, + "restart": { + "type": "boolean", + "description": "gRPC server restart" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "gRPC server service port", + "maximum": 65535, + "minimum": 0 + } + } + }, + "rest": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "REST server enabled" + }, + "host": { "type": "string", "description": "REST server host" }, + "port": { + "type": "integer", + "description": "REST server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "REST server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "REST server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "REST server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "REST server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "REST server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "REST server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "REST server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "REST server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "REST server service port", + "maximum": 65535, + "minimum": 0 + } + } + } + } + } + } + }, "target": { "type": "object", "description": "target cluster location", diff --git a/charts/vald-benchmark-operator/scenario-values.schema.json b/charts/vald-benchmark-operator/scenario-values.schema.json index c4aadae03c6..7817e50c38b 100644 --- a/charts/vald-benchmark-operator/scenario-values.schema.json +++ b/charts/vald-benchmark-operator/scenario-values.schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Values", "type": "object", "properties": { diff --git a/charts/vald-benchmark-operator/schemas/job-values.yaml b/charts/vald-benchmark-operator/schemas/job-values.yaml index 021418e9f56..8edc3b474e2 100644 --- a/charts/vald-benchmark-operator/schemas/job-values.yaml +++ b/charts/vald-benchmark-operator/schemas/job-values.yaml @@ -373,3 +373,449 @@ concurrency_limit: 200 # @schema {"name": "ttl_seconds_after_finished", "type": "integer", "minimum": 0, "maximum": 65535} # ttl_seconds_after_finished -- limits the lifetime of a Job that has finished execution. ttl_seconds_after_finished: 10 +# @schema {"name": "global_config", "type": "object"} +global_config: + # @schema {"name": "global_config.version", "type": "string", "default": "v0.0.1"} + # version -- version info + version: "v0.0.1" + # @schema {"name": "global_config.time_zone", "type": "string"} + # time_zone -- Time zone + time_zone: UTC + # @schema {"name": "global_config.logging", "type": "object", "anchor": "logging"} + logging: + # @schema {"name": "global_config.logging.logger", "type": "string", "enum": ["glg", "zap"]} + # logging.logger -- logger name. + # currently logger must be `glg` or `zap`. + logger: glg + # @schema {"name": "global_config.logging.level", "type": "string", "enum": ["debug", "info", "warn", "error", "fatal"]} + # logging.level -- logging level. + # logging level must be `debug`, `info`, `warn`, `error` or `fatal`. + level: debug + # @schema {"name": "global_config.logging.format", "type": "string", "enum": ["raw", "json"]} + # logging.format -- logging format. + # logging format must be `raw` or `json` + format: raw +# @schema {"name": "server_config", "type": "object", "anchor": "server_config"} +server_config: + # @schema {"name": "server_config.servers", "type": "object"} + servers: + # @schema {"name": "server_config.servers.rest", "type": "object"} + rest: + # @schema {"name": "server_config.servers.rest.enabled", "type": "boolean"} + # server_config.servers.rest.enabled -- REST server enabled + enabled: false + # @schema {"name": "server_config.servers.rest.host", "type": "string"} + # server_config.servers.rest.host -- REST server host + host: 0.0.0.0 + # @schema {"name": "server_config.servers.rest.port", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.servers.rest.port -- REST server port + port: 8080 + # @schema {"name": "server_config.servers.rest.servicePort", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.servers.rest.servicePort -- REST server service port + servicePort: 8080 + # @schema {"name": "server_config.servers.rest.server", "type": "object"} + server: + # @schema {"name": "server_config.servers.rest.server.mode", "type": "string"} + # server_config.servers.rest.server.mode -- REST server server mode + mode: REST + # @schema {"name": "server_config.servers.rest.server.probe_wait_time", "type": "string"} + # server_config.servers.rest.server.probe_wait_time -- REST server probe wait time + probe_wait_time: 3s + # @schema {"name": "server_config.servers.rest.server.network", "type": "string", "enum": ["tcp", "tcp4", "tcp6", "udp", "udp4", "udp6", "unix", "unixgram", "unixpacket"]} + # server_config.servers.rest.server.network -- mysql network + network: tcp + # @schema {"name": "server_config.servers.rest.server.socket_path", "type": "string"} + # server_config.servers.rest.server.socket_path -- mysql socket_path + socket_path: "" + # @schema {"name": "server_config.servers.rest.server.http", "type": "object"} + http: + # @schema {"name": "server_config.servers.rest.server.http.shutdown_duration", "type": "string"} + # server_config.servers.rest.server.http.shutdown_duration -- REST server shutdown duration + shutdown_duration: 5s + # @schema {"name": "server_config.servers.rest.server.http.handler_timeout", "type": "string"} + # server_config.servers.rest.server.http.handler_timeout -- REST server handler timeout + handler_timeout: 5s + # @schema {"name": "server_config.servers.rest.server.http.idle_timeout", "type": "string"} + # server_config.servers.rest.server.http.idle_timeout -- REST server idle timeout + idle_timeout: 2s + # @schema {"name": "server_config.servers.rest.server.http.read_header_timeout", "type": "string"} + # server_config.servers.rest.server.http.read_header_timeout -- REST server read header timeout + read_header_timeout: 1s + # @schema {"name": "server_config.servers.rest.server.http.read_timeout", "type": "string"} + # server_config.servers.rest.server.http.read_timeout -- REST server read timeout + read_timeout: 1s + # @schema {"name": "server_config.servers.rest.server.http.write_timeout", "type": "string"} + # server_config.servers.rest.server.http.write_timeout -- REST server write timeout + write_timeout: 1s + # @schema {"name": "server_config.servers.rest.server.socket_option", "type": "object", "anchor": "socket_option"} + socket_option: + # @schema {"name": "server_config.servers.rest.server.socket_option.reuse_port", "type": "boolean"} + # server_config.servers.rest.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality + reuse_port: true + # @schema {"name": "server_config.servers.rest.server.socket_option.reuse_addr", "type": "boolean"} + # server_config.servers.rest.server.socket_option.reuse_addr -- server listen socket option for reuse_addr functionality + reuse_addr: true + # @schema {"name": "server_config.servers.rest.server.socket_option.tcp_fast_open", "type": "boolean"} + # server_config.servers.rest.server.socket_option.tcp_fast_open -- server listen socket option for tcp_fast_open functionality + tcp_fast_open: true + # @schema {"name": "server_config.servers.rest.server.socket_option.tcp_no_delay", "type": "boolean"} + # server_config.servers.rest.server.socket_option.tcp_no_delay -- server listen socket option for tcp_no_delay functionality + tcp_no_delay: true + # @schema {"name": "server_config.servers.rest.server.socket_option.tcp_cork", "type": "boolean"} + # server_config.servers.rest.server.socket_option.tcp_cork -- server listen socket option for tcp_cork functionality + tcp_cork: false + # @schema {"name": "server_config.servers.rest.server.socket_option.tcp_quick_ack", "type": "boolean"} + # server_config.servers.rest.server.socket_option.tcp_quick_ack -- server listen socket option for tcp_quick_ack functionality + tcp_quick_ack: true + # @schema {"name": "server_config.servers.rest.server.socket_option.tcp_defer_accept", "type": "boolean"} + # server_config.servers.rest.server.socket_option.tcp_defer_accept -- server listen socket option for tcp_defer_accept functionality + tcp_defer_accept: true + # @schema {"name": "server_config.servers.rest.server.socket_option.ip_transparent", "type": "boolean"} + # server_config.servers.rest.server.socket_option.ip_transparent -- server listen socket option for ip_transparent functionality + ip_transparent: false + # @schema {"name": "server_config.servers.rest.server.socket_option.ip_recover_destination_addr", "type": "boolean"} + # server_config.servers.rest.server.socket_option.ip_recover_destination_addr -- server listen socket option for ip_recover_destination_addr functionality + ip_recover_destination_addr: false + # @schema {"name": "server_config.servers.grpc", "type": "object"} + grpc: + # @schema {"name": "server_config.servers.grpc.enabled", "type": "boolean"} + # server_config.servers.grpc.enabled -- gRPC server enabled + enabled: true + # @schema {"name": "server_config.servers.grpc.host", "type": "string"} + # server_config.servers.grpc.host -- gRPC server host + host: 0.0.0.0 + # @schema {"name": "server_config.servers.grpc.port", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.servers.grpc.port -- gRPC server port + port: 8081 + # @schema {"name": "server_config.servers.grpc.servicePort", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.servers.grpc.servicePort -- gRPC server service port + servicePort: 8081 + # @schema {"name": "server_config.servers.grpc.server", "type": "object"} + server: + # @schema {"name": "server_config.servers.grpc.server.mode", "type": "string"} + # server_config.servers.grpc.server.mode -- gRPC server server mode + mode: GRPC + # @schema {"name": "server_config.servers.grpc.server.probe_wait_time", "type": "string"} + # server_config.servers.grpc.server.probe_wait_time -- gRPC server probe wait time + probe_wait_time: "3s" + # @schema {"name": "server_config.servers.grpc.server.network", "type": "string", "enum": ["tcp", "tcp4", "tcp6", "udp", "udp4", "udp6", "unix", "unixgram", "unixpacket"]} + # server_config.servers.grpc.server.network -- mysql network + network: tcp + # @schema {"name": "server_config.servers.grpc.server.socket_path", "type": "string"} + # server_config.servers.grpc.server.socket_path -- mysql socket_path + socket_path: "" + # @schema {"name": "server_config.servers.grpc.server.grpc", "type": "object"} + grpc: + # @schema {"name": "server_config.servers.grpc.server.grpc.bidirectional_stream_concurrency", "type": "integer"} + # server_config.servers.grpc.server.grpc.bidirectional_stream_concurrency -- gRPC server bidirectional stream concurrency + bidirectional_stream_concurrency: 20 + # @schema {"name": "server_config.servers.grpc.server.grpc.max_receive_message_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.max_receive_message_size -- gRPC server max receive message size + max_receive_message_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.max_send_message_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.max_send_message_size -- gRPC server max send message size + max_send_message_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.initial_window_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.initial_window_size -- gRPC server initial window size + initial_window_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.initial_conn_window_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.initial_conn_window_size -- gRPC server initial connection window size + initial_conn_window_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive", "type": "object"} + keepalive: + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.max_conn_idle", "type": "string"} + # server_config.servers.grpc.server.grpc.keepalive.max_conn_idle -- gRPC server keep alive max connection idle + max_conn_idle: "" + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.max_conn_age", "type": "string"} + # server_config.servers.grpc.server.grpc.keepalive.max_conn_age -- gRPC server keep alive max connection age + max_conn_age: "" + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.max_conn_age_grace", "type": "string"} + # server_config.servers.grpc.server.grpc.keepalive.max_conn_age_grace -- gRPC server keep alive max connection age grace + max_conn_age_grace: "" + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.time", "type": "string"} + # server_config.servers.grpc.server.grpc.keepalive.time -- gRPC server keep alive time + time: "120s" + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.timeout", "type": "string"} + # server_config.servers.grpc.server.grpc.keepalive.timeout -- gRPC server keep alive timeout + timeout: "30s" + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.min_time", "type": "string"} + # server_config.servers.grpc.server.grpc.keepalive.min_time -- gRPC server keep alive min_time + min_time: "60s" + # @schema {"name": "server_config.servers.grpc.server.grpc.keepalive.permit_without_stream", "type": "boolean"} + # server_config.servers.grpc.server.grpc.keepalive.permit_without_stream -- gRPC server keep alive permit_without_stream + permit_without_stream: true + # @schema {"name": "server_config.servers.grpc.server.grpc.write_buffer_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.write_buffer_size -- gRPC server write buffer size + write_buffer_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.read_buffer_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.read_buffer_size -- gRPC server read buffer size + read_buffer_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.connection_timeout", "type": "string"} + # server_config.servers.grpc.server.grpc.connection_timeout -- gRPC server connection timeout + connection_timeout: "" + # @schema {"name": "server_config.servers.grpc.server.grpc.max_header_list_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.max_header_list_size -- gRPC server max header list size + max_header_list_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.header_table_size", "type": "integer"} + # server_config.servers.grpc.server.grpc.header_table_size -- gRPC server header table size + header_table_size: 0 + # @schema {"name": "server_config.servers.grpc.server.grpc.interceptors", "type": "array", "items": {"type": "string", "enum": ["RecoverInterceptor", "AccessLogInterceptor", "TraceInterceptor", "MetricInterceptor"]}} + # server_config.servers.grpc.server.grpc.interceptors -- gRPC server interceptors + interceptors: + - "RecoverInterceptor" + # @schema {"name": "server_config.servers.grpc.server.grpc.enable_reflection", "type": "boolean"} + # server_config.servers.grpc.server.grpc.enable_reflection -- gRPC server reflection option + enable_reflection: true + # @schema {"name": "server_config.servers.grpc.server.socket_option", "alias": "socket_option"} + socket_option: + # server_config.servers.grpc.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality + reuse_port: true + # server_config.servers.grpc.server.socket_option.reuse_addr -- server listen socket option for reuse_addr functionality + reuse_addr: true + # server_config.servers.grpc.server.socket_option.tcp_fast_open -- server listen socket option for tcp_fast_open functionality + tcp_fast_open: true + # server_config.servers.grpc.server.socket_option.tcp_no_delay -- server listen socket option for tcp_no_delay functionality + tcp_no_delay: true + # server_config.servers.grpc.server.socket_option.tcp_cork -- server listen socket option for tcp_cork functionality + tcp_cork: false + # server_config.servers.grpc.server.socket_option.tcp_quick_ack -- server listen socket option for tcp_quick_ack functionality + tcp_quick_ack: true + # server_config.servers.grpc.server.socket_option.tcp_defer_accept -- server listen socket option for tcp_defer_accept functionality + tcp_defer_accept: true + # server_config.servers.grpc.server.socket_option.ip_transparent -- server listen socket option for ip_transparent functionality + ip_transparent: false + # server_config.servers.grpc.server.socket_option.ip_recover_destination_addr -- server listen socket option for ip_recover_destination_addr functionality + ip_recover_destination_addr: false + # @schema {"name": "server_config.servers.grpc.server.restart", "type": "boolean"} + # server_config.servers.grpc.server.restart -- gRPC server restart + restart: true + # @schema {"name": "server_config.healths", "type": "object"} + healths: + # @schema {"name": "server_config.healths.startup", "type": "object"} + startup: + # @schema {"name": "server_config.healths.startup.enabled", "type": "boolean"} + # server_config.healths.startup.enabled -- startup server enabled + enabled: true + # @schema {"name": "server_config.healths.startup.port", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.healths.startup.port -- startup server port + port: 3000 + # @schema {"name": "server_config.healths.startup.startupProbe", "type": "object"} + startupProbe: + # @schema {"name": "server_config.healths.startup.startupProbe.httpGet", "type": "object"} + httpGet: + # @schema {"name": "server_config.healths.startup.startupProbe.httpGet.path", "type": "string"} + # server_config.healths.startup.startupProbe.httpGet.path -- startup probe path + path: /liveness + # @schema {"name": "server_config.healths.startup.startupProbe.httpGet.port", "type": "string"} + # server_config.healths.startup.startupProbe.httpGet.port -- startup probe port + port: liveness + # @schema {"name": "server_config.healths.startup.startupProbe.httpGet.scheme", "type": "string"} + # server_config.healths.startup.startupProbe.httpGet.scheme -- startup probe scheme + scheme: HTTP + # @schema {"name": "server_config.healths.startup.startupProbe.initialDelaySeconds", "type": "integer"} + # server_config.healths.startup.startupProbe.initialDelaySeconds -- startup probe initial delay seconds + initialDelaySeconds: 5 + # @schema {"name": "server_config.healths.startup.startupProbe.timeoutSeconds", "type": "integer"} + # server_config.healths.startup.startupProbe.timeoutSeconds -- startup probe timeout seconds + timeoutSeconds: 2 + # @schema {"name": "server_config.healths.startup.startupProbe.successThreshold", "type": "integer"} + # server_config.healths.startup.startupProbe.successThreshold -- startup probe success threshold + successThreshold: 1 + # @schema {"name": "server_config.healths.startup.startupProbe.failureThreshold", "type": "integer"} + # server_config.healths.startup.startupProbe.failureThreshold -- startup probe failure threshold + failureThreshold: 30 + # @schema {"name": "server_config.healths.startup.startupProbe.periodSeconds", "type": "integer"} + # server_config.healths.startup.startupProbe.periodSeconds -- startup probe period seconds + periodSeconds: 5 + # @schema {"name": "server_config.healths.liveness", "type": "object"} + liveness: + # @schema {"name": "server_config.healths.liveness.enabled", "type": "boolean"} + # server_config.healths.liveness.enabled -- liveness server enabled + enabled: true + # @schema {"name": "server_config.healths.liveness.host", "type": "string"} + # server_config.healths.liveness.host -- liveness server host + host: 0.0.0.0 + # @schema {"name": "server_config.healths.liveness.port", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.healths.liveness.port -- liveness server port + port: 3000 + # @schema {"name": "server_config.healths.liveness.servicePort", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.healths.liveness.servicePort -- liveness server service port + servicePort: 3000 + # @schema {"name": "server_config.healths.liveness.livenessProbe", "type": "object"} + livenessProbe: + # @schema {"name": "server_config.healths.liveness.livenessProbe.httpGet", "type": "object"} + httpGet: + # @schema {"name": "server_config.healths.liveness.livenessProbe.httpGet.path", "type": "string"} + # server_config.healths.liveness.livenessProbe.httpGet.path -- liveness probe path + path: /liveness + # @schema {"name": "server_config.healths.liveness.livenessProbe.httpGet.port", "type": "string"} + # server_config.healths.liveness.livenessProbe.httpGet.port -- liveness probe port + port: liveness + # @schema {"name": "server_config.healths.liveness.livenessProbe.httpGet.scheme", "type": "string"} + # server_config.healths.liveness.livenessProbe.httpGet.scheme -- liveness probe scheme + scheme: HTTP + # @schema {"name": "server_config.healths.liveness.livenessProbe.initialDelaySeconds", "type": "integer"} + # server_config.healths.liveness.livenessProbe.initialDelaySeconds -- liveness probe initial delay seconds + initialDelaySeconds: 5 + # @schema {"name": "server_config.healths.liveness.livenessProbe.timeoutSeconds", "type": "integer"} + # server_config.healths.liveness.livenessProbe.timeoutSeconds -- liveness probe timeout seconds + timeoutSeconds: 2 + # @schema {"name": "server_config.healths.liveness.livenessProbe.successThreshold", "type": "integer"} + # server_config.healths.liveness.livenessProbe.successThreshold -- liveness probe success threshold + successThreshold: 1 + # @schema {"name": "server_config.healths.liveness.livenessProbe.failureThreshold", "type": "integer"} + # server_config.healths.liveness.livenessProbe.failureThreshold -- liveness probe failure threshold + failureThreshold: 2 + # @schema {"name": "server_config.healths.liveness.livenessProbe.periodSeconds", "type": "integer"} + # server_config.healths.liveness.livenessProbe.periodSeconds -- liveness probe period seconds + periodSeconds: 3 + # @schema {"name": "server_config.healths.liveness.server", "type": "object"} + server: + # @schema {"name": "server_config.healths.liveness.server.mode", "type": "string"} + # server_config.healths.liveness.server.mode -- liveness server mode + mode: "" + # @schema {"name": "server_config.healths.liveness.server.probe_wait_time", "type": "string"} + # server_config.healths.liveness.server.probe_wait_time -- liveness server probe wait time + probe_wait_time: "3s" + # @schema {"name": "server_config.healths.liveness.server.network", "type": "string", "enum": ["tcp", "tcp4", "tcp6", "udp", "udp4", "udp6", "unix", "unixgram", "unixpacket"]} + # server_config.healths.liveness.server.network -- mysql network + network: tcp + # @schema {"name": "server_config.healths.liveness.server.socket_path", "type": "string"} + # server_config.healths.liveness.server.socket_path -- mysql socket_path + socket_path: "" + # @schema {"name": "server_config.healths.liveness.server.http", "type": "object"} + http: + # @schema {"name": "server_config.healths.liveness.server.http.shutdown_duration", "type": "string"} + # server_config.healths.liveness.server.http.shutdown_duration -- liveness server shutdown duration + shutdown_duration: "5s" + # @schema {"name": "server_config.healths.liveness.server.http.handler_timeout", "type": "string"} + # server_config.healths.liveness.server.http.handler_timeout -- liveness server handler timeout + handler_timeout: "" + # @schema {"name": "server_config.healths.liveness.server.http.idle_timeout", "type": "string"} + # server_config.healths.liveness.server.http.idle_timeout -- liveness server idle timeout + idle_timeout: "" + # @schema {"name": "server_config.healths.liveness.server.http.read_header_timeout", "type": "string"} + # server_config.healths.liveness.server.http.read_header_timeout -- liveness server read header timeout + read_header_timeout: "" + # @schema {"name": "server_config.healths.liveness.server.http.read_timeout", "type": "string"} + # server_config.healths.liveness.server.http.read_timeout -- liveness server read timeout + read_timeout: "" + # @schema {"name": "server_config.healths.liveness.server.http.write_timeout", "type": "string"} + # server_config.healths.liveness.server.http.write_timeout -- liveness server write timeout + write_timeout: "" + # @schema {"name": "server_config.healths.liveness.server.socket_option", "alias": "socket_option"} + socket_option: + # server_config.healths.liveness.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality + reuse_port: true + # server_config.healths.liveness.server.socket_option.reuse_addr -- server listen socket option for reuse_addr functionality + reuse_addr: true + # server_config.healths.liveness.server.socket_option.tcp_fast_open -- server listen socket option for tcp_fast_open functionality + tcp_fast_open: true + # server_config.healths.liveness.server.socket_option.tcp_no_delay -- server listen socket option for tcp_no_delay functionality + tcp_no_delay: true + # server_config.healths.liveness.server.socket_option.tcp_cork -- server listen socket option for tcp_cork functionality + tcp_cork: false + # server_config.healths.liveness.server.socket_option.tcp_quick_ack -- server listen socket option for tcp_quick_ack functionality + tcp_quick_ack: true + # server_config.healths.liveness.server.socket_option.tcp_defer_accept -- server listen socket option for tcp_defer_accept functionality + tcp_defer_accept: true + # server_config.healths.liveness.server.socket_option.ip_transparent -- server listen socket option for ip_transparent functionality + ip_transparent: false + # server_config.healths.liveness.server.socket_option.ip_recover_destination_addr -- server listen socket option for ip_recover_destination_addr functionality + ip_recover_destination_addr: false + # @schema {"name": "server_config.healths.readiness", "type": "object"} + readiness: + # @schema {"name": "server_config.healths.readiness.enabled", "type": "boolean"} + # server_config.healths.readiness.enabled -- readiness server enabled + enabled: true + # @schema {"name": "server_config.healths.readiness.host", "type": "string"} + # server_config.healths.readiness.host -- readiness server host + host: 0.0.0.0 + # @schema {"name": "server_config.healths.readiness.port", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.healths.readiness.port -- readiness server port + port: 3001 + # @schema {"name": "server_config.healths.readiness.servicePort", "type": "integer", "minimum": 0, "maximum": 65535} + # server_config.healths.readiness.servicePort -- readiness server service port + servicePort: 3001 + # @schema {"name": "server_config.healths.readiness.readinessProbe", "type": "object"} + readinessProbe: + # @schema {"name": "server_config.healths.readiness.readinessProbe.httpGet", "type": "object"} + httpGet: + # @schema {"name": "server_config.healths.readiness.readinessProbe.httpGet.path", "type": "string"} + # server_config.healths.readiness.readinessProbe.httpGet.path -- readiness probe path + path: /readiness + # @schema {"name": "server_config.healths.readiness.readinessProbe.httpGet.port", "type": "string"} + # server_config.healths.readiness.readinessProbe.httpGet.port -- readiness probe port + port: readiness + # @schema {"name": "server_config.healths.readiness.readinessProbe.httpGet.scheme", "type": "string"} + # server_config.healths.readiness.readinessProbe.httpGet.scheme -- readiness probe scheme + scheme: HTTP + # @schema {"name": "server_config.healths.readiness.readinessProbe.initialDelaySeconds", "type": "integer"} + # server_config.healths.readiness.readinessProbe.initialDelaySeconds -- readiness probe initial delay seconds + initialDelaySeconds: 10 + # @schema {"name": "server_config.healths.readiness.readinessProbe.timeoutSeconds", "type": "integer"} + # server_config.healths.readiness.readinessProbe.timeoutSeconds -- readiness probe timeout seconds + timeoutSeconds: 2 + # @schema {"name": "server_config.healths.readiness.readinessProbe.successThreshold", "type": "integer"} + # server_config.healths.readiness.readinessProbe.successThreshold -- readiness probe success threshold + successThreshold: 1 + # @schema {"name": "server_config.healths.readiness.readinessProbe.failureThreshold", "type": "integer"} + # server_config.healths.readiness.readinessProbe.failureThreshold -- readiness probe failure threshold + failureThreshold: 2 + # @schema {"name": "server_config.healths.readiness.readinessProbe.periodSeconds", "type": "integer"} + # server_config.healths.readiness.readinessProbe.periodSeconds -- readiness probe period seconds + periodSeconds: 3 + # @schema {"name": "server_config.healths.readiness.server", "type": "object"} + server: + # @schema {"name": "server_config.healths.readiness.server.mode", "type": "string"} + # server_config.healths.readiness.server.mode -- readiness server mode + mode: "" + # @schema {"name": "server_config.healths.readiness.server.probe_wait_time", "type": "string"} + # server_config.healths.readiness.server.probe_wait_time -- readiness server probe wait time + probe_wait_time: "3s" + # @schema {"name": "server_config.healths.readiness.server.network", "type": "string", "enum": ["tcp", "tcp4", "tcp6", "udp", "udp4", "udp6", "unix", "unixgram", "unixpacket"]} + # server_config.healths.readiness.server.network -- mysql network + network: tcp + # @schema {"name": "server_config.healths.readiness.server.socket_path", "type": "string"} + # server_config.healths.readiness.server.socket_path -- mysql socket_path + socket_path: "" + # @schema {"name": "server_config.healths.readiness.server.http", "type": "object"} + http: + # @schema {"name": "server_config.healths.readiness.server.http.shutdown_duration", "type": "string"} + # server_config.healths.readiness.server.http.shutdown_duration -- readiness server shutdown duration + shutdown_duration: "0s" + # @schema {"name": "server_config.healths.readiness.server.http.handler_timeout", "type": "string"} + # server_config.healths.readiness.server.http.handler_timeout -- readiness server handler timeout + handler_timeout: "" + # @schema {"name": "server_config.healths.readiness.server.http.idle_timeout", "type": "string"} + # server_config.healths.readiness.server.http.idle_timeout -- readiness server idle timeout + idle_timeout: "" + # @schema {"name": "server_config.healths.readiness.server.http.read_header_timeout", "type": "string"} + # server_config.healths.readiness.server.http.read_header_timeout -- readiness server read header timeout + read_header_timeout: "" + # @schema {"name": "server_config.healths.readiness.server.http.read_timeout", "type": "string"} + # server_config.healths.readiness.server.http.read_timeout -- readiness server read timeout + read_timeout: "" + # @schema {"name": "server_config.healths.readiness.server.http.write_timeout", "type": "string"} + # server_config.healths.readiness.server.http.write_timeout -- readiness server write timeout + write_timeout: "" + # @schema {"name": "server_config.healths.readiness.server.socket_option", "alias": "socket_option"} + socket_option: + # server_config.healths.readiness.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality + reuse_port: true + # server_config.healths.readiness.server.socket_option.reuse_addr -- server listen socket option for reuse_addr functionality + reuse_addr: true + # server_config.healths.readiness.server.socket_option.tcp_fast_oepn -- server listen socket option for tcp_fast_open functionality + tcp_fast_open: true + # server_config.healths.readiness.server.socket_option.tcp_no_delay -- server listen socket option for tcp_no_delay functionality + tcp_no_delay: true + # server_config.healths.readiness.server.socket_option.tcp_cork -- server listen socket option for tcp_cork functionality + tcp_cork: false + # server_config.healths.readiness.server.socket_option.tcp_quick_ack -- server listen socket option for tcp_quick_ack functionality + tcp_quick_ack: true + # server_config.healths.readiness.server.socket_option.tcp_defer_accept -- server listen socket option for tcp_defer_accept functionality + tcp_defer_accept: true + # server_config.healths.readiness.server.socket_option.ip_transparent -- server listen socket option for ip_transparent functionality + ip_transparent: false + # server_config.healths.readiness.server.socket_option.ip_recover_destination_addr -- server listen socket option for ip_recover_destination_addr functionality + ip_recover_destination_addr: false diff --git a/charts/vald-benchmark-operator/values.schema.json b/charts/vald-benchmark-operator/values.schema.json index 57fb5cd25bb..8e4ded670dd 100644 --- a/charts/vald-benchmark-operator/values.schema.json +++ b/charts/vald-benchmark-operator/values.schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Values", "type": "object", "properties": { diff --git a/internal/k8s/vald/benchmark/api/v1/job_types.go b/internal/k8s/vald/benchmark/api/v1/job_types.go index 729bf276299..db70bff5bc1 100644 --- a/internal/k8s/vald/benchmark/api/v1/job_types.go +++ b/internal/k8s/vald/benchmark/api/v1/job_types.go @@ -23,23 +23,25 @@ import ( ) type BenchmarkJobSpec struct { - Target *BenchmarkTarget `json:"target,omitempty"` - Dataset *BenchmarkDataset `json:"dataset,omitempty"` - Dimension int `json:"dimension,omitempty"` - Replica int `json:"replica,omitempty"` - Repetition int `json:"repetition,omitempty"` - JobType string `json:"job_type,omitempty"` - InsertConfig *config.InsertConfig `json:"insert_config,omitempty"` - UpdateConfig *config.UpdateConfig `json:"update_config,omitempty"` - UpsertConfig *config.UpsertConfig `json:"upsert_config,omitempty"` - SearchConfig *config.SearchConfig `json:"search_config,omitempty"` - RemoveConfig *config.RemoveConfig `json:"remove_config,omitempty"` - ObjectConfig *config.ObjectConfig `json:"object_config,omitempty"` - ClientConfig *config.GRPCClient `json:"client_config,omitempty"` - Rules []*config.BenchmarkJobRule `json:"rules,omitempty"` - RPS int `json:"rps,omitempty"` - ConcurrencyLimit int `json:"concurrency_limit,omitempty"` - TTLSecondsAfterFinished int `json:"ttl_seconds_after_finished,omitempty"` + Target *BenchmarkTarget `json:"target,omitempty" yaml:"target"` + Dataset *BenchmarkDataset `json:"dataset,omitempty" yaml:"dataset"` + Dimension int `json:"dimension,omitempty" yaml:"dimension"` + Replica int `json:"replica,omitempty" yaml:"replica"` + Repetition int `json:"repetition,omitempty" yaml:"repetition"` + JobType string `json:"job_type,omitempty" yaml:"job_type"` + InsertConfig *config.InsertConfig `json:"insert_config,omitempty" yaml:"insert_config"` + UpdateConfig *config.UpdateConfig `json:"update_config,omitempty" yaml:"update_config"` + UpsertConfig *config.UpsertConfig `json:"upsert_config,omitempty" yaml:"upsert_config"` + SearchConfig *config.SearchConfig `json:"search_config,omitempty" yaml:"search_config"` + RemoveConfig *config.RemoveConfig `json:"remove_config,omitempty" yaml:"remove_config"` + ObjectConfig *config.ObjectConfig `json:"object_config,omitempty" yaml:"object_config"` + ClientConfig *config.GRPCClient `json:"client_config,omitempty" yaml:"client_config"` + Rules []*config.BenchmarkJobRule `json:"rules,omitempty" yaml:"rules"` + RPS int `json:"rps,omitempty" yaml:"rps"` + ConcurrencyLimit int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit"` + TTLSecondsAfterFinished int `json:"ttl_seconds_after_finished,omitempty" yaml:"ttl_seconds_after_finished"` + GlobalConfig *config.GlobalConfig `json:"global_config,omitempty" yaml:"global_config"` + ServerConfig *config.Servers `json:"server_config,omitempty" yaml:"server_config"` } type BenchmarkJobStatus string diff --git a/k8s/tools/benchmark/operator/clusterrole.yaml b/k8s/tools/benchmark/operator/clusterrole.yaml index 4d3fc54948e..31f10cf1825 100644 --- a/k8s/tools/benchmark/operator/clusterrole.yaml +++ b/k8s/tools/benchmark/operator/clusterrole.yaml @@ -19,7 +19,7 @@ kind: ClusterRole metadata: creationTimestamp: null name: vald-benchmark-operator - namespace: default + namespace: benchmark rules: - apiGroups: - apps diff --git a/k8s/tools/benchmark/operator/clusterrolebinding.yaml b/k8s/tools/benchmark/operator/clusterrolebinding.yaml index ded68d52046..5d7f814011f 100644 --- a/k8s/tools/benchmark/operator/clusterrolebinding.yaml +++ b/k8s/tools/benchmark/operator/clusterrolebinding.yaml @@ -18,11 +18,11 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vald-benchmark-operator - namespace: default + namespace: benchmark subjects: - kind: ServiceAccount name: vald-benchmark-operator - namespace: default + namespace: benchmark roleRef: kind: ClusterRole name: vald-benchmark-operator diff --git a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml index d85ea0c12b7..f5669958ee6 100644 --- a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml +++ b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml @@ -280,6 +280,34 @@ spec: dimension: type: integer minimum: 1 + global_config: + type: object + properties: + logging: + type: object + properties: + format: + type: string + enum: + - raw + - json + level: + type: string + enum: + - debug + - info + - warn + - error + - fatal + logger: + type: string + enum: + - glg + - zap + time_zone: + type: string + version: + type: string insert_config: type: object properties: @@ -349,6 +377,416 @@ spec: type: number timeout: type: string + server_config: + type: object + properties: + healths: + type: object + properties: + liveness: + type: object + properties: + enabled: + type: boolean + host: + type: string + livenessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + readiness: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + readinessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + startup: + type: object + properties: + enabled: + type: boolean + port: + type: integer + maximum: 65535 + minimum: 0 + startupProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + servers: + type: object + properties: + grpc: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + grpc: + type: object + properties: + bidirectional_stream_concurrency: + type: integer + connection_timeout: + type: string + enable_reflection: + type: boolean + header_table_size: + type: integer + initial_conn_window_size: + type: integer + initial_window_size: + type: integer + interceptors: + type: array + items: + type: string + enum: + - RecoverInterceptor + - AccessLogInterceptor + - TraceInterceptor + - MetricInterceptor + keepalive: + type: object + properties: + max_conn_age: + type: string + max_conn_age_grace: + type: string + max_conn_idle: + type: string + min_time: + type: string + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_header_list_size: + type: integer + max_receive_message_size: + type: integer + max_send_message_size: + type: integer + read_buffer_size: + type: integer + write_buffer_size: + type: integer + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + restart: + type: boolean + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + rest: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 target: type: object properties: diff --git a/k8s/tools/benchmark/operator/deployment.yaml b/k8s/tools/benchmark/operator/deployment.yaml index 520609518bc..6447be46144 100644 --- a/k8s/tools/benchmark/operator/deployment.yaml +++ b/k8s/tools/benchmark/operator/deployment.yaml @@ -18,7 +18,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: vald-benchmark-operator - namespace: default + namespace: benchmark labels: app: vald-benchmark-operator app.kubernetes.io/name: vald-benchmark-operator diff --git a/k8s/tools/benchmark/operator/serviceaccount.yaml b/k8s/tools/benchmark/operator/serviceaccount.yaml index cf7ac3908db..a67c176c364 100644 --- a/k8s/tools/benchmark/operator/serviceaccount.yaml +++ b/k8s/tools/benchmark/operator/serviceaccount.yaml @@ -18,4 +18,4 @@ apiVersion: v1 kind: ServiceAccount metadata: name: vald-benchmark-operator - namespace: default + namespace: benchmark diff --git a/pkg/tools/benchmark/job/config/config.go b/pkg/tools/benchmark/job/config/config.go index 3b1492e25d0..248b442da6b 100644 --- a/pkg/tools/benchmark/job/config/config.go +++ b/pkg/tools/benchmark/job/config/config.go @@ -102,6 +102,15 @@ func NewConfig(ctx context.Context, path string) (cfg *Config, err error) { if err != nil { log.Warn(err.Error()) } else { + // GlobalConfig + if jobResource.Spec.GlobalConfig != nil { + cfg.GlobalConfig = *jobResource.Spec.GlobalConfig + } + // ServerConfig + if jobResource.Spec.ServerConfig != nil { + cfg.Server = jobResource.Spec.ServerConfig + } + // Job cfg.Job.Target = (*config.BenchmarkTarget)(jobResource.Spec.Target) cfg.Job.Dataset = (*config.BenchmarkDataset)(jobResource.Spec.Dataset) cfg.Job.Dimension = jobResource.Spec.Dimension @@ -109,13 +118,27 @@ func NewConfig(ctx context.Context, path string) (cfg *Config, err error) { cfg.Job.Repetition = jobResource.Spec.Repetition cfg.Job.JobType = jobResource.Spec.JobType cfg.Job.Rules = jobResource.Spec.Rules - cfg.Job.InsertConfig = jobResource.Spec.InsertConfig - cfg.Job.UpdateConfig = jobResource.Spec.UpdateConfig - cfg.Job.UpsertConfig = jobResource.Spec.UpsertConfig - cfg.Job.SearchConfig = jobResource.Spec.SearchConfig - cfg.Job.RemoveConfig = jobResource.Spec.RemoveConfig - cfg.Job.ObjectConfig = jobResource.Spec.ObjectConfig - cfg.Job.ClientConfig = jobResource.Spec.ClientConfig + if jobResource.Spec.InsertConfig != nil { + cfg.Job.InsertConfig = jobResource.Spec.InsertConfig.Bind() + } + if jobResource.Spec.UpdateConfig != nil { + cfg.Job.UpdateConfig = jobResource.Spec.UpdateConfig.Bind() + } + if jobResource.Spec.UpsertConfig != nil { + cfg.Job.UpsertConfig = jobResource.Spec.UpsertConfig.Bind() + } + if jobResource.Spec.SearchConfig != nil { + cfg.Job.SearchConfig = jobResource.Spec.SearchConfig.Bind() + } + if jobResource.Spec.RemoveConfig != nil { + cfg.Job.RemoveConfig = jobResource.Spec.RemoveConfig.Bind() + } + if jobResource.Spec.ObjectConfig != nil { + cfg.Job.ObjectConfig = jobResource.Spec.ObjectConfig.Bind() + } + if jobResource.Spec.ClientConfig != nil { + cfg.Job.ClientConfig = jobResource.Spec.ClientConfig.Bind() + } cfg.Job.RPS = jobResource.Spec.RPS cfg.Job.ConcurrencyLimit = jobResource.Spec.ConcurrencyLimit if annotations := jobResource.GetAnnotations(); annotations != nil { diff --git a/pkg/tools/benchmark/job/handler/grpc/handler.go b/pkg/tools/benchmark/job/handler/grpc/handler.go index 7febc72d11d..e3ec51c25d9 100644 --- a/pkg/tools/benchmark/job/handler/grpc/handler.go +++ b/pkg/tools/benchmark/job/handler/grpc/handler.go @@ -21,7 +21,6 @@ import ( "context" "github.com/vdaas/vald/apis/grpc/v1/benchmark" - "github.com/vdaas/vald/internal/singleflight" "github.com/vdaas/vald/pkg/tools/benchmark/job/service" ) @@ -33,8 +32,7 @@ type Benchmark interface { type server struct { benchmark.UnimplementedJobServer - job service.Job - group singleflight.Group + job service.Job } func New(opts ...Option) (bm Benchmark, err error) { @@ -47,8 +45,6 @@ func New(opts ...Option) (bm Benchmark, err error) { } } - b.group = singleflight.New() - return b, nil } diff --git a/pkg/tools/benchmark/job/service/job.go b/pkg/tools/benchmark/job/service/job.go index 2f115ea157d..c2869045516 100644 --- a/pkg/tools/benchmark/job/service/job.go +++ b/pkg/tools/benchmark/job/service/job.go @@ -33,6 +33,7 @@ import ( "github.com/vdaas/vald/internal/k8s/client" v1 "github.com/vdaas/vald/internal/k8s/vald/benchmark/api/v1" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/rand" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/test/data/hdf5" "github.com/vdaas/vald/internal/timeutil/rate" @@ -345,3 +346,11 @@ func calcRecall(linearRes, searchRes *payload.Search_Response) (recall float64) } return recall / float64(len(lres)) } + +// TODO: apply many object type +func addNoiseToVec(oVec []float32) []float32 { + noise := rand.Float32() + vec := oVec + vec[0] += noise + return vec +} diff --git a/pkg/tools/benchmark/job/service/search.go b/pkg/tools/benchmark/job/service/search.go index d8fb0d25f97..dc79b856261 100644 --- a/pkg/tools/benchmark/job/service/search.go +++ b/pkg/tools/benchmark/job/service/search.go @@ -92,6 +92,9 @@ func (j *job) search(ctx context.Context, ech chan error) error { } if res != nil && j.searchConfig.EnableLinearSearch { sres[iter-j.dataset.Range.Start] = res + log.Debugf("[benchmark job] Finish search: iter = %d, len = %d", iter, len(res.Results)) + } else { + log.Debugf("[benchmark job] Finish search: iter = %d, res = %v", iter, res) } log.Debugf("[benchmark job] Finish search: iter = %d, len = %d", iter, len(res.Results)) return nil diff --git a/pkg/tools/benchmark/job/service/update.go b/pkg/tools/benchmark/job/service/update.go index 2c7ab12c49f..653ec64b3e6 100644 --- a/pkg/tools/benchmark/job/service/update.go +++ b/pkg/tools/benchmark/job/service/update.go @@ -63,7 +63,7 @@ func (j *job) update(ctx context.Context, ech chan error) error { res, err := j.client.Update(egctx, &payload.Update_Request{ Vector: &payload.Object_Vector{ Id: strconv.Itoa(iter), - Vector: vecs[idx], + Vector: addNoiseToVec(vecs[idx]), }, Config: cfg, }) diff --git a/pkg/tools/benchmark/job/service/upsert.go b/pkg/tools/benchmark/job/service/upsert.go index a31f3379727..d9eeaecaee4 100644 --- a/pkg/tools/benchmark/job/service/upsert.go +++ b/pkg/tools/benchmark/job/service/upsert.go @@ -62,7 +62,7 @@ func (j *job) upsert(ctx context.Context, ech chan error) error { res, err := j.client.Upsert(egctx, &payload.Upsert_Request{ Vector: &payload.Object_Vector{ Id: strconv.Itoa(iter), - Vector: vecs[idx], + Vector: addNoiseToVec(vecs[idx]), }, Config: cfg, }) diff --git a/pkg/tools/benchmark/operator/handler/grpc/handler.go b/pkg/tools/benchmark/operator/handler/grpc/handler.go index a11cf4133fa..7809bc394cd 100644 --- a/pkg/tools/benchmark/operator/handler/grpc/handler.go +++ b/pkg/tools/benchmark/operator/handler/grpc/handler.go @@ -21,7 +21,6 @@ import ( "context" "github.com/vdaas/vald/apis/grpc/v1/benchmark" - "github.com/vdaas/vald/internal/singleflight" "github.com/vdaas/vald/pkg/tools/benchmark/operator/service" ) @@ -34,7 +33,6 @@ type server struct { benchmark.UnimplementedJobServer operator service.Operator - group singleflight.Group } func New(opts ...Option) (bm Benchmark, err error) { @@ -47,8 +45,6 @@ func New(opts ...Option) (bm Benchmark, err error) { } } - b.group = singleflight.New() - return b, nil }