feat: Add o11y supports

all-in-one/Dockerfile

@@ -3,13 +3,21 @@ ARG BASE_VERSION=2022-10-27T19-02-22
 ARG CORE_VERSION=2.0.4
 ARG CONSOLE_VERSION=1.4.6
 ARG APISERVER_VERSION=0.0.16
+ARG PROMETHEUS_VERSION=v2.40.7
+ARG PROMTAIL_VERSION=2.9.4
+ARG LOKI_VERSION=2.9.4
+ARG GRAFANA_VERSION=9.3.6
 
-FROM ${HUB}/api-server:${APISERVER_VERSION} as apiserver
-FROM ${HUB}/higress:${CORE_VERSION} as controller
-FROM ${HUB}/pilot:${CORE_VERSION} as pilot
-FROM ${HUB}/gateway:${CORE_VERSION} as gateway
-FROM ${HUB}/console:${CONSOLE_VERSION} as console
-FROM eclipse-temurin:21-jre as jdk
+FROM ${HUB}/api-server:${APISERVER_VERSION} AS apiserver
+FROM ${HUB}/higress:${CORE_VERSION} AS controller
+FROM ${HUB}/pilot:${CORE_VERSION} AS pilot
+FROM ${HUB}/gateway:${CORE_VERSION} AS gateway
+FROM ${HUB}/console:${CONSOLE_VERSION} AS console
+FROM ${HUB}/prometheus:${PROMETHEUS_VERSION} AS prometheus
+FROM ${HUB}/promtail:${PROMTAIL_VERSION} AS promtail
+FROM ${HUB}/loki:${LOKI_VERSION} AS loki
+FROM ${HUB}/grafana:${GRAFANA_VERSION} AS grafana
+FROM ${HUB}/eclipse-temurin:21-jre AS jdk
 
 FROM ${HUB}/base:${BASE_VERSION}
 
@@ -21,12 +29,17 @@ COPY --from=controller /usr/local/bin/higress /usr/local/bin/higress
 
 # Install pilot
 COPY --from=pilot /usr/local/bin/pilot-discovery /usr/local/bin/pilot-discovery
-COPY --from=pilot /var/lib/istio/envoy/*.json /var/lib/istio/envoy/
 COPY --from=pilot /usr/local/bin/higress-pilot-start.sh /usr/local/bin/higress-pilot-start.sh
 
 # Install gateway
+COPY --from=gateway /var/lib/istio/envoy/*.json /var/lib/istio/envoy/
 COPY --from=gateway /usr/local/bin/pilot-agent /usr/local/bin/pilot-agent
 COPY --from=gateway /usr/local/bin/envoy /usr/local/bin/envoy
+COPY --from=gateway /usr/local/bin/higress-proxy-*.sh /usr/local/bin/
+RUN chmod a+x /usr/local/bin/higress-proxy-container-init.sh; \
+  sed -i 's/1337/0/g' /usr/local/bin/higress-proxy-container-init.sh; \
+  /usr/local/bin/higress-proxy-container-init.sh
+COPY --from=gateway /usr/local/bin/supercronic* /usr/local/bin/
 
 # Install console
 COPY --from=console /app /app
@@ -36,16 +49,28 @@ ENV JAVA_HOME=/opt/java/openjdk
 COPY --from=jdk $JAVA_HOME $JAVA_HOME
 ENV PATH="${JAVA_HOME}/bin:${PATH}"
 
-# Install supervisord and initialize related folders
-RUN apt-get update && \
-  apt-get install --no-install-recommends -y \
-  supervisor \
+# Install Prometheus
+COPY --from=prometheus /bin/prometheus /usr/local/bin/prometheus
+
+# Install Promtail
+COPY --from=promtail /usr/bin/promtail /usr/local/bin/promtail
+
+# Install Loki
+COPY --from=loki /usr/bin/loki /usr/local/bin/loki
+
+# Install Grafana
+COPY --from=grafana /usr/share/grafana /usr/share/grafana
+COPY --from=grafana /run.sh /usr/local/bin/grafana.sh
+
+# Install supervisord, logrotate, cron and initialize related folders
+RUN apt-get update --allow-unauthenticated && \
+  apt-get install --no-install-recommends -y --allow-unauthenticated \
+  supervisor logrotate cron \
+  && apt-get upgrade -y --allow-unauthenticated \
   && apt-get clean \
   && rm -rf /var/log/*log /var/lib/apt/lists/* /var/log/apt/* /var/lib/dpkg/*-old /var/cache/debconf/*-old \
   && mkdir -p /var/log/higress \
-  && chown 1337:1337 -R /var/log/higress \
-  && mkdir /data \
-  && chown 1337:1337 -R /data
+  && mkdir /data
 COPY ./supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
 # Initialize configurations
@@ -54,6 +79,10 @@ COPY ./gateway/podinfo /etc/istio/pod
 COPY ./scripts /usr/local/bin
 COPY ./apiserver/config /app/kubeconfig
 COPY ./config /opt/data/defaultConfig
+COPY ./prometheus /etc/prometheus
+COPY ./promtail /etc/promtail
+COPY ./loki /etc/loki
+COPY ./grafana /etc/grafana
 
 EXPOSE 8080 8443 8001
 

all-in-one/config/configmaps/higress-config.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: higress-config
+  namespace: higress-system
+  creationTimestamp: "2000-01-01T00:00:00Z"
+  resourceVersion: "1"
+data:
+  higress: |-
+    downstream:
+      connectionBufferLimits: 32768
+      http2:
+        initialConnectionWindowSize: 1048576
+        initialStreamWindowSize: 65535
+        maxConcurrentStreams: 100
+      idleTimeout: 180
+      maxRequestHeadersKb: 60
+      routeTimeout: 0
+    upstream:
+      connectionBufferLimits: 10485760
+      idleTimeout: 10

all-in-one/get-ai-gateway.sh

@@ -156,7 +156,7 @@ runConfigWizard() {
     "Zhipu AI|ZHIPUAI"
     "Ollama|OLLAMA|configureOllamaProvider"
     "Claude|CLAUDE|configureClaudeProvider"
-    "Baidu AI Cloud|BAIDU"
+    # "Baidu AI Cloud|BAIDU"
     # "Tencent Hunyuan|HUNYUAN"
     "Stepfun|STEPFUN"
     "Minimax|MINIMAX|configureMinimaxProvider"

all-in-one/grafana/grafana.ini

@@ -0,0 +1,21 @@
+[server]
+protocol=http
+domain=localhost
+root_url="%(protocol)s://%(domain)s/grafana"
+serve_from_sub_path=true
+
+[auth]
+disable_login_form=true
+disable_signout_menu=true
+
+[auth.anonymous]
+enabled=true
+org_name=Main Org.
+org_role=Viewer
+
+[users]
+default_theme=light
+viewers_can_edit=true
+
+[security]
+allow_embedding=true

all-in-one/loki/config.yaml

@@ -0,0 +1,49 @@
+auth_enabled: false
+common:
+  compactor_address: 'loki'
+  path_prefix: /var/loki
+  replication_factor: 1
+  storage:
+    filesystem:
+      chunks_directory: /var/loki/chunks
+      rules_directory: /var/loki/rules
+frontend:
+  scheduler_address: ""
+frontend_worker:
+  scheduler_address: ""
+index_gateway:
+  mode: simple
+limits_config:
+  max_cache_freshness_per_query: 10m
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  split_queries_by_interval: 15m
+memberlist:
+  join_members:
+  - localhost
+query_range:
+  align_queries_with_step: true
+ruler:
+  storage:
+    type: local
+runtime_config:
+  file: /etc/loki/runtime-config.yaml
+schema_config:
+  configs:
+  - from: "2022-01-11"
+    index:
+      period: 24h
+      prefix: loki_index_
+    object_store: filesystem
+    schema: v12
+    store: boltdb-shipper
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+storage_config:
+  hedging:
+    at: 250ms
+    max_per_second: 20
+    up_to: 3
+tracing:
+  enabled: false

all-in-one/loki/runtime-config.yaml

@@ -0,0 +1 @@
+{}

all-in-one/meshConfig/mesh

@@ -16,6 +16,9 @@ dnsRefreshRate: 200s
 enableAutoMtls: false
 enablePrometheusMerge: true
 ingressControllerMode: "OFF"
+mseIngressGlobalConfig:
+  enableH3: false
+  enableProxyProtocol: false
 protocolDetectionTimeout: 100ms
 rootNamespace: higress-system
 trustDomain: cluster.local

all-in-one/prometheus/prometheus.yaml

@@ -0,0 +1,17 @@
+global:
+  scrape_interval:     15s 
+  evaluation_interval: 15s
+scrape_configs:
+  - job_name: 'prometheus'
+    metrics_path: /prometheus/metrics
+    static_configs:
+    - targets: ['localhost:9090']
+  - job_name: 'gateway'
+    metrics_path: /stats/prometheus
+    static_configs:
+    - targets: ['localhost:15020']
+      labels:
+        container: 'higress-gateway'
+        namespace: 'higress-system'
+        higress: 'higress-system-higress-gateway'
+        pod: 'higress'

all-in-one/promtail/promtail.yaml

@@ -0,0 +1,54 @@
+server:
+  log_level: info
+  http_listen_port: 3101
+  grpc_listen_port: 9096
+
+clients:
+- url: http://localhost:3100/loki/api/v1/push
+
+positions:
+  filename: /var/promtail/promtail-positions.yaml
+target_config:
+  sync_period: 10s
+scrape_configs:
+- job_name: access-logs
+  static_configs:
+  - targets:
+    - localhost
+    labels:
+      __path__: /var/log/proxy/access.log
+  pipeline_stages:
+  - json:
+      expressions:
+        authority:
+        method:
+        path:
+        protocol:
+        request_id:
+        response_code:
+        response_flags:
+        route_name:
+        trace_id:
+        upstream_cluster:
+        upstream_host:
+        upstream_transport_failure_reason:
+        user_agent:
+        x_forwarded_for:
+  - labels:
+      authority:
+      method:
+      path:
+      protocol:
+      request_id:
+      response_code:
+      response_flags:
+      route_name:
+      trace_id:
+      upstream_cluster:
+      upstream_host:
+      upstream_transport_failure_reason:
+      user_agent:
+      x_forwarded_for:
+  - timestamp:
+      source: timestamp
+      format: RFC3339Nano

all-in-one/scripts/base.sh

@@ -27,7 +27,6 @@ function readinessCheck() {
 
 function createDir() {
     sudo mkdir -p "$1"
-    sudo chown 1337:1337 "$1"
 }
 
 case $MODE in
@@ -39,9 +38,19 @@ case $MODE in
         MODE=full
         ;;
 esac
-
 echo "Mode=$MODE"
 
+case $O11Y in
+    true|TRUE|on|ON|yes|YES)
+        O11Y=on
+        ;;
+    *)
+        # Default to full mode
+        O11Y=off
+        ;;
+esac
+echo "O11Y=$O11Y"
+
 CONSOLE_USED_MARKER='/data/.console-used'
 CONSOLE_USED='false'
 if [ -f "$CONSOLE_USED_MARKER" ]; then

all-in-one/scripts/config-template/ai-gateway.sh

@@ -21,7 +21,7 @@ function initializeLlmProviderConfigs() {
   initializeLlmProviderConfig yi yi YI api.lingyiwanwu.com
   initializeLlmProviderConfig deepseek deepseek DEEPSEEK api.deepseek.com
   initializeLlmProviderConfig zhipuai zhipuai ZHIPUAI open.bigmodel.cn
-  initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com
+  # initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com
   # initializeLlmProviderConfig hunyuan hunyuan HUNYUAN hunyuan.tencentcloudapi.com 443 "https" "" "${EXTRA_CONFIGS[@]}"
   initializeLlmProviderConfig stepfun stepfun STEPFUN api.stepfun.com
   # initializeLlmProviderConfig cloudflare cloudflare CLOUDFLARE api.cloudflare.com 443 "https" "" "${EXTRA_CONFIGS[@]}"

all-in-one/scripts/start-console.sh

@@ -24,5 +24,12 @@ touch "$CONSOLE_USED_MARKER"
 
 set -e
 
-HIGRESS_CONSOLE_KUBE_CONFIG="/app/kubeconfig" SERVER_PORT="$CONSOLE_PORT" \
+if [ "$O11Y" == "on" ]; then
+    export HIGRESS_CONSOLE_DASHBOARD_BASE_URL="http://localhost:3000/grafana"
+    export HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_PROM_URL="http://localhost:9090/prometheus"
+    export HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_LOKI_URL="http://localhost:3100"
+fi
+
+HIGRESS_CONSOLE_KUBE_CONFIG="/app/kubeconfig" \
+    SERVER_PORT="$CONSOLE_PORT" \
     bash /app/start.sh

all-in-one/scripts/start-gateway.sh

@@ -17,8 +17,16 @@ set -e
 
 createDir /etc/istio/proxy
 createDir /var/lib/istio/data
+createDir /var/log/proxy
+touch /var/log/proxy/access.log
 
-/usr/local/bin/pilot-agent proxy router \
+if [ "$O11Y" == "on" ]; then
+    sed -i -E 's/^accessLogFile: .+$/accessLogFile: \/var\/log\/proxy\/access.log/' /etc/istio/config/mesh
+else
+    sed -i -E 's/^accessLogFile: .+$/accessLogFile: \/dev\/stdout/' /etc/istio/config/mesh
+fi
+
+/usr/local/bin/higress-proxy-start.sh proxy router \
     --domain=higress-system.svc.cluster.local \
     --proxyLogLevel=${GATEWAY_LOG_LEVEL:-warning} \
     --proxyComponentLogLevel=${GATEWAY_COMPONENT_LOG_LEVEL:-misc:error} \

all-in-one/scripts/start-grafana.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+cd "$(dirname -- "$0")"
+ROOT=$(pwd)
+cd - >/dev/null
+source $ROOT/base.sh
+
+if [ "$O11Y" != "on" ]; then
+    echo "Grafana won't run when o11y is not turned on."
+    sleep 2
+    exit 0
+fi
+
+set -e
+
+createDir /var/lib/grafana
+createDir /var/log/grafana
+
+GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \
+    GF_PATHS_DATA="/var/lib/grafana" \
+    GF_PATHS_HOME="/usr/share/grafana" \
+    GF_PATHS_LOGS="/var/log/grafana" \
+    GF_PATHS_PLUGINS="/var/lib/grafana/plugins" \
+    GF_PATHS_PROVISIONING="/etc/grafana/provisioning" \
+    PATH="/usr/share/grafana/bin:$PATH" \
+    bash /usr/local/bin/grafana.sh

all-in-one/scripts/start-loki.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+cd "$(dirname -- "$0")"
+ROOT=$(pwd)
+cd - >/dev/null
+source $ROOT/base.sh
+
+if [ "$O11Y" != "on" ]; then
+    echo "Loki won't run when o11y is not turned on."
+    sleep 2
+    exit 0
+fi
+
+set -e
+
+createDir /var/loki/chunks
+createDir /var/loki/rules
+
+/usr/local/bin/loki \
+    -config.file=/etc/loki/config.yaml \
+    -target=all

all-in-one/scripts/start-pilot.sh

@@ -95,4 +95,6 @@ initCerts
     --log_output_level=default:info \
     --domain=cluster.local \
     --keepaliveMaxServerConnectionAge=30m \
-    --caCertFile=/etc/certs/ca-cert.pem
+    --caCertFile=/etc/certs/ca-cert.pem \
+    --meshConfig=/etc/istio/config/mesh \
+    --networksConfig=/etc/istio/config/meshNetworks

all-in-one/scripts/start-prometheus.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+cd "$(dirname -- "$0")"
+ROOT=$(pwd)
+cd - >/dev/null
+source $ROOT/base.sh
+
+if [ "$O11Y" != "on" ]; then
+    echo "Prometheus won't run when o11y is not turned on."
+    sleep 2
+    exit 0
+fi
+
+set -e
+
+createDir /prometheus
+
+/usr/local/bin/prometheus \
+      --config.file=/etc/prometheus/prometheus.yaml \
+      --web.external-url=/prometheus \
+      --storage.tsdb.path=/prometheus \
+      --storage.tsdb.retention=6h

all-in-one/scripts/start-promtail.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cd "$(dirname -- "$0")"
+ROOT=$(pwd)
+cd - >/dev/null
+source $ROOT/base.sh
+
+if [ "$O11Y" != "on" ]; then
+    echo "Promtail won't run when o11y is not turned on."
+    sleep 2
+    exit 0
+fi
+
+set -e
+
+createDir /var/promtail
+
+/usr/local/bin/promtail -config.file=/etc/promtail/promtail.yaml

all-in-one/supervisord/supervisord.conf

@@ -7,6 +7,7 @@ loglevel=info
 pidfile=/var/log/higress/supervisord.pid
 
 [program:apiserver]
+directory=/
 command=bash /usr/local/bin/start-apiserver.sh
 priority=0
 autostart=true
@@ -18,6 +19,7 @@ redirect_stderr=true
 environment=
 
 [program:controller]
+directory=/
 command=bash /usr/local/bin/start-controller.sh
 priority=100
 autostart=true
@@ -26,9 +28,10 @@ startsecs=1
 stdout_logfile=/var/log/higress/controller.log
 stdout_logfile_maxbytes=10MB
 redirect_stderr=true
-environment=CONTROLLER_KEEP_XDS_CONFIG_LABELS="false",CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS="false"
+environment=CONTROLLER_KEEP_XDS_CONFIG_LABELS="false",CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS="false",PILOT_ENABLE_GATEWAY_API="false",PILOT_ENABLE_ALPHA_GATEWAY_API="false"
 
 [program:pilot]
+directory=/
 command=bash /usr/local/bin/start-pilot.sh
 priority=200
 autostart=true
@@ -37,9 +40,10 @@ startsecs=1
 stdout_logfile=/var/log/higress/pilot.log
 stdout_logfile_maxbytes=10MB
 redirect_stderr=true
-environment=REVISION="default",JWT_POLICY="none",POD_NAME="higress-pilot",POD_NAMESPACE="higress-system",ROOT_CA_DIR="/etc/certs",PILOT_CERT_PROVIDER="istiod",KUBECONFIG="/app/kubeconfig",PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY="false",PILOT_ENABLE_METADATA_EXCHANGE="false",PILOT_SCOPE_GATEWAY_TO_NAMESPACE="true",VALIDATION_ENABLED="false",PILOT_TRACE_SAMPLING="1",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND="true",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND="true",PILOT_ENABLE_ANALYSIS="false",PILOT_ENABLE_GATEWAY_API="true",PILOT_ENABLE_GATEWAY_API_STATUS="false",PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER="false",CLUSTER_ID="Kubernetes",CUSTOM_CA_CERT_NAME="higress-ca-root-cert",DEBUG_AUTH="false"
+environment=REVISION="default",JWT_POLICY="none",POD_NAME="higress-pilot",POD_NAMESPACE="higress-system",ROOT_CA_DIR="/etc/certs",PILOT_CERT_PROVIDER="istiod",KUBECONFIG="/app/kubeconfig",PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY="false",PILOT_ENABLE_METADATA_EXCHANGE="false",PILOT_SCOPE_GATEWAY_TO_NAMESPACE="true",VALIDATION_ENABLED="false",PILOT_TRACE_SAMPLING="1",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND="true",PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND="true",PILOT_ENABLE_ANALYSIS="false",PILOT_ENABLE_GATEWAY_API="false",PILOT_ENABLE_ALPHA_GATEWAY_API="false",PILOT_ENABLE_GATEWAY_API_STATUS="false",PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER="false",PILOT_ENABLE_ALPN_FILTER="true",VALIDATION_WEBHOOK_CONFIG_NAME="",ISTIO_DUAL_STACK="false",ENABLE_OPTIMIZED_CONFIG_REBUILD="false",PILOT_ENABLE_K8S_SELECT_WORKLOAD_ENTRIES="false",DEFAULT_UPSTREAM_CONCURRENCY_THRESHOLD="10000",ISTIO_GPRC_MAXRECVMSGSIZE="104857600",ENBALE_SCOPED_RDS="true",ON_DEMAND_RDS="false",HOST_RDS_MERGE_SUBSET="false",PILOT_FILTER_GATEWAY_CLUSTER_CONFIG="true",PRIORITIZED_LEADER_ELECTION="false",INJECT_ENABLED="false",CLUSTER_ID="Kubernetes",CUSTOM_CA_CERT_NAME="higress-ca-root-cert",DEBUG_AUTH="false"
 
 [program:gateway]
+directory=/
 command=bash /usr/local/bin/start-gateway.sh
 priority=300
 autostart=true
@@ -48,9 +52,10 @@ startsecs=1
 stdout_logfile=/var/log/higress/gateway.log
 stdout_logfile_maxbytes=10MB
 redirect_stderr=true
-environment=JWT_POLICY="none",CA_ROOT_CA="/etc/certs/root-cert.pem",XDS_ROOT_CA="/etc/certs/root-cert.pem",PROV_CERT="/etc/certs",PILOT_XDS_SEND_TIMEOUT="60s",PROXY_XDS_VIA_AGENT="true",ENABLE_INGRESS_GATEWAY_SDS="false",POD_NAME="higress-gateway",POD_NAMESPACE="higress-system",ISTIO_META_CLUSTER_ID="Kubernetes",ISTIO_META_HTTP10="1",INSTANCE_NAME="higress-gateway"
+environment=JWT_POLICY="none",CA_ROOT_CA="/etc/certs/root-cert.pem",XDS_ROOT_CA="/etc/certs/root-cert.pem",PROV_CERT="/etc/certs",PILOT_XDS_SEND_TIMEOUT="60s",PROXY_XDS_VIA_AGENT="true",ENABLE_INGRESS_GATEWAY_SDS="false",POD_NAME="higress-gateway",POD_NAMESPACE="higress-system",ISTIO_META_CLUSTER_ID="Kubernetes",ISTIO_META_HTTP10="1",INSTANCE_NAME="higress-gateway",LITE_METRICS="on"
 
 [program:console]
+directory=/
 command=bash /usr/local/bin/start-console.sh
 priority=900
 autostart=true
@@ -59,4 +64,52 @@ startsecs=1
 stdout_logfile=/var/log/higress/console.log
 stdout_logfile_maxbytes=10MB
 redirect_stderr=true
+environment=
+
+[program:prometheus]
+directory=/
+command=bash /usr/local/bin/start-prometheus.sh
+priority=2000
+autostart=true
+autorestart=unexpected
+startsecs=1
+stdout_logfile=/var/log/higress/prometheus.log
+stdout_logfile_maxbytes=10MB
+redirect_stderr=true
+environment=
+
+[program:promtail]
+directory=/
+command=bash /usr/local/bin/start-promtail.sh
+priority=2100
+autostart=true
+autorestart=unexpected
+startsecs=1
+stdout_logfile=/var/log/higress/promtail.log
+stdout_logfile_maxbytes=10MB
+redirect_stderr=true
+environment=HOSTNAME="higress-gateway"
+
+[program:loki]
+directory=/
+command=bash /usr/local/bin/start-loki.sh
+priority=2200
+autostart=true
+autorestart=unexpected
+startsecs=1
+stdout_logfile=/var/log/higress/loki.log
+stdout_logfile_maxbytes=10MB
+redirect_stderr=true
+environment=
+
+[program:grafana]
+directory=/
+command=bash /usr/local/bin/start-grafana.sh
+priority=3000
+autostart=true
+autorestart=unexpected
+startsecs=1
+stdout_logfile=/var/log/higress/grafana.log
+stdout_logfile_maxbytes=10MB
+redirect_stderr=true
 environment=

bin/configure.sh

@@ -614,6 +614,10 @@ HIGRESS_CONTROLLER_TAG='${HIGRESS_CONTROLLER_TAG}'
 HIGRESS_PILOT_TAG='${HIGRESS_PILOT_TAG}'
 HIGRESS_GATEWAY_TAG='${HIGRESS_GATEWAY_TAG}'
 HIGRESS_CONSOLE_TAG='${HIGRESS_CONSOLE_TAG}'
+PROMETHEUS_TAG='${PROMETHEUS_TAG}'
+PROMTAIL_TAG='${PROMTAIL_TAG}'
+LOKI_TAG='${LOKI_TAG}'
+GRAFANA_TAG='${GRAFANA_TAG}'
 NACOS_HTTP_PORT='${NACOS_HTTP_PORT}'
 NACOS_GRPC_PORT='${NACOS_GRPC_PORT}'
 GATEWAY_HTTP_PORT='${GATEWAY_HTTP_PORT}'

compose/.env

@@ -8,11 +8,15 @@ NACOS_PASSWORD=''
 NACOS_DATA_ENC_KEY=''
 NACOS_SERVER_TAG='v2.2.3'
 HIGRESS_RUNNER_TAG='0.0.3'
-HIGRESS_API_SERVER_TAG='0.0.16'
+HIGRESS_API_SERVER_TAG='0.0.18'
 HIGRESS_CONTROLLER_TAG='2.0.4'
 HIGRESS_PILOT_TAG='2.0.4'
 HIGRESS_GATEWAY_TAG='2.0.4'
 HIGRESS_CONSOLE_TAG='1.4.6'
+PROMETHEUS_TAG='v2.40.7'
+PROMTAIL_TAG='2.9.4'
+LOKI_TAG='2.9.4'
+GRAFANA_TAG='9.3.6'
 NACOS_HTTP_PORT='8848'
 NACOS_GRPC_PORT='9848'
 GATEWAY_HTTP_PORT='80'

compose/docker-compose.yml

@@ -203,6 +203,7 @@ services:
       - ./volumes/gateway/certs:/etc/certs:ro
       - ./volumes/gateway/podinfo:/etc/istio/pod:ro
       - ./volumes/gateway/istio/data:/var/lib/istio/data:rw
+      - ./volumes/gateway/log:/var/log/proxy:rw
 
   console:
     image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/console:${HIGRESS_CONSOLE_TAG}
@@ -230,6 +231,126 @@ services:
     volumes:
       - ./volumes/kube:/home/higress/.kube:ro
 
+  prometheus:
+    image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/prometheus:${PROMETHEUS_TAG}
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yaml"
+      - "--web.external-url=/prometheus"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention=6h"
+    depends_on:
+      gateway:
+        condition: service_healthy
+    networks:
+      - higress-net
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "wget",
+          "--no-verbose",
+          "--tries=1",
+          "--spider",
+          "http://127.0.0.1:9090/prometheus/-/healthy/"
+        ]
+      interval: 1s
+      timeout: 3s
+      retries: 60
+    restart: always
+    volumes:
+      - ./volumes/prometheus/config:/etc/prometheus:ro
+      - ./volumes/prometheus/data:/prometheus:rw
+
+  promtail:
+    image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/promtail:${PROMTAIL_TAG}
+    command:
+      - -config.file=/etc/promtail/promtail.yaml
+    env_file:
+      - ./env/promtail.env
+    depends_on:
+      gateway:
+        condition: service_healthy
+      loki:
+        condition: service_healthy
+    networks:
+      - higress-net
+    ports:
+      - "3101:3101/tcp"
+    healthcheck:
+      disable: true
+      # There is no curl or wget in the promtail image. We need to find another way to perform health check.
+      # test:
+      #   [
+      #     "CMD",
+      #     "curl",
+      #     "-fk",
+      #     "http://127.0.0.1:3101/ready"
+      #   ]
+      # interval: 1s
+      # timeout: 3s
+      # retries: 60
+    restart: always
+    volumes:
+      - ./volumes/promtail/config:/etc/promtail:ro
+      - ./volumes/promtail/data:/var/promtail:rw
+      - ./volumes/gateway/log:/var/log/proxy:ro
+
+  loki:
+    image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/loki:${LOKI_TAG}
+    command:
+      - -config.file=/etc/loki/config/config.yaml
+      - -target=all
+    networks:
+      - higress-net
+    ports:
+      - "3100:3100/tcp"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "wget",
+          "--no-verbose",
+          "--tries=1",
+          "--spider",
+          "http://127.0.0.1:3100/ready"
+        ]
+      interval: 1s
+      timeout: 3s
+      retries: 60
+    restart: always
+    volumes:
+      - ./volumes/loki/config:/etc/loki/config:ro
+      - ./volumes/loki/data:/var/loki:rw
+
+  grafana:
+    image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/grafana:${GRAFANA_TAG}
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      loki:
+        condition: service_healthy
+    networks:
+      - higress-net
+    ports:
+      - "3000:3000/tcp" 
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "wget",
+          "--no-verbose",
+          "--tries=1",
+          "--spider",
+          "http://127.0.0.1:3000/api/health"
+        ]
+      interval: 1s
+      timeout: 3s
+      retries: 60
+    restart: always
+    volumes:
+      - ./volumes/grafana/config:/etc/grafana:ro
+      - ./volumes/grafana/lib:/var/lib/grafana:rw
+
   postcheck:
     image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/runner:${HIGRESS_RUNNER_TAG}
     command:
@@ -247,5 +368,14 @@ services:
         condition: service_healthy
       console:
         condition: service_healthy
+      # promtail doesn't support health check at the moment.
+      # promtail:
+      #   condition: service_healthy
+      prometheus:
+        condition: service_healthy
+      loki:
+        condition: service_healthy
+      grafana:
+        condition: service_healthy
     volumes:
       - ./scripts:/workspace:ro

compose/env/console.env

@@ -2,4 +2,7 @@ JAVA_HOME=/usr/local/openjdk-18
 LANG=C.UTF-8
 HIGRESS_CONSOLE_KUBE_CONFIG=/home/higress/.kube/config
 HIGRESS_CONSOLE_CONTROLLER_SERVICE_HOST=pilot
-HIGRESS_CONSOLE_CONTROLLER_ACCESS_TOKEN=
+HIGRESS_CONSOLE_CONTROLLER_ACCESS_TOKEN=
+HIGRESS_CONSOLE_DASHBOARD_BASE_URL=http://grafana:3000/grafana
+HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_PROM_URL=http://prometheus:9090/prometheus
+HIGRESS_CONSOLE_DASHBOARD_DATASOURCE_LOKI_URL=http://loki:3100

compose/env/controller.env

@@ -1,2 +1,4 @@
 CONTROLLER_KEEP_XDS_CONFIG_LABELS=false
-CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS=false
+CONTROLLER_KEEP_XDS_CONFIG_ANNOTATIONS=false
+PILOT_ENABLE_GATEWAY_API=false
+PILOT_ENABLE_ALPHA_GATEWAY_API=false

compose/env/gateway.env

@@ -9,4 +9,5 @@ POD_NAME=higress-gateway
 POD_NAMESPACE=higress-system
 ISTIO_META_CLUSTER_ID=Kubernetes
 ISTIO_META_HTTP10=1
-INSTANCE_NAME=higress-gateway
+INSTANCE_NAME=higress-gateway
+LITE_METRICS=on

compose/env/pilot.env

@@ -1,22 +1,36 @@
 HIGRESS_CONTROLLER_SVC=controller
-HIGRESS_CONTROLLER_PORT="15051"
+HIGRESS_CONTROLLER_PORT=15051
 REVISION=default
 JWT_POLICY=none
 POD_NAME=higress-pilot
 POD_NAMESPACE=higress-system
 PILOT_CERT_PROVIDER=istiod
 KUBECONFIG=/var/run/secrets/remote/config
-PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY="false"
-PILOT_ENABLE_METADATA_EXCHANGE="false"
-PILOT_SCOPE_GATEWAY_TO_NAMESPACE="true"
-VALIDATION_ENABLED="false"
-PILOT_TRACE_SAMPLING="1"
-PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND="true"
-PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND="true"
-PILOT_ENABLE_ANALYSIS="false"
-PILOT_ENABLE_GATEWAY_API=true
+PILOT_ENABLE_CROSS_CLUSTER_WORKLOAD_ENTRY=false
+PILOT_ENABLE_METADATA_EXCHANGE=false
+PILOT_SCOPE_GATEWAY_TO_NAMESPACE=true
+VALIDATION_ENABLED=false
+PILOT_TRACE_SAMPLING=1
+PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND=true
+PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND=true
+PILOT_ENABLE_ANALYSIS=false
+PILOT_ENABLE_GATEWAY_API=false
+PILOT_ENABLE_ALPHA_GATEWAY_API=false
 PILOT_ENABLE_GATEWAY_API_STATUS=false
 PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER=false
+PILOT_ENABLE_ALPN_FILTER=true
+VALIDATION_WEBHOOK_CONFIG_NAME=
+ISTIO_DUAL_STACK=false
+ENABLE_OPTIMIZED_CONFIG_REBUILD=false
+PILOT_ENABLE_K8S_SELECT_WORKLOAD_ENTRIES=false
+DEFAULT_UPSTREAM_CONCURRENCY_THRESHOLD=10000
+ISTIO_GPRC_MAXRECVMSGSIZE=104857600
+ENBALE_SCOPED_RDS=true
+ON_DEMAND_RDS=false
+HOST_RDS_MERGE_SUBSET=false
+PILOT_FILTER_GATEWAY_CLUSTER_CONFIG=true
+PRIORITIZED_LEADER_ELECTION=false
+INJECT_ENABLED=false
 CLUSTER_ID=Kubernetes
 CUSTOM_CA_CERT_NAME=higress-ca-root-cert
 DEBUG_AUTH=false

compose/env/promtail.env

@@ -0,0 +1 @@
+HOSTNAME=higress-gateway

compose/scripts/init.sh

@@ -299,10 +299,208 @@ higress="higress-system-higress-gateway"
 EOF
 
   mkdir -p $VOLUMES_ROOT/gateway/istio/data
+
+  mkdir -p $VOLUMES_ROOT/gateway/log
+  touch $VOLUMES_ROOT/gateway/log/access.log
+}
+
+initializePrometheus() {
+  echo "Initializing Prometheus configurations..."
+
+  mkdir -p $VOLUMES_ROOT/prometheus/config && cd "$_"
+  cat <<EOF >./prometheus.yaml
+global:
+  scrape_interval:     15s 
+  evaluation_interval: 15s
+scrape_configs:
+  - job_name: 'prometheus'
+    metrics_path: /prometheus/metrics
+    static_configs:
+    - targets: ['localhost:9090']
+  - job_name: 'gateway'
+    metrics_path: /stats/prometheus
+    static_configs:
+    - targets: ['gateway:15020']
+      labels:
+        container: 'higress-gateway'
+        namespace: 'higress-system'
+        higress: 'higress-system-higress-gateway'
+        pod: 'higress'
+EOF
+
+  mkdir -p $VOLUMES_ROOT/prometheus/data
+  chmod a+rwx $VOLUMES_ROOT/prometheus/data
+}
+
+initializePromtail() {
+  echo "Initializing Promtail configurations..."
+
+  mkdir -p $VOLUMES_ROOT/promtail/config && cd "$_"
+  cat <<EOF >./promtail.yaml
+server:
+  log_level: info
+  http_listen_port: 3101
+
+clients:
+- url: http://loki:3100/loki/api/v1/push
+
+positions:
+  filename: /var/promtail/promtail-positions.yaml
+target_config:
+  sync_period: 10s
+scrape_configs:
+- job_name: access-logs
+  static_configs:
+  - targets:
+    - localhost
+    labels:
+      __path__: /var/log/proxy/access.log
+  pipeline_stages:
+  - json:
+      expressions:
+        authority:
+        method:
+        path:
+        protocol:
+        request_id:
+        response_code:
+        response_flags:
+        route_name:
+        trace_id:
+        upstream_cluster:
+        upstream_host:
+        upstream_transport_failure_reason:
+        user_agent:
+        x_forwarded_for:
+  - labels:
+      authority:
+      method:
+      path:
+      protocol:
+      request_id:
+      response_code:
+      response_flags:
+      route_name:
+      trace_id:
+      upstream_cluster:
+      upstream_host:
+      upstream_transport_failure_reason:
+      user_agent:
+      x_forwarded_for:
+  - timestamp:
+      source: timestamp
+      format: RFC3339Nano
+EOF
+
+  mkdir -p $VOLUMES_ROOT/promtail/data
+  chmod a+rwx $VOLUMES_ROOT/promtail/data
+}
+
+initializeLoki() {
+  echo "Initializing Loki configurations..."
+
+  mkdir -p $VOLUMES_ROOT/loki/config && cd "$_"
+  cat <<EOF >./config.yaml
+auth_enabled: false
+common:
+  compactor_address: 'loki'
+  path_prefix: /var/loki
+  replication_factor: 1
+  storage:
+    filesystem:
+      chunks_directory: /var/loki/chunks
+      rules_directory: /var/loki/rules
+frontend:
+  scheduler_address: ""
+frontend_worker:
+  scheduler_address: ""
+index_gateway:
+  mode: ring
+limits_config:
+  max_cache_freshness_per_query: 10m
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  split_queries_by_interval: 15m
+memberlist:
+  join_members:
+  - loki
+query_range:
+  align_queries_with_step: true
+ruler:
+  storage:
+    type: local
+runtime_config:
+  file: /etc/loki/config/runtime-config.yaml
+schema_config:
+  configs:
+  - from: "2022-01-11"
+    index:
+      period: 24h
+      prefix: loki_index_
+    object_store: filesystem
+    schema: v12
+    store: boltdb-shipper
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+storage_config:
+  hedging:
+    at: 250ms
+    max_per_second: 20
+    up_to: 3
+tracing:
+  enabled: false
+EOF
+  cat <<EOF >./runtime-config.yaml
+{}
+EOF
+
+  mkdir -p $VOLUMES_ROOT/loki/data/
+  chmod a+rwx $VOLUMES_ROOT/loki/data/
+}
+
+initializeGrafana() {
+  echo "Initializing Grafana configurations..."
+
+  mkdir -p $VOLUMES_ROOT/grafana/config && cd "$_"
+  cat <<EOF >./grafana.ini
+[server]
+protocol=http
+domain=localhost
+root_url="%(protocol)s://%(domain)s/grafana"
+serve_from_sub_path=true
+
+[auth]
+disable_login_form=true
+disable_signout_menu=true
+
+[auth.anonymous]
+enabled=true
+org_name=Main Org.
+org_role=Viewer
+
+[users]
+default_theme=light
+viewers_can_edit=true
+
+[security]
+allow_embedding=true
+EOF
+
+  mkdir -p $VOLUMES_ROOT/grafana/lib
+  chmod a+rwx $VOLUMES_ROOT/grafana/lib
+}
+
+initializeO11y() {
+  initializePrometheus
+  initializePromtail
+  initializeLoki
+  initializeGrafana
 }
 
 initializeConfigStorage
 initializeApiServer
 initializeController
 initializePilot
 initializeGateway
+initializeO11y

compose/scripts/prepare.sh

@@ -33,7 +33,7 @@ checkConfigExists() {
   if [[ "$configGroupVersion" == *"/"* ]]; then
     uriPrefix="/apis"
   fi
-  local url;
+  local url
   if [ -z "$namespace" ]; then
     url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}/${configName}"
   else
@@ -65,7 +65,7 @@ getConfig() {
   if [[ "$configGroupVersion" == *"/"* ]]; then
     uriPrefix="/apis"
   fi
-  local url;
+  local url
   if [ -z "$namespace" ]; then
     url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}/${configName}"
   else
@@ -102,7 +102,7 @@ publishConfig() {
   if [[ "$configGroupVersion" == *"/"* ]]; then
     uriPrefix="/apis"
   fi
-  local url;
+  local url
   if [ -z "$namespace" ]; then
     url="${API_SERVER_BASE_URL}${uriPrefix}/${configGroupVersion}/${configType}"
   else
@@ -226,9 +226,22 @@ metadata:
   name: higress-config
   namespace: higress-system
 data:
+  higress: |-
+    downstream:
+      connectionBufferLimits: 32768
+      http2:
+        initialConnectionWindowSize: 1048576
+        initialStreamWindowSize: 65535
+        maxConcurrentStreams: 100
+      idleTimeout: 180
+      maxRequestHeadersKb: 60
+      routeTimeout: 0
+    upstream:
+      connectionBufferLimits: 10485760
+      idleTimeout: 10
   mesh: |-
     accessLogEncoding: TEXT
-    accessLogFile: /dev/stdout
+    accessLogFile: /var/log/proxy/access.log
     accessLogFormat: |
       {"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"}
     configSources:
@@ -244,6 +257,9 @@ data:
     enableAutoMtls: false
     enablePrometheusMerge: true
     ingressControllerMode: "OFF"
+    mseIngressGlobalConfig:
+      enableH3: false
+      enableProxyProtocol: false
     protocolDetectionTimeout: 100ms
     rootNamespace: higress-system
     trustDomain: cluster.local
@@ -396,8 +412,72 @@ EOF
   fi
 }
 
+checkPrometheus() {
+  echo "Checking Prometheus configurations..."
+
+  if [ ! -d "$VOLUMES_ROOT/prometheus/config/" ]; then
+    echo "  The config folder of Prometheus is missing."
+    exit -1
+  fi
+  if [ ! -f "$VOLUMES_ROOT/prometheus/config/prometheus.yaml" ] ; then
+    echo "  Prometheus config file prometheus.yaml is missing."
+    exit -1
+  fi
+}
+
+checkPromtail() {
+  echo "Checking Promtail configurations..."
+
+  if [ ! -d "$VOLUMES_ROOT/promtail/config/" ]; then
+    echo "  The config folder of Promtail is missing."
+    exit -1
+  fi
+  if [ ! -f "$VOLUMES_ROOT/promtail/config/promtail.yaml" ] ; then
+    echo "  Promtail config file promtail.yaml is missing."
+    exit -1
+  fi
+}
+
+checkLoki() {
+  echo "Checking Loki configurations..."
+
+  if [ ! -d "$VOLUMES_ROOT/loki/config/" ]; then
+    echo "  The config folder of Loki is missing."
+    exit -1
+  fi
+  if [ ! -f "$VOLUMES_ROOT/loki/config/config.yaml" ] ; then
+    echo "  Loki config file config.yaml is missing."
+    exit -1
+  fi
+  if [ ! -f "$VOLUMES_ROOT/loki/config/runtime-config.yaml" ] ; then
+    echo "  Loki config file runtime-config.yaml is missing."
+    exit -1
+  fi
+}
+
+checkGrafana() {
+  echo "Checking Grafana configurations..."
+
+  if [ ! -d "$VOLUMES_ROOT/grafana/config/" ]; then
+    echo "  The config folder of Grafana is missing."
+    exit -1
+  fi
+  if [ ! -f "$VOLUMES_ROOT/grafana/config/grafana.ini" ] ; then
+    echo "  Grafana config file grafana.ini is missing."
+    exit -1
+  fi
+}
+
+checkO11y() {
+  checkPrometheus
+  checkPromtail
+  checkLoki
+  checkGrafana
+}
+
 checkStorage
 checkPilot
 checkGateway
 checkConsole
 checkGatewayApi
+checkO11y

src/apiserver/.gitignore

@@ -22,6 +22,7 @@ vendor/
 
 # editor and IDE paraphernalia
 .idea
+.run
 *.swp
 *.swo
 *~