feat: use ebpf for redirecting client connections to the emulator

Signed-off-by: Matheus Pimenta <[email protected]>

Author: matheuscscp

.dockerignore

@@ -1,2 +1,4 @@
 **/*_test.go
 internal/testing/
+internal/redirect/redirect_bpf*
+ebpf/vmlinux.h

.gitattributes

@@ -1,4 +1,5 @@
 timoni/gke-metadata-server/cue.mod/** linguist-vendored
+ebpf/vmlinux.h linguist-vendored
 **/*.yml linguist-detectable=true
 **/*.yml linguist-language=YAML
 **/*.yaml linguist-detectable=true

.gitignore

@@ -6,3 +6,4 @@ terraform.*
 *.tgz
 Chart.yaml
 config.cue
+internal/redirect/redirect_bpf*

Dockerfile

@@ -22,13 +22,19 @@
 
 FROM golang:1.23.5-alpine3.21 AS builder
 
+RUN apk add --no-cache clang llvm bpftool libbpf-dev
+
 WORKDIR /app
 
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./internal/ ./internal/
+COPY ./ebpf/ ./ebpf/
+
+RUN bpftool btf dump file /sys/kernel/btf/vmlinux format c > ebpf/vmlinux.h
+RUN go generate ./internal/redirect
 
 # CGO_ENABLED=0 to build a statically-linked binary
 # -ldflags '-w -s' to strip debugging information for smaller size

Dockerfile.test

@@ -22,12 +22,18 @@
 
 FROM golang:1.23.5-alpine3.21
 
+RUN apk add --no-cache clang llvm bpftool libbpf-dev
+
 WORKDIR /app
 
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./internal/ ./internal/
+COPY ./ebpf/ ./ebpf/
+
+RUN bpftool btf dump file /sys/kernel/btf/vmlinux format c > ebpf/vmlinux.h
+RUN go generate ./internal/redirect
 
 CMD [ "go", "test", "-v", "./..." ]

Makefile

@@ -25,7 +25,7 @@ SHELL := /bin/bash
 TEST_IMAGE := ghcr.io/matheuscscp/gke-metadata-server/test
 
 .PHONY: dev
-dev: tidy dev-cluster build dev-test
+dev: tidy gen-ebpf dev-cluster build dev-test
 
 .PHONY: clean
 clean:
@@ -42,14 +42,19 @@ tidy:
 	git status
 
 .PHONY: gen
-gen: timoni-gen
+gen: gen-timoni
 
-.PHONY: timoni-gen
-timoni-gen:
+.PHONY: gen-timoni
+gen-timoni:
 	cd timoni/gke-metadata-server; cue get go k8s.io/api/rbac/v1
 	cd timoni/gke-metadata-server; cue get go k8s.io/api/admissionregistration/v1
 	cd timoni/gke-metadata-server; cue get go github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1
 
+.PHONY: gen-ebpf
+gen-ebpf:
+	bpftool btf dump file /sys/kernel/btf/vmlinux format c > ebpf/vmlinux.h
+	go generate ./internal/redirect
+
 .PHONY: dev-cluster
 dev-cluster:
 	kind delete cluster -n gke-metadata-server || true

cmd/server.go

@@ -24,8 +24,8 @@ package main
 
 import (
 	"fmt"
+	"net/netip"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/matheuscscp/gke-metadata-server/internal/googlecredentials"
@@ -35,6 +35,7 @@ import (
 	watchnode "github.com/matheuscscp/gke-metadata-server/internal/node/watch"
 	listpods "github.com/matheuscscp/gke-metadata-server/internal/pods/list"
 	watchpods "github.com/matheuscscp/gke-metadata-server/internal/pods/watch"
+	"github.com/matheuscscp/gke-metadata-server/internal/redirect"
 	"github.com/matheuscscp/gke-metadata-server/internal/server"
 	"github.com/matheuscscp/gke-metadata-server/internal/serviceaccounts"
 	getserviceaccount "github.com/matheuscscp/gke-metadata-server/internal/serviceaccounts/get"
@@ -48,7 +49,7 @@ import (
 
 func newServerCommand() *cobra.Command {
 	var (
-		serverAddr                          string
+		serverPort                          int
 		webhookAddr                         string
 		workloadIdentityProvider            string
 		defaultNodeServiceAccountName       string
@@ -77,6 +78,17 @@ func newServerCommand() *cobra.Command {
 			if nodeName == "" {
 				return fmt.Errorf("NODE_NAME environment variable must be specified")
 			}
+			podIP := os.Getenv("POD_IP")
+			if podIP == "" {
+				return fmt.Errorf("POD_IP environment variable must be specified")
+			}
+			emulatorIP, err := netip.ParseAddr(podIP)
+			if err != nil {
+				return fmt.Errorf("error parsing POD_IP environment variable: %w", err)
+			}
+			if !emulatorIP.Is4() {
+				return fmt.Errorf("POD_IP environment variable must be an IPv4 address")
+			}
 			if defaultNodeServiceAccountName == "" {
 				return fmt.Errorf("--default-node-service-account-name argument must be specified")
 			}
@@ -104,6 +116,13 @@ func newServerCommand() *cobra.Command {
 				}
 			}()
 
+			// install ebpf redirect program
+			redirBPF, err := redirect.LoadAndAttachBPF(emulatorIP, serverPort, logging.Debug())
+			if err != nil {
+				return fmt.Errorf("error loading eBPF redirect program: %w", err)
+			}
+			defer redirBPF.Close()
+
 			// create clients
 			kubeClient, err := createKubernetesClient(ctx)
 			if err != nil {
@@ -213,7 +232,7 @@ func newServerCommand() *cobra.Command {
 			}
 			s := server.New(ctx, server.ServerOptions{
 				NodeName:                  nodeName,
-				ServerAddr:                serverAddr,
+				ServerPort:                serverPort,
 				Pods:                      pods,
 				Node:                      node,
 				ServiceAccounts:           serviceAccounts,
@@ -226,7 +245,7 @@ func newServerCommand() *cobra.Command {
 			webhookServer := webhook.New(ctx, webhook.ServerOptions{
 				ServerAddr:       webhookAddr,
 				InitNetworkImage: webhookInitNetworkImage,
-				DaemonSetPort:    strings.Split(serverAddr, ":")[1],
+				DaemonSetPort:    serverPort,
 				MetricsRegistry:  metricsRegistry,
 			})
 
@@ -243,7 +262,7 @@ func newServerCommand() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVar(&serverAddr, "server-addr", ":8080",
+	cmd.Flags().IntVar(&serverPort, "server-port", 8080,
 		"Network address where the metadata server must listen on")
 	cmd.Flags().StringVar(&webhookAddr, "webhook-addr", ":8081",
 		"Network address where the webhook server must listen on")

ebpf/redirect.c

@@ -0,0 +1,88 @@
+// MIT License
+//
+// Copyright (c) 2025 Matheus Pimenta
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "vmlinux.h"
+
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_endian.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_helpers.h>
+
+
+#define AF_INET 2 // IPv4 family
+
+struct Config {
+	__u32 emulator_ip;
+	__u16 emulator_port;
+	__u16 debug;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, struct Config);
+} map_config SEC(".maps");
+
+
+// Hooks to connect() syscalls. Redirects connections targeting
+// the GKE metadata server to the emulator.
+SEC("cgroup/connect4")
+int redirect_connect4(struct bpf_sock_addr *ctx) {
+	// Only forward IPv4 TCP connections
+	if (ctx->user_family != AF_INET) {
+		return 1;
+	}
+	if (ctx->protocol != IPPROTO_TCP) {
+		return 1;
+	}
+
+	const __u32 dst = bpf_ntohl(ctx->user_ip4);
+	const __u32 dst_port = bpf_ntohs(ctx->user_port);
+
+	// 0xA9FEA9FE is 169.254.169.254, the GKE Metadata Server IP address
+	if (dst != 0xA9FEA9FE || dst_port != 80) {
+		return 1;
+	}
+
+	// Fetch emulator configuration
+	const __u32 key = 0;
+	struct Config *conf = bpf_map_lookup_elem(&map_config, &key);
+	if (!conf) {
+		bpf_printk("Error: redirect_connect4 called without configuration");
+		return 1;
+	}
+
+	// Redirect the connection to the emulator
+	ctx->user_ip4 = bpf_htonl(conf->emulator_ip);
+	ctx->user_port = bpf_htons(conf->emulator_port);
+
+	if (conf->debug) {
+		const __u32 emu = ctx->user_ip4;
+		bpf_printk("Redirecting connection to %pI4:%d", &emu, conf->emulator_port);
+	}
+
+	return 1;
+}
+
+char __LICENSE[] SEC("license") = "GPL";

go.mod

@@ -6,6 +6,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.6.0
 	cloud.google.com/go/storage v1.50.0
 	github.com/cert-manager/cert-manager v1.16.3
+	github.com/cilium/ebpf v0.17.2
 	github.com/coreos/go-oidc/v3 v3.12.0
 	github.com/go-logr/logr v1.4.2
 	github.com/golang-jwt/jwt/v5 v5.2.1
@@ -15,7 +16,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.10.0
 	golang.org/x/oauth2 v0.25.0
-	google.golang.org/api v0.218.0
+	google.golang.org/api v0.219.0
 	k8s.io/api v0.32.1
 	k8s.io/apimachinery v0.32.1
 	k8s.io/client-go v0.32.1
@@ -25,7 +26,7 @@ require (
 )
 
 require (
-	cel.dev/expr v0.18.0 // indirect
+	cel.dev/expr v0.19.0 // indirect
 	cloud.google.com/go v0.116.0 // indirect
 	cloud.google.com/go/auth v0.14.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect
@@ -75,14 +76,14 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.31.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.32.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
-	go.opentelemetry.io/otel v1.31.0 // indirect
-	go.opentelemetry.io/otel/metric v1.31.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.31.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect
-	go.opentelemetry.io/otel/trace v1.31.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
 	golang.org/x/crypto v0.32.0 // indirect
 	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
@@ -92,9 +93,9 @@ require (
 	golang.org/x/time v0.9.0 // indirect
 	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
-	google.golang.org/grpc v1.69.4 // indirect
-	google.golang.org/protobuf v1.36.3 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect
+	google.golang.org/grpc v1.70.0 // indirect
+	google.golang.org/protobuf v1.36.4 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect