link: implement kprobe_multi link type

As of Linux 5.18, or commit 5a5c11ee3e65 ("Merge branch 'bpf: Add kprobe
multi link'"), attaching multiple k(ret)probes using a single system call
is now possible.

This commit adds support for this through the KprobeMulti() and
KretprobeMulti() APIs in package link.

Co-authored-by: Timo Beckers <[email protected]>

Author: mmat11

attachtype_string.go

@@ -1153,8 +1153,9 @@ func getProgType(sectionName string) (ProgramType, AttachType, uint32, string) {
 		{"cgroup/setsockopt", CGroupSockopt, AttachCGroupSetsockopt, 0},
 		{"struct_ops+", StructOps, AttachNone, 0},
 		{"sk_lookup/", SkLookup, AttachSkLookup, 0},
-
 		{"seccomp", SocketFilter, AttachNone, 0},
+		{"kprobe.multi", Kprobe, AttachTraceKprobeMulti, 0},
+		{"kretprobe.multi", Kprobe, AttachTraceKprobeMulti, 0},
 	}
 
 	for _, t := range types {

elf_reader.go

@@ -36,3 +36,17 @@ func NewStringPointer(str string) Pointer {
 
 	return Pointer{ptr: unsafe.Pointer(p)}
 }
+
+// NewStringSlicePointer allocates an array of Pointers to each string in the
+// given slice of strings and returns a 64-bit pointer to the start of the
+// resulting array.
+//
+// Use this function to pass arrays of strings as syscall arguments.
+func NewStringSlicePointer(strings []string) Pointer {
+	sp := make([]Pointer, 0, len(strings))
+	for _, s := range strings {
+		sp = append(sp, NewStringPointer(s))
+	}
+
+	return Pointer{ptr: unsafe.Pointer(&sp[0])}
+}

internal/sys/ptr.go

@@ -0,0 +1,176 @@
+package link
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"unsafe"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/asm"
+	"github.com/cilium/ebpf/internal"
+	"github.com/cilium/ebpf/internal/sys"
+	"github.com/cilium/ebpf/internal/unix"
+)
+
+// KprobeMultiOptions defines additional parameters that will be used
+// when opening a KprobeMulti Link.
+type KprobeMultiOptions struct {
+	// Symbols takes a list of kernel symbol names to attach an ebpf program to.
+	//
+	// Mutually exclusive with Addresses.
+	Symbols []string
+
+	// Addresses takes a list of kernel symbol addresses in case they can not
+	// be referred to by name.
+	//
+	// Note that only start addresses can be specified, since the fprobe API
+	// limits the attach point to the function entry or return.
+	//
+	// Mutually exclusive with Symbols.
+	Addresses []uint64
+
+	// Cookies specifies arbitrary values that can be fetched from an eBPF
+	// program via `bpf_get_attach_cookie()`.
+	//
+	// If set, its length should be equal to the length of Symbols or Addresses.
+	// Each Cookie is assigned to the Symbol or Address specified at the
+	// corresponding slice index.
+	Cookies []uint64
+}
+
+// KprobeMulti attaches the given eBPF program to the entry point of a given set
+// of kernel symbols.
+//
+// The difference with Kprobe() is that multi-kprobe accomplishes this in a
+// single system call, making it significantly faster than attaching many
+// probes one at a time.
+//
+// Requires at least Linux 5.18.
+func KprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions) (Link, error) {
+	return kprobeMulti(prog, opts, 0)
+}
+
+// KretprobeMulti attaches the given eBPF program to the return point of a given
+// set of kernel symbols.
+//
+// The difference with Kretprobe() is that multi-kprobe accomplishes this in a
+// single system call, making it significantly faster than attaching many
+// probes one at a time.
+//
+// Requires at least Linux 5.18.
+func KretprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions) (Link, error) {
+	return kprobeMulti(prog, opts, unix.BPF_F_KPROBE_MULTI_RETURN)
+}
+
+func kprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions, flags uint32) (Link, error) {
+	if prog == nil {
+		return nil, errors.New("cannot attach a nil program")
+	}
+
+	syms := uint32(len(opts.Symbols))
+	addrs := uint32(len(opts.Addresses))
+	cookies := uint32(len(opts.Cookies))
+
+	if syms == 0 && addrs == 0 {
+		return nil, fmt.Errorf("one of Symbols or Addresses is required: %w", errInvalidInput)
+	}
+	if syms != 0 && addrs != 0 {
+		return nil, fmt.Errorf("Symbols and Addresses are mutually exclusive: %w", errInvalidInput)
+	}
+	if cookies > 0 && cookies != syms && cookies != addrs {
+		return nil, fmt.Errorf("Cookies must be exactly Symbols or Addresses in length: %w", errInvalidInput)
+	}
+
+	if err := haveBPFLinkKprobeMulti(); err != nil {
+		return nil, err
+	}
+
+	attr := &sys.LinkCreateKprobeMultiAttr{
+		ProgFd:           uint32(prog.FD()),
+		AttachType:       sys.BPF_TRACE_KPROBE_MULTI,
+		KprobeMultiFlags: flags,
+	}
+
+	switch {
+	case syms != 0:
+		attr.Count = syms
+		attr.Syms = sys.NewStringSlicePointer(opts.Symbols)
+
+	case addrs != 0:
+		attr.Count = addrs
+		attr.Addrs = sys.NewPointer(unsafe.Pointer(&opts.Addresses[0]))
+	}
+
+	if cookies != 0 {
+		attr.Cookies = sys.NewPointer(unsafe.Pointer(&opts.Cookies[0]))
+	}
+
+	fd, err := sys.LinkCreateKprobeMulti(attr)
+	if errors.Is(err, unix.ESRCH) {
+		return nil, fmt.Errorf("couldn't find one or more symbols: %w", os.ErrNotExist)
+	}
+	if errors.Is(err, unix.EINVAL) {
+		return nil, fmt.Errorf("%w (missing kernel symbol or prog's AttachType not AttachTraceKprobeMulti?)", err)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &kprobeMultiLink{RawLink{fd, ""}}, nil
+}
+
+type kprobeMultiLink struct {
+	RawLink
+}
+
+var _ Link = (*kprobeMultiLink)(nil)
+
+func (kml *kprobeMultiLink) Update(prog *ebpf.Program) error {
+	return fmt.Errorf("update kprobe_multi: %w", ErrNotSupported)
+}
+
+func (kml *kprobeMultiLink) Pin(string) error {
+	return fmt.Errorf("pin kprobe_multi: %w", ErrNotSupported)
+}
+
+func (kml *kprobeMultiLink) Unpin() error {
+	return fmt.Errorf("unpin kprobe_multi: %w", ErrNotSupported)
+}
+
+var haveBPFLinkKprobeMulti = internal.FeatureTest("bpf_link_kprobe_multi", "5.18", func() error {
+	prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{
+		Name: "probe_kpm_link",
+		Type: ebpf.Kprobe,
+		Instructions: asm.Instructions{
+			asm.Mov.Imm(asm.R0, 0),
+			asm.Return(),
+		},
+		AttachType: ebpf.AttachTraceKprobeMulti,
+		License:    "MIT",
+	})
+	if errors.Is(err, unix.E2BIG) {
+		// Kernel doesn't support AttachType field.
+		return internal.ErrNotSupported
+	}
+	if err != nil {
+		return err
+	}
+	defer prog.Close()
+
+	fd, err := sys.LinkCreateKprobeMulti(&sys.LinkCreateKprobeMultiAttr{
+		ProgFd:     uint32(prog.FD()),
+		AttachType: sys.BPF_TRACE_KPROBE_MULTI,
+		Count:      1,
+		Syms:       sys.NewStringSlicePointer([]string{"vprintk"}),
+	})
+	if errors.Is(err, unix.EINVAL) {
+		return internal.ErrNotSupported
+	}
+	if err != nil {
+		return err
+	}
+	fd.Close()
+
+	return nil
+})

link/kprobe_multi.go

@@ -0,0 +1,131 @@
+package link
+
+import (
+	"errors"
+	"math"
+	"os"
+	"testing"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/internal/testutils"
+	"github.com/cilium/ebpf/internal/unix"
+)
+
+var kprobeMultiSyms = []string{"vprintk", "inet6_release"}
+
+func TestKprobeMulti(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti, "")
+
+	km, err := KprobeMulti(prog, KprobeMultiOptions{Symbols: kprobeMultiSyms})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer km.Close()
+
+	testLink(t, km, prog)
+}
+
+func TestKprobeMultiInput(t *testing.T) {
+	// Program type that loads on all kernels. Not expected to link successfully.
+	prog := mustLoadProgram(t, ebpf.SocketFilter, 0, "")
+
+	// One of Symbols or Addresses must be given.
+	_, err := KprobeMulti(prog, KprobeMultiOptions{})
+	if !errors.Is(err, errInvalidInput) {
+		t.Fatalf("expected errInvalidInput, got: %v", err)
+	}
+
+	// Symbols and Addresses are mutually exclusive.
+	_, err = KprobeMulti(prog, KprobeMultiOptions{
+		Symbols:   []string{"foo"},
+		Addresses: []uint64{1},
+	})
+	if !errors.Is(err, errInvalidInput) {
+		t.Fatalf("expected errInvalidInput, got: %v", err)
+	}
+
+	// One Symbol, two cookies..
+	_, err = KprobeMulti(prog, KprobeMultiOptions{
+		Symbols: []string{"one"},
+		Cookies: []uint64{2, 3},
+	})
+	if !errors.Is(err, errInvalidInput) {
+		t.Fatalf("expected errInvalidInput, got: %v", err)
+	}
+}
+
+func TestKprobeMultiErrors(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti, "")
+
+	// Nonexistent kernel symbol.
+	_, err := KprobeMulti(prog, KprobeMultiOptions{Symbols: []string{"bogus"}})
+	if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, unix.EINVAL) {
+		t.Fatalf("expected ErrNotExist or EINVAL, got: %s", err)
+	}
+
+	// Only have a negative test for addresses as it would be hard to maintain a
+	// proper one.
+	if _, err := KprobeMulti(prog, KprobeMultiOptions{
+		Addresses: []uint64{math.MaxUint64},
+	}); !errors.Is(err, unix.EINVAL) {
+		t.Fatalf("expected EINVAL, got: %s", err)
+	}
+}
+
+func TestKprobeMultiCookie(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti, "")
+
+	if _, err := KprobeMulti(prog, KprobeMultiOptions{
+		Symbols: kprobeMultiSyms,
+		Cookies: []uint64{0, 1},
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestKprobeMultiProgramCall(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkKprobeMulti())
+
+	m, p := newUpdaterMapProg(t, ebpf.Kprobe, ebpf.AttachTraceKprobeMulti)
+
+	// For simplicity, just assert the increment happens with any symbol in the array.
+	opts := KprobeMultiOptions{
+		Symbols: []string{"__do_sys_getpid"},
+	}
+	km, err := KprobeMulti(p, opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Trigger ebpf program call.
+	unix.Getpid()
+
+	// Assert that the value at index 0 has been updated to 1.
+	assertMapValue(t, m, 0, 1)
+
+	// Close the link.
+	if err := km.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Reset map value to 0 at index 0.
+	if err := m.Update(uint32(0), uint32(0), ebpf.UpdateExist); err != nil {
+		t.Fatal(err)
+	}
+
+	// Retrigger the ebpf program call.
+	unix.Getpid()
+
+	// Assert that this time the value has not been updated.
+	assertMapValue(t, m, 0, 0)
+}
+
+func TestHaveBPFLinkKprobeMulti(t *testing.T) {
+	testutils.CheckFeatureTest(t, haveBPFLinkKprobeMulti)
+}

link/kprobe_multi_test.go

@@ -308,7 +308,7 @@ func TestKprobeTraceFSGroup(t *testing.T) {
 }
 
 func TestKprobeProgramCall(t *testing.T) {
-	m, p := newUpdaterMapProg(t, ebpf.Kprobe)
+	m, p := newUpdaterMapProg(t, ebpf.Kprobe, 0)
 
 	// Open Kprobe on `sys_getpid` and attach it
 	// to the ebpf program created above.
@@ -340,7 +340,7 @@ func TestKprobeProgramCall(t *testing.T) {
 	assertMapValue(t, m, 0, 0)
 }
 
-func newUpdaterMapProg(t *testing.T, typ ebpf.ProgramType) (*ebpf.Map, *ebpf.Program) {
+func newUpdaterMapProg(t *testing.T, typ ebpf.ProgramType, attach ebpf.AttachType) (*ebpf.Map, *ebpf.Program) {
 	// Create ebpf map. Will contain only one key with initial value 0.
 	m, err := ebpf.NewMap(&ebpf.MapSpec{
 		Type:       ebpf.Array,
@@ -378,7 +378,8 @@ func newUpdaterMapProg(t *testing.T, typ ebpf.ProgramType) (*ebpf.Map, *ebpf.Pro
 			asm.Mov.Imm(asm.R0, 0),
 			asm.Return(),
 		},
-		License: "Dual MIT/GPL",
+		AttachType: attach,
+		License:    "Dual MIT/GPL",
 	})
 	if err != nil {
 		t.Fatal(err)

link/kprobe_test.go

@@ -286,7 +286,8 @@ func (l *RawLink) Info() (*Info, error) {
 		extra = &TracingInfo{}
 	case XDPType:
 		extra = &XDPInfo{}
-	case RawTracepointType, IterType, PerfEventType:
+	case RawTracepointType, IterType,
+		PerfEventType, KprobeMultiType:
 		// Extra metadata not supported.
 	default:
 		return nil, fmt.Errorf("unknown link info type: %d", info.Type)
@@ -296,7 +297,7 @@ func (l *RawLink) Info() (*Info, error) {
 		buf := bytes.NewReader(info.Extra[:])
 		err := binary.Read(buf, internal.NativeEndian, extra)
 		if err != nil {
-			return nil, fmt.Errorf("can not read extra link info: %w", err)
+			return nil, fmt.Errorf("cannot read extra link info: %w", err)
 		}
 	}