adding/updating ci files

.github/workflows/test.yml

@@ -0,0 +1,170 @@
+name: bpf-ci
+
+on:
+  pull_request:
+
+concurrency: 
+  group: ci-test-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  llvm-toolchain:
+    runs-on: ubuntu-latest
+    outputs:
+      llvm: ${{ steps.llvm-toolchain-impl.outputs.version }}
+    steps:
+      - id: llvm-version
+        uses: libbpf/ci/get-llvm-version@master
+      - id: llvm-toolchain-impl
+        shell: bash
+        run: echo "::set-output name=version::llvm-${{ steps.llvm-version.outputs.version }}"
+  set-matrix:
+    needs: llvm-toolchain
+    runs-on: ubuntu-latest
+    outputs:
+      build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }}
+      test-matrix: ${{ steps.set-matrix-impl.outputs.test_matrix }}
+    steps:
+      - id: set-matrix-impl
+        shell: python3 -I {0}
+        run: |
+          from json import dumps
+
+          matrix = [
+            {"kernel": "LATEST", "runs_on": ["ubuntu-latest", "self-hosted"], "arch": "x86_64", "toolchain": "gcc"},
+            {"kernel": "LATEST", "runs_on": ["ubuntu-latest", "self-hosted"], "arch": "x86_64", "toolchain": "${{ needs.llvm-toolchain.outputs.llvm }}"},
+            {"kernel": "LATEST", "runs_on": ["z15", "self-hosted"], "arch": "s390x", "toolchain": "gcc"},
+          ]
+          self_hosted_repos = [
+            "kernel-patches/bpf",
+            "kernel-patches/vmtest",
+          ]
+
+          # Only a few repository within "kernel-patches" use self-hosted runners.
+          if "${{ github.repository_owner }}" != "kernel-patches" or "${{ github.repository }}" not in self_hosted_repos:
+            # Outside of those repositories, remove the self-hosted label and skip
+            # any testing on s390x, as no suitable runners will be available.
+            for idx in range(len(matrix) - 1, -1, -1):
+              if "z15" in matrix[idx]["runs_on"]:
+                del matrix[idx]
+              else:
+                matrix[idx]["runs_on"].remove("self-hosted")
+
+          build_matrix = {"include": matrix}
+          print(f"::set-output name=build_matrix::{dumps(build_matrix)}")
+
+          tests = ["test_progs", "test_progs_no_alu32", "test_maps", "test_verifier"]
+          test_matrix = {"include": [{**config, **{"test": test}}
+                                      for config in matrix
+                                        for test in tests]}
+          print(f"::set-output name=test_matrix::{dumps(test_matrix)}")
+  build:
+    name: build for ${{ matrix.arch }} with ${{ matrix.toolchain }}
+    needs: set-matrix
+    runs-on: ${{ matrix.runs_on }}
+    timeout-minutes: 100
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }}
+    env:
+      KERNEL: ${{ matrix.kernel }}
+      REPO_ROOT: ${{ github.workspace }}
+      REPO_PATH: ""
+    steps:
+      - uses: actions/checkout@v2
+      - if: ${{ github.repository == 'kernel-patches/vmtest' }}
+        name: Download bpf-next tree
+        uses: libbpf/ci/get-linux-source@master
+        with:
+          dest: '.kernel'
+      - if: ${{ github.repository == 'kernel-patches/vmtest' }}
+        name: Move linux source in place
+        shell: bash
+        run: |
+          rm -rf .kernel/.git
+          cp -rf .kernel/. .
+          rm -rf .kernel
+      - uses: libbpf/ci/patch-kernel@master
+        with:
+          patches-root: '${{ github.workspace }}/travis-ci/diffs'
+          repo-root: '${{ github.workspace }}'
+      - name: Setup build environment
+        uses: libbpf/ci/setup-build-env@master
+      - name: Build kernel image
+        uses: libbpf/ci/build-linux@master
+        with:
+          arch: ${{ matrix.arch }}
+          toolchain: ${{ matrix.toolchain }}
+      - name: Build selftests
+        uses: libbpf/ci/build-selftests@master
+        with:
+          vmlinux_btf: ${{ github.workspace }}/vmlinux
+          toolchain: ${{ matrix.toolchain }}
+      - name: Build samples
+        uses: libbpf/ci/build-samples@master
+        with:
+          vmlinux_btf: ${{ github.workspace }}/vmlinux
+          toolchain: ${{ matrix.toolchain }}
+      - name: Tar artifacts
+        run: |
+          file_list=""
+          if [ "${{ github.repository }}" == "kernel-patches/vmtest" ]; then
+            # Package up a bunch of additional infrastructure to support running
+            # 'make kernelrelease' and bpf tool checks later on.
+            file_list="$(find . -iname Makefile | xargs) \
+              scripts/ \
+              tools/testing/selftests/bpf/ \
+              tools/include/ \
+              tools/bpf/bpftool/";
+          fi
+
+          tar -czf vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.gz \
+            .config \
+            arch/*/boot/bzImage \
+            include/config/auto.conf \
+            include/generated/autoconf.h \
+            ${file_list} \
+            --exclude '*.h' \
+            selftests/bpf/ \
+            vmlinux
+      - uses: actions/upload-artifact@v3
+        with:
+          name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}
+          if-no-files-found: error
+          path: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.gz
+  test:
+    name: ${{ matrix.test }} on ${{ matrix.arch }} with ${{ matrix.toolchain }}
+    needs: [set-matrix, build]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.set-matrix.outputs.test-matrix) }}
+    runs-on: ${{ matrix.runs_on }}
+    timeout-minutes: 100
+    env:
+      KERNEL: ${{ matrix.kernel }}
+      REPO_ROOT: ${{ github.workspace }}
+      REPO_PATH: ""
+    steps:
+      - uses: actions/checkout@main
+      - uses: actions/download-artifact@v3
+        with:
+          name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}
+          path: .
+      - name: Untar artifacts
+        run: tar -xzf vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.gz
+      - name: Prepare rootfs
+        uses: libbpf/ci/prepare-rootfs@master
+        with:
+          project-name: 'libbpf'
+          arch: ${{ matrix.arch }}
+          kernel: ${{ matrix.kernel }}
+          kernel-root: '.'
+          image-output: '/tmp/root.img'
+          test: ${{ matrix.test }}
+      - name: Run selftests
+        uses: libbpf/ci/run-qemu@master
+        with:
+          arch: ${{ matrix.arch}}
+          img: '/tmp/root.img'
+          vmlinuz: '${{ github.workspace }}/vmlinuz'
+          kernel-root: '.'

README

@@ -1,18 +0,0 @@
-Linux kernel
-============
-
-There are several guides for kernel developers and users. These guides can
-be rendered in a number of formats, like HTML and PDF. Please read
-Documentation/admin-guide/README.rst first.
-
-In order to build the documentation, use ``make htmldocs`` or
-``make pdfdocs``.  The formatted documentation can also be read online at:
-
-    https://www.kernel.org/doc/html/latest/
-
-There are various text files in the Documentation/ subdirectory,
-several of them using the Restructured Text markup notation.
-
-Please read the Documentation/process/changes.rst file, as it contains the
-requirements for building and running the kernel, and information about
-the problems which may result by upgrading your kernel.

travis-ci/vmtest/configs/DENYLIST

@@ -0,0 +1,6 @@
+# TEMPORARY
+btf_dump/btf_dump: syntax
+kprobe_multi_test/bench_attach
+core_reloc/enum64val
+core_reloc/size___diff_sz
+core_reloc/type_based___diff_sz

travis-ci/vmtest/configs/DENYLIST.s390x

@@ -0,0 +1,4 @@
+tc_redirect/tc_redirect_dtime            # very flaky
+lru_bug                                  # not yet in bpf-next denylist
+usdt/basic                               # failing verifier due to bounds check after LLVM update
+usdt/multispec                           # same as above

travis-ci/vmtest/helpers.sh

@@ -0,0 +1,36 @@
+# $1 - start or end
+# $2 - fold identifier, no spaces
+# $3 - fold section description
+foldable() {
+  local YELLOW='\033[1;33m'
+  local NOCOLOR='\033[0m'
+  if [ $1 = "start" ]; then
+    line="::group::$2"
+    if [ ! -z "${3:-}" ]; then
+      line="$line - ${YELLOW}$3${NOCOLOR}"
+    fi
+  else
+    line="::endgroup::"
+  fi
+  echo -e "$line"
+}
+
+__print() {
+  local TITLE=""
+  if [[ -n $2 ]]; then
+      TITLE=" title=$2"
+  fi
+  echo "::$1${TITLE}::$3"
+}
+
+# $1 - title
+# $2 - message
+print_error() {
+  __print error $1 $2
+}
+
+# $1 - title
+# $2 - message
+print_notice() {
+  __print notice $1 $2
+}

travis-ci/vmtest/run_selftests.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source $(cd $(dirname $0) && pwd)/helpers.sh
+
+ARCH=$(uname -m)
+
+STATUS_FILE=/exitstatus
+
+read_lists() {
+	(for path in "$@"; do
+		if [[ -s "$path" ]]; then
+			cat "$path"
+		fi;
+	done) | cut -d'#' -f1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '\n' ','
+}
+
+TEST_PROGS_ARGS=""
+# Disabled due to issue
+# if [[ "$(nproc)" -gt 2 ]]; then
+#   TEST_PROGS_ARGS="-j"
+# fi
+
+test_progs() {
+  foldable start test_progs "Testing test_progs"
+  # "&& true" does not change the return code (it is not executed
+  # if the Python script fails), but it prevents exiting on a
+  # failure due to the "set -e".
+  ./test_progs ${DENYLIST:+-d"$DENYLIST"} ${ALLOWLIST:+-a"$ALLOWLIST"} ${TEST_PROGS_ARGS} && true
+  echo "test_progs:$?" >>"${STATUS_FILE}"
+  foldable end test_progs
+}
+
+test_progs_no_alu32() {
+  foldable start test_progs-no_alu32 "Testing test_progs-no_alu32"
+  ./test_progs-no_alu32 ${DENYLIST:+-d"$DENYLIST"} ${ALLOWLIST:+-a"$ALLOWLIST"} ${TEST_PROGS_ARGS} && true
+  echo "test_progs-no_alu32:$?" >>"${STATUS_FILE}"
+  foldable end test_progs-no_alu32
+}
+
+test_maps() {
+  foldable start test_maps "Testing test_maps"
+  taskset 0xF ./test_maps && true
+  echo "test_maps:$?" >>"${STATUS_FILE}"
+  foldable end test_maps
+}
+
+test_verifier() {
+  foldable start test_verifier "Testing test_verifier"
+  ./test_verifier && true
+  echo "test_verifier:$?" >>"${STATUS_FILE}"
+  foldable end test_verifier
+}
+
+foldable end vm_init
+
+foldable start kernel_config "Kconfig"
+
+zcat /proc/config.gz
+
+foldable end kernel_config
+
+configs_path=${PROJECT_NAME}/selftests/bpf
+local_configs_path=${PROJECT_NAME}/vmtest/configs
+DENYLIST=$(read_lists \
+	"$configs_path/DENYLIST" \
+	"$configs_path/DENYLIST.${ARCH}" \
+	"$local_configs_path/DENYLIST" \
+	"$local_configs_path/DENYLIST.${ARCH}" \
+)
+ALLOWLIST=$(read_lists \
+	"$configs_path/ALLOWLIST" \
+	"$configs_path/ALLOWLIST.${ARCH}" \
+	"$local_configs_path/ALLOWLIST" \
+	"$local_configs_path/ALLOWLIST.${ARCH}" \
+)
+
+echo "DENYLIST: ${DENYLIST}"
+echo "ALLOWLIST: ${ALLOWLIST}"
+
+cd ${PROJECT_NAME}/selftests/bpf
+
+if [ $# -eq 0 ]; then
+	test_progs
+	test_progs_no_alu32
+	test_maps
+	test_verifier
+else
+	for test_name in "$@"; do
+		"${test_name}"
+	done
+fi