Pilot executes cassandra directly, bypassing the container image entry point

Makefile

@@ -18,6 +18,9 @@ CMDS := controller apiserver pilot-elasticsearch pilot-cassandra
 GOPATH ?= /tmp/go
 GOFLAGS ?= "-a"
 
+# Path of the Navigator Cassandra Pilot for integration tests
+TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA ?= "${CURDIR}/navigator-pilot-cassandra_linux_amd64"
+
 help:
 	# all       - runs verify, build and docker_build targets
 	# test      - runs go_test target
@@ -32,7 +35,7 @@ help:
 
 all: verify build docker_build
 
-test: go_test
+test: go_test test_integration
 
 .run_e2e:
 	${HACK_DIR}/prepare-e2e.sh
@@ -44,7 +47,7 @@ build: $(CMDS)
 
 generate: .generate_files
 
-verify: .hack_verify dep_verify go_verify helm_verify
+verify: .hack_verify dep_verify go_verify helm_verify test_integration
 
 .hack_verify:
 	@echo Running repo-infra verify scripts
@@ -90,7 +93,7 @@ $(CMDS):
 go_build: $(CMDS)
 
 go_test:
-	go test -v $$(go list ./... | grep -v '/vendor/')
+	go test -v $$(go list ./... | grep -v -e '/vendor/'  -e 'github.com/jetstack/navigator/test/')
 
 go_fmt:
 	./hack/verify-lint.sh
@@ -105,3 +108,11 @@ go_fmt:
 # Helm targets
 helm_verify:
 	helm lint contrib/charts/*
+
+.download_integration_test_binaries:
+	$(HACK_DIR)/download-integration-test-binaries.sh
+	touch	.download_integration_test_binaries
+
+test_integration: .download_integration_test_binaries apiserver pilot-cassandra
+	TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA=$(TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA) \
+	go test $(GO_TEST_ARGS) -v ./test/integration/...

hack/download-integration-test-binaries.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Download the binaries needed by the sigs.k8s.io/testing_frameworks/integration package.
+# etcd, kube-apiserver, kubectl
+# XXX: There is already a script to do this:
+# * sigs.k8s.io/testing_frameworks/integration/scripts/download-binaries.sh
+# But it currently downloads kube-apiserver v1.10.0-alpha.1 which doesn't support ``CustomResourceSubresources``.
+# See https://github.com/kubernetes-sigs/testing_frameworks/issues/44
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Close stdin
+exec 0<&-
+
+ROOT_DIR="$(git rev-parse --show-toplevel)"
+
+ETCD_VERSION=v3.2.10
+ETCD_URL="https://storage.googleapis.com/etcd"
+
+KUBE_VERSION_URL="https://storage.googleapis.com/kubernetes-release/release/stable-1.10.txt"
+KUBE_VERSION=$(curl --fail --silent "${KUBE_VERSION_URL}")
+KUBE_BIN_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64"
+
+ASSETS_DIR="${ROOT_DIR}/vendor/sigs.k8s.io/testing_frameworks/integration/assets/bin"
+
+mkdir -p "${ASSETS_DIR}"
+
+curl --fail --silent ${ETCD_URL}/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | \
+    tar --extract --gzip --directory="${ASSETS_DIR}" --strip-components=1 --wildcards '*/etcd'
+curl --fail --silent --output "${ASSETS_DIR}/kube-apiserver" "${KUBE_BIN_URL}/kube-apiserver"
+curl --fail --silent --output "${ASSETS_DIR}/kubectl" "${KUBE_BIN_URL}/kubectl"
+
+chmod +x ${ASSETS_DIR}/*

test/integration/pilot_test.go

@@ -0,0 +1,200 @@
+package integration_test
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/onsi/gomega/gbytes"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/testing_frameworks/integration"
+
+	"github.com/jetstack/navigator/internal/test/util/testfs"
+)
+
+// MakeKubeConfig creates a kube/config file which is suitable for communicating
+// with the API server started by ``integration.ControlPlane``.
+func MakeKubeConfig(f *integration.ControlPlane, path string) error {
+	ctl := f.KubeCtl()
+	ctl.Opts = []string{"--kubeconfig", path}
+
+	_, _, err := ctl.Run(
+		"config",
+		"set-credentials",
+		"user1",
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create user")
+	}
+
+	_, _, err = ctl.Run(
+		"config",
+		"set-cluster",
+		"integration1",
+		"--server",
+		f.APIURL().String(),
+		"--certificate-authority",
+		filepath.Join(f.APIServer.CertDir, "apiserver.crt"),
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create cluster")
+	}
+
+	_, _, err = ctl.Run(
+		"config",
+		"set-context",
+		"default",
+		"--cluster", "integration1",
+		"--user", "user1",
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to create context")
+	}
+
+	_, _, err = ctl.Run(
+		"config",
+		"use-context",
+		"default",
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to use context")
+	}
+	return nil
+}
+
+func TestPilotCassandra(t *testing.T) {
+	// Start the API server with CustomResourceSubresources feature enabled.
+	// Navigator pilot calls on UpdateStatus on Pilot resources for example.
+	cp := &integration.ControlPlane{
+		APIServer: &integration.APIServer{
+			Args: []string{
+				"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
+				"--cert-dir={{ .CertDir }}",
+				"--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}",
+				"--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}",
+				"--secure-port=0",
+				"--feature-gates=CustomResourceSubresources=true",
+				"-v=4", "--alsologtostderr",
+			},
+			// Out: os.Stdout,
+			// Err: os.Stderr,
+		},
+	}
+	err := cp.Start()
+	require.NoError(t, err)
+	defer func() {
+		err := cp.Stop()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	cli := cp.KubeCtl()
+
+	// Create a CRD for the Navigator Pilot resource type.
+	// This allows us to test the Pilot against the Kubernetes API server,
+	// without having to start the Navigator API server and configure aggregation etc.
+	crdPath, err := filepath.Abs("testdata/pilot_crd.yaml")
+	require.NoError(t, err)
+	stdout, stderr, err := cli.Run(
+		"apply",
+		"--filename",
+		crdPath,
+	)
+	t.Log("stdout", stdout)
+	t.Log("stderr", stderr)
+	require.NoError(t, err)
+
+	// Create a Pilot resource for our Pilot process to watch and update.
+	stdout, stderr, err = cli.Run(
+		"create",
+		"namespace",
+		"ns1",
+	)
+	t.Log("stdout", stdout)
+	t.Log("stderr", stderr)
+	require.NoError(t, err)
+	pilotResourcePath, err := filepath.Abs("testdata/pilot.yaml")
+	require.NoError(t, err)
+	stdout, stderr, err = cli.Run(
+		"apply",
+		"--filename",
+		pilotResourcePath,
+	)
+	t.Log("stdout", stdout)
+	t.Log("stderr", stderr)
+	require.NoError(t, err)
+
+	// Temporary configuration directories.
+	tfs := testfs.New(t)
+	kubeConfig := tfs.TempPath("kube.config")
+	cassConfig := tfs.TempDir("etc_cassandra")
+	pilotConfigDir := tfs.TempDir("etc/pilot")
+	// A fake cassandra binary for the Pilot to execute.
+	cassPath, err := filepath.Abs("testdata/fake_cassandra")
+	require.NoError(t, err)
+	err = MakeKubeConfig(cp, kubeConfig)
+	require.NoError(t, err)
+
+	pilotPath, pilotPathFound := os.LookupEnv("TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA")
+	if !pilotPathFound {
+		t.Fatal(
+			"Please set environment variable TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA " +
+				"with the path to the navigator-pilot-cassandra binary under test.")
+	}
+
+	expectedClusterName := "cluster1"
+	cmd := exec.Command(
+		pilotPath,
+		"--pilot-name", "pilot1",
+		"--pilot-namespace", "ns1",
+		"--kubeconfig", kubeConfig,
+		"--v=4", "--alsologtostderr",
+		"--leader-elect=false",
+		"--cassandra-cluster-name", expectedClusterName,
+		"--cassandra-config-path", cassConfig,
+		"--cassandra-path", cassPath,
+		"--cassandra-rack", "rack-for-test",
+		"--cassandra-dc", "dc-for-test",
+		"--config-dir", pilotConfigDir,
+	)
+	startDetectStream := gbytes.NewBuffer()
+	// The fake Cassandra script echos "FAKE CASSANDRA" in a loop
+	ready := startDetectStream.Detect("FAKE CASSANDRA")
+	cmd.Stdout = startDetectStream
+	cmd.Stderr = os.Stderr
+	// This will return when the executable writes "FAKE CASSANDRA" to stdout.
+	err = cmd.Start()
+	require.NoError(t, err)
+	defer func() {
+		// XXX: Pilot doesn't respond to SIGKILL
+		err := cmd.Process.Signal(os.Interrupt)
+		require.NoError(t, err)
+		err = cmd.Wait()
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				if status.ExitStatus() == 255 {
+					// XXX Pilot should return 0 if it exits cleanly after a signal.
+					return
+				}
+			}
+		}
+		require.NoError(t, err)
+	}()
+	select {
+	case <-ready:
+	case <-time.After(time.Second * 5):
+		t.Fatal("timeout waiting for process to start")
+	}
+	// The pilot has executed the Cassandra sub-process, so it should already
+	// have written the configuration files as part of the pre-start hook
+	// mechanism.
+	// We did not provide an existing cassandra.yaml or cassandra-rackdc.properties.
+	// So we expect thos files to now exist.
+	assert.FileExists(t, cassConfig+"/cassandra.yaml")
+	assert.FileExists(t, cassConfig+"/cassandra-rackdc.properties")
+}

test/integration/testdata/cassandra.yaml

@@ -0,0 +1,2 @@
+foo:
+  bar: org.apache.cassandra.locator.SimpleSeedProvider

test/integration/testdata/fake_cassandra

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -o errexit
+set -o pipefail
+
+while true; do
+    echo "FAKE CASSANDRA"
+    sleep 1
+done

test/integration/testdata/pilot.yaml

@@ -0,0 +1,11 @@
+apiVersion: navigator.jetstack.io/v1alpha1
+kind: Pilot
+metadata:
+  name: pilot1
+  namespace: ns1
+  ownerReferences:
+  - apiVersion: navigator.jetstack.io/v1alpha1
+    kind: CassandraCluster
+    name: cluster1
+    uid: 23c88696-cf7b-11e7-9ec9-0a580a200927
+    controller: true

test/integration/testdata/pilot_crd.yaml

@@ -0,0 +1,17 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: pilots.navigator.jetstack.io
+spec:
+  group: navigator.jetstack.io
+  version: v1alpha1
+  scope: Namespaced
+  names:
+    plural: pilots
+    singular: pilot
+    kind: Pilot
+    # categories is a list of grouped resources the custom resource belongs to.
+    categories:
+    - all
+  subresources:
+    status: {}