Override Cassandra configuration and launch Cassandra directly

* Bypass the entrypoint script of the chosen Docker image.
* Instead, make direct changes to the Cassandra.yaml file.
* And execute ``cassandra`` directly.
* Add a pilot integration test

Author: wallrj

.gitignore

@@ -6,3 +6,4 @@
 .generate_exes
 .get_deps
 bin/
+**/.test/

Makefile

@@ -18,6 +18,9 @@ CMDS := controller apiserver pilot-elasticsearch pilot-cassandra
 GOPATH ?= /tmp/go
 GOFLAGS ?= "-a"
 
+# Path of the Navigator Cassandra Pilot for integration tests
+TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA ?= "${CURDIR}/navigator-pilot-cassandra_linux_amd64"
+
 help:
 	# all       - runs verify, build and docker_build targets
 	# test      - runs go_test target
@@ -32,7 +35,7 @@ help:
 
 all: verify build docker_build
 
-test: go_test
+test: go_test test_integration
 
 .run_e2e:
 	${HACK_DIR}/prepare-e2e.sh
@@ -44,7 +47,7 @@ build: $(CMDS)
 
 generate: .generate_files
 
-verify: .hack_verify dep_verify go_verify helm_verify
+verify: .hack_verify dep_verify go_verify helm_verify test_integration
 
 .hack_verify:
 	@echo Running repo-infra verify scripts
@@ -90,7 +93,7 @@ $(CMDS):
 go_build: $(CMDS)
 
 go_test:
-	go test -v $$(go list ./... | grep -v '/vendor/')
+	go test -v $$(go list ./... | grep -v -e '/vendor/'  -e 'github.com/jetstack/navigator/test/')
 
 go_fmt:
 	./hack/verify-lint.sh
@@ -105,3 +108,11 @@ go_fmt:
 # Helm targets
 helm_verify:
 	helm lint contrib/charts/*
+
+.download_integration_test_binaries:
+	$(HACK_DIR)/download-integration-test-binaries.sh
+	touch	.download_integration_test_binaries
+
+test_integration: .download_integration_test_binaries apiserver pilot-cassandra
+	TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA=$(TEST_ASSET_NAVIGATOR_PILOT_CASSANDRA) \
+	go test $(GO_TEST_ARGS) -v ./test/integration/...

docs/cassandra.rst

@@ -25,6 +25,8 @@ All the C* nodes (pods) in a ``nodepool`` have the same configuration and the fo
 
 .. include:: configure-scheduler.rst
 
+.. _availability-zones-cassandra:
+
 Cassandra Across Multiple Availability Zones
 --------------------------------------------
 
@@ -240,6 +242,37 @@ Navigator will add C* nodes, one at a time, until the desired number of nodes is
 
 You can look at ``CassandraCluster.Status.NodePools[<nodepoolname>].ReadyReplicas`` to see the current number of healthy C* nodes in each ``nodepool``.
 
+Pilots and Cassandra Docker Images
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, Navigator will use the `Cassandra Docker images from DockerHub <https://hub.docker.com/_/cassandra/>`_.
+It will use an image with a tag matching the supplied ``CassandraCluster.Spec.Version`` field.
+If you prefer to use your own container image you should configure the ``CassandraCluster.Spec.Image`` fields.
+
+Navigator installs a ``navigator-pilot-cassandra`` executable into each Pod at the path ``/pilot``.
+This ``pilot`` process connects to the API server to:
+get extra configuration settings,
+report the status of this C* node, and to
+perform leader election of a single pilot in the cluster.
+
+The ``pilot`` overrides the following keys in the default ``/etc/cassandra/cassandra.yaml`` file:
+
+* ``cluster_name``: This will be set to match the name of the corresponding ``CassandraCluster`` resource in the API server.
+* ``listen_address`` / ``listen_interface`` / ``broadcast_address`` / ``rpc_address`` / ``broadcast_rpc_address``:  These keys and the corresponding values will be removed from the default configuration.
+  This ensures that Cassandra process listens and communicates using the IP address currently associated with the fully qualified domain name of the Pod.
+  This is important if the Pod is moved to another node and is assigned a different IP address.
+  Removing these settings from the Configuration file ensures that Cassandra uses the most recent IP address that Kubernetes has assigned to the Pod and that other C* nodes in the cluster are notified of the change of IP address.
+* ``seed_provider``: This is set to ``io.jetstack.cassandra.KubernetesSeedProvider`` which allows Cassandra to look up the seed IP addresses from a Kubernetes service.
+  The ``seed_provider.*.seeds`` sub key will be removed.
+  This is to avoid the risk of nodes mistakenly joining the cluster as seeds if the seed provider service is temporarily unavailable.
+
+The ``pilot`` also overwrites ``cassandra-rackdc.properties`` with values derived from the ``CassandraCluster.Spec.Nodepools`` (see :ref:`availability-zones-cassandra`).
+
+Finally the ``pilot`` executes ``/usr/bin/cassandra`` directly.
+
+.. note::
+   The default entry point (e.g. `/docker-entrypoint.sh <https://github.com/docker-library/cassandra/blob/master/3.11/docker-entrypoint.sh>`_ is ignored.
+
 Supported Versions
 ------------------
 

docs/quick-start/cassandra-cluster.yaml

@@ -31,3 +31,5 @@ spec:
   pilotImage:
     repository: "quay.io/jetstack/navigator-pilot-cassandra"
     tag: "v0.1.0"
+  securityContext:
+    runAsUser: 999

hack/download-integration-test-binaries.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Close stdin
+exec 0<&-
+
+ROOT_DIR="$(git rev-parse --show-toplevel)"
+
+ETCD_VERSION=v3.2.10
+ETCD_URL="https://storage.googleapis.com/etcd"
+
+KUBE_VERSION_URL="https://storage.googleapis.com/kubernetes-release/release/stable-1.10.txt"
+KUBE_VERSION=$(curl --fail --silent "${KUBE_VERSION_URL}")
+KUBE_BIN_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64"
+
+ASSETS_DIR="${ROOT_DIR}/vendor/sigs.k8s.io/testing_frameworks/integration/assets/bin"
+
+mkdir -p "${ASSETS_DIR}"
+
+curl --fail --silent ${ETCD_URL}/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | \
+    tar --extract --gzip --directory="${ASSETS_DIR}" --strip-components=1 '*/etcd'
+curl --fail --silent --output "${ASSETS_DIR}/kube-apiserver" "${KUBE_BIN_URL}/kube-apiserver"
+curl --fail --silent --output "${ASSETS_DIR}/kubectl" "${KUBE_BIN_URL}/kubectl"
+
+chmod +x ${ASSETS_DIR}/*

hack/testdata/cass-cluster-test.template.yaml

@@ -24,3 +24,5 @@ spec:
     repository: "${NAVIGATOR_IMAGE_REPOSITORY}/navigator-pilot-cassandra"
     tag: "${NAVIGATOR_IMAGE_TAG}"
     pullPolicy: "${NAVIGATOR_IMAGE_PULLPOLICY}"
+  securityContext:
+    runAsUser: 999

internal/test/util/testfs/testfs.go

@@ -0,0 +1,53 @@
+package testfs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type TestFs struct {
+	t *testing.T
+	d string
+}
+
+func New(t *testing.T) *TestFs {
+	d := fmt.Sprintf(".test/%s", t.Name())
+	d, err := filepath.Abs(d)
+	require.NoError(t, err)
+	err = os.RemoveAll(d)
+	if err != nil && !os.IsNotExist(err) {
+		t.Fatalf("Error while removing old test directory: %s", err)
+	}
+
+	err = os.MkdirAll(d, os.ModePerm)
+	require.NoError(t, err)
+
+	return &TestFs{
+		t: t,
+		d: d,
+	}
+}
+
+func (tfs *TestFs) TempPath(name string) string {
+	outPath := path.Join(tfs.d, name)
+	tmpFile, err := ioutil.TempFile(tfs.d, name)
+	require.NoError(tfs.t, err)
+	err = tmpFile.Close()
+	require.NoError(tfs.t, err)
+	err = os.Rename(tmpFile.Name(), outPath)
+	require.NoError(tfs.t, err)
+	return outPath
+}
+
+func (tfs *TestFs) TempDir(name string) string {
+	outPath := path.Join(tfs.d, name)
+	err := os.MkdirAll(outPath, os.ModePerm)
+	require.NoError(tfs.t, err)
+	return outPath
+}

pkg/config/config.go

@@ -0,0 +1,59 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+	shutil "github.com/termie/go-shutil"
+)
+
+const (
+	BackupSuffix = ".navigator_original"
+)
+
+type config struct {
+	*viper.Viper
+}
+
+type Interface interface {
+	WriteConfig() error
+	Set(key string, value interface{})
+	AllSettings() map[string]interface{}
+}
+
+var _ Interface = &config{}
+
+func newFrom(path, format string) (Interface, error) {
+	path, err := filepath.Abs(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to read absolute path")
+	}
+	backupPath := path + BackupSuffix
+	err = shutil.CopyFile(path, backupPath, false)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "unable to create backup file")
+	}
+	c := &config{viper.New()}
+	c.SetConfigFile(path)
+	c.SetConfigType(format)
+	f, err := os.Open(path)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "unable to open file")
+	}
+	defer f.Close()
+	err = c.ReadConfig(f)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to read file")
+	}
+	return c, nil
+}
+
+func NewFromYaml(path string) (Interface, error) {
+	return newFrom(path, "yaml")
+}
+
+func NewFromProperties(path string) (Interface, error) {
+	return newFrom(path, "properties")
+}

pkg/config/config_test.go

@@ -0,0 +1,47 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	shutil "github.com/termie/go-shutil"
+
+	"github.com/jetstack/navigator/internal/test/util/testfs"
+	"github.com/jetstack/navigator/pkg/config"
+)
+
+func TestRoundTrip(t *testing.T) {
+	tfs := testfs.New(t)
+
+	inPath := tfs.TempPath("config.yaml")
+	err := shutil.CopyFile("testdata/config1.yaml", inPath, false)
+	require.NoError(t, err)
+
+	c1, err := config.NewFromYaml(inPath)
+	require.NoError(t, err)
+
+	err = c1.WriteConfig()
+	require.NoError(t, err)
+
+	c2, err := config.NewFromYaml(inPath)
+	require.NoError(t, err)
+
+	assert.Equal(t, c1.AllSettings(), c2.AllSettings())
+}
+
+func TestBackup(t *testing.T) {
+	tfs := testfs.New(t)
+	path := tfs.TempPath("conf1.yaml")
+	_, err := config.NewFromYaml(path)
+	require.NoError(t, err)
+	assert.FileExists(t, path+".navigator_original")
+}
+
+func TestNoSource(t *testing.T) {
+	tfs := testfs.New(t)
+	dir := tfs.TempDir("conf1")
+	path := dir + "/conf1.yaml"
+	_, err := config.NewFromYaml(path)
+	require.NoError(t, err)
+}

pkg/config/testdata/config1.yaml

@@ -0,0 +1,3 @@
+foo:
+- bar
+- baz