Update to New Operator API Version (#145)

* Update to new operator API version

* Add check that initcontainer is attached by operator

* Misc test fixes

* Makefile refactor (adds verbosity)

* Update test app deployments for all agents

.github/workflows/test.yml

@@ -82,6 +82,7 @@ jobs:
             --set=controllerManager.manager.image.version=${{ inputs.K8S_OPERATOR_IMAGE_TAG }}
           sleep 5
           kubectl wait --for=condition=Ready -n default --all pods
+          sleep 30
 
       - name: Build init container
         run: |

tests/Makefile

@@ -2,12 +2,9 @@
 MAKEFILE_DIR:=$(dir $(realpath $(firstword ${MAKEFILE_LIST})))
 REPO_ROOT:=$(realpath ${MAKEFILE_DIR}../)
 
-# Initcontainer language under test
-INITCONTAINER_LANGUAGE:=${INITCONTAINER_LANGUAGE}
-
 # Path to the local copy of the repo newrelic/k8s-agents-operator.
 # Only required if building and testing the operator from source.
-LOCAL_OPERATOR_REPO_PATH:=${REPO_ROOT}/../k8s-agents-operator
+LOCAL_OPERATOR_REPO_PATH?=${REPO_ROOT}/../k8s-agents-operator
 
 .PHONY: default
 default: all
@@ -19,79 +16,108 @@ all: minikube operator test
 # Start minikube
 .PHONY: minikube
 minikube:
-	minikube start --driver=docker
-	helm repo update
+	@echo "===== Starting minikube cluster. ====="
+	@minikube start --driver=docker
 
 # Install official k8s-agents-operator repo
 .PHONY: operator
 operator:
-	helm uninstall k8s-agents-operator --ignore-not-found
-	helm repo add k8s-agents-operator https://newrelic.github.io/k8s-agents-operator
-	helm upgrade --install k8s-agents-operator k8s-agents-operator/k8s-agents-operator \
+	@echo "===== Removing any existing operator deployments. ====="
+	@helm uninstall k8s-agents-operator --ignore-not-found
+	@sleep 1
+	@echo "===== Adding and updating operator helm chart. ====="
+	@helm repo add --force-update k8s-agents-operator https://newrelic.github.io/k8s-agents-operator
+	@echo "===== Deploying operator to minikube cluster. ====="
+	@helm upgrade --install k8s-agents-operator k8s-agents-operator/k8s-agents-operator \
 		--set=licenseKey=${NEW_RELIC_LICENSE_KEY} \
 		--set=controllerManager.manager.image.version=edge
-	sleep 5
-	kubectl wait --for=condition=Ready pods $$(kubectl get pods | grep k8s-agents-operator | cut -d" " -f1)
+	@sleep 5
+	@kubectl wait --for=condition=Ready pods $$(kubectl get pods | grep k8s-agents-operator | cut -d" " -f1)
+	@echo "===== Giving operator time to generate certificates. ====="
+	@sleep 30
 
 # Build and install local copy of k8s-agents-operator
 .PHONY: operator-local
 operator-local:
-	eval $$(minikube docker-env --shell=bash) && \
+	@echo "===== Building operator container image. ====="
+	@eval $$(minikube docker-env --shell=bash) && \
 		docker build ${LOCAL_OPERATOR_REPO_PATH}/ -t e2e/k8s-agents-operator:e2e
-	helm uninstall k8s-agents-operator --ignore-not-found
-	sleep 1
-	helm upgrade --install k8s-agents-operator ${LOCAL_OPERATOR_REPO_PATH}/charts/k8s-agents-operator/ \
+	@echo "===== Removing any existing operator deployments. ====="
+	@helm uninstall k8s-agents-operator --ignore-not-found
+	@sleep 1
+	@echo "===== Deploying operator to minikube cluster. ====="
+	@helm upgrade --install k8s-agents-operator ${LOCAL_OPERATOR_REPO_PATH}/charts/k8s-agents-operator/ \
 		--set=licenseKey=${NEW_RELIC_LICENSE_KEY} \
 		--set=newRelicHost=${NEW_RELIC_HOST} \
 		--set=controllerManager.manager.image.pullPolicy=Never \
 		--set=controllerManager.manager.image.repository=e2e/k8s-agents-operator \
 		--set=controllerManager.manager.image.version=e2e
-	sleep 5
-	kubectl wait --for=condition=Ready pods $$(kubectl get pods | grep k8s-agents-operator | cut -d" " -f1)
+	@sleep 5
+	@kubectl wait --for=condition=Ready pods $$(kubectl get pods | grep k8s-agents-operator | cut -d" " -f1)
+	@echo "===== Giving operator time to generate certificates. ====="
+	@sleep 30
 
 # Set license key in default namespace
 .PHONY: license_key_secret
 license_key_secret:
-	kubectl delete secret newrelic-key-secret --ignore-not-found
-	kubectl create secret generic newrelic-key-secret \
+	@echo "===== Setting license key secret. ====="
+	@kubectl delete secret newrelic-key-secret --ignore-not-found
+	@kubectl create secret generic newrelic-key-secret \
 		--from-literal="new_relic_license_key=${NEW_RELIC_LICENSE_KEY}"
 
 # Build local initcontainer image
 .PHONY: build-initcontainer
 build-initcontainer: check-language-arg
-	eval $$(minikube docker-env --shell=bash) && \
+	@echo "===== Building initcontainer image. ====="
+	@eval $$(minikube docker-env --shell=bash) && \
 		docker build -t e2e/newrelic-${INITCONTAINER_LANGUAGE}-init:e2e ${REPO_ROOT}/src/${INITCONTAINER_LANGUAGE}/
 
 # Build local test app image
 .PHONY: build-testapp
 build-testapp: check-language-arg
-	eval $$(minikube docker-env --shell=bash) && \
+	@echo "===== Building test app container image. ====="
+	@eval $$(minikube docker-env --shell=bash) && \
 		docker build -t e2e/test-app-${INITCONTAINER_LANGUAGE}:e2e ${REPO_ROOT}/tests/${INITCONTAINER_LANGUAGE}/
 
 # Deploy and open test app in browser
 .PHONY: test
 test: build-initcontainer build-testapp check-language-arg
-	helm uninstall test-deployment --ignore-not-found
-	sleep 1
-	helm install test-deployment ${REPO_ROOT}/tests/${INITCONTAINER_LANGUAGE}/chart/
-	sleep 3
-	kubectl wait --for=condition=Ready -n default --all pods
-	minikube service test-app-${INITCONTAINER_LANGUAGE}-service -n default
+	@echo "===== Removing any existing test app deployments. ====="
+	@helm uninstall test-deployment --ignore-not-found
+	@sleep 1
+	@echo "===== Deploying test app to minikube cluster. ====="
+	@helm install test-deployment ${REPO_ROOT}/tests/${INITCONTAINER_LANGUAGE}/chart/
+	@sleep 5
+	@kubectl wait --for=condition=Ready -n default --all pods
+	@echo "===== Recreating test app pod to ensure instrumentation resource is used. ====="
+	@kubectl delete pods -l "app=test-app-${INITCONTAINER_LANGUAGE}"
+	@sleep 5
+	@kubectl wait --for=condition=Ready -n default --all pods
+# If yq is installed, check for the initcontainer to be visible on the pod spec
+	@which -s yq && \
+		echo "===== Ensuring initcontainer was successfully attached by operator. ====="; \
+		kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" \
+			&& echo "===== Initcontainer successfully attached. =====" \
+			|| { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+	@echo "===== Opening tunnel to test app and opening web browser. ====="
+	@minikube service test-app-${INITCONTAINER_LANGUAGE}-service -n default
 
 # View test app container logs
 .PHONY: get-pods
 get-pods:
-	kubectl get pods -n default
+	@kubectl get pods -n default
 
 # View test app container logs
 .PHONY: logs-testapp
 logs-testapp:
-	kubectl logs $$(kubectl get pods -n default | grep test-app- | cut -d" " -f1)
+	@echo "===== Test app logs: ====="
+	@kubectl logs $$(kubectl get pods -n default | grep test-app- | cut -d" " -f1)
 
 # View test app container logs
 .PHONY: logs-operator
 logs-operator:
-	kubectl logs $$(kubectl get pods | grep k8s-agents-operator | cut -d" " -f1)
+	@echo "===== Operator logs: ====="
+	@kubectl logs $$(kubectl get pods | grep k8s-agents-operator | cut -d" " -f1)
 
 .PHONY: check-language-arg
 check-language-arg:

tests/dotnet/chart/templates/instrumentation.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: newrelic.com/v1alpha2
+apiVersion: newrelic.com/v1beta1
 kind: Instrumentation
 metadata:
   name: newrelic-instrumentation-dotnet

tests/dotnet/test-specs.yml

@@ -1,12 +1,21 @@
-description: End-to-end tests for dotnet initcontainer
+description: End-to-end tests for .NET initcontainer
 custom_test_key: tags.testKey
 scenarios:
   - description: This scenario will verify that a transaction is reported by the test app after a curl request
     before:
+      # Deploy test app chart
       - helm install test-dotnet ./chart/ --set=scenarioTag="${SCENARIO_TAG}" -n default
       - sleep 5
-      - kubectl wait --for=condition=Ready -n default --all pods
-      - sleep 15 # give the agent time to connect
+      # Recreate test app pod to ensure instrumentation resource is used
+      - kubectl delete pods -l "app=test-app-dotnet"
+      # Wait for pods to pass healthchecks
+      - sleep 5
+      - kubectl wait --for=condition=Ready -n default --all pods --timeout=2m
+      # Ensure initcontainer was attached by operator
+      - kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" || { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+      # Give the agent time to connect
+      - sleep 15
+      # Send traffic to test app to generate transactions
       - curl --fail-with-body $(minikube service test-app-dotnet-service --url -n default)/WeatherForecast
     tests:
       nrqls:

tests/java/chart/templates/instrumentation.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: newrelic.com/v1alpha2
+apiVersion: newrelic.com/v1beta1
 kind: Instrumentation
 metadata:
   name: newrelic-instrumentation-java

tests/java/test-specs.yml

@@ -1,11 +1,19 @@
-description: End-to-end tests for java initcontainer
+description: End-to-end tests for Java initcontainer
 custom_test_key: tags.testKey
 scenarios:
   - description: This scenario will verify that a transaction is reported by the test app after a curl request
     before:
+      # Deploy test app chart
       - helm install test-java ./chart/ --set=scenarioTag="${SCENARIO_TAG}" -n default
+      - sleep 5
+      # Recreate test app pod to ensure instrumentation resource is used
+      - kubectl delete pods -l "app=test-app-java"
+      # Wait for pods to pass healthchecks
       - sleep 120
-      - kubectl wait --for=condition=Ready -n default --all pods
+      - kubectl wait --for=condition=Ready -n default --all pods --timeout=2m
+      # Ensure initcontainer was attached by operator
+      - kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" || { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+      # Send traffic to test app to generate transactions
       - curl --fail-with-body $(minikube service test-app-java-service --url -n default)
     tests:
       nrqls:

tests/nodejs/chart/templates/instrumentation.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: newrelic.com/v1alpha2
+apiVersion: newrelic.com/v1beta1
 kind: Instrumentation
 metadata:
   name: newrelic-instrumentation-nodejs

tests/nodejs/test-specs.yml

@@ -3,9 +3,17 @@ custom_test_key: tags.testKey
 scenarios:
   - description: This scenario will verify that a transaction is reported by the test app after a curl request
     before:
+      # Deploy test app chart
       - helm install test-nodejs ./chart/ --set=scenarioTag="${SCENARIO_TAG}" -n default
       - sleep 5
-      - kubectl wait --for=condition=Ready -n default --all pods
+      # Recreate test app pod to ensure instrumentation resource is used
+      - kubectl delete pods -l "app=test-app-nodejs"
+      # Wait for pods to pass healthchecks
+      - sleep 5
+      - kubectl wait --for=condition=Ready -n default --all pods --timeout=2m
+      # Ensure initcontainer was attached by operator
+      - kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" || { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+      # Send traffic to test app to generate transactions
       - curl --fail-with-body $(minikube service test-app-nodejs-service --url -n default)
     tests:
       nrqls:

tests/php/chart/templates/instrumentation.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: newrelic.com/v1alpha2
+apiVersion: newrelic.com/v1beta1
 kind: Instrumentation
 metadata:
   name: newrelic-instrumentation-php

tests/php/test-specs.yml

@@ -1,11 +1,19 @@
-description: End-to-end tests for php initcontainer
+description: End-to-end tests for PHP initcontainer
 custom_test_key: tags.testKey
 scenarios:
   - description: This scenario will verify that a transaction is reported by the test app after a curl request
     before:
+      # Deploy test app chart
       - helm install test-php ./chart/ --set=scenarioTag="${SCENARIO_TAG}" ${TEST_APP_HELM_CHART_ARGS} -n default
       - sleep 5
+      # Recreate test app pod to ensure instrumentation resource is used
+      - kubectl delete pods -l "app=test-app-php"
+      # Wait for pods to pass healthchecks
+      - sleep 5
       - kubectl wait --for=condition=Ready -n default --all pods --timeout=2m
+      # Ensure initcontainer was attached by operator
+      - kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" || { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+      # Send traffic to test app to generate transactions
       - curl --fail-with-body $(minikube service test-app-php-service --url -n default)
       - sleep 5
       - curl --fail-with-body $(minikube service test-app-php-service --url -n default)

tests/python/chart/templates/instrumentation.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: newrelic.com/v1alpha2
+apiVersion: newrelic.com/v1beta1
 kind: Instrumentation
 metadata:
   name: newrelic-instrumentation-python

tests/python/test-specs.yml

@@ -1,11 +1,19 @@
-description: End-to-end tests for python initcontainer
+description: End-to-end tests for Python initcontainer
 custom_test_key: tags.testKey
 scenarios:
   - description: This scenario will verify that a transaction is reported by the test app after a curl request
     before:
+      # Deploy test app chart
       - helm install test-python ./chart/ --set=scenarioTag="${SCENARIO_TAG}" -n default
       - sleep 5
+      # Recreate test app pod to ensure instrumentation resource is used
+      - kubectl delete pods -l "app=test-app-python"
+      # Wait for pods to pass healthchecks
+      - sleep 5
       - kubectl wait --for=condition=Ready -n default --all pods --timeout=2m
+      # Ensure initcontainer was attached by operator
+      - kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" || { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+      # Send traffic to test app to generate transactions
       - curl --fail-with-body $(minikube service test-app-python-service --url -n default)
     tests:
       nrqls:

tests/ruby/chart/templates/instrumentation.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: newrelic.com/v1alpha2
+apiVersion: newrelic.com/v1beta1
 kind: Instrumentation
 metadata:
   name: newrelic-instrumentation-ruby

tests/ruby/test-specs.yml

@@ -1,11 +1,19 @@
-description: End-to-end tests for ruby initcontainer
+description: End-to-end tests for Ruby initcontainer
 custom_test_key: tags.testKey
 scenarios:
   - description: This scenario will verify that a transaction is reported by the test app after a curl request
     before:
+      # Deploy test app chart
       - helm install test-ruby ./chart/ --set=scenarioTag="${SCENARIO_TAG}" -n default
       - sleep 5
-      - kubectl wait --for=condition=Ready -n default --all pods
+      # Recreate test app pod to ensure instrumentation resource is used
+      - kubectl delete pods -l "app=test-app-ruby"
+      # Wait for pods to pass healthchecks
+      - sleep 5
+      - kubectl wait --for=condition=Ready -n default --all pods --timeout=2m
+      # Ensure initcontainer was attached by operator
+      - kubectl get pods --output yaml | yq '.items[].spec.initContainers[].name' | grep -q "newrelic-instrumentation-" || { echo -e "===== Operator failed to attach initcontainer. =====" >&2; exit 1; }
+      # Send traffic to test app to generate transactions
       - curl --fail-with-body $(minikube service test-app-ruby-service --url -n default)/
     tests:
       nrqls: