diff --git a/sample-operators/controller-namespace-deletion/README.md b/sample-operators/controller-namespace-deletion/README.md new file mode 100644 index 0000000000..3ea02d1d36 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/README.md @@ -0,0 +1,8 @@ +This sample demonstrates the workaround for problem when a namespace +is being deleted with a running controller, that watches resources +in its own namespace. If the pod or other underlying resources (role, +role binding, service account) are deleted before the cleanup of +the custom resource the namespace deletion is stuck. + +see also: https://github.com/operator-framework/java-operator-sdk/pull/2528 + diff --git a/sample-operators/controller-namespace-deletion/k8s/operator.yaml b/sample-operators/controller-namespace-deletion/k8s/operator.yaml new file mode 100644 index 0000000000..bc9eeb84ed --- /dev/null +++ b/sample-operators/controller-namespace-deletion/k8s/operator.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + finalizers: + - controller.deletion/finalizer + +--- +apiVersion: v1 +kind: Pod +metadata: + name: operator +spec: + serviceAccountName: operator + containers: + - name: operator + image: controller-namespace-deletion-operator + imagePullPolicy: Never + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + terminationGracePeriodSeconds: 30 + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operator + finalizers: + - controller.deletion/finalizer +subjects: + - kind: ServiceAccount + name: operator +roleRef: + kind: Role + name: operator + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator + finalizers: + - controller.deletion/finalizer +rules: + - apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - '*' + - apiGroups: + - "namespacedeletion.io" + resources: + - controllernamespacedeletioncustomresources + - controllernamespacedeletioncustomresources/status + verbs: + - '*' + diff --git a/sample-operators/controller-namespace-deletion/pom.xml b/sample-operators/controller-namespace-deletion/pom.xml new file mode 100644 index 0000000000..4979eec0d1 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + + + io.javaoperatorsdk + sample-operators + 5.0.0-SNAPSHOT + + + sample-controller-namespace-deletion + jar + Operator SDK - Samples - Controller Namespace Deletion + Deleting namespace with controller and custom resources + + + + + io.javaoperatorsdk + operator-framework-bom + ${project.version} + pom + import + + + + + + + io.javaoperatorsdk + operator-framework + + + io.fabric8 + crd-generator-apt + provided + + + org.apache.logging.log4j + log4j-slf4j2-impl + compile + + + org.apache.logging.log4j + log4j-core + compile + + + org.takes + takes + 1.24.4 + + + org.awaitility + awaitility + compile + + + io.javaoperatorsdk + operator-framework-junit-5 + test + + + org.junit.jupiter + junit-jupiter-params + test + + + + + + com.google.cloud.tools + jib-maven-plugin + ${jib-maven-plugin.version} + + + gcr.io/distroless/java17-debian11 + + + controller-namespace-deletion-operator + + + + + + + diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java new file mode 100644 index 0000000000..ae0f1034ee --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java @@ -0,0 +1,14 @@ +package io.javaoperatorsdk.operator.sample; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("namespacedeletion.io") +@Version("v1") +public class ControllerNamespaceDeletionCustomResource + extends CustomResource + implements Namespaced { + +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java new file mode 100644 index 0000000000..5364852467 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java @@ -0,0 +1,49 @@ +package io.javaoperatorsdk.operator.sample; + +import java.time.LocalTime; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.client.KubernetesClientBuilder; +import io.javaoperatorsdk.operator.Operator; +import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider; + +import static java.time.temporal.ChronoUnit.SECONDS; + +public class ControllerNamespaceDeletionOperator { + + private static final Logger log = + LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class); + + public static void main(String[] args) { + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + log.info("Shutting down..."); + boolean allResourcesDeleted = waitUntilResourcesDeleted(); + log.info("All resources within timeout: {}", allResourcesDeleted); + })); + + Operator operator = new Operator(); + operator.register(new ControllerNamespaceDeletionReconciler(), + ControllerConfigurationOverrider::watchingOnlyCurrentNamespace); + operator.start(); + } + + private static boolean waitUntilResourcesDeleted() { + try (var client = new KubernetesClientBuilder().build()) { + var startTime = LocalTime.now(); + while (startTime.until(LocalTime.now(), SECONDS) < 20) { + var items = + client.resources(ControllerNamespaceDeletionCustomResource.class) + .inNamespace(client.getConfiguration().getNamespace()) + .list().getItems(); + log.info("Custom resource in namespace: {}", items); + if (items.isEmpty()) { + return true; + } + } + return false; + } + } +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java new file mode 100644 index 0000000000..7261f269b4 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java @@ -0,0 +1,59 @@ +package io.javaoperatorsdk.operator.sample; + +import java.time.Duration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; + +public class ControllerNamespaceDeletionReconciler + implements Reconciler, + Cleaner { + + private static final Logger log = + LoggerFactory.getLogger(ControllerNamespaceDeletionReconciler.class); + + public static final Duration CLEANUP_DELAY = Duration.ofSeconds(10); + + @Override + public UpdateControl reconcile( + ControllerNamespaceDeletionCustomResource resource, + Context context) { + log.info("Reconciling: {} in namespace: {}", resource.getMetadata().getName(), + resource.getMetadata().getNamespace()); + + var response = createResponseResource(resource); + response.getStatus().setValue(resource.getSpec().getValue()); + + return UpdateControl.patchStatus(response); + } + + private ControllerNamespaceDeletionCustomResource createResponseResource( + ControllerNamespaceDeletionCustomResource resource) { + var res = new ControllerNamespaceDeletionCustomResource(); + res.setMetadata(new ObjectMetaBuilder() + .withName(resource.getMetadata().getName()) + .withNamespace(resource.getMetadata().getNamespace()) + .build()); + res.setStatus(new ControllerNamespaceDeletionStatus()); + return res; + } + + @Override + public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource, + Context context) { + log.info("Cleaning up resource"); + try { + Thread.sleep(CLEANUP_DELAY.toMillis()); + return DeleteControl.defaultDelete(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java new file mode 100644 index 0000000000..dc5092e7e5 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.sample; + + +public class ControllerNamespaceDeletionSpec { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java new file mode 100644 index 0000000000..732fa7d626 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.sample; + + +public class ControllerNamespaceDeletionStatus { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml b/sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..0ec69bf713 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java new file mode 100644 index 0000000000..36c7f132ab --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java @@ -0,0 +1,144 @@ +package io.javaoperatorsdk.operator.sample; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.rbac.RoleBinding; +import io.fabric8.kubernetes.client.ConfigBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; + +import static io.javaoperatorsdk.operator.junit.AbstractOperatorExtension.CRD_READY_WAIT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + + +class ControllerNamespaceDeletionE2E { + + private static final Logger log = LoggerFactory.getLogger(ControllerNamespaceDeletionE2E.class); + + public static final String TEST_RESOURCE_NAME = "test1"; + public static final String INITIAL_VALUE = "initial value"; + public static final String ROLE_ROLE_BINDING_FINALIZER = "controller.deletion/finalizer"; + public static final String RESOURCE_NAME = "operator"; + + String namespace; + KubernetesClient client; + + // not for local mode by design + @EnabledIfSystemProperty(named = "test.deployment", matches = "remote") + @Test + void customResourceCleanedUpOnNamespaceDeletion() { + deployController(); + client.resource(testResource()).serverSideApply(); + + await().untilAsserted(() -> { + var res = client.resources(ControllerNamespaceDeletionCustomResource.class) + .inNamespace(namespace).withName(TEST_RESOURCE_NAME).get(); + assertThat(res.getStatus()).isNotNull(); + assertThat(res.getStatus().getValue()).isEqualTo(INITIAL_VALUE); + }); + + client.namespaces().withName(namespace).delete(); + + await().timeout(Duration.ofSeconds(20)).untilAsserted(() -> { + var ns = client.resources(ControllerNamespaceDeletionCustomResource.class) + .inNamespace(namespace).withName(TEST_RESOURCE_NAME).get(); + assertThat(ns).isNull(); + }); + + log.info("Removing finalizers from role and role bing and service account"); + removeRoleAndRoleBindingFinalizers(); + + await().timeout(Duration.ofSeconds(20)).untilAsserted(() -> { + var ns = client.namespaces().withName(namespace).get(); + assertThat(ns).isNull(); + }); + } + + private void removeRoleAndRoleBindingFinalizers() { + var rolebinding = + client.rbac().roleBindings().inNamespace(namespace).withName(RESOURCE_NAME).get(); + rolebinding.getFinalizers().clear(); + client.resource(rolebinding).update(); + + var role = client.rbac().roles().inNamespace(namespace).withName(RESOURCE_NAME).get(); + role.getFinalizers().clear(); + client.resource(role).update(); + + var sa = client.serviceAccounts().inNamespace(namespace).withName(RESOURCE_NAME).get(); + sa.getMetadata().getFinalizers().clear(); + client.resource(sa).update(); + } + + ControllerNamespaceDeletionCustomResource testResource() { + var cr = new ControllerNamespaceDeletionCustomResource(); + cr.setMetadata(new ObjectMetaBuilder() + .withName(TEST_RESOURCE_NAME) + .withNamespace(namespace) + .build()); + cr.setSpec(new ControllerNamespaceDeletionSpec()); + cr.getSpec().setValue(INITIAL_VALUE); + return cr; + } + + + @BeforeEach + void setup() { + namespace = "controller-namespace-" + UUID.randomUUID(); + client = new KubernetesClientBuilder().withConfig(new ConfigBuilder() + .withNamespace(namespace) + .build()).build(); + applyCRD(); + client.namespaces().resource(new NamespaceBuilder().withNewMetadata().withName(namespace) + .endMetadata().build()).create(); + } + + void deployController() { + try { + List resources = client.load(new FileInputStream("k8s/operator.yaml")).items(); + resources.forEach(hm -> { + hm.getMetadata().setNamespace(namespace); + if (hm.getKind().equalsIgnoreCase("rolebinding")) { + var crb = (RoleBinding) hm; + for (var subject : crb.getSubjects()) { + subject.setNamespace(namespace); + } + } + }); + client.resourceList(resources) + .inNamespace(namespace) + .createOrReplace(); + + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } + + void applyCRD() { + String path = + "target/classes/META-INF/fabric8/controllernamespacedeletioncustomresources.namespacedeletion.io-v1.yml"; + try (InputStream is = new FileInputStream(path)) { + final var crd = client.load(is); + crd.serverSideApply(); + Thread.sleep(CRD_READY_WAIT); + log.debug("Applied CRD with name: {}", crd.get().get(0).getMetadata().getName()); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml b/sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..2b7fdd3479 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/sample-operators/leader-election/pom.xml b/sample-operators/leader-election/pom.xml index 894b96d988..d686634ad7 100644 --- a/sample-operators/leader-election/pom.xml +++ b/sample-operators/leader-election/pom.xml @@ -76,11 +76,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - io.fabric8 diff --git a/sample-operators/mysql-schema/pom.xml b/sample-operators/mysql-schema/pom.xml index bc8207a92e..46d555966c 100644 --- a/sample-operators/mysql-schema/pom.xml +++ b/sample-operators/mysql-schema/pom.xml @@ -102,11 +102,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - diff --git a/sample-operators/pom.xml b/sample-operators/pom.xml index 6d09f9a3ad..478508f9d5 100644 --- a/sample-operators/pom.xml +++ b/sample-operators/pom.xml @@ -17,5 +17,6 @@ webpage mysql-schema leader-election + controller-namespace-deletion diff --git a/sample-operators/tomcat-operator/pom.xml b/sample-operators/tomcat-operator/pom.xml index 7af9893609..d22260c614 100644 --- a/sample-operators/tomcat-operator/pom.xml +++ b/sample-operators/tomcat-operator/pom.xml @@ -104,11 +104,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - diff --git a/sample-operators/webpage/pom.xml b/sample-operators/webpage/pom.xml index 9982da49c4..34b6846b94 100644 --- a/sample-operators/webpage/pom.xml +++ b/sample-operators/webpage/pom.xml @@ -75,11 +75,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 -