feat: e2e test to demonstrate namespace deletion problem fix (#2528)

Signed-off-by: Attila Mészáros <[email protected]>

sample-operators/controller-namespace-deletion/README.md

@@ -0,0 +1,8 @@
+This sample demonstrates the workaround for problem when a namespace
+is being deleted with a running controller, that watches resources
+in its own namespace. If the pod or other underlying resources (role,
+role binding, service account) are deleted before the cleanup of
+the custom resource the namespace deletion is stuck.
+
+see also: https://github.com/operator-framework/java-operator-sdk/pull/2528
+

sample-operators/controller-namespace-deletion/k8s/operator.yaml

@@ -0,0 +1,62 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: operator
+  finalizers:
+    - controller.deletion/finalizer
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator
+spec:
+  serviceAccountName: operator
+  containers:
+    - name: operator
+      image: controller-namespace-deletion-operator
+      imagePullPolicy: Never
+      env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+  terminationGracePeriodSeconds: 30
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: operator
+  finalizers:
+    - controller.deletion/finalizer
+subjects:
+  - kind: ServiceAccount
+    name: operator
+roleRef:
+  kind: Role
+  name: operator
+  apiGroup: rbac.authorization.k8s.io
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: operator
+  finalizers:
+    - controller.deletion/finalizer
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - '*'
+  - apiGroups:
+      - "namespacedeletion.io"
+    resources:
+      - controllernamespacedeletioncustomresources
+      - controllernamespacedeletioncustomresources/status
+    verbs:
+      - '*'
+

sample-operators/controller-namespace-deletion/pom.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>io.javaoperatorsdk</groupId>
+    <artifactId>sample-operators</artifactId>
+    <version>5.0.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>sample-controller-namespace-deletion</artifactId>
+  <packaging>jar</packaging>
+  <name>Operator SDK - Samples - Controller Namespace Deletion</name>
+  <description>Deleting namespace with controller and custom resources</description>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>io.javaoperatorsdk</groupId>
+        <artifactId>operator-framework-bom</artifactId>
+        <version>${project.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.javaoperatorsdk</groupId>
+      <artifactId>operator-framework</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>crd-generator-apt</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j2-impl</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.takes</groupId>
+      <artifactId>takes</artifactId>
+      <version>1.24.4</version>
+    </dependency>
+    <dependency>
+      <groupId>org.awaitility</groupId>
+      <artifactId>awaitility</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.javaoperatorsdk</groupId>
+      <artifactId>operator-framework-junit-5</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.google.cloud.tools</groupId>
+        <artifactId>jib-maven-plugin</artifactId>
+        <version>${jib-maven-plugin.version}</version>
+        <configuration>
+          <from>
+            <image>gcr.io/distroless/java17-debian11</image>
+          </from>
+          <to>
+            <image>controller-namespace-deletion-operator</image>
+          </to>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java

@@ -0,0 +1,14 @@
+package io.javaoperatorsdk.operator.sample;
+
+import io.fabric8.kubernetes.api.model.Namespaced;
+import io.fabric8.kubernetes.client.CustomResource;
+import io.fabric8.kubernetes.model.annotation.Group;
+import io.fabric8.kubernetes.model.annotation.Version;
+
+@Group("namespacedeletion.io")
+@Version("v1")
+public class ControllerNamespaceDeletionCustomResource
+    extends CustomResource<ControllerNamespaceDeletionSpec, ControllerNamespaceDeletionStatus>
+    implements Namespaced {
+
+}

sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java

@@ -0,0 +1,49 @@
+package io.javaoperatorsdk.operator.sample;
+
+import java.time.LocalTime;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.fabric8.kubernetes.client.KubernetesClientBuilder;
+import io.javaoperatorsdk.operator.Operator;
+import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider;
+
+import static java.time.temporal.ChronoUnit.SECONDS;
+
+public class ControllerNamespaceDeletionOperator {
+
+  private static final Logger log =
+      LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class);
+
+  public static void main(String[] args) {
+
+    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+      log.info("Shutting down...");
+      boolean allResourcesDeleted = waitUntilResourcesDeleted();
+      log.info("All resources within timeout: {}", allResourcesDeleted);
+    }));
+
+    Operator operator = new Operator();
+    operator.register(new ControllerNamespaceDeletionReconciler(),
+        ControllerConfigurationOverrider::watchingOnlyCurrentNamespace);
+    operator.start();
+  }
+
+  private static boolean waitUntilResourcesDeleted() {
+    try (var client = new KubernetesClientBuilder().build()) {
+      var startTime = LocalTime.now();
+      while (startTime.until(LocalTime.now(), SECONDS) < 20) {
+        var items =
+            client.resources(ControllerNamespaceDeletionCustomResource.class)
+                .inNamespace(client.getConfiguration().getNamespace())
+                .list().getItems();
+        log.info("Custom resource in namespace: {}", items);
+        if (items.isEmpty()) {
+          return true;
+        }
+      }
+      return false;
+    }
+  }
+}

sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java

@@ -0,0 +1,59 @@
+package io.javaoperatorsdk.operator.sample;
+
+import java.time.Duration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.javaoperatorsdk.operator.api.reconciler.Cleaner;
+import io.javaoperatorsdk.operator.api.reconciler.Context;
+import io.javaoperatorsdk.operator.api.reconciler.DeleteControl;
+import io.javaoperatorsdk.operator.api.reconciler.Reconciler;
+import io.javaoperatorsdk.operator.api.reconciler.UpdateControl;
+
+public class ControllerNamespaceDeletionReconciler
+    implements Reconciler<ControllerNamespaceDeletionCustomResource>,
+    Cleaner<ControllerNamespaceDeletionCustomResource> {
+
+  private static final Logger log =
+      LoggerFactory.getLogger(ControllerNamespaceDeletionReconciler.class);
+
+  public static final Duration CLEANUP_DELAY = Duration.ofSeconds(10);
+
+  @Override
+  public UpdateControl<ControllerNamespaceDeletionCustomResource> reconcile(
+      ControllerNamespaceDeletionCustomResource resource,
+      Context<ControllerNamespaceDeletionCustomResource> context) {
+    log.info("Reconciling: {} in namespace: {}", resource.getMetadata().getName(),
+        resource.getMetadata().getNamespace());
+
+    var response = createResponseResource(resource);
+    response.getStatus().setValue(resource.getSpec().getValue());
+
+    return UpdateControl.patchStatus(response);
+  }
+
+  private ControllerNamespaceDeletionCustomResource createResponseResource(
+      ControllerNamespaceDeletionCustomResource resource) {
+    var res = new ControllerNamespaceDeletionCustomResource();
+    res.setMetadata(new ObjectMetaBuilder()
+        .withName(resource.getMetadata().getName())
+        .withNamespace(resource.getMetadata().getNamespace())
+        .build());
+    res.setStatus(new ControllerNamespaceDeletionStatus());
+    return res;
+  }
+
+  @Override
+  public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource,
+      Context<ControllerNamespaceDeletionCustomResource> context) {
+    log.info("Cleaning up resource");
+    try {
+      Thread.sleep(CLEANUP_DELAY.toMillis());
+      return DeleteControl.defaultDelete();
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java

@@ -0,0 +1,15 @@
+package io.javaoperatorsdk.operator.sample;
+
+
+public class ControllerNamespaceDeletionSpec {
+
+  private String value;
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+}

sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java

@@ -0,0 +1,15 @@
+package io.javaoperatorsdk.operator.sample;
+
+
+public class ControllerNamespaceDeletionStatus {
+
+  private String value;
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+}

sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d %-30c{1.} [%-5level] %msg%n%throwable"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="debug">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>