Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: e2e test to demonstrate namespace deletion problem fix #2528

Merged
merged 6 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions sample-operators/controller-namespace-deletion/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
This sample demonstrates the workaround for problem when a namespace
is being deleted with a running controller, that watches resources
in its own namespace. If the pod or other underlying resources (role,
role binding, service account) are deleted before the cleanup of
the custom resource the namespace deletion is stuck.

see also: https://github.com/operator-framework/java-operator-sdk/pull/2528

62 changes: 62 additions & 0 deletions sample-operators/controller-namespace-deletion/k8s/operator.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: operator
finalizers:
- controller.deletion/finalizer

---
apiVersion: v1
kind: Pod
metadata:
name: operator
spec:
serviceAccountName: operator
containers:
- name: operator
image: controller-namespace-deletion-operator
imagePullPolicy: Never
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
terminationGracePeriodSeconds: 30

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: operator
finalizers:
- controller.deletion/finalizer
subjects:
- kind: ServiceAccount
name: operator
roleRef:
kind: Role
name: operator
apiGroup: rbac.authorization.k8s.io

---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: operator
finalizers:
- controller.deletion/finalizer
rules:
- apiGroups:
- "apiextensions.k8s.io"
resources:
- customresourcedefinitions
verbs:
- '*'
- apiGroups:
- "namespacedeletion.io"
resources:
- controllernamespacedeletioncustomresources
- controllernamespacedeletioncustomresources/status
verbs:
- '*'

87 changes: 87 additions & 0 deletions sample-operators/controller-namespace-deletion/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>io.javaoperatorsdk</groupId>
<artifactId>sample-operators</artifactId>
<version>5.0.0-SNAPSHOT</version>
</parent>

<artifactId>sample-controller-namespace-deletion</artifactId>
<packaging>jar</packaging>
<name>Operator SDK - Samples - Controller Namespace Deletion</name>
<description>Deleting namespace with controller and custom resources</description>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.javaoperatorsdk</groupId>
<artifactId>operator-framework-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<dependency>
<groupId>io.javaoperatorsdk</groupId>
<artifactId>operator-framework</artifactId>
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>crd-generator-apt</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.takes</groupId>
<artifactId>takes</artifactId>
<version>1.24.4</version>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.javaoperatorsdk</groupId>
<artifactId>operator-framework-junit-5</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>com.google.cloud.tools</groupId>
<artifactId>jib-maven-plugin</artifactId>
<version>${jib-maven-plugin.version}</version>
<configuration>
<from>
<image>gcr.io/distroless/java17-debian11</image>
</from>
<to>
<image>controller-namespace-deletion-operator</image>
</to>
</configuration>
</plugin>
</plugins>
</build>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package io.javaoperatorsdk.operator.sample;

import io.fabric8.kubernetes.api.model.Namespaced;
import io.fabric8.kubernetes.client.CustomResource;
import io.fabric8.kubernetes.model.annotation.Group;
import io.fabric8.kubernetes.model.annotation.Version;

@Group("namespacedeletion.io")
@Version("v1")
public class ControllerNamespaceDeletionCustomResource
extends CustomResource<ControllerNamespaceDeletionSpec, ControllerNamespaceDeletionStatus>
implements Namespaced {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package io.javaoperatorsdk.operator.sample;

import java.time.LocalTime;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.fabric8.kubernetes.client.KubernetesClientBuilder;
import io.javaoperatorsdk.operator.Operator;
import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider;

import static java.time.temporal.ChronoUnit.SECONDS;

public class ControllerNamespaceDeletionOperator {

private static final Logger log =
LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class);

public static void main(String[] args) {

Runtime.getRuntime().addShutdownHook(new Thread(() -> {
log.info("Shutting down...");
boolean allResourcesDeleted = waitUntilResourcesDeleted();
log.info("All resources within timeout: {}", allResourcesDeleted);
}));

Operator operator = new Operator();
operator.register(new ControllerNamespaceDeletionReconciler(),
ControllerConfigurationOverrider::watchingOnlyCurrentNamespace);
operator.start();
}

private static boolean waitUntilResourcesDeleted() {
try (var client = new KubernetesClientBuilder().build()) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we try to delay shutting down the informers and use them here instead of creating a new client?

Copy link
Collaborator Author

@csviri csviri Sep 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is just a demonstration, of how the problems could be solved, thus without a change into to operator impl for now.
In a next iteration will come with a comprehensive solution. Note that to cover the whole cycle we need to delay of stop of reconciliation thus the whole operator so the finalizers are correctly removed.

var startTime = LocalTime.now();
while (startTime.until(LocalTime.now(), SECONDS) < 20) {
var items =
client.resources(ControllerNamespaceDeletionCustomResource.class)
.inNamespace(client.getConfiguration().getNamespace())
.list().getItems();
log.info("Custom resource in namespace: {}", items);
if (items.isEmpty()) {
return true;
}
}
return false;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package io.javaoperatorsdk.operator.sample;

import java.time.Duration;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.javaoperatorsdk.operator.api.reconciler.Cleaner;
import io.javaoperatorsdk.operator.api.reconciler.Context;
import io.javaoperatorsdk.operator.api.reconciler.DeleteControl;
import io.javaoperatorsdk.operator.api.reconciler.Reconciler;
import io.javaoperatorsdk.operator.api.reconciler.UpdateControl;

public class ControllerNamespaceDeletionReconciler
implements Reconciler<ControllerNamespaceDeletionCustomResource>,
Cleaner<ControllerNamespaceDeletionCustomResource> {

private static final Logger log =
LoggerFactory.getLogger(ControllerNamespaceDeletionReconciler.class);

public static final Duration CLEANUP_DELAY = Duration.ofSeconds(10);

@Override
public UpdateControl<ControllerNamespaceDeletionCustomResource> reconcile(
ControllerNamespaceDeletionCustomResource resource,
Context<ControllerNamespaceDeletionCustomResource> context) {
log.info("Reconciling: {} in namespace: {}", resource.getMetadata().getName(),
resource.getMetadata().getNamespace());

var response = createResponseResource(resource);
response.getStatus().setValue(resource.getSpec().getValue());

return UpdateControl.patchStatus(response);
}

private ControllerNamespaceDeletionCustomResource createResponseResource(
ControllerNamespaceDeletionCustomResource resource) {
var res = new ControllerNamespaceDeletionCustomResource();
res.setMetadata(new ObjectMetaBuilder()
.withName(resource.getMetadata().getName())
.withNamespace(resource.getMetadata().getNamespace())
.build());
res.setStatus(new ControllerNamespaceDeletionStatus());
return res;
}

@Override
public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource,
Context<ControllerNamespaceDeletionCustomResource> context) {
log.info("Cleaning up resource");
try {
Thread.sleep(CLEANUP_DELAY.toMillis());
return DeleteControl.defaultDelete();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package io.javaoperatorsdk.operator.sample;


public class ControllerNamespaceDeletionSpec {

private String value;

public String getValue() {
return value;
}

public void setValue(String value) {
this.value = value;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package io.javaoperatorsdk.operator.sample;


public class ControllerNamespaceDeletionStatus {

private String value;

public String getValue() {
return value;
}

public void setValue(String value) {
this.value = value;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d %-30c{1.} [%-5level] %msg%n%throwable"/>
</Console>
</Appenders>
<Loggers>
<Root level="debug">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
Loading
Loading