Merge pull request #49 from ocadotechnology/update-apis

feat: upgrade to more recent API and client

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6-alpine
+FROM python:3.8-alpine3.12
 RUN apk --no-cache add curl
 COPY . ./app
 WORKDIR /app

mirroroperator/operator.py

@@ -41,23 +41,37 @@ def __init__(self, env_vars):
     def watch_registry_mirrors(self):
         watcher = kubernetes.watch.Watch()
         try:
-            for event in watcher.stream(self.object_api.list_cluster_custom_object, CRD_GROUP, CRD_VERSION, CRD_PLURAL):
+            for event in watcher.stream(
+                self.object_api.list_cluster_custom_object,
+                CRD_GROUP,
+                CRD_VERSION,
+                CRD_PLURAL
+            ):
                 registry_mirror_kwargs = event['object'].copy()
                 registry_mirror_kwargs.update(self.registry_mirror_vars)
-                mirror = RegistryMirror(event_type=event['type'], **registry_mirror_kwargs)
+                LOGGER.debug("RM kwargs: {}".format(registry_mirror_kwargs))
+                mirror = RegistryMirror(
+                    event_type=event['type'], **registry_mirror_kwargs
+                )
                 mirror.apply()
         except ApiException as e:
             status = HTTPStatus(e.status)
             if status == HTTPStatus.NOT_FOUND:
-                raise NoCRDException("CRD not found. Please ensure you create a CRD with group - %s,"
-                                     "version - %s and plural - %s before this operator can run.",
-                                     CRD_GROUP, CRD_VERSION, CRD_PLURAL)
+                raise NoCRDException(
+                    "CRD not found. Please ensure you create a CRD with group"
+                    " - %s, version - %s and plural - %s before this operator"
+                    " can run.",
+                    CRD_GROUP, CRD_VERSION, CRD_PLURAL)
             else:
-                LOGGER.exception("Error watching custom object events", exc_info=True)
+                LOGGER.exception(
+                    "Error watching custom object events",
+                    exc_info=True
+                )
 
 
 def safely_eval_env(env_var):
-    return ast.literal_eval(os.environ.get(env_var)) if os.environ.get(env_var) is not None else None
+    return ast.literal_eval(os.environ.get(env_var)
+                            ) if os.environ.get(env_var) is not None else None
 
 
 if __name__ == '__main__':
@@ -67,7 +81,8 @@ def safely_eval_env(env_var):
     env_vars = dict(
         namespace=os.environ.get("NAMESPACE", "default"),
         # optional to allow for image to be pulled from elsewhere
-        hostess_docker_registry=os.environ.get("HOSTESS_DOCKER_REGISTRY", "docker.io"),
+        hostess_docker_registry=os.environ.get(
+            "HOSTESS_DOCKER_REGISTRY", "docker.io"),
         hostess_docker_image=os.environ.get("HOSTESS_DOCKER_IMAGE",
                                             "ocadotechnology/mirror-hostess"),
         hostess_docker_tag=os.environ.get("HOSTESS_DOCKER_TAG", "1.1.0"),

mirroroperator/registrymirror.py

@@ -36,7 +36,8 @@ def __init__(self, event_type, namespace, hostess_docker_registry,
         self.apiVersion = kwargs.get("apiVersion")
         upstream_url = kwargs.get("spec", {}).get("upstreamUrl")
 
-        self.masquerade_url = kwargs.get("spec", {}).get("masqueradeUrl", "mirror-"+upstream_url)
+        self.masquerade_url = kwargs.get("spec", {}).get(
+            "masqueradeUrl", "mirror-"+upstream_url)
 
         self.credentials_secret_name = kwargs.get(
             "spec", {}).get("credentialsSecret")
@@ -50,16 +51,18 @@ def __init__(self, event_type, namespace, hostess_docker_registry,
         self.image_pull_secrets = kwargs["image_pull_secrets"] or ""
         self.ca_certificate_bundle = kwargs["ca_certificate_bundle"]
 
-        self.volume_claim_spec = client.V1PersistentVolumeClaimSpec(**kwargs.get(
-            "spec",
-            {},
-        ).get(
-            "volumeClaimTemplate",
-            {},
-        ).get(
-            "spec",
-            {},
-        ))
+        self.volume_claim_spec = client.V1PersistentVolumeClaimSpec(
+            **kwargs.get(
+                "spec",
+                {},
+            ).get(
+                "volumeClaimTemplate",
+                {},
+            ).get(
+                "spec",
+                {},
+            )
+        )
         if not self.volume_claim_spec.access_modes:
             self.volume_claim_spec.access_modes = ["ReadWriteOnce"]
         if not self.volume_claim_spec.resources:
@@ -148,8 +151,7 @@ def __init__(self, event_type, namespace, hostess_docker_registry,
         )
 
         self.core_api = client.CoreV1Api()
-        self.apps_api = client.AppsV1beta1Api()
-        self.ext_api = client.ExtensionsV1beta1Api()
+        self.apps_api = client.AppsV1Api()
 
     def apply(self):
         if self.event_type != "DELETED":
@@ -172,7 +174,7 @@ def apply(self):
             )
 
             daemon_set = self.run_action_and_parse_error(
-                self.ext_api.read_namespaced_daemon_set,
+                self.apps_api.read_namespaced_daemon_set,
                 self.daemon_set_name,
                 self.namespace
             )
@@ -224,9 +226,10 @@ def generate_daemon_set(self, daemon_set):
         ds_labels.update(self.ss_ds_labels)
         daemon_set.metadata = copy.deepcopy(self.metadata)
         daemon_set.metadata.name = self.daemon_set_name
-        daemon_set.metadata.labels = ds_labels 
-        daemon_set.spec = client.V1beta1DaemonSetSpec(
+        daemon_set.metadata.labels = ds_labels
+        daemon_set.spec = client.V1DaemonSetSpec(
                 min_ready_seconds=10,
+                selector=self.labels,
                 template=client.V1PodTemplateSpec(
                     metadata=client.V1ObjectMeta(
                         labels=ds_pod_labels
@@ -345,7 +348,7 @@ def generate_daemon_set(self, daemon_set):
                                  )]
                     )
                 ),
-                update_strategy=client.V1beta1DaemonSetUpdateStrategy(
+                update_strategy=client.V1DaemonSetUpdateStrategy(
                     type="RollingUpdate"
                 )
             )
@@ -380,12 +383,14 @@ def get_upstream_credentials(self):
         return (the_user, the_pass)
 
     def generate_stateful_set(self):
-        stateful_set = client.V1beta1StatefulSet(
+        stateful_set = client.V1StatefulSet(
             metadata=self.metadata,
-            spec=client.V1beta1StatefulSetSpec(
+            spec=client.V1StatefulSetSpec(
                 # we can't update service name or pod management policy
                 service_name=self.full_name + "-headless",
                 pod_management_policy="Parallel",
+                selector=self.labels,
+                template=client.V1PodTemplateSpec(),
                 # we can't update volume claim templates
                 volume_claim_templates=[client.V1PersistentVolumeClaim(
                     metadata=client.V1ObjectMeta(
@@ -560,7 +565,7 @@ def generate_stateful_set(self):
                         volumes=volumes,
                     )
                 )
-        stateful_set.spec.update_strategy = client.V1beta1StatefulSetUpdateStrategy(type="RollingUpdate",)
+        stateful_set.spec.update_strategy = client.V1StatefulSetUpdateStrategy(type="RollingUpdate",)
         stateful_set.metadata.labels = ss_labels
         return stateful_set
 
@@ -586,8 +591,9 @@ def update_services(self, service, service_headless):
             )
         )
         if not service:
-            self.run_action_and_parse_error(self.core_api.create_namespaced_service,
-                                            self.namespace, empty_service)
+            self.run_action_and_parse_error(
+                self.core_api.create_namespaced_service,
+                self.namespace, empty_service)
             LOGGER.info("Service created")
 
         else:
@@ -644,15 +650,15 @@ def update_secret(self, secret):
                                             secret)
 
     def update_daemon_set(self, daemon_set):
-        empty_daemon_set = client.V1beta1DaemonSet()
+        empty_daemon_set = client.V1DaemonSet()
         if not daemon_set:
             daemon_set = self.generate_daemon_set(empty_daemon_set)
-            self.run_action_and_parse_error(self.ext_api.create_namespaced_daemon_set,
+            self.run_action_and_parse_error(self.apps_api.create_namespaced_daemon_set,
                                             self.namespace, daemon_set)
             LOGGER.info("Daemon set created")
         else:
             daemon_set = self.generate_daemon_set(daemon_set)
             self.run_action_and_parse_error(
-                self.ext_api.replace_namespaced_daemon_set,
+                self.apps_api.replace_namespaced_daemon_set,
                 daemon_set.metadata.name, self.namespace, daemon_set)
             LOGGER.info("Daemon set replaced")

requirements.txt

@@ -1,3 +1,3 @@
 bitmath==1.3.3.1
-kubernetes==3.0.0a1
+kubernetes==11.0.0
 statsd==3.2.1

setup.py

@@ -14,7 +14,7 @@
 ]
 dependencies = [
     'bitmath==1.3.3.1',
-    'kubernetes==3.0.0a1',
+    'kubernetes==11.0.0',
     'statsd==3.2.1',
     ]
 
@@ -46,6 +46,7 @@ def run_tests(self):
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.8',
     ],
     entry_points={
     },

tests/kubernetes_mock_responses.py

@@ -1,14 +1,14 @@
 import base64
 import json
 
-EMPTY_SERVICE = """{"api_version": "v1",
+EMPTY_SERVICE = """{"apiVersion": "v1",
      "kind": "Service",
      "metadata": {
                   "creation_timestamp": "2017-09-12T13:23:47Z",
                   "labels": {"app": "docker-registry", "mirror": "hub"},
                   "name": "registry-mirror-hub",
                   "namespace": "default",
-                  "ownerReferences": [{"api_version": "k8s.osp.tech/v1",
+                  "ownerReferences": [{"apiVersion": "k8s.osp.tech/v1",
                                         "kind": "RegistryMirror",
                                         "name": "hub",
                                         "uid": "c7137776-97b7-11e7-a6e5-0800276be3ff"}],
@@ -27,11 +27,11 @@
 
 EMPTY_DAEMON_SET = """{
     "kind":"DaemonSet",
-    "apiVersion":"extensions/v1beta1",
+    "apiVersion":"apps/v1",
     "metadata": {
                 "name":"registry-mirror-hub-utils",
                 "namespace":"default",
-                "selfLink":"/apis/extensions/v1beta1/namespaces/default/daemonsets/registry-mirror-hub-utils",
+                "selfLink":"/apis/apps/v1/namespaces/default/daemonsets/registry-mirror-hub-utils",
                 "uid":"dff71ac8-97c2-11e7-a6e5-0800276be3ff",
                 "resourceVersion":"18519",
                 "generation":1,
@@ -99,11 +99,11 @@
 
 EMPTY_STATEFUL_SET = """{
     "kind":"StatefulSet",
-    "apiVersion":"apps/v1beta1",
+    "apiVersion":"apps/v1",
     "metadata": {
         "name":"registry-mirror-hub",
         "namespace":"default",
-        "selfLink":"/apis/apps/v1beta1/namespaces/default/statefulsets/registry-mirror-hub",
+        "selfLink":"/apis/apps/v1/namespaces/default/statefulsets/registry-mirror-hub",
         "uid":"512e7799-97c4-11e7-a6e5-0800276be3ff",
         "resourceVersion":"20254",
         "generation":1,

tests/kubernetes_test_case.py

@@ -1,5 +1,7 @@
 from unittest import TestCase
-from kubernetes.config.incluster_config import SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
+from kubernetes.config.incluster_config import (
+    SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
+)
 import os
 import kubernetes
 import json
@@ -21,8 +23,12 @@ def check_calls(self, methods, calls, exp_metadata):
             if exp_call != "GET":
                 body = json.loads(request.body)
                 self.assertIn(exp_metadata.name, body['metadata']['name'])
-                self.assertDictContainsSubset(exp_metadata.labels, body['metadata']['labels'])
-                self.assertEqual(body['metadata']['ownerReferences'][0]['name'], exp_metadata.owner_references[0].name)
+                self.assertDictContainsSubset(
+                    exp_metadata.labels,
+                    body['metadata']['labels'])
+                self.assertEqual(
+                    body['metadata']['ownerReferences'][0]['name'],
+                    exp_metadata.owner_references[0].name)
 
     def tearDown(self):
         os.remove("blip")