diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index f5f226a01..11cf5fdb9 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -32,6 +32,7 @@ generate_appwrapper, ) from ..utils.kube_api_helpers import _kube_api_error_handling +from ..utils.generate_yaml import is_openshift_cluster from ..utils.openshift_oauth import ( create_openshift_oauth_objects, delete_openshift_oauth_objects, @@ -415,25 +416,48 @@ def cluster_dashboard_uri(self) -> str: """ Returns a string containing the cluster's dashboard URI. """ - try: - config_check() - api_instance = client.NetworkingV1Api(api_config_handler()) - ingresses = api_instance.list_namespaced_ingress(self.config.namespace) - except Exception as e: # pragma no cover - return _kube_api_error_handling(e) + config_check() + if is_openshift_cluster(): + try: + api_instance = client.CustomObjectsApi(api_config_handler()) + routes = api_instance.list_namespaced_custom_object( + group="route.openshift.io", + version="v1", + namespace=self.config.namespace, + plural="routes", + ) + except Exception as e: # pragma: no cover + return _kube_api_error_handling(e) + + for route in routes["items"]: + if route["metadata"][ + "name" + ] == f"ray-dashboard-{self.config.name}" or route["metadata"][ + "name" + ].startswith( + f"{self.config.name}-ingress" + ): + protocol = "https" if route["spec"].get("tls") else "http" + return f"{protocol}://{route['spec']['host']}" + else: + try: + api_instance = client.NetworkingV1Api(api_config_handler()) + ingresses = api_instance.list_namespaced_ingress(self.config.namespace) + except Exception as e: # pragma no cover + return _kube_api_error_handling(e) - for ingress in ingresses.items: - annotations = ingress.metadata.annotations - protocol = "http" - if ( - ingress.metadata.name == f"ray-dashboard-{self.config.name}" - or ingress.metadata.name.startswith(f"{self.config.name}-ingress") - ): - if annotations == None: - protocol = "http" - elif "route.openshift.io/termination" in annotations: - protocol = "https" - return f"{protocol}://{ingress.spec.rules[0].host}" + for ingress in ingresses.items: + annotations = ingress.metadata.annotations + protocol = "http" + if ( + ingress.metadata.name == f"ray-dashboard-{self.config.name}" + or ingress.metadata.name.startswith(f"{self.config.name}-ingress") + ): + if annotations == None: + protocol = "http" + elif "route.openshift.io/termination" in annotations: + protocol = "https" + return f"{protocol}://{ingress.spec.rules[0].host}" return "Dashboard ingress not available yet, have you run cluster.up()?" def list_jobs(self) -> List: @@ -532,6 +556,14 @@ def _component_resources_up( plural="rayclusters", body=resource, ) + elif resource["kind"] == "Ingress": + api_instance.create_namespaced_custom_object( + group="networking.k8s.io", + version="v1", + namespace=namespace, + plural="ingresses", + body=resource, + ) elif resource["kind"] == "Route": api_instance.create_namespaced_custom_object( group="route.openshift.io", @@ -561,6 +593,15 @@ def _component_resources_down( plural="rayclusters", name=self.app_wrapper_name, ) + elif resource["kind"] == "Ingress": + name = resource["metadata"]["name"] + api_instance.delete_namespaced_custom_object( + group="networking.k8s.io", + version="v1", + namespace=namespace, + plural="ingresses", + name=name, + ) elif resource["kind"] == "Route": name = resource["metadata"]["name"] api_instance.delete_namespaced_custom_object( @@ -663,29 +704,48 @@ def _check_aw_exists(name: str, namespace: str) -> bool: ) except Exception as e: # pragma: no cover return _kube_api_error_handling(e, print_error=False) - for aw in aws["items"]: if aw["metadata"]["name"] == name: return True return False -# Cant test this until get_current_namespace is fixed +# Cant test this until get_current_namespace is fixed and placed in this function over using `self` def _get_ingress_domain(self): # pragma: no cover - try: - config_check() - api_client = client.NetworkingV1Api(api_config_handler()) - if self.config.namespace != None: - namespace = self.config.namespace - else: - namespace = get_current_namespace() - ingresses = api_client.list_namespaced_ingress(namespace) - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) + config_check() + + if self.config.namespace != None: + namespace = self.config.namespace + else: + namespace = get_current_namespace() domain = None - for ingress in ingresses.items: - if ingress.spec.rules[0].http.paths[0].backend.service.port.number == 10001: - domain = ingress.spec.rules[0].host + + if is_openshift_cluster(): + try: + api_instance = client.CustomObjectsApi(api_config_handler()) + + routes = api_instance.list_namespaced_custom_object( + group="route.openshift.io", + version="v1", + namespace=namespace, + plural="routes", + ) + except Exception as e: # pragma: no cover + return _kube_api_error_handling(e) + + for route in routes["items"]: + if route["spec"]["port"]["targetPort"] == "client": + domain = route["spec"]["host"] + else: + try: + api_client = client.NetworkingV1Api(api_config_handler()) + ingresses = api_client.list_namespaced_ingress(namespace) + except Exception as e: # pragma: no cover + return _kube_api_error_handling(e) + + for ingress in ingresses.items: + if ingress.spec.rules[0].http.paths[0].backend.service.port.number == 10001: + domain = ingress.spec.rules[0].host return domain diff --git a/src/codeflare_sdk/templates/base-template.yaml b/src/codeflare_sdk/templates/base-template.yaml index c98f53c99..8e6fd0e9e 100644 --- a/src/codeflare_sdk/templates/base-template.yaml +++ b/src/codeflare_sdk/templates/base-template.yaml @@ -289,7 +289,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ray-dashboard-raytest + name: ray-dashboard-deployment-ingress namespace: default annotations: annotations-example:annotations-example @@ -306,12 +306,28 @@ spec: pathType: Prefix path: / host: ray-dashboard-raytest. + - replicas: 1 + generictemplate: + kind: Route + apiVersion: route.openshift.io/v1 + metadata: + name: ray-dashboard-deployment-route + namespace: default + labels: + # allows me to return name of service that Ray operator creates + odh-ray-cluster-service: deployment-name-head-svc + spec: + to: + kind: Service + name: deployment-name-head-svc + port: + targetPort: dashboard - replicas: 1 generictemplate: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: rayclient-deployment-name + name: rayclient-deployment-ingress namespace: default annotations: annotations-example:annotations-example @@ -330,6 +346,24 @@ spec: path: '' pathType: ImplementationSpecific host: rayclient-raytest. + - replicas: 1 + generictemplate: + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: rayclient-deployment-route + namespace: default + labels: + # allows me to return name of service that Ray operator creates + odh-ray-cluster-service: deployment-name-head-svc + spec: + port: + targetPort: client + tls: + termination: passthrough + to: + kind: Service + name: deployment-name-head-svc - replicas: 1 generictemplate: apiVersion: v1 diff --git a/src/codeflare_sdk/utils/generate_yaml.py b/src/codeflare_sdk/utils/generate_yaml.py index 3ffbefb57..95c17cc28 100755 --- a/src/codeflare_sdk/utils/generate_yaml.py +++ b/src/codeflare_sdk/utils/generate_yaml.py @@ -29,8 +29,6 @@ from base64 import b64encode from urllib3.util import parse_url -from kubernetes import client, config - from .kube_api_helpers import _get_api_host @@ -56,22 +54,60 @@ def gen_dashboard_ingress_name(cluster_name): return f"ray-dashboard-{cluster_name}" -# Check if the ingress api cluster resource exists +# Check if the routes api exists def is_openshift_cluster(): try: config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) - api_instance.get_cluster_custom_object( - "config.openshift.io", "v1", "ingresses", "cluster" - ) - - return True - except client.ApiException as e: # pragma: no cover - if e.status == 404 or e.status == 403: - return False + for api in client.ApisApi(api_config_handler()).get_api_versions().groups: + for v in api.versions: + if "route.openshift.io/v1" in v.group_version: + return True else: - print(f"Error detecting cluster type defaulting to Kubernetes: {e}") return False + except client.ApiException as e: # pragma: no cover + print(f"Error detecting cluster type defaulting to Kubernetes: {e}") + return False + + +def update_dashboard_route(route_item, cluster_name, namespace): + metadata = route_item.get("generictemplate", {}).get("metadata") + metadata["name"] = gen_dashboard_ingress_name(cluster_name) + metadata["namespace"] = namespace + metadata["labels"]["odh-ray-cluster-service"] = f"{cluster_name}-head-svc" + spec = route_item.get("generictemplate", {}).get("spec") + spec["to"]["name"] = f"{cluster_name}-head-svc" + + +# ToDo: refactor the update_x_route() functions +def update_rayclient_route(route_item, cluster_name, namespace): + metadata = route_item.get("generictemplate", {}).get("metadata") + metadata["name"] = f"rayclient-{cluster_name}" + metadata["namespace"] = namespace + metadata["labels"]["odh-ray-cluster-service"] = f"{cluster_name}-head-svc" + spec = route_item.get("generictemplate", {}).get("spec") + spec["to"]["name"] = f"{cluster_name}-head-svc" + + +def update_dashboard_exposure( + ingress_item, route_item, cluster_name, namespace, ingress_options, ingress_domain +): + if is_openshift_cluster(): + update_dashboard_route(route_item, cluster_name, namespace) + else: + update_dashboard_ingress( + ingress_item, cluster_name, namespace, ingress_options, ingress_domain + ) + + +def update_rayclient_exposure( + client_route_item, client_ingress_item, cluster_name, namespace, ingress_domain +): + if is_openshift_cluster(): + update_rayclient_route(client_route_item, cluster_name, namespace) + else: + update_rayclient_ingress( + client_ingress_item, cluster_name, namespace, ingress_domain + ) def update_dashboard_ingress( @@ -123,25 +159,15 @@ def update_dashboard_ingress( "name" ] = f"{cluster_name}-head-svc" else: - metadata["name"] = f"ray-dashboard-{cluster_name}" + spec["ingressClassName"] = "nginx" + metadata["name"] = gen_dashboard_ingress_name(cluster_name) metadata["namespace"] = namespace spec["rules"][0]["http"]["paths"][0]["backend"]["service"][ "name" ] = f"{cluster_name}-head-svc" - if is_openshift_cluster(): - try: - config_check() - api_client = client.CustomObjectsApi(api_config_handler()) - ingress = api_client.get_cluster_custom_object( - "config.openshift.io", "v1", "ingresses", "cluster" - ) - del spec["ingressClassName"] - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) - domain = ingress["spec"]["domain"] - elif ingress_domain is None: + if ingress_domain is None: raise ValueError( - "ingress_domain is invalid. For Kubernetes Clusters please specify an ingress domain" + "ingress_domain is invalid. Please specify an ingress domain" ) else: domain = ingress_domain @@ -162,38 +188,19 @@ def update_rayclient_ingress( "name" ] = f"{cluster_name}-head-svc" - if is_openshift_cluster(): - try: - config_check() - api_client = client.CustomObjectsApi(api_config_handler()) - ingress = api_client.get_cluster_custom_object( - "config.openshift.io", "v1", "ingresses", "cluster" - ) - ingressClassName = "openshift-default" - annotations = { - "nginx.ingress.kubernetes.io/rewrite-target": "/", - "nginx.ingress.kubernetes.io/ssl-redirect": "true", - "route.openshift.io/termination": "passthrough", - } - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) - domain = ingress["spec"]["domain"] - elif ingress_domain is None: - raise ValueError( - "ingress_domain is invalid. For Kubernetes Clusters please specify an ingress domain" - ) - else: - domain = ingress_domain + if ingress_domain is not None: ingressClassName = "nginx" annotations = { "nginx.ingress.kubernetes.io/rewrite-target": "/", "nginx.ingress.kubernetes.io/ssl-redirect": "true", "nginx.ingress.kubernetes.io/ssl-passthrough": "true", } + else: + raise ValueError("ingress_domain is invalid. Please specify a domain") metadata["annotations"] = annotations spec["ingressClassName"] = ingressClassName - spec["rules"][0]["host"] = f"rayclient-{cluster_name}-{namespace}.{domain}" + spec["rules"][0]["host"] = f"rayclient-{cluster_name}-{namespace}.{ingress_domain}" def update_names(yaml, item, appwrapper_name, cluster_name, namespace): @@ -397,8 +404,9 @@ def update_ca_secret(ca_secret_item, cluster_name, namespace): def enable_local_interactive(resources, cluster_name, namespace, ingress_domain): - rayclient_ingress_item = resources["resources"].get("GenericItems")[2] - ca_secret_item = resources["resources"].get("GenericItems")[3] + rayclient_ingress_item = resources["resources"].get("GenericItems")[3] + rayclient_route_item = resources["resources"].get("GenericItems")[4] + ca_secret_item = resources["resources"].get("GenericItems")[5] item = resources["resources"].get("GenericItems")[0] update_ca_secret(ca_secret_item, cluster_name, namespace) # update_ca_secret_volumes @@ -422,26 +430,21 @@ def enable_local_interactive(resources, cluster_name, namespace, ingress_domain) command = command.replace("deployment-name", cluster_name) - if is_openshift_cluster(): - # We can try get the domain through checking ingresses.config.openshift.io - try: - config_check() - api_client = client.CustomObjectsApi(api_config_handler()) - ingress = api_client.get_cluster_custom_object( - "config.openshift.io", "v1", "ingresses", "cluster" - ) - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) - domain = ingress["spec"]["domain"] - elif ingress_domain is None: + if ingress_domain is None: raise ValueError( - "ingress_domain is invalid. For Kubernetes Clusters please specify an ingress domain" + "ingress_domain is invalid. For creating the client route/ingress please specify an ingress domain" ) else: domain = ingress_domain command = command.replace("server-name", domain) - update_rayclient_ingress(rayclient_ingress_item, cluster_name, namespace, domain) + update_rayclient_exposure( + rayclient_route_item, + rayclient_ingress_item, + cluster_name, + namespace, + ingress_domain, + ) item["generictemplate"]["spec"]["headGroupSpec"]["template"]["spec"][ "initContainers" @@ -484,7 +487,9 @@ def disable_raycluster_tls(resources): updated_items = [] for i in resources["GenericItems"][:]: - if "rayclient-deployment-name" in i["generictemplate"]["metadata"]["name"]: + if "rayclient-deployment-ingress" in i["generictemplate"]["metadata"]["name"]: + continue + if "rayclient-deployment-route" in i["generictemplate"]["metadata"]["name"]: continue if "ca-secret-deployment-name" in i["generictemplate"]["metadata"]["name"]: continue @@ -493,6 +498,26 @@ def disable_raycluster_tls(resources): resources["GenericItems"] = updated_items +def delete_route_or_ingress(resources): + if is_openshift_cluster(): + client_to_remove_name = "rayclient-deployment-ingress" + dashboard_to_remove_name = "ray-dashboard-deployment-ingress" + else: + client_to_remove_name = "rayclient-deployment-route" + dashboard_to_remove_name = "ray-dashboard-deployment-route" + + updated_items = [] + for i in resources["GenericItems"][:]: + if dashboard_to_remove_name in i["generictemplate"]["metadata"]["name"]: + continue + elif client_to_remove_name in i["generictemplate"]["metadata"]["name"]: + continue + + updated_items.append(i) + + resources["GenericItems"] = updated_items + + def write_user_appwrapper(user_yaml, output_file_name): # Create the directory if it doesn't exist directory_path = os.path.dirname(output_file_name) @@ -626,6 +651,7 @@ def generate_appwrapper( resources = user_yaml.get("spec", "resources") item = resources["resources"].get("GenericItems")[0] ingress_item = resources["resources"].get("GenericItems")[1] + route_item = resources["resources"].get("GenericItems")[2] update_names(user_yaml, item, appwrapper_name, cluster_name, namespace) update_labels(user_yaml, instascale, instance_types) update_priority(user_yaml, item, dispatch_priority, priority_val) @@ -658,14 +684,21 @@ def generate_appwrapper( head_memory, head_gpus, ) - update_dashboard_ingress( - ingress_item, cluster_name, namespace, ingress_options, ingress_domain + update_dashboard_exposure( + ingress_item, + route_item, + cluster_name, + namespace, + ingress_options, + ingress_domain, ) if local_interactive: enable_local_interactive(resources, cluster_name, namespace, ingress_domain) else: disable_raycluster_tls(resources["resources"]) + delete_route_or_ingress(resources["resources"]) + if openshift_oauth: enable_openshift_oauth(user_yaml, cluster_name, namespace) diff --git a/tests/test-case-no-mcad.yamls b/tests/test-case-no-mcad.yamls index b8993a7f0..484636bc2 100644 --- a/tests/test-case-no-mcad.yamls +++ b/tests/test-case-no-mcad.yamls @@ -145,6 +145,7 @@ metadata: name: ray-dashboard-unit-test-cluster-ray namespace: ns spec: + ingressClassName: nginx rules: - host: ray-dashboard-unit-test-cluster-ray-ns.apps.cluster.awsroute.org http: diff --git a/tests/test-case-prio.yaml b/tests/test-case-prio.yaml index 6051104ac..70b68e971 100644 --- a/tests/test-case-prio.yaml +++ b/tests/test-case-prio.yaml @@ -178,6 +178,7 @@ spec: name: ray-dashboard-prio-test-cluster namespace: ns spec: + ingressClassName: nginx rules: - host: ray-dashboard-prio-test-cluster-ns.apps.cluster.awsroute.org http: diff --git a/tests/test-case.yaml b/tests/test-case.yaml index 7c649c5ff..920459c40 100644 --- a/tests/test-case.yaml +++ b/tests/test-case.yaml @@ -175,6 +175,7 @@ spec: name: ray-dashboard-unit-test-cluster namespace: ns spec: + ingressClassName: nginx rules: - host: ray-dashboard-unit-test-cluster-ns.apps.cluster.awsroute.org http: diff --git a/tests/unit_test.py b/tests/unit_test.py index c33b95ab9..7ad0d08d7 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -262,6 +262,7 @@ def test_config_creation(): def test_cluster_creation(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") cluster = createClusterWithConfig(mocker) assert cluster.app_wrapper_yaml == f"{aw_dir}unit-test-cluster.yaml" assert cluster.app_wrapper_name == "unit-test-cluster" @@ -286,6 +287,7 @@ def test_create_app_wrapper_raises_error_with_no_image(): def test_cluster_creation_no_mcad(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", return_value={"spec": {"domain": "apps.cluster.awsroute.org"}}, @@ -304,6 +306,7 @@ def test_cluster_creation_no_mcad(mocker): def test_cluster_creation_priority(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( "kubernetes.client.CustomObjectsApi.list_cluster_custom_object", @@ -327,17 +330,15 @@ def test_cluster_creation_priority(mocker): def test_default_cluster_creation(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "codeflare_sdk.cluster.cluster.get_current_namespace", return_value="opendatahub", ) - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) default_config = ClusterConfiguration( name="unit-test-default-cluster", image="quay.io/project-codeflare/ray:latest-py39-cu118", + ingress_domain="apps.cluster.awsroute.org", ) cluster = Cluster(default_config) @@ -382,6 +383,14 @@ def arg_check_apply_effect(group, version, namespace, plural, body, *args): for resource in yamls: if resource["kind"] == "RayCluster": assert body == resource + elif plural == "ingresses": + assert group == "networking.k8s.io" + assert version == "v1" + with open(f"{aw_dir}unit-test-cluster-ray.yaml") as f: + yamls = yaml.load_all(f, Loader=yaml.FullLoader) + for resource in yamls: + if resource["kind"] == "Ingress": + assert body == resource elif plural == "routes": assert group == "route.openshift.io" assert version == "v1" @@ -412,6 +421,7 @@ def arg_check_del_effect(group, version, namespace, plural, name, *args): def test_cluster_up_down(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", @@ -435,6 +445,7 @@ def test_cluster_up_down(mocker): def test_cluster_up_down_no_mcad(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", @@ -469,14 +480,14 @@ def arg_check_list_effect(group, version, plural, name, *args): return {"spec": {"domain": "test"}} -""" -def test_get_ingress_domain(self, mocker): +""" We need to fix get_current_namespace in order to reuse this test. +def test_get_ingress_domain(mocker): mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", side_effect=arg_check_list_effect, ) - domain = _get_ingress_domain(self) + domain = _get_ingress_domain() assert domain == "test" """ @@ -541,6 +552,7 @@ def test_delete_openshift_oauth_objects(mocker): def test_cluster_uris(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( "codeflare_sdk.cluster.cluster._get_ingress_domain", @@ -640,6 +652,7 @@ def ingress_retrieval(port, annotations=None): def test_ray_job_wrapping(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") cluster = cluster = createClusterWithConfig(mocker) cluster.config.image = "quay.io/project-codeflare/ray:latest-py39-cu118" mocker.patch( @@ -734,10 +747,7 @@ def test_print_appwrappers(capsys): def test_ray_details(mocker, capsys): - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) + mocker.patch("kubernetes.client.ApisApi.get_api_versions") ray1 = RayCluster( name="raytest1", status=RayClusterStatus.READY, @@ -765,6 +775,7 @@ def test_ray_details(mocker, capsys): name="raytest2", namespace="ns", image="quay.io/project-codeflare/ray:latest-py39-cu118", + ingress_domain="apps.cluster.awsroute.org", ) ) captured = capsys.readouterr() @@ -1770,16 +1781,17 @@ def get_aw_obj(group, version, namespace, plural): def test_get_cluster(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) mocker.patch( "kubernetes.client.CustomObjectsApi.list_namespaced_custom_object", side_effect=get_ray_obj, ) - cluster = get_cluster("quicktest") + mocker.patch( + "codeflare_sdk.utils.generate_yaml.is_openshift_cluster", + return_value=True, + ) + cluster = get_cluster(cluster_name="quicktest") cluster_config = cluster.config assert cluster_config.name == "quicktest" and cluster_config.namespace == "ns" assert ( @@ -1877,11 +1889,8 @@ def test_list_queue(mocker, capsys): def test_cluster_status(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) fake_aw = AppWrapper( "test", AppWrapperStatus.FAILED, can_run=True, job_state="unused" ) @@ -1904,6 +1913,7 @@ def test_cluster_status(mocker): name="test", namespace="ns", image="quay.io/project-codeflare/ray:latest-py39-cu118", + ingress_domain="apps.cluster.awsroute.org", ) ) mocker.patch("codeflare_sdk.cluster.cluster._app_wrapper_status", return_value=None) @@ -1969,10 +1979,7 @@ def test_cluster_status(mocker): def test_wait_ready(mocker, capsys): - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "kubernetes.client.NetworkingV1Api.list_namespaced_ingress", return_value=ingress_retrieval(8265), @@ -2000,6 +2007,7 @@ def test_wait_ready(mocker, capsys): name="test", namespace="ns", image="quay.io/project-codeflare/ray:latest-py39-cu118", + ingress_domain="apps.cluster.awsroute.org", ) ) try: @@ -2032,6 +2040,7 @@ def test_wait_ready(mocker, capsys): def test_jobdefinition_coverage(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", return_value={"spec": {"domain": ""}}, @@ -2048,7 +2057,8 @@ def test_job_coverage(): abstract.logs() -def test_DDPJobDefinition_creation(): +def test_DDPJobDefinition_creation(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") ddp = createTestDDP() assert ddp.script == "test.py" assert ddp.m == None @@ -2072,6 +2082,7 @@ def test_DDPJobDefinition_dry_run(mocker: MockerFixture): that the attributes of the returned object are of the correct type, and that the values from cluster and job definition are correctly passed. """ + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( "codeflare_sdk.cluster.cluster.Cluster.cluster_dashboard_uri", @@ -2108,7 +2119,7 @@ def test_DDPJobDefinition_dry_run_no_cluster(mocker): that the attributes of the returned object are of the correct type, and that the values from cluster and job definition are correctly passed. """ - + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "codeflare_sdk.job.jobs.get_current_namespace", return_value="opendatahub", @@ -2147,6 +2158,7 @@ def test_DDPJobDefinition_dry_run_no_resource_args(mocker): Test that the dry run correctly gets resources from the cluster object when the job definition does not specify resources. """ + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch.object(Cluster, "job_client") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", @@ -2186,6 +2198,7 @@ def test_DDPJobDefinition_dry_run_no_cluster_no_resource_args(mocker): that the attributes of the returned object are of the correct type, and that the values from cluster and job definition are correctly passed. """ + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "codeflare_sdk.job.jobs.get_current_namespace", @@ -2240,6 +2253,7 @@ def test_DDPJobDefinition_submit(mocker: MockerFixture): Tests that the submit method returns the correct type: DDPJob And that the attributes of the returned object are of the correct type """ + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mock_schedule = MagicMock() mocker.patch.object(Runner, "schedule", mock_schedule) mock_schedule.return_value = "fake-dashboard-url" @@ -2270,6 +2284,7 @@ def test_DDPJobDefinition_submit(mocker: MockerFixture): def test_DDPJob_creation(mocker: MockerFixture): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch.object(Cluster, "job_client") mock_schedule = MagicMock() mocker.patch.object(Runner, "schedule", mock_schedule) @@ -2295,6 +2310,7 @@ def test_DDPJob_creation(mocker: MockerFixture): def test_DDPJob_creation_no_cluster(mocker: MockerFixture): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") ddp_def = createTestDDP() ddp_def.image = "fake-image" mocker.patch( @@ -2320,6 +2336,7 @@ def test_DDPJob_creation_no_cluster(mocker: MockerFixture): def test_DDPJob_status(mocker: MockerFixture): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") # Setup the neccesary mock patches mock_status = MagicMock() mocker.patch.object(Runner, "status", mock_status) @@ -2334,6 +2351,7 @@ def test_DDPJob_status(mocker: MockerFixture): def test_DDPJob_logs(mocker: MockerFixture): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mock_log = MagicMock() mocker.patch.object(Runner, "log_lines", mock_log) # Setup the neccesary mock patches @@ -2380,7 +2398,8 @@ def parse_j(cmd): return f"{worker}x{gpu}" -def test_AWManager_creation(): +def test_AWManager_creation(mocker): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") testaw = AWManager(f"{aw_dir}test.yaml") assert testaw.name == "test" assert testaw.namespace == "ns" @@ -2421,6 +2440,7 @@ def arg_check_aw_del_effect(group, version, namespace, plural, name, *args): def test_AWManager_submit_remove(mocker, capsys): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") testaw = AWManager(f"{aw_dir}test.yaml") testaw.remove() captured = capsys.readouterr() @@ -2483,21 +2503,6 @@ def secret_ca_retreival(secret_name, namespace): return client.models.V1Secret(data=data) -def test_is_openshift_cluster(mocker): - mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") - mocker.patch.object( - client.CustomObjectsApi, - "get_cluster_custom_object", - side_effect=client.ApiException(status=404), - ) - assert is_openshift_cluster() == False - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) - assert is_openshift_cluster() == True - - def test_generate_tls_cert(mocker): """ test the function codeflare_sdk.utils.generate_ca_cert generates the correct outputs @@ -2548,6 +2553,7 @@ def test_enable_local_interactive(mocker): cluster_name = "test-enable-local" namespace = "default" ingress_domain = "mytest.domain" + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "codeflare_sdk.utils.generate_yaml.is_openshift_cluster", return_value=False ) @@ -2581,7 +2587,7 @@ def test_enable_local_interactive(mocker): worker_group_spec = aw_spec["resources"]["GenericItems"][0]["generictemplate"][ "spec" ]["workerGroupSpecs"] - ca_secret = aw_spec["resources"]["GenericItems"][3]["generictemplate"] + ca_secret = aw_spec["resources"]["GenericItems"][5]["generictemplate"] # At a minimal, make sure the following items are presented in the appwrapper spec.resources. # 1. headgroup has the initContainers command to generated TLS cert from the mounted CA cert. # Note: In this particular command, the DNS.5 in [alt_name] must match the exposed local_client_url: rayclient-{cluster_name}.{namespace}.{ingress_domain} @@ -2637,7 +2643,7 @@ def test_enable_local_interactive(mocker): assert ca_secret["metadata"]["namespace"] == namespace # 5. Rayclient ingress - Kind - rayclient_ingress = aw_spec["resources"]["GenericItems"][2]["generictemplate"] + rayclient_ingress = aw_spec["resources"]["GenericItems"][3]["generictemplate"] paths = [ { "backend": { @@ -2663,47 +2669,6 @@ def test_enable_local_interactive(mocker): "host": f"rayclient-{cluster_name}-{namespace}.{ingress_domain}", "http": {"paths": paths}, } - # 5.1 Rayclient ingress - OCP - user_yaml = read_template(template) - aw_spec = user_yaml.get("spec", None) - cluster_name = "test-ocp-enable-local" - namespace = "default" - ocp_cluster_domain = {"spec": {"domain": "mytest.ocp.domain"}} - ingress_domain = ocp_cluster_domain["spec"]["domain"] - mocker.patch( - "codeflare_sdk.utils.generate_yaml.is_openshift_cluster", return_value=True - ) - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value=ocp_cluster_domain, - ) - paths = [ - { - "backend": { - "service": { - "name": f"{cluster_name}-head-svc", - "port": {"number": 10001}, - } - }, - "path": "", - "pathType": "ImplementationSpecific", - } - ] - enable_local_interactive(aw_spec, cluster_name, namespace, ingress_domain) - rayclient_ocp_ingress = aw_spec["resources"]["GenericItems"][2]["generictemplate"] - assert rayclient_ocp_ingress["kind"] == "Ingress" - assert rayclient_ocp_ingress["metadata"]["annotations"] == { - "nginx.ingress.kubernetes.io/rewrite-target": "/", - "nginx.ingress.kubernetes.io/ssl-redirect": "true", - "route.openshift.io/termination": "passthrough", - } - assert rayclient_ocp_ingress["metadata"]["name"] == f"rayclient-{cluster_name}" - assert rayclient_ocp_ingress["metadata"]["namespace"] == namespace - assert rayclient_ocp_ingress["spec"]["ingressClassName"] == "openshift-default" - assert rayclient_ocp_ingress["spec"]["rules"][0] == { - "host": f"rayclient-{cluster_name}-{namespace}.{ingress_domain}", - "http": {"paths": paths}, - } def test_create_openshift_oauth(mocker: MockerFixture): @@ -2816,6 +2781,7 @@ def test_replace_openshift_oauth(mocker: MockerFixture): def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( "codeflare_sdk.utils.generate_yaml._get_api_host", return_value="foo.com" ) @@ -2823,10 +2789,6 @@ def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): "codeflare_sdk.cluster.cluster.get_current_namespace", return_value="opendatahub", ) - mocker.patch( - "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", - return_value={"spec": {"domain": ""}}, - ) write_user_appwrapper = MagicMock() mocker.patch( "codeflare_sdk.utils.generate_yaml.write_user_appwrapper", write_user_appwrapper @@ -2836,6 +2798,7 @@ def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): "test_cluster", openshift_oauth=True, image="quay.io/project-codeflare/ray:latest-py39-cu118", + ingress_domain="apps.cluster.awsroute.org", ) ) user_yaml = write_user_appwrapper.call_args.args[0]