Pipeline Helm Charts #1191

Workflow file for this run

.github/workflows/full_kubeflow_integration_test.yaml at d02f072

	name: Test End-to-End Integration
	on:
	workflow_dispatch:
	push:
	branches:
	- master
	pull_request:
	branches:
	- master

	permissions:
	contents: read
	actions: read

	env:
	KF_PROFILE: kubeflow-user-example-com

	jobs:
	kubeflow-integration:
	name: Kubeflow Installation and Testing
	if: ${{ github.repository == 'kubeflow/manifests' }}
	runs-on:
	labels: oracle-vm-16cpu-64gb-x86-64
	# labels: ubuntu-latest-16-cores
	timeout-minutes: 45
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Install KinD, Create KinD cluster and Install kustomize
	run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh

	- name: Install kubectl
	run: ./tests/kubectl_install.sh

	- name: Create Kubeflow Namespace
	run: kustomize build common/kubeflow-namespace/base \| kubectl apply -f -

	- name: Install Certificate Manager
	run: ./tests/cert_manager_install.sh

	- name: Install Istio CNI
	run: ./tests/istio-cni_install.sh

	- name: Install OAuth2 Proxy
	run: ./tests/oauth2-proxy_install.sh

	- name: Install Kubeflow Istio Resources
	run: kustomize build common/istio/kubeflow-istio-resources/base \| kubectl apply -f -

	- name: Install Multi-Tenancy
	run: ./tests/multi_tenancy_install.sh

	- name: Install Dex
	run: ./tests/dex_install.sh

	- name: Install Central Dashboard
	run: ./tests/central_dashboard_install.sh

	- name: Install Knative
	run: ./tests/knative_install.sh

	- name: Install KServe
	run: ./tests/kserve_install.sh

	#- name: Install Pipelines
	# run: ./tests/pipelines_install.sh

	- name: Install Pipelines with SeaweedFS
	run: ./tests/pipelines_swfs_install.sh

	- name: Create KF Profile
	run: ./tests/kubeflow_profile_install.sh

	- name: Install Jupyter Web Application
	run: kustomize build applications/jupyter/jupyter-web-app/upstream/overlays/istio/ \| kubectl apply -f -

	- name: Install Notebook Controller
	run: kustomize build applications/jupyter/notebook-controller/upstream/overlays/kubeflow/ \| kubectl apply -f -

	- name: Install Admission Webhook
	run: kustomize build applications/admission-webhook/upstream/overlays/cert-manager \| kubectl apply -f -

	- name: Install Volumes Web Application
	run: ./tests/volumes_web_application_install.sh

	- name: Install Katib
	run: ./tests/katib_install.sh

	- name: Install Training Operator
	run: ./tests/training_operator_install.sh

	- name: Install Trainer
	run: ./tests/trainer_install.sh

	- name: Install Ray
	run: \|
	cd experimental/ray/
	kustomize build kuberay-operator/overlays/kubeflow \| kubectl -n kubeflow apply --server-side -f -
	kubectl -n kubeflow wait --for=condition=available --timeout=60s deploy/kuberay-operator


	- name: Install Model Registry
	run: \|
	kustomize build applications/model-registry/upstream/overlays/db \| kubectl apply -n kubeflow -f -
	kustomize build applications/model-registry/upstream/options/istio \| kubectl apply -n kubeflow -f -

	- name: Install Model Registry UI
	run: \|
	kustomize build applications/model-registry/upstream/options/ui/overlays/istio \| kubectl apply -n kubeflow -f -

	- name: Install Spark
	run: chmod u+x tests/*.sh && ./tests/spark_install.sh

	- name: Wait for All Pods to be Ready
	run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 90s --field-selector=status.phase!=Succeeded

	- name: Port-forward the istio-ingress gateway
	run: ./tests/port_forward_gateway.sh

	# name: Setup OAuth2 and Dex Credentials
	# run: chmod +x tests/oauth2_dex_credentials.sh && ./tests/oauth2_dex_credentials.sh

	- name: Run Training Operator Test
	run: ./tests/training_operator_test.sh "${KF_PROFILE}"

	- name: Run Trainer Test
	run: ./tests/trainer_test.sh "${KF_PROFILE}"

	- name: Test Dex Login
	run: \|
	pip3 install -q requests
	python3 tests/dex_login_test.py
	echo "Dex login test completed successfully."

	- name: Verify Pipeline Integration
	run: \|
	KF_PROFILE=kubeflow-user-example-com
	if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then
	echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE"
	exit 1
	fi
	kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json \| jq -r '.data \| keys[] as $k \| "\($k): \(. \| .[$k] \| @base64d)"' \| tr '\n' ' '

	- name: Install lib2to3 for KFP V1 SDK with Python3.12 (lib2to3 has been removed from python 3.12)
	run: \|
	sudo apt-get update
	sudo apt-get install -y python3-lib2to3

	- name: V1 Pipeline Test
	run: \|
	pip3 install "kfp>=1.8.23,<2.0.0"
	TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
	python3 tests/pipeline_v1_test.py "${TOKEN}" "${KF_PROFILE}"

	- name: V2 Pipeline Test
	run: \|
	pip3 install -U "kfp>=2.14.3"
	TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
	python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}"

	- name: Test Pipeline Access with Unauthorized Token
	run: \|
	kubectl create namespace test-unauthorized
	kubectl create serviceaccount test-unauthorized -n test-unauthorized
	UNAUTHORIZED_TOKEN=$(kubectl -n test-unauthorized create token test-unauthorized)
	python3 tests/pipeline_v2_test.py test_unauthorized_access "$UNAUTHORIZED_TOKEN" "${KF_PROFILE}"

	- name: Test SeaweedFS Namespace Isolation
	run: ./tests/swfs_namespace_isolation_test.sh

	- name: Test Volumes Web Application API
	run: ./tests/volumes_web_application_test.sh "${KF_PROFILE}"

	- name: Apply PodDefault for Pipeline Access Token
	run: sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml \| kubectl apply -f -

	- name: Create Test Notebook
	run: \|
	sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/notebook.test.kubeflow-user-example.com.yaml \| kubectl apply -f -
	kubectl wait --for=condition=Ready pod -l app=test -n $KF_PROFILE --timeout=180s

	- name: Copy and execute the pipeline run script in KF Notebook
	run: \|
	cp tests/pipeline_run_and_wait_kubeflow.py /tmp/run_pipeline_temp.py
	sed -i "s/experiment_namespace = \"kubeflow-user-example-com\"/experiment_namespace = \"$KF_PROFILE\"/g" /tmp/run_pipeline_temp.py
	sed -i 's/except Exception:/except Exception as e:/g' /tmp/run_pipeline_temp.py
	sed -i 's/logger.info("Experiment not found, trying to create experiment.")/logger.info("Experiment not found, trying to create experiment. Error: " + str(e))/g' /tmp/run_pipeline_temp.py

	kubectl -n $KF_PROFILE cp /tmp/run_pipeline_temp.py test-0:/home/jovyan/pipeline_run_and_wait_kubeflow.py

	kubectl -n $KF_PROFILE exec test-0 -- python /home/jovyan/pipeline_run_and_wait_kubeflow.py

	- name: Run Katib Test
	run: ./tests/katib_test.sh "${KF_PROFILE}"

	- name: Run KServe Test
	run: \|
	./tests/kserve_test.sh ${KF_PROFILE}

	- name: Run Spark Test
	run: chmod u+x tests/*.sh && ./tests/spark_test.sh "${KF_PROFILE}"

	- name: Run Ray Test
	run: \|
	cd experimental/ray/
	./test.sh ${KF_PROFILE}

	- name: Test Model Registry Deployment
	run: \|
	kubectl wait --for=condition=available -n kubeflow deployment/model-registry-db --timeout=60s
	kubectl wait --for=condition=available -n kubeflow deployment/model-registry-deployment --timeout=60s

	- name: Test Model Registry UI Deployment
	run: kubectl wait --for=condition=available -n kubeflow deployment/model-registry-ui --timeout=60s

	- name: Test Model Registry API
	run: \|
	export KF_TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"

	nohup kubectl port-forward svc/model-registry-service -n kubeflow 8082:8080 &
	sleep 5
	curl -s -X 'GET' \
	'http://localhost:8082/api/model_registry/v1alpha3/registered_models?pageSize=100&orderBy=ID&sortOrder=DESC' \
	-H 'accept: application/json'

	curl -s --fail \
	"localhost:8080/model-registry/api/v1/model_registry?namespace=${KF_PROFILE}" \
	-H "Authorization: Bearer ${KF_TOKEN}"

	- name: Test Model Registry API with Unauthorized Token
	run: \|
	UNAUTHORIZED_TOKEN=$(kubectl -n test-unauthorized create token test-unauthorized \|\| kubectl -n test-unauthorized create token default)

	STATUS_CODE=$(curl -s \
	--output /dev/stderr --write-out "%{http_code}" \
	"localhost:8080/model-registry/api/v1/model_registry?namespace=${KF_PROFILE}" \
	-H "Authorization: Bearer ${UNAUTHORIZED_TOKEN}")

	if test $STATUS_CODE -ne 403; then
	echo "Error: Unauthorized access was not correctly rejected. Got status code: ${STATUS_CODE}"
	exit 1
	fi

	- name: Apply Pod Security Standards Restricted
	run: ./tests/PSS_enable.sh restricted

	- name: Verify Components
	run: kubectl get pods --all-namespaces \| grep -E '(Error\|CrashLoopBackOff)' && exit 1 \|\| true

	- name: Install Metrics Server
	run: ./tests/metrics-server_install.sh

	- name: Check Pod Resource Usage
	run: \|
	echo "==== Resource Usage Table ===="
	pip3 install -q PyYAML
	python3 tests/metrics-server_resource_table.py

	- name: Collect Logs on Failure
	if: failure()
	run: \|
	mkdir -p logs
	kubectl get all --all-namespaces > logs/resources.txt
	kubectl get events --all-namespaces --sort-by=.metadata.creationTimestamp > logs/events.txt
	for namespace in kubeflow kubeflow-system istio-system cert-manager auth kubeflow-user-example-com; do
	kubectl describe pods -n $namespace > logs/$namespace-pods.txt
	for pod in $(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}'); do
	kubectl logs -n $namespace $pod --tail=100 > logs/$namespace-$pod.txt 2>&1 \|\| true
	done
	done

	- name: Upload Diagnostic Logs
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: kubeflow-test-logs
	path: logs/

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Pipeline Helm Charts #1191

Workflow file

Pipeline Helm Charts #1191

Uh oh!

Jobs

Run details

Workflow file for this run