feat(backend): replace MLMD with native artifact/task storage

Remove ML Metadata (MLMD) service dependency and implement artifact
and task tracking directly in the KFP database via the API server.

This architectural change eliminates the external MLMD service (metadata-grpc,
metadata-writer) and consolidates all metadata operations through the KFP API.

Major changes:
- Add v2beta1 artifact service API with storage layer implementation
- Extend run service with task CRUD endpoints and ViewMode
- Extend run response object with detailed task information
- Refactor driver/launcher to use KFP API client instead of MLMD client
- Remove all MLMD-related deployments and manifests
- Remove object store session info storage in metadata layer
- Add comprehensive test coverage for new storage and API layers

This simplifies deployment, reduces operational complexity, and provides
better control over metadata storage performance and schema.

Signed-off-by: Humair Khan <[email protected]>

Author: HumairAK

.github/actions/deploy/action.yml

@@ -64,7 +64,7 @@ runs:
     - name: Load Docker Images
       shell: bash
       run: |
-        APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent" "frontend" "metadata-writer")
+        APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent" "frontend")
         for app in "${APPS[@]}"; do
           docker image load -i ${{ inputs.image_path }}/$app/$app.tar
           docker push ${{ inputs.image_registry }}/$app:${{ inputs.image_tag }}

.github/actions/kfp-k8s/action.yml

@@ -1,5 +1,5 @@
-name: "Install kfp & kfp-kubernetes"
-description: "Install kfp & kfp-kubernetes"
+name: "Install kfp-server-api, kfp & kfp-kubernetes"
+description: "Install kfp-server-api, kfp & kfp-kubernetes from source"
 inputs:
   build_version:
     required: true
@@ -16,6 +16,13 @@ runs:
       shell: bash
       run: pip install build==${{inputs.build_version}}
 
+    - name: Build kfp-server-api dist
+      id: build-kfp-server-api
+      shell: bash
+      working-directory: backend/api/v2beta1/python_http_client
+      run: |
+        python -m build .
+
     - name: Build kfp dist
       id: install-kfp
       shell: bash
@@ -37,15 +44,15 @@ runs:
       working-directory: ./kubernetes_platform
       run: make golang
 
-    # kfp is installed transitively
-    # --find-links ensures pip first looks in the sdk/python/dist folder
-    # outputted from generate-kfp-kubernetes-proto-files step before looking at pypi
+    # kfp and kfp-server-api are installed transitively
+    # --find-links ensures pip first looks in the dist folders before looking at pypi
+    # for kfp-server-api and kfp packages
     - name: Install kfp & kfp-kubernetes from source
       id: install-kfp-kubernetes
       shell: bash
       if: ${{ steps.generate-kfp-kubernetes-proto-files.outcome == 'success' }}
       run: |
-        pip install -e ./kubernetes_platform/python[dev] --find-links=sdk/python/dist
+        pip install -e ./kubernetes_platform/python[dev] --find-links=backend/api/v2beta1/python_http_client/dist --find-links=sdk/python/dist
 
     # testing reinstalling kfp package from source with no deps
     - name: Reinstall kfp from source with no deps

.github/resources/manifests/base/grpc-specs.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/kubernetes-native/default/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -63,11 +56,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml

@@ -18,10 +18,6 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
-
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
@@ -31,10 +27,6 @@ patches:
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -51,11 +43,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/default/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/minio/kustomization.yaml

@@ -15,4 +15,4 @@ spec:
             - name: HTTPS_PROXY
               value: "http://squid.squid.svc.cluster.local:3128"
             - name: NO_PROXY
-              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,metadata-grpc-service,metadata-grpc-service.kubeflow,ml-pipeline.kubeflow"
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,ml-pipeline.kubeflow"

.github/resources/manifests/standalone/cache-disabled-proxy-minio/apiserver-env.yaml

@@ -15,6 +15,6 @@ spec:
             - name: HTTPS_PROXY
               value: "http://squid.squid.svc.cluster.local:3128"
             - name: NO_PROXY
-              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,metadata-grpc-service,metadata-grpc-service.kubeflow,ml-pipeline.kubeflow"
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,ml-pipeline.kubeflow"
             - name: OBJECTSTORECONFIG_HOST
               value: "minio-service.kubeflow.svc.cluster.local"