create parquet metadata cache service

charts/operator-wandb/Chart.lock

@@ -32,6 +32,9 @@ dependencies:
 - name: wandb-base
   repository: file://../wandb-base
   version: 0.11.6
+- name: wandb-base
+  repository: file://../wandb-base
+  version: 0.11.6
 - name: mysql
   repository: file://charts/mysql
   version: 0.1.0
@@ -92,5 +95,5 @@ dependencies:
 - name: clickhouse
   repository: file://charts/clickhouse
   version: 9.1.1
-digest: sha256:29ab2278caa8a43a445f702ce7f807c684896bf8f67b31f4e25281f588b9840e
-generated: "2025-12-19T05:05:35.702691+09:00"
+digest: sha256:07c52b0edf2e2b81f45e5bdd02e0c1e020356fc237503a4381fa99bda0797a63
+generated: "2025-12-22T11:00:21.134404-06:00"

charts/operator-wandb/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.38.9
+version: 0.38.10
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 
@@ -67,6 +67,11 @@ dependencies:
     version: "0.11.6"
     repository: file://../wandb-base
     condition: parquet.install
+  - name: wandb-base
+    alias: parquet-metadata-cache
+    version: "0.11.6"
+    repository: file://../wandb-base
+    condition: parquet-metadata-cache.install
   - name: mysql
     version: "*.*.*"
     repository: file://charts/mysql

charts/operator-wandb/templates/parquet-metadata-cache.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-parquet-metadata-cache-configmap
+  labels:
+    {{- include "wandb.commonLabels" . | nindent 4 }}
+data:
+  # Port configuration
+  GORILLA_PARQUET_METADATA_CACHE_PORT: "9091"
+
+  GORILLA_CACHE_SIZE: "10737418240"
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-parquet-metadata-cache-secret
+  labels:
+    {{- include "wandb.commonLabels" . | nindent 4 }}
+data: {}

charts/operator-wandb/templates/parquet.yaml

@@ -18,6 +18,9 @@ data:
   GORILLA_PARQUET_ARROW_BUFFER_SIZE: "2147483648" # 2GB
   GORILLA_COLLECT_AUDIT_LOGS: "true"
 
+  # Parquet metadata cache configuration
+  GORILLA_PARQUET_READER_REMOTE_METADATA_CACHE_ADDR: "{{ .Release.Name }}-parquet-metadata-cache:9091"
+
 ---
 apiVersion: v1
 kind: Secret

charts/operator-wandb/values.yaml

@@ -299,13 +299,13 @@
     enabled: false
     rateLimits:
       enabled: false
-      filestreamCount: 20  # number of request counts per second (project level)
-      filestreamSize: 40  # number of MB per second (project level)
-      fileStreamPerRunCount: 1  # number of request counts per run per second (project level)
-      runUpdateCount: 20  # number of run metadata update requests per second (project level)
-      sdkGraphqlQuerySeconds: 4  # request query seconds over DB per second (project level)
-      createArtifacts: 833  # number of create artifact requests per second (project level)
-      createArtifactsTimeWindow: 120  # 2 minutes - time window for create artifacts rate limit
+      filestreamCount: 20 # number of request counts per second (project level)
+      filestreamSize: 40 # number of MB per second (project level)
+      fileStreamPerRunCount: 1 # number of request counts per run per second (project level)
+      runUpdateCount: 20 # number of run metadata update requests per second (project level)
+      sdkGraphqlQuerySeconds: 4 # request query seconds over DB per second (project level)
+      createArtifacts: 833 # number of create artifact requests per second (project level)
+      createArtifactsTimeWindow: 120 # 2 minutes - time window for create artifacts rate limit
     additionalPaths:
       proxy: false
       files: false
@@ -1892,6 +1892,119 @@
           maxReplicas: 4
           minReplicas: 2
 
+parquet-metadata-cache:
+  install: true
+  deploymentPostfix: "bc"
+  serviceAccount:
+    create: false
+    name: wandb-parquet
+  image:
+    repository: wandb/megabinary
+    tag: latest
+  service:
+    enabled: true
+    ports:
+      - port: 9091
+        protocol: TCP
+        name: grpc
+  envFrom:
+    "{{ .Release.Name }}-bucket-configmap": "configMapRef"
+    "{{ .Release.Name }}-redis-configmap": "configMapRef"
+    "{{ .Release.Name }}-kafka-configmap": "configMapRef"
+    "{{ .Release.Name }}-global-configmap": "configMapRef"
+    "{{ .Release.Name }}-global-secret": "secretRef"
+    "{{ .Release.Name }}-parquet-metadata-cache-configmap": "configMapRef"
+  envTpls:
+    - '{{ include "wandb.downwardEnvs" . }}'
+    - '{{ include "wandb.bucket.cwIdentity" . }}'
+    - '{{ include "wandb.bucketEnvs" . }}'
+    - '{{ include "wandb.mysqlEnvs" . }}'
+    - '{{ include "wandb.redisEnvs" . }}'
+    - '{{ include "wandb.queueEnvs" . }}'
+    - '{{ include "wandb.historyStoreEnvs" . }}'
+    - '{{ include "wandb.observabilityEnvs" . }}'
+    - '{{ include "wandb.statsigEnvs" . }}'
+    - '{{ include "wandb.license" . }}'
+  containers:
+    parquet-metadata-cache:
+      args: ["parquet-metadata-cache"]
+      ports:
+        - name: grpc
+          containerPort: 9091
+          protocol: TCP
+      volumeMountsTpls:
+        - '{{ include "wandb.caCertsVolumeMounts" . }}'
+  volumesTpls:
+    - '{{ include "wandb.caCertsVolumes" . }}'
+  size: ""
+  sizing:
+    small:
+      resources:
+        limits:
+          cpu: "6"
+          memory: 16Gi
+        requests:
+          cpu: "4"
+          memory: 16Gi
+      autoscaling:
+        horizontal:
+          enabled: false
+          maxReplicas: 1
+          minReplicas: 1
+    medium:
+      resources:
+        limits:
+          cpu: "15"
+          memory: 64Gi
+        requests:
+          cpu: "15"
+          memory: 64Gi
+      autoscaling:
+        horizontal:
+          enabled: false
+          maxReplicas: 1
+          minReplicas: 1
+    large:
+      resources:
+        limits:
+          cpu: "15"
+          memory: 64Gi
+        requests:
+          cpu: "15"
+          memory: 64Gi
+      autoscaling:
+        horizontal:
+          enabled: false
+          maxReplicas: 1
+          minReplicas: 1
+    # At larger t-shirt sizes, reserve an entire node for parquet.
+    xlarge:
+      resources:
+        limits:
+          cpu: "15"
+          memory: 64Gi
+        requests:
+          cpu: "15"
+          memory: 64Gi
+      autoscaling:
+        horizontal:
+          enabled: false
+          maxReplicas: 1
+          minReplicas: 1
+    xxlarge:
+      resources:
+        limits:
+          cpu: "15"
+          memory: 64Gi
+        requests:
+          cpu: "15"
+          memory: 64Gi
+      autoscaling:
+        horizontal:
+          enabled: false
+          maxReplicas: 1
+          minReplicas: 1
+
 settingsMigrationJob:
   install: false
   image: