Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use helm for pko #1073

Merged
merged 8 commits into from
Jan 20, 2025
Merged

Use helm for pko #1073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
36b9640
Use helm for pko
janboll Jan 9, 2025
4c0452a
Make Materialize
janboll Jan 17, 2025
cc6cb46
Fix lint
janboll Jan 17, 2025
01ec5cd
Add PKO build steps
janboll Jan 17, 2025
dcb7eff
Add ACR login
janboll Jan 17, 2025
8094b0b
Remove global roles
janboll Jan 20, 2025
44660a3
Remove obsolete config
janboll Jan 20, 2025
575c0c1
Remove PKO UAMI, not requried
janboll Jan 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .github/workflows/services-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,3 +157,30 @@
run: |
cd tooling/image-sync
make push

build_push_package_operator:
permissions:
id-token: 'write'
contents: 'read'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1

- name: "install azure-cli"
if: inputs.push == true
uses: "Azure/ARO-HCP@main"

- name: 'Az CLI login'
if: inputs.push == true
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

- name: Build package operator container image from [email protected]:package-operator/package-operator.git
run: |
cd pko
make image
20 changes: 20 additions & 0 deletions config/config.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,26 @@
"regionalSubdomain"
]
},
"pko": {
"type": "object",
"properties": {
"image": {
"type": "string"
},
"imageManager": {
"type": "string"
},
"imageTag": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"image",
"imageManager",
"imageTag"
]
},
"clusterService": {
"type": "object",
"properties": {
Expand Down
5 changes: 5 additions & 0 deletions config/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ defaults:
consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro

pko:
image: arohcpsvcdev.azurecr.io/package-operator/package-operator-package
imageManager: arohcpsvcdev.azurecr.io/package-operator/package-operator-manager
imageTag: v1.15.0

# Cluster Service
clusterService:
acrRG: global
Expand Down
5 changes: 5 additions & 0 deletions config/public-cloud-cs-pr.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,11 @@
},
"ocpAcrName": "arohcpocpdev",
"oidcStorageAccountName": "arohcpoidccspr",
"pko": {
"image": "arohcpsvcdev.azurecr.io/package-operator/package-operator-package",
"imageManager": "arohcpsvcdev.azurecr.io/package-operator/package-operator-manager",
"imageTag": "v1.15.0"
},
"region": "westus3",
"regionRG": "hcp-underlay-cspr",
"serviceKeyVault": {
Expand Down
5 changes: 5 additions & 0 deletions config/public-cloud-dev.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,11 @@
},
"ocpAcrName": "arohcpocpdev",
"oidcStorageAccountName": "arohcpoidcdev",
"pko": {
"image": "arohcpsvcdev.azurecr.io/package-operator/package-operator-package",
"imageManager": "arohcpsvcdev.azurecr.io/package-operator/package-operator-manager",
"imageTag": "v1.15.0"
},
"region": "westus3",
"regionRG": "hcp-underlay-dev",
"serviceKeyVault": {
Expand Down
5 changes: 5 additions & 0 deletions config/public-cloud-personal-dev.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,11 @@
},
"ocpAcrName": "arohcpocpdev",
"oidcStorageAccountName": "arohcpoidcusw3tst",
"pko": {
"image": "arohcpsvcdev.azurecr.io/package-operator/package-operator-package",
"imageManager": "arohcpsvcdev.azurecr.io/package-operator/package-operator-manager",
"imageTag": "v1.15.0"
},
"region": "westus3",
"regionRG": "hcp-underlay-usw3tst",
"serviceKeyVault": {
Expand Down
35 changes: 33 additions & 2 deletions pko/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,37 @@
SHELL = /bin/bash
-include ../setup-env.mk
-include ../helm-cmd.mk
HELM_CMD ?= helm upgrade --install

NAMESPACE ?= package-operator-system
ARO_HCP_IMAGE_REGISTRY ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io
ARO_HCP_IMAGE_REPOSITORY ?= package-operator/package-operator-package

deploy:
kubectl apply -f https://github.com/package-operator/package-operator/releases/download/v1.15.0/self-bootstrap-job.yaml
@kubectl create namespace ${NAMESPACE} --dry-run=client -o json | kubectl apply -f -
IMAGE_PULLER_MI_CLIENT_ID=$$(az identity show \
-g ${RESOURCEGROUP} \
-n image-puller \
--query clientId -o tsv) && \
IMAGE_PULLER_MI_TENANT_ID=$$(az identity show \
-g ${RESOURCEGROUP} \
-n image-puller \
--query tenantId -o tsv) && \
${HELM_CMD} package-operator ./helm \
--namespace ${NAMESPACE} \
--set pkoImage=${PKO_IMAGE} \
--set pkoImageManager=${PKO_IMAGE_MANAGER} \
--set pkoImageTag=${PKO_IMAGE_TAG} \
--set pullBinding.workloadIdentityClientId="$${IMAGE_PULLER_MI_CLIENT_ID}" \
--set pullBinding.workloadIdentityTenantId="$${IMAGE_PULLER_MI_TENANT_ID}" \
--set pullBinding.registry=${ARO_HCP_IMAGE_REGISTRY} \
--set pullBinding.scope='repository:*:pull'

image:
az acr login --name ${ARO_HCP_IMAGE_ACR} && \
cd $$(mktemp -d) && \
git clone https://github.com/package-operator/package-operator.git && \
cd package-operator && \
git checkout ${PKO_IMAGE_TAG} && \
IMAGE_REGISTRY=${ARO_HCP_IMAGE_REGISTRY}/package-operator ./do ci:release

.PHONY: deploy
7 changes: 7 additions & 0 deletions pko/helm/Chart.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
apiVersion: v2
name: package-operator
description: A Helm chart for package-operator
type: application

version: 0.1.0
appVersion: "1.0.0"
15 changes: 15 additions & 0 deletions pko/helm/templates/acrpullbinding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: acrpull.microsoft.com/v1beta2
kind: AcrPullBinding
metadata:
name: pull-binding
spec:
acr:
environment: PublicCloud
server: {{ .Values.pullBinding.registry }}
scope: {{ .Values.pullBinding.scope }}
auth:
workloadIdentity:
serviceAccountRef: package-operator
clientID: {{ .Values.pullBinding.workloadIdentityClientId }}
tenantID: {{ .Values.pullBinding.workloadIdentityTenantId }}
serviceAccountName: package-operator
14 changes: 14 additions & 0 deletions pko/helm/templates/clusterrolebinding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: package-operator
labels:
package-operator.run/cache: "True"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a more fine grained role we could use?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used the same role used by the PKO team: https://github.com/package-operator/package-operator/blob/f564bbba288a9e3748c871c0eaec329de2ec4cd2/config/self-bootstrap-job.yaml.tpl#L16

So I'll have to ask, if we can reduce the access.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think it also comes down to what packages this PKO instance will manage. as we develop a feeling about what kind of resources RVMO will bring in, we could narrow the role towards that.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, cause the service account assigned to is used by the operator during runtime.

subjects:
- kind: ServiceAccount
name: package-operator
namespace: package-operator-system
37 changes: 37 additions & 0 deletions pko/helm/templates/job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
apiVersion: batch/v1
kind: Job
metadata:
name: package-operator-bootstrap
namespace: package-operator-system
spec:
# delete right after completion
ttlSecondsAfterFinished: 0
# set deadline to 30min
activeDeadlineSeconds: 1800
template:
spec:
restartPolicy: OnFailure
serviceAccountName: package-operator
containers:
- name: package-operator
image: "{{ .Values.pkoImageManager }}:{{ .Values.pkoImageTag }}"
args: ["-self-bootstrap={{ .Values.pkoImage }}:{{ .Values.pkoImageTag }}"]
imagePullPolicy: Always
env:
- name: PKO_REGISTRY_HOST_OVERRIDES
value: ''
- name: PKO_CONFIG
value: ''
- name: PKO_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PKO_SERVICE_ACCOUNT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PKO_SERVICE_ACCOUNT_NAME
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
backoffLimit: 3
7 changes: 7 additions & 0 deletions pko/helm/templates/serviceaccount.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: package-operator
namespace: package-operator-system
labels:
package-operator.run/cache: "True"
8 changes: 8 additions & 0 deletions pko/helm/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
pkoImage: ""
pkoImageManager: ""
pkoImageTag: ""
pullBinding:
registry: ""
scope: ""
workloadIdentityClientId: ""
workloadIdentityTenantId: ""
26 changes: 26 additions & 0 deletions pko/pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.RP.PKO
rolloutName: RP PKO Rollout
resourceGroups:
- name: {{ .mgmt.rg }}
subscription: {{ .svc.subscription }}
aksCluster: {{ .aksName }}
steps:
- name: deploy
action: Shell
command: make deploy
dryRun:
variables:
- name: DRY_RUN
value: "true"
variables:
- name: ARO_HCP_IMAGE_ACR
configRef: svcAcrName
- name: PKO_IMAGE
configRef: pko.image
- name: PKO_IMAGE_MANAGER
configRef: pko.imageManager
- name: PKO_IMAGE_TAG
configRef: pko.imageTag
- name: RESOURCEGROUP
configRef: mgmt.rg
Loading