Skip to content

🌱 Use IRSO in e2e tests #2776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adilGhaffarDev wants to merge 1 commit into
base: main
Choose a base branch
from
Open

🌱 Use IRSO in e2e tests #2776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 48 additions & 31 deletions scripts/ci-e2e.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,14 @@ export KUBERNETES_VERSION=${KUBERNETES_VERSION}
export IMAGE_OS=${IMAGE_OS}
export FORCE_REPO_UPDATE="false"
export SKIP_NODE_IMAGE_PREPULL="true"
export USE_IRSO="${USE_IRSO:-false}"
export IPA_BASEURI=https://artifactory.nordix.org/artifactory/openstack-remote-cache/ironic-python-agent/dib
EOF

# Set USE_IRSO only when IMAGE_OS is not ubuntu
if [[ "${IMAGE_OS}" != "ubuntu" ]]; then
echo 'export USE_IRSO="true"' >> "${M3_DEV_ENV_PATH}/config_${USER}.sh"
fi

# Always set DATE variable for nightly builds because it is needed to form
# the URL for CAPI nightly build components in e2e_conf.yaml even if not used.
DATE=$(date '+%Y%m%d' -d '1 day ago')
Expand Down Expand Up @@ -168,44 +173,63 @@ kustomize_envsubst() {
echo "envsubst applied to ${file}"
}

yaml_envsubst() {
local kustomize_dir="$1"

for file in "${kustomize_dir}"/*.yaml; do
if [[ -f "${file}" ]]; then
local tmp_file
tmp_file=$(mktemp)
envsubst < "${file}" > "${tmp_file}" && mv "${tmp_file}" "${file}"
fi
done
}

# Generate credentials
BMO_OVERLAYS=(
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.9"
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.10"
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.11"
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/pr-test"
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-latest"
)
IRONIC_OVERLAYS=(
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-27.0"
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-29.0"
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-31.0"
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-32.0"
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/pr-test"
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-latest"
IRSO_IRONIC_OVERLAYS=(
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/release-31.0"
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/release-32.0"
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/pr-test"
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/latest"
)

# Update BMO and Ironic images in kustomization.yaml files to use the same image that was used before pivot in the metal3-dev-env
case "${REPO_NAME:-}" in
baremetal-operator)
# shellcheck disable=SC2034
BARE_METAL_OPERATOR_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
export BARE_METAL_OPERATOR_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
;;

ironic-image)
# shellcheck disable=SC2034
IRONIC_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
export IRONIC_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
;;
esac

IRONIC_IMAGE="${IRONIC_IMAGE:-quay.io/metal3-io/ironic:main}"
update_kustomize_image quay.io/metal3-io/baremetal-operator BARE_METAL_OPERATOR_IMAGE "${REPO_ROOT}"/test/e2e/data/bmo-deployment/overlays/pr-test
update_kustomize_image quay.io/metal3-io/ironic IRONIC_IMAGE "${REPO_ROOT}"/test/e2e/data/ironic-deployment/overlays/pr-test

# Apply envsubst to kustomization.yaml files in BMO and Ironic overlays
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/bmo-deployment/overlays/pr-test
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-deployment/overlays/pr-test
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base/
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/operator
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/overlays/pr-test/
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/overlays/latest/

for overlay in "${IRSO_IRONIC_OVERLAYS[@]}"; do
kustomize_envsubst "${overlay}"
done

# Create usernames and passwords and other files related to ironi basic auth if they
# Create usernames and passwords and other files related to ironic basic auth if they
# are missing
if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
IRONIC_AUTH_DIR="${IRONIC_AUTH_DIR:-${IRONIC_DATA_DIR}/auth}"
Expand All @@ -228,13 +252,19 @@ if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
IRONIC_PASSWORD="$(cat "${IRONIC_AUTH_DIR}/ironic-password")"
fi
fi
IRONIC_INSPECTOR_USERNAME="${IRONIC_INSPECTOR_USERNAME:-${IRONIC_USERNAME}}"
IRONIC_INSPECTOR_PASSWORD="${IRONIC_INSPECTOR_PASSWORD:-${IRONIC_PASSWORD}}"

export IRONIC_USERNAME
export IRONIC_PASSWORD
export IRONIC_INSPECTOR_USERNAME
export IRONIC_INSPECTOR_PASSWORD
fi

if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
echo "${IRONIC_USERNAME}" > "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/ironic-username
echo "${IRONIC_PASSWORD}" > "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/ironic-password
fi

if [[ "${IRONIC_TLS_SETUP}" == "true" ]]; then
cp "${IRONIC_KEY_FILE}" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
cp "${IRONIC_CERT_FILE}" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
fi

for overlay in "${BMO_OVERLAYS[@]}"; do
Expand All @@ -246,19 +276,6 @@ for overlay in "${BMO_OVERLAYS[@]}"; do
fi
done

for overlay in "${IRONIC_OVERLAYS[@]}"; do
echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
"${overlay}/ironic-htpasswd"
envsubst < "${REPO_ROOT}/test/e2e/data/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
"${overlay}/ironic-auth-config"
IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl
envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
"${overlay}/ironic-inspector-auth-config"
echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
"${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd"
done

# run e2e tests
if [[ -n "${CLUSTER_TOPOLOGY:-}" ]]; then
export CLUSTER_TOPOLOGY=true
Expand Down
248 changes: 248 additions & 0 deletions test/e2e/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
infrav1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1"
ipamv1 "github.com/metal3-io/ip-address-manager/api/v1alpha1"
irsov1alpha1 "github.com/metal3-io/ironic-standalone-operator/api/v1alpha1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
Expand Down Expand Up @@ -1410,3 +1411,250 @@ func UpgradeControlPlane(ctx context.Context, inputGetter func() UpgradeControlP
Intervals: e2eConfig.GetIntervals(specName, "wait-machine-running"),
})
}

type InstallIRSOInput struct {
E2EConfig *clusterctl.E2EConfig
ClusterProxy framework.ClusterProxy
IronicNamespace string
ClusterName string
IrsoOperatorKustomize string
IronicKustomize string
}

func InstallIRSO(ctx context.Context, input InstallIRSOInput) error {
By("Create Ironic namespace")
targetClusterClientSet := input.ClusterProxy.GetClientSet()
ironicNamespaceObj := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: input.IronicNamespace,
},
}
_, err := targetClusterClientSet.CoreV1().Namespaces().Create(ctx, ironicNamespaceObj, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
Logf("Ironic namespace %q already exists, continuing", input.IronicNamespace)
} else {
Expect(err).ToNot(HaveOccurred(), "Unable to create the Ironic namespace")
}
}

irsoDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "ironic-deploy-logs", input.ClusterProxy.GetName())
By(fmt.Sprintf("Installing IRSO from kustomization %s on the target cluster", input.IrsoOperatorKustomize))
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: input.IrsoOperatorKustomize,
ClusterProxy: input.ClusterProxy,
WaitForDeployment: true,
WatchDeploymentLogs: true,
LogPath: irsoDeployLogFolder,
DeploymentName: "ironic-standalone-operator-controller-manager",
DeploymentNamespace: IRSOControllerNameSpace,
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
})
Expect(err).NotTo(HaveOccurred())

By("Waiting for Ironic CRD to be available")
Eventually(func(g Gomega) {
crd := &apiextensionsv1.CustomResourceDefinition{}
err = input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{
Name: "ironics.ironic.metal3.io",
}, crd)
g.Expect(err).ToNot(HaveOccurred(), "Ironic CRD not found")
// Check if CRD is established
established := false
for _, cond := range crd.Status.Conditions {
if cond.Type == apiextensionsv1.Established && cond.Status == apiextensionsv1.ConditionTrue {
established = true
break
}
}
g.Expect(established).To(BeTrue(), "Ironic CRD is not established yet")
}, input.E2EConfig.GetIntervals("default", "wait-deployment")...).Should(Succeed())
Logf("Ironic CRD is available and established")

// Retry applying Ironic CR until it's successfully created
Eventually(func(g Gomega) {
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: input.IronicKustomize,
ClusterProxy: input.ClusterProxy,
WaitForDeployment: false,
WatchDeploymentLogs: false,
})
g.Expect(err).NotTo(HaveOccurred(), "Failed to apply Ironic CR")
// Verify Ironic CR was actually created
ironic := &irsov1alpha1.Ironic{}
err = input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{
Name: "ironic",
Namespace: input.IronicNamespace,
}, ironic)
g.Expect(err).NotTo(HaveOccurred(), "Ironic CR was not created")
Logf("Ironic CR successfully created")
}, input.E2EConfig.GetIntervals("default", "wait-deployment")...).Should(Succeed())

return nil
}

// WaitForIronicReady waits until the given Ironic resource has Ready condition = True.
func WaitForIronicReady(ctx context.Context, input WaitForIronicInput) {
Logf("Waiting for Ironic %q to be Ready", input.Name)

Eventually(func(g Gomega) {
ironic := &irsov1alpha1.Ironic{}
err := input.Client.Get(ctx, client.ObjectKey{
Namespace: input.Namespace,
Name: input.Name,
}, ironic)
g.Expect(err).ToNot(HaveOccurred())

ready := false
for _, cond := range ironic.Status.Conditions {
if cond.Type == string(irsov1alpha1.IronicStatusReady) && cond.Status == metav1.ConditionTrue && ironic.Status.InstalledVersion != "" {
ready = true
break
}
}
g.Expect(ready).To(BeTrue(), "Ironic %q is not Ready yet", input.Name)
}, input.Intervals...).Should(Succeed())

Logf("Ironic %q is Ready", input.Name)
}

// WaitForIronicInput bundles the parameters for WaitForIronicReady.
type WaitForIronicInput struct {
Client client.Client
Name string
Namespace string
Intervals []interface{} // e.g. []interface{}{time.Minute * 15, time.Second * 5}
}

// InstallBMOInput bundles parameters for InstallBMO.
type InstallBMOInput struct {
E2EConfig *clusterctl.E2EConfig
ClusterProxy framework.ClusterProxy
Namespace string // Namespace where BMO will run (shared with Ironic)
BmoKustomization string // Kustomization path or URL for BMO manifests
LogFolder string // Optional explicit log folder; if empty a default is derived
WaitIntervals []any // Optional override; if nil uses default e2e config intervals
WatchLogs bool // Whether to watch deployment logs
}

// InstallBMO installs the Baremetal Operator (BMO) in the target cluster similar to InstallIRSO.
func InstallBMO(ctx context.Context, input InstallBMOInput) error {
By("Ensure BMO namespace exists")
clientset := input.ClusterProxy.GetClientSet()
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: input.Namespace}}
_, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
Logf("Namespace %q already exists, continuing", input.Namespace)
} else {
return fmt.Errorf("failed creating namespace %q: %w", input.Namespace, err)
}
}

// Determine log folder
logFolder := input.LogFolder
if logFolder == "" {
logFolder = filepath.Join(os.TempDir(), "target_cluster_logs", "bmo-deploy-logs", input.ClusterProxy.GetName())
}
intervals := input.WaitIntervals
if intervals == nil {
intervals = input.E2EConfig.GetIntervals("default", "wait-deployment")
}

By(fmt.Sprintf("Installing BMO from kustomization %s on the target cluster", input.BmoKustomization))
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
Kustomization: input.BmoKustomization,
ClusterProxy: input.ClusterProxy,
WaitForDeployment: true,
WatchDeploymentLogs: input.WatchLogs,
LogPath: logFolder,
DeploymentName: "baremetal-operator-controller-manager",
DeploymentNamespace: input.Namespace,
WaitIntervals: intervals,
})
if err != nil {
return fmt.Errorf("failed installing BMO: %w", err)
}

By("BMO deployment applied and available")
return nil
}

type UninstallIRSOAndIronicResourcesInput struct {
E2EConfig *clusterctl.E2EConfig
ClusterProxy framework.ClusterProxy
IronicNamespace string
IrsoOperatorKustomize string
IronicKustomization string
IsDevEnvUninstall bool
}

// UninstallIRSOAndIronicResources removes the IRSO deployment, Ironic CR, IronicDatabase CR (if present), and related secrets.
func UninstallIRSOAndIronicResources(ctx context.Context, input UninstallIRSOAndIronicResourcesInput) error {
if input.IsDevEnvUninstall {
ironicObj := &irsov1alpha1.Ironic{
ObjectMeta: metav1.ObjectMeta{
Name: "ironic",
Namespace: input.IronicNamespace,
},
}
err := input.ClusterProxy.GetClient().Delete(ctx, ironicObj)
Expect(err).ToNot(HaveOccurred(), "Failed to delete Ironic")
} else {
By("Remove Ironic CR in the cluster " + input.ClusterProxy.GetName())
err := BuildAndRemoveKustomization(ctx, input.IronicKustomization, input.ClusterProxy)
Expect(err).NotTo(HaveOccurred())
}

By("Remove Ironic Service Deployment in the cluster " + input.ClusterProxy.GetName())
RemoveDeployment(ctx, func() RemoveDeploymentInput {
return RemoveDeploymentInput{
ManagementCluster: input.ClusterProxy,
Namespace: input.IronicNamespace,
Name: "ironic-service",
}
})

if input.IsDevEnvUninstall {
By("Remove Ironic Standalone Operator Deployment in the cluster " + input.ClusterProxy.GetName())
RemoveDeployment(ctx, func() RemoveDeploymentInput {
return RemoveDeploymentInput{
ManagementCluster: input.ClusterProxy,
Namespace: IRSOControllerNameSpace,
Name: "ironic-standalone-operator-controller-manager",
}
})
} else {
By("Uninstalling IRSO operator via kustomize")
err := BuildAndRemoveKustomization(ctx, input.IrsoOperatorKustomize, input.ClusterProxy)
Expect(err).NotTo(HaveOccurred())
}

clusterClient := input.ClusterProxy.GetClient()

// Delete secrets
secretNames := []string{"ironic-auth", "ironic-cert", "ironic-cacert"}
for _, s := range secretNames {
Byf("Deleting secret %s", s)
secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: s, Namespace: input.IronicNamespace}}
err := clusterClient.Delete(ctx, secret)
if err != nil {
Logf("Failed to delete secret %s: %v", s, err)
}
}

// Wait for secrets to be deleted
By("Waiting for Ironic secrets to be deleted")
Eventually(func() bool {
for _, s := range secretNames {
errS := clusterClient.Get(ctx, client.ObjectKey{Name: s, Namespace: input.IronicNamespace}, &corev1.Secret{})
if errS == nil || !apierrors.IsNotFound(errS) {
return false
}
}
return true
}, input.E2EConfig.GetIntervals("default", "wait-delete-ironic")...).Should(BeTrue(), "IRSO/Ironic resources not fully deleted")

By("IRSO and Ironic resources uninstalled")
return nil
}
Loading