Skip to content

Commit 5367d71

Browse files
Use IRSO in e2e tests
Signed-off-by: Muhammad Adil Ghaffar <[email protected]>
1 parent 3ab0aaf commit 5367d71

40 files changed

+666
-837
lines changed

scripts/ci-e2e.sh

Lines changed: 48 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,14 @@ export KUBERNETES_VERSION=${KUBERNETES_VERSION}
5353
export IMAGE_OS=${IMAGE_OS}
5454
export FORCE_REPO_UPDATE="false"
5555
export SKIP_NODE_IMAGE_PREPULL="true"
56-
export USE_IRSO="${USE_IRSO:-false}"
56+
export IPA_BASEURI=https://artifactory.nordix.org/artifactory/openstack-remote-cache/ironic-python-agent/dib
5757
EOF
5858

59+
# Set USE_IRSO only when IMAGE_OS is not ubuntu
60+
if [[ "${IMAGE_OS}" != "ubuntu" ]]; then
61+
echo 'export USE_IRSO="true"' >> "${M3_DEV_ENV_PATH}/config_${USER}.sh"
62+
fi
63+
5964
# Always set DATE variable for nightly builds because it is needed to form
6065
# the URL for CAPI nightly build components in e2e_conf.yaml even if not used.
6166
DATE=$(date '+%Y%m%d' -d '1 day ago')
@@ -168,44 +173,63 @@ kustomize_envsubst() {
168173
echo "envsubst applied to ${file}"
169174
}
170175

176+
yaml_envsubst() {
177+
local kustomize_dir="$1"
178+
179+
for file in "${kustomize_dir}"/*.yaml; do
180+
if [[ -f "${file}" ]]; then
181+
local tmp_file
182+
tmp_file=$(mktemp)
183+
envsubst < "${file}" > "${tmp_file}" && mv "${tmp_file}" "${file}"
184+
fi
185+
done
186+
}
187+
171188
# Generate credentials
172189
BMO_OVERLAYS=(
173-
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.9"
174190
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.10"
175191
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.11"
176192
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/pr-test"
177193
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-latest"
178194
)
179-
IRONIC_OVERLAYS=(
180-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-27.0"
181-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-29.0"
182-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-31.0"
183-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-32.0"
184-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/pr-test"
185-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-latest"
195+
IRSO_IRONIC_OVERLAYS=(
196+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/release-31.0"
197+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/release-32.0"
198+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/pr-test"
199+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/latest"
186200
)
187201

188202
# Update BMO and Ironic images in kustomization.yaml files to use the same image that was used before pivot in the metal3-dev-env
189203
case "${REPO_NAME:-}" in
190204
baremetal-operator)
191205
# shellcheck disable=SC2034
192-
BARE_METAL_OPERATOR_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
206+
export BARE_METAL_OPERATOR_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
193207
;;
194208

195209
ironic-image)
196210
# shellcheck disable=SC2034
197-
IRONIC_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
211+
export IRONIC_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
198212
;;
199213
esac
200214

215+
IRONIC_IMAGE="${IRONIC_IMAGE:-quay.io/metal3-io/ironic:main}"
201216
update_kustomize_image quay.io/metal3-io/baremetal-operator BARE_METAL_OPERATOR_IMAGE "${REPO_ROOT}"/test/e2e/data/bmo-deployment/overlays/pr-test
202-
update_kustomize_image quay.io/metal3-io/ironic IRONIC_IMAGE "${REPO_ROOT}"/test/e2e/data/ironic-deployment/overlays/pr-test
203217

204218
# Apply envsubst to kustomization.yaml files in BMO and Ironic overlays
205219
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/bmo-deployment/overlays/pr-test
206-
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-deployment/overlays/pr-test
220+
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base
221+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base/
222+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/
223+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
224+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/operator
225+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/overlays/pr-test/
226+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/overlays/latest/
227+
228+
for overlay in "${IRSO_IRONIC_OVERLAYS[@]}"; do
229+
kustomize_envsubst "${overlay}"
230+
done
207231

208-
# Create usernames and passwords and other files related to ironi basic auth if they
232+
# Create usernames and passwords and other files related to ironic basic auth if they
209233
# are missing
210234
if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
211235
IRONIC_AUTH_DIR="${IRONIC_AUTH_DIR:-${IRONIC_DATA_DIR}/auth}"
@@ -228,13 +252,19 @@ if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
228252
IRONIC_PASSWORD="$(cat "${IRONIC_AUTH_DIR}/ironic-password")"
229253
fi
230254
fi
231-
IRONIC_INSPECTOR_USERNAME="${IRONIC_INSPECTOR_USERNAME:-${IRONIC_USERNAME}}"
232-
IRONIC_INSPECTOR_PASSWORD="${IRONIC_INSPECTOR_PASSWORD:-${IRONIC_PASSWORD}}"
233255

234256
export IRONIC_USERNAME
235257
export IRONIC_PASSWORD
236-
export IRONIC_INSPECTOR_USERNAME
237-
export IRONIC_INSPECTOR_PASSWORD
258+
fi
259+
260+
if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
261+
echo "${IRONIC_USERNAME}" > "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/ironic-username
262+
echo "${IRONIC_PASSWORD}" > "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/ironic-password
263+
fi
264+
265+
if [[ "${IRONIC_TLS_SETUP}" == "true" ]]; then
266+
cp "${IRONIC_KEY_FILE}" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
267+
cp "${IRONIC_CERT_FILE}" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
238268
fi
239269

240270
for overlay in "${BMO_OVERLAYS[@]}"; do
@@ -246,19 +276,6 @@ for overlay in "${BMO_OVERLAYS[@]}"; do
246276
fi
247277
done
248278

249-
for overlay in "${IRONIC_OVERLAYS[@]}"; do
250-
echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
251-
"${overlay}/ironic-htpasswd"
252-
envsubst < "${REPO_ROOT}/test/e2e/data/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
253-
"${overlay}/ironic-auth-config"
254-
IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
255-
curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl
256-
envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
257-
"${overlay}/ironic-inspector-auth-config"
258-
echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
259-
"${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd"
260-
done
261-
262279
# run e2e tests
263280
if [[ -n "${CLUSTER_TOPOLOGY:-}" ]]; then
264281
export CLUSTER_TOPOLOGY=true

test/e2e/common.go

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
2525
infrav1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1"
2626
ipamv1 "github.com/metal3-io/ip-address-manager/api/v1alpha1"
27+
irsov1alpha1 "github.com/metal3-io/ironic-standalone-operator/api/v1alpha1"
2728
. "github.com/onsi/ginkgo/v2"
2829
. "github.com/onsi/gomega"
2930
"github.com/pkg/errors"
@@ -1227,3 +1228,250 @@ func IsMetal3DataCountEqualToMachineCount(ctx context.Context, c client.Client,
12271228

12281229
return len(m3DataList.Items) == len(machineList.Items)
12291230
}
1231+
1232+
type InstallIRSOInput struct {
1233+
E2EConfig *clusterctl.E2EConfig
1234+
ClusterProxy framework.ClusterProxy
1235+
IronicNamespace string
1236+
ClusterName string
1237+
IrsoOperatorKustomize string
1238+
IronicKustomize string
1239+
}
1240+
1241+
func InstallIRSO(ctx context.Context, input InstallIRSOInput) error {
1242+
By("Create Ironic namespace")
1243+
targetClusterClientSet := input.ClusterProxy.GetClientSet()
1244+
ironicNamespaceObj := &corev1.Namespace{
1245+
ObjectMeta: metav1.ObjectMeta{
1246+
Name: input.IronicNamespace,
1247+
},
1248+
}
1249+
_, err := targetClusterClientSet.CoreV1().Namespaces().Create(ctx, ironicNamespaceObj, metav1.CreateOptions{})
1250+
if err != nil {
1251+
if apierrors.IsAlreadyExists(err) {
1252+
Logf("Ironic namespace %q already exists, continuing", input.IronicNamespace)
1253+
} else {
1254+
Expect(err).ToNot(HaveOccurred(), "Unable to create the Ironic namespace")
1255+
}
1256+
}
1257+
1258+
irsoDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "ironic-deploy-logs", input.ClusterProxy.GetName())
1259+
By(fmt.Sprintf("Installing IRSO from kustomization %s on the target cluster", input.IrsoOperatorKustomize))
1260+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
1261+
Kustomization: input.IrsoOperatorKustomize,
1262+
ClusterProxy: input.ClusterProxy,
1263+
WaitForDeployment: true,
1264+
WatchDeploymentLogs: true,
1265+
LogPath: irsoDeployLogFolder,
1266+
DeploymentName: "ironic-standalone-operator-controller-manager",
1267+
DeploymentNamespace: IRSOControllerNameSpace,
1268+
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
1269+
})
1270+
Expect(err).NotTo(HaveOccurred())
1271+
1272+
By("Waiting for Ironic CRD to be available")
1273+
Eventually(func(g Gomega) {
1274+
crd := &apiextensionsv1.CustomResourceDefinition{}
1275+
err = input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{
1276+
Name: "ironics.ironic.metal3.io",
1277+
}, crd)
1278+
g.Expect(err).ToNot(HaveOccurred(), "Ironic CRD not found")
1279+
// Check if CRD is established
1280+
established := false
1281+
for _, cond := range crd.Status.Conditions {
1282+
if cond.Type == apiextensionsv1.Established && cond.Status == apiextensionsv1.ConditionTrue {
1283+
established = true
1284+
break
1285+
}
1286+
}
1287+
g.Expect(established).To(BeTrue(), "Ironic CRD is not established yet")
1288+
}, input.E2EConfig.GetIntervals("default", "wait-deployment")...).Should(Succeed())
1289+
Logf("Ironic CRD is available and established")
1290+
1291+
// Retry applying Ironic CR until it's successfully created
1292+
Eventually(func(g Gomega) {
1293+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
1294+
Kustomization: input.IronicKustomize,
1295+
ClusterProxy: input.ClusterProxy,
1296+
WaitForDeployment: false,
1297+
WatchDeploymentLogs: false,
1298+
})
1299+
g.Expect(err).NotTo(HaveOccurred(), "Failed to apply Ironic CR")
1300+
// Verify Ironic CR was actually created
1301+
ironic := &irsov1alpha1.Ironic{}
1302+
err = input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{
1303+
Name: "ironic",
1304+
Namespace: input.IronicNamespace,
1305+
}, ironic)
1306+
g.Expect(err).NotTo(HaveOccurred(), "Ironic CR was not created")
1307+
Logf("Ironic CR successfully created")
1308+
}, input.E2EConfig.GetIntervals("default", "wait-deployment")...).Should(Succeed())
1309+
1310+
return nil
1311+
}
1312+
1313+
// WaitForIronicReady waits until the given Ironic resource has Ready condition = True.
1314+
func WaitForIronicReady(ctx context.Context, input WaitForIronicInput) {
1315+
Logf("Waiting for Ironic %q to be Ready", input.Name)
1316+
1317+
Eventually(func(g Gomega) {
1318+
ironic := &irsov1alpha1.Ironic{}
1319+
err := input.Client.Get(ctx, client.ObjectKey{
1320+
Namespace: input.Namespace,
1321+
Name: input.Name,
1322+
}, ironic)
1323+
g.Expect(err).ToNot(HaveOccurred())
1324+
1325+
ready := false
1326+
for _, cond := range ironic.Status.Conditions {
1327+
if cond.Type == string(irsov1alpha1.IronicStatusReady) && cond.Status == metav1.ConditionTrue && ironic.Status.InstalledVersion != "" {
1328+
ready = true
1329+
break
1330+
}
1331+
}
1332+
g.Expect(ready).To(BeTrue(), "Ironic %q is not Ready yet", input.Name)
1333+
}, input.Intervals...).Should(Succeed())
1334+
1335+
Logf("Ironic %q is Ready", input.Name)
1336+
}
1337+
1338+
// WaitForIronicInput bundles the parameters for WaitForIronicReady.
1339+
type WaitForIronicInput struct {
1340+
Client client.Client
1341+
Name string
1342+
Namespace string
1343+
Intervals []interface{} // e.g. []interface{}{time.Minute * 15, time.Second * 5}
1344+
}
1345+
1346+
// InstallBMOInput bundles parameters for InstallBMO.
1347+
type InstallBMOInput struct {
1348+
E2EConfig *clusterctl.E2EConfig
1349+
ClusterProxy framework.ClusterProxy
1350+
Namespace string // Namespace where BMO will run (shared with Ironic)
1351+
BmoKustomization string // Kustomization path or URL for BMO manifests
1352+
LogFolder string // Optional explicit log folder; if empty a default is derived
1353+
WaitIntervals []any // Optional override; if nil uses default e2e config intervals
1354+
WatchLogs bool // Whether to watch deployment logs
1355+
}
1356+
1357+
// InstallBMO installs the Baremetal Operator (BMO) in the target cluster similar to InstallIRSO.
1358+
func InstallBMO(ctx context.Context, input InstallBMOInput) error {
1359+
By("Ensure BMO namespace exists")
1360+
clientset := input.ClusterProxy.GetClientSet()
1361+
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: input.Namespace}}
1362+
_, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
1363+
if err != nil {
1364+
if apierrors.IsAlreadyExists(err) {
1365+
Logf("Namespace %q already exists, continuing", input.Namespace)
1366+
} else {
1367+
return fmt.Errorf("failed creating namespace %q: %w", input.Namespace, err)
1368+
}
1369+
}
1370+
1371+
// Determine log folder
1372+
logFolder := input.LogFolder
1373+
if logFolder == "" {
1374+
logFolder = filepath.Join(os.TempDir(), "target_cluster_logs", "bmo-deploy-logs", input.ClusterProxy.GetName())
1375+
}
1376+
intervals := input.WaitIntervals
1377+
if intervals == nil {
1378+
intervals = input.E2EConfig.GetIntervals("default", "wait-deployment")
1379+
}
1380+
1381+
By(fmt.Sprintf("Installing BMO from kustomization %s on the target cluster", input.BmoKustomization))
1382+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
1383+
Kustomization: input.BmoKustomization,
1384+
ClusterProxy: input.ClusterProxy,
1385+
WaitForDeployment: true,
1386+
WatchDeploymentLogs: input.WatchLogs,
1387+
LogPath: logFolder,
1388+
DeploymentName: "baremetal-operator-controller-manager",
1389+
DeploymentNamespace: input.Namespace,
1390+
WaitIntervals: intervals,
1391+
})
1392+
if err != nil {
1393+
return fmt.Errorf("failed installing BMO: %w", err)
1394+
}
1395+
1396+
By("BMO deployment applied and available")
1397+
return nil
1398+
}
1399+
1400+
type UninstallIRSOAndIronicResourcesInput struct {
1401+
E2EConfig *clusterctl.E2EConfig
1402+
ClusterProxy framework.ClusterProxy
1403+
IronicNamespace string
1404+
IrsoOperatorKustomize string
1405+
IronicKustomization string
1406+
IsDevEnvUninstall bool
1407+
}
1408+
1409+
// UninstallIRSOAndIronicResources removes the IRSO deployment, Ironic CR, IronicDatabase CR (if present), and related secrets.
1410+
func UninstallIRSOAndIronicResources(ctx context.Context, input UninstallIRSOAndIronicResourcesInput) error {
1411+
if input.IsDevEnvUninstall {
1412+
ironicObj := &irsov1alpha1.Ironic{
1413+
ObjectMeta: metav1.ObjectMeta{
1414+
Name: "ironic",
1415+
Namespace: input.IronicNamespace,
1416+
},
1417+
}
1418+
err := input.ClusterProxy.GetClient().Delete(ctx, ironicObj)
1419+
Expect(err).ToNot(HaveOccurred(), "Failed to delete Ironic")
1420+
} else {
1421+
By("Remove Ironic CR in the cluster " + input.ClusterProxy.GetName())
1422+
err := BuildAndRemoveKustomization(ctx, input.IronicKustomization, input.ClusterProxy)
1423+
Expect(err).NotTo(HaveOccurred())
1424+
}
1425+
1426+
By("Remove Ironic Service Deployment in the cluster " + input.ClusterProxy.GetName())
1427+
RemoveDeployment(ctx, func() RemoveDeploymentInput {
1428+
return RemoveDeploymentInput{
1429+
ManagementCluster: input.ClusterProxy,
1430+
Namespace: input.IronicNamespace,
1431+
Name: "ironic-service",
1432+
}
1433+
})
1434+
1435+
if input.IsDevEnvUninstall {
1436+
By("Remove Ironic Standalone Operator Deployment in the cluster " + input.ClusterProxy.GetName())
1437+
RemoveDeployment(ctx, func() RemoveDeploymentInput {
1438+
return RemoveDeploymentInput{
1439+
ManagementCluster: input.ClusterProxy,
1440+
Namespace: IRSOControllerNameSpace,
1441+
Name: "ironic-standalone-operator-controller-manager",
1442+
}
1443+
})
1444+
} else {
1445+
By("Uninstalling IRSO operator via kustomize")
1446+
err := BuildAndRemoveKustomization(ctx, input.IrsoOperatorKustomize, input.ClusterProxy)
1447+
Expect(err).NotTo(HaveOccurred())
1448+
}
1449+
1450+
clusterClient := input.ClusterProxy.GetClient()
1451+
1452+
// Delete secrets
1453+
secretNames := []string{"ironic-auth", "ironic-cert", "ironic-cacert"}
1454+
for _, s := range secretNames {
1455+
Byf("Deleting secret %s", s)
1456+
secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: s, Namespace: input.IronicNamespace}}
1457+
err := clusterClient.Delete(ctx, secret)
1458+
if err != nil {
1459+
Logf("Failed to delete secret %s: %v", s, err)
1460+
}
1461+
}
1462+
1463+
// Wait for secrets to be deleted
1464+
By("Waiting for Ironic secrets to be deleted")
1465+
Eventually(func() bool {
1466+
for _, s := range secretNames {
1467+
errS := clusterClient.Get(ctx, client.ObjectKey{Name: s, Namespace: input.IronicNamespace}, &corev1.Secret{})
1468+
if errS == nil || !apierrors.IsNotFound(errS) {
1469+
return false
1470+
}
1471+
}
1472+
return true
1473+
}, input.E2EConfig.GetIntervals("default", "wait-delete-ironic")...).Should(BeTrue(), "IRSO/Ironic resources not fully deleted")
1474+
1475+
By("IRSO and Ironic resources uninstalled")
1476+
return nil
1477+
}

0 commit comments

Comments
 (0)