Skip to content

Commit 69c2f70

Browse files
Use IRSO in e2e tests
Signed-off-by: Muhammad Adil Ghaffar <[email protected]>
1 parent 3ab0aaf commit 69c2f70

40 files changed

+663
-837
lines changed

scripts/ci-e2e.sh

Lines changed: 47 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,14 @@ export KUBERNETES_VERSION=${KUBERNETES_VERSION}
5353
export IMAGE_OS=${IMAGE_OS}
5454
export FORCE_REPO_UPDATE="false"
5555
export SKIP_NODE_IMAGE_PREPULL="true"
56-
export USE_IRSO="${USE_IRSO:-false}"
56+
export IPA_BASEURI=https://artifactory.nordix.org/artifactory/openstack-remote-cache/ironic-python-agent/dib
5757
EOF
5858

59+
# Set USE_IRSO only when IMAGE_OS is not ubuntu
60+
if [[ "${IMAGE_OS}" != "ubuntu" ]]; then
61+
echo 'export USE_IRSO="true"' >> "${M3_DEV_ENV_PATH}/config_${USER}.sh"
62+
fi
63+
5964
# Always set DATE variable for nightly builds because it is needed to form
6065
# the URL for CAPI nightly build components in e2e_conf.yaml even if not used.
6166
DATE=$(date '+%Y%m%d' -d '1 day ago')
@@ -168,44 +173,63 @@ kustomize_envsubst() {
168173
echo "envsubst applied to ${file}"
169174
}
170175

176+
yaml_envsubst() {
177+
local kustomize_dir="$1"
178+
179+
for file in "${kustomize_dir}"/*.yaml; do
180+
if [[ -f "${file}" ]]; then
181+
local tmp_file
182+
tmp_file=$(mktemp)
183+
envsubst < "${file}" > "${tmp_file}" && mv "${tmp_file}" "${file}"
184+
fi
185+
done
186+
}
187+
171188
# Generate credentials
172189
BMO_OVERLAYS=(
173-
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.9"
174190
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.10"
175191
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.11"
176192
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/pr-test"
177193
"${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-latest"
178194
)
179-
IRONIC_OVERLAYS=(
180-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-27.0"
181-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-29.0"
182-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-31.0"
183-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-32.0"
184-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/pr-test"
185-
"${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-latest"
195+
IRSO_IRONIC_OVERLAYS=(
196+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/release-31.0"
197+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/release-32.0"
198+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/pr-test"
199+
"${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/ironic/overlays/latest"
186200
)
187201

188202
# Update BMO and Ironic images in kustomization.yaml files to use the same image that was used before pivot in the metal3-dev-env
189203
case "${REPO_NAME:-}" in
190204
baremetal-operator)
191205
# shellcheck disable=SC2034
192-
BARE_METAL_OPERATOR_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
206+
export BARE_METAL_OPERATOR_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
193207
;;
194208

195209
ironic-image)
196210
# shellcheck disable=SC2034
197-
IRONIC_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
211+
export IRONIC_IMAGE="${REGISTRY}/localimages/tested_repo:latest"
198212
;;
199213
esac
200214

215+
IRONIC_IMAGE="${IRONIC_IMAGE:-quay.io/metal3-io/ironic:main}"
201216
update_kustomize_image quay.io/metal3-io/baremetal-operator BARE_METAL_OPERATOR_IMAGE "${REPO_ROOT}"/test/e2e/data/bmo-deployment/overlays/pr-test
202-
update_kustomize_image quay.io/metal3-io/ironic IRONIC_IMAGE "${REPO_ROOT}"/test/e2e/data/ironic-deployment/overlays/pr-test
203217

204218
# Apply envsubst to kustomization.yaml files in BMO and Ironic overlays
205219
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/bmo-deployment/overlays/pr-test
206-
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-deployment/overlays/pr-test
220+
kustomize_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base
221+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/base/
222+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/
223+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
224+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/operator
225+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/overlays/pr-test/
226+
yaml_envsubst "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/overlays/latest/
227+
228+
for overlay in "${IRSO_IRONIC_OVERLAYS[@]}"; do
229+
kustomize_envsubst "${overlay}"
230+
done
207231

208-
# Create usernames and passwords and other files related to ironi basic auth if they
232+
# Create usernames and passwords and other files related to ironic basic auth if they
209233
# are missing
210234
if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
211235
IRONIC_AUTH_DIR="${IRONIC_AUTH_DIR:-${IRONIC_DATA_DIR}/auth}"
@@ -228,15 +252,17 @@ if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
228252
IRONIC_PASSWORD="$(cat "${IRONIC_AUTH_DIR}/ironic-password")"
229253
fi
230254
fi
231-
IRONIC_INSPECTOR_USERNAME="${IRONIC_INSPECTOR_USERNAME:-${IRONIC_USERNAME}}"
232-
IRONIC_INSPECTOR_PASSWORD="${IRONIC_INSPECTOR_PASSWORD:-${IRONIC_PASSWORD}}"
233255

234256
export IRONIC_USERNAME
235257
export IRONIC_PASSWORD
236-
export IRONIC_INSPECTOR_USERNAME
237-
export IRONIC_INSPECTOR_PASSWORD
238258
fi
239259

260+
echo "${IRONIC_USERNAME}" > "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/ironic-username
261+
echo "${IRONIC_PASSWORD}" > "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/basic-auth/ironic-password
262+
263+
cp "${IRONIC_KEY_FILE}" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
264+
cp "${IRONIC_CERT_FILE}" "${REPO_ROOT}"/test/e2e/data/ironic-standalone-operator/ironic/components/tls/
265+
240266
for overlay in "${BMO_OVERLAYS[@]}"; do
241267
echo "${IRONIC_USERNAME}" > "${overlay}/ironic-username"
242268
echo "${IRONIC_PASSWORD}" > "${overlay}/ironic-password"
@@ -246,19 +272,6 @@ for overlay in "${BMO_OVERLAYS[@]}"; do
246272
fi
247273
done
248274

249-
for overlay in "${IRONIC_OVERLAYS[@]}"; do
250-
echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
251-
"${overlay}/ironic-htpasswd"
252-
envsubst < "${REPO_ROOT}/test/e2e/data/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
253-
"${overlay}/ironic-auth-config"
254-
IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
255-
curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl
256-
envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
257-
"${overlay}/ironic-inspector-auth-config"
258-
echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
259-
"${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd"
260-
done
261-
262275
# run e2e tests
263276
if [[ -n "${CLUSTER_TOPOLOGY:-}" ]]; then
264277
export CLUSTER_TOPOLOGY=true
@@ -267,3 +280,6 @@ else
267280
export EXP_MACHINE_TAINT_PROPAGATION=true
268281
make e2e-tests
269282
fi
283+
284+
mkdir -p "/tmp/manifests/irso_yamls"
285+
cp -r "${REPO_ROOT}/test/e2e/data/ironic-standalone-operator/" "/tmp/manifests/irso_yamls/"

test/e2e/common.go

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
2525
infrav1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1"
2626
ipamv1 "github.com/metal3-io/ip-address-manager/api/v1alpha1"
27+
irsov1alpha1 "github.com/metal3-io/ironic-standalone-operator/api/v1alpha1"
2728
. "github.com/onsi/ginkgo/v2"
2829
. "github.com/onsi/gomega"
2930
"github.com/pkg/errors"
@@ -1227,3 +1228,247 @@ func IsMetal3DataCountEqualToMachineCount(ctx context.Context, c client.Client,
12271228

12281229
return len(m3DataList.Items) == len(machineList.Items)
12291230
}
1231+
1232+
type InstallIRSOInput struct {
1233+
E2EConfig *clusterctl.E2EConfig
1234+
ClusterProxy framework.ClusterProxy
1235+
IronicNamespace string
1236+
ClusterName string
1237+
IrsoOperatorKustomize string
1238+
IronicKustomize string
1239+
}
1240+
1241+
func InstallIRSO(ctx context.Context, input InstallIRSOInput) error {
1242+
By("Create Ironic namespace")
1243+
targetClusterClientSet := input.ClusterProxy.GetClientSet()
1244+
ironicNamespaceObj := &corev1.Namespace{
1245+
ObjectMeta: metav1.ObjectMeta{
1246+
Name: input.IronicNamespace,
1247+
},
1248+
}
1249+
_, err := targetClusterClientSet.CoreV1().Namespaces().Create(ctx, ironicNamespaceObj, metav1.CreateOptions{})
1250+
if err != nil {
1251+
if apierrors.IsAlreadyExists(err) {
1252+
Logf("Ironic namespace %q already exists, continuing", input.IronicNamespace)
1253+
} else {
1254+
Expect(err).ToNot(HaveOccurred(), "Unable to create the Ironic namespace")
1255+
}
1256+
}
1257+
1258+
irsoDeployLogFolder := filepath.Join(os.TempDir(), "target_cluster_logs", "ironic-deploy-logs", input.ClusterProxy.GetName())
1259+
By(fmt.Sprintf("Installing IRSO from kustomization %s on the target cluster", input.IrsoOperatorKustomize))
1260+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
1261+
Kustomization: input.IrsoOperatorKustomize,
1262+
ClusterProxy: input.ClusterProxy,
1263+
WaitForDeployment: true,
1264+
WatchDeploymentLogs: true,
1265+
LogPath: irsoDeployLogFolder,
1266+
DeploymentName: "ironic-standalone-operator-controller-manager",
1267+
DeploymentNamespace: IRSOControllerNameSpace,
1268+
WaitIntervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
1269+
})
1270+
Expect(err).NotTo(HaveOccurred())
1271+
1272+
By("Waiting for Ironic CRD to be available")
1273+
Eventually(func(g Gomega) {
1274+
crd := &apiextensionsv1.CustomResourceDefinition{}
1275+
err = input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{
1276+
Name: "ironics.ironic.metal3.io",
1277+
}, crd)
1278+
g.Expect(err).ToNot(HaveOccurred(), "Ironic CRD not found")
1279+
// Check if CRD is established
1280+
established := false
1281+
for _, cond := range crd.Status.Conditions {
1282+
if cond.Type == apiextensionsv1.Established && cond.Status == apiextensionsv1.ConditionTrue {
1283+
established = true
1284+
break
1285+
}
1286+
}
1287+
g.Expect(established).To(BeTrue(), "Ironic CRD is not established yet")
1288+
}, input.E2EConfig.GetIntervals("default", "wait-deployment")...).Should(Succeed())
1289+
Logf("Ironic CRD is available and established")
1290+
1291+
By("Install Ironic CR in the target cluster")
1292+
By(fmt.Sprintf("Installing Ironic from kustomization %s on the target cluster", input.IronicKustomize))
1293+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
1294+
Kustomization: input.IronicKustomize,
1295+
ClusterProxy: input.ClusterProxy,
1296+
WaitForDeployment: false,
1297+
WatchDeploymentLogs: false,
1298+
})
1299+
Expect(err).NotTo(HaveOccurred())
1300+
1301+
WaitForIronicReady(ctx, WaitForIronicInput{
1302+
Client: input.ClusterProxy.GetClient(),
1303+
Name: "ironic",
1304+
Namespace: input.IronicNamespace,
1305+
Intervals: input.E2EConfig.GetIntervals("default", "wait-deployment"),
1306+
})
1307+
return nil
1308+
}
1309+
1310+
// WaitForIronicReady waits until the given Ironic resource has Ready condition = True.
1311+
func WaitForIronicReady(ctx context.Context, input WaitForIronicInput) {
1312+
Logf("Waiting for Ironic %q to be Ready", input.Name)
1313+
1314+
Eventually(func(g Gomega) {
1315+
ironic := &irsov1alpha1.Ironic{}
1316+
err := input.Client.Get(ctx, client.ObjectKey{
1317+
Namespace: input.Namespace,
1318+
Name: input.Name,
1319+
}, ironic)
1320+
g.Expect(err).ToNot(HaveOccurred())
1321+
1322+
ready := false
1323+
for _, cond := range ironic.Status.Conditions {
1324+
if cond.Type == string(irsov1alpha1.IronicStatusReady) && cond.Status == metav1.ConditionTrue && ironic.Status.InstalledVersion != "" {
1325+
ready = true
1326+
break
1327+
}
1328+
}
1329+
g.Expect(ready).To(BeTrue(), "Ironic %q is not Ready yet", input.Name)
1330+
}, input.Intervals...).Should(Succeed())
1331+
1332+
Logf("Ironic %q is Ready", input.Name)
1333+
}
1334+
1335+
// WaitForIronicInput bundles the parameters for WaitForIronicReady.
1336+
type WaitForIronicInput struct {
1337+
Client client.Client
1338+
Name string
1339+
Namespace string
1340+
Intervals []interface{} // e.g. []interface{}{time.Minute * 15, time.Second * 5}
1341+
}
1342+
1343+
// InstallBMOInput bundles parameters for InstallBMO.
1344+
type InstallBMOInput struct {
1345+
E2EConfig *clusterctl.E2EConfig
1346+
ClusterProxy framework.ClusterProxy
1347+
Namespace string // Namespace where BMO will run (shared with Ironic)
1348+
BmoKustomization string // Kustomization path or URL for BMO manifests
1349+
LogFolder string // Optional explicit log folder; if empty a default is derived
1350+
WaitIntervals []any // Optional override; if nil uses default e2e config intervals
1351+
WatchLogs bool // Whether to watch deployment logs
1352+
}
1353+
1354+
// InstallBMO installs the Baremetal Operator (BMO) in the target cluster similar to InstallIRSO.
1355+
func InstallBMO(ctx context.Context, input InstallBMOInput) error {
1356+
By("Ensure BMO namespace exists")
1357+
clientset := input.ClusterProxy.GetClientSet()
1358+
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: input.Namespace}}
1359+
_, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
1360+
if err != nil {
1361+
if apierrors.IsAlreadyExists(err) {
1362+
Logf("Namespace %q already exists, continuing", input.Namespace)
1363+
} else {
1364+
return fmt.Errorf("failed creating namespace %q: %w", input.Namespace, err)
1365+
}
1366+
}
1367+
1368+
// Determine log folder
1369+
logFolder := input.LogFolder
1370+
if logFolder == "" {
1371+
logFolder = filepath.Join(os.TempDir(), "target_cluster_logs", "bmo-deploy-logs", input.ClusterProxy.GetName())
1372+
}
1373+
intervals := input.WaitIntervals
1374+
if intervals == nil {
1375+
intervals = input.E2EConfig.GetIntervals("default", "wait-deployment")
1376+
}
1377+
1378+
By(fmt.Sprintf("Installing BMO from kustomization %s on the target cluster", input.BmoKustomization))
1379+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
1380+
Kustomization: input.BmoKustomization,
1381+
ClusterProxy: input.ClusterProxy,
1382+
WaitForDeployment: true,
1383+
WatchDeploymentLogs: input.WatchLogs,
1384+
LogPath: logFolder,
1385+
DeploymentName: "baremetal-operator-controller-manager",
1386+
DeploymentNamespace: input.Namespace,
1387+
WaitIntervals: intervals,
1388+
})
1389+
if err != nil {
1390+
return fmt.Errorf("failed installing BMO: %w", err)
1391+
}
1392+
1393+
By("BMO deployment applied and available")
1394+
return nil
1395+
}
1396+
1397+
type UninstallIRSOAndIronicResourcesInput struct {
1398+
E2EConfig *clusterctl.E2EConfig
1399+
ClusterProxy framework.ClusterProxy
1400+
IronicNamespace string
1401+
IrsoOperatorKustomize string
1402+
IronicKustomization string
1403+
IsDevEnvUninstall bool
1404+
}
1405+
1406+
// UninstallIRSOAndIronicResources removes the IRSO deployment, Ironic CR, IronicDatabase CR (if present), and related secrets.
1407+
func UninstallIRSOAndIronicResources(ctx context.Context, input UninstallIRSOAndIronicResourcesInput) error {
1408+
if input.IsDevEnvUninstall {
1409+
ironicObj := &irsov1alpha1.Ironic{
1410+
ObjectMeta: metav1.ObjectMeta{
1411+
Name: "ironic",
1412+
Namespace: input.IronicNamespace,
1413+
},
1414+
}
1415+
err := input.ClusterProxy.GetClient().Delete(ctx, ironicObj)
1416+
Expect(err).ToNot(HaveOccurred(), "Failed to delete Ironic")
1417+
} else {
1418+
By("Remove Ironic CR in the cluster " + input.ClusterProxy.GetName())
1419+
err := BuildAndRemoveKustomization(ctx, input.IronicKustomization, input.ClusterProxy)
1420+
Expect(err).NotTo(HaveOccurred())
1421+
}
1422+
1423+
By("Remove Ironic Service Deployment in the cluster " + input.ClusterProxy.GetName())
1424+
RemoveDeployment(ctx, func() RemoveDeploymentInput {
1425+
return RemoveDeploymentInput{
1426+
ManagementCluster: input.ClusterProxy,
1427+
Namespace: input.IronicNamespace,
1428+
Name: "ironic-service",
1429+
}
1430+
})
1431+
1432+
if input.IsDevEnvUninstall {
1433+
By("Remove Ironic Standalone Operator Deployment in the cluster " + input.ClusterProxy.GetName())
1434+
RemoveDeployment(ctx, func() RemoveDeploymentInput {
1435+
return RemoveDeploymentInput{
1436+
ManagementCluster: input.ClusterProxy,
1437+
Namespace: IRSOControllerNameSpace,
1438+
Name: "ironic-standalone-operator-controller-manager",
1439+
}
1440+
})
1441+
} else {
1442+
By("Uninstalling IRSO operator via kustomize")
1443+
err := BuildAndRemoveKustomization(ctx, input.IrsoOperatorKustomize, input.ClusterProxy)
1444+
Expect(err).NotTo(HaveOccurred())
1445+
}
1446+
1447+
clusterClient := input.ClusterProxy.GetClient()
1448+
1449+
// Delete secrets
1450+
secretNames := []string{"ironic-auth", "ironic-cert", "ironic-cacert"}
1451+
for _, s := range secretNames {
1452+
Byf("Deleting secret %s", s)
1453+
secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: s, Namespace: input.IronicNamespace}}
1454+
err := clusterClient.Delete(ctx, secret)
1455+
if err != nil {
1456+
Logf("Failed to delete secret %s: %v", s, err)
1457+
}
1458+
}
1459+
1460+
// Wait for secrets to be deleted
1461+
By("Waiting for Ironic secrets to be deleted")
1462+
Eventually(func() bool {
1463+
for _, s := range secretNames {
1464+
errS := clusterClient.Get(ctx, client.ObjectKey{Name: s, Namespace: input.IronicNamespace}, &corev1.Secret{})
1465+
if errS == nil || !apierrors.IsNotFound(errS) {
1466+
return false
1467+
}
1468+
}
1469+
return true
1470+
}, input.E2EConfig.GetIntervals("default", "wait-delete-ironic")...).Should(BeTrue(), "IRSO/Ironic resources not fully deleted")
1471+
1472+
By("IRSO and Ironic resources uninstalled")
1473+
return nil
1474+
}

0 commit comments

Comments
 (0)