Skip to content

Commit adbfb48

Browse files
adilGhaffarDevadilGhaffarDev
committed
Improve logging in e2e tests
Signed-off-by: Muhammad Adil Ghaffar <[email protected]>
1 parent 838b26d commit adbfb48

File tree

5 files changed

+54
-97
lines changed

5 files changed

+54
-97
lines changed

‎scripts/fetch_manifests.sh‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎test/e2e/common.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ func DumpSpecResourcesAndCleanup(ctx context.Context, specName string, bootstrap
142142
bootstrapClusterProxy.CollectWorkloadClusterLogs(ctx, namespace, clusterName, artifactFolder)
143143

144144
By("Fetch logs from target cluster")
145-
err := FetchClusterLogs(targetClusterProxy, clusterLogCollectionBasePath)
145+
err := FetchClusterLogs(targetClusterProxy, filepath.Join(artifactFolder, workloadClusterLogCollectionBasePath, targetClusterProxy.GetName(), "final-logs"))
146146
if err != nil {
147147
Logf("Error: %v", err)
148148
}

‎test/e2e/logcollector.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@ func (Metal3LogCollector) CollectMachinePoolLog(_ context.Context, _ client.Clie
108108
// FetchManifests fetches relevant Metal3, CAPI, and Kubernetes core resources
109109
// and dumps them to a file.
110110
func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) error {
111-
outputPath = filepath.Join(outputPath, clusterProxy.GetName())
112111
ctx := context.Background()
113112
restConfig := clusterProxy.GetRESTConfig()
114113
dynamicClient, err := dynamic.NewForConfig(restConfig)
@@ -159,6 +158,7 @@ func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) erro
159158
"m3data",
160159
"m3dataclaim",
161160
"m3datatemplate",
161+
"ironic",
162162
}
163163
client := clusterProxy.GetClient()
164164

@@ -194,9 +194,8 @@ func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) erro
194194
// to files.
195195
func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) error {
196196
ctx := context.Background()
197-
baseDir := filepath.Join(outputPath, clusterProxy.GetName())
198197
// Ensure the base directory exists
199-
if err := os.MkdirAll(baseDir, 0o750); err != nil {
198+
if err := os.MkdirAll(outputPath, 0o750); err != nil {
200199
return fmt.Errorf("couldn't create directory: %w", err)
201200
}
202201

@@ -206,7 +205,7 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
206205
// Print the Pods' information to file
207206
// This does the same thing as:
208207
// kubectl --kubeconfig="${KUBECONFIG_WORKLOAD}" get pods -A
209-
outputFile := filepath.Join(baseDir, "pods.log")
208+
outputFile := filepath.Join(outputPath, "pods.log")
210209
file, err := os.Create(outputFile)
211210
if err != nil {
212211
return fmt.Errorf("failed to create output file: %w", err)
@@ -270,7 +269,7 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
270269
}
271270

272271
machineName := pod.Spec.NodeName
273-
podDir := filepath.Join(baseDir, "machines", machineName, namespace.Name, pod.Name)
272+
podDir := filepath.Join(outputPath, "machines", machineName, namespace.Name, pod.Name)
274273
if err = os.MkdirAll(podDir, 0o750); err != nil {
275274
return fmt.Errorf("couldn't create directory: %w", err)
276275
}

‎test/e2e/pivoting.go‎

Lines changed: 42 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
. "github.com/onsi/gomega"
1717
corev1 "k8s.io/api/core/v1"
1818
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19-
"k8s.io/apimachinery/pkg/runtime"
2019
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
2120
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
2221
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
@@ -26,17 +25,18 @@ import (
2625
)
2726

2827
const (
29-
bmoPath = "BMOPATH"
30-
ironicTLSSetup = "IRONIC_TLS_SETUP"
31-
ironicBasicAuth = "IRONIC_BASIC_AUTH"
32-
ironicKeepalived = "IRONIC_KEEPALIVED"
33-
ironicMariadb = "IRONIC_USE_MARIADB"
34-
Kind = "kind"
35-
NamePrefix = "NAMEPREFIX"
36-
restartContainerCertUpdate = "RESTART_CONTAINER_CERTIFICATE_UPDATED"
37-
ironicNamespace = "IRONIC_NAMESPACE"
38-
clusterLogCollectionBasePath = "/tmp/target_cluster_logs"
39-
Metal3ipamProviderName = "metal3"
28+
bmoPath = "BMOPATH"
29+
ironicTLSSetup = "IRONIC_TLS_SETUP"
30+
ironicBasicAuth = "IRONIC_BASIC_AUTH"
31+
ironicKeepalived = "IRONIC_KEEPALIVED"
32+
ironicMariadb = "IRONIC_USE_MARIADB"
33+
Kind = "kind"
34+
NamePrefix = "NAMEPREFIX"
35+
restartContainerCertUpdate = "RESTART_CONTAINER_CERTIFICATE_UPDATED"
36+
ironicNamespace = "IRONIC_NAMESPACE"
37+
workloadClusterLogCollectionBasePath = "workload_cluster_logs"
38+
bootstrapClusterLogCollectionBasePath = "bootstrap_cluster_logs"
39+
Metal3ipamProviderName = "metal3"
4040
)
4141

4242
type PivotingInput struct {
@@ -64,7 +64,13 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
6464
ListNodes(ctx, input.TargetCluster.GetClient())
6565

6666
By("Fetch logs from target cluster before pivot")
67-
err := FetchClusterLogs(input.TargetCluster, filepath.Join(clusterLogCollectionBasePath, "beforePivot"))
67+
err := FetchClusterLogs(input.TargetCluster, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.TargetCluster.GetName(), "beforePivot"))
68+
if err != nil {
69+
Logf("Error: %v", err)
70+
}
71+
72+
By("Fetch logs from bootstrap cluster before pivot")
73+
err = FetchClusterLogs(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, "beforePivot"))
6874
if err != nil {
6975
Logf("Error: %v", err)
7076
}
@@ -83,17 +89,20 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
8389
}
8490

8591
By("Fetch container logs")
86-
bootstrapCluster := os.Getenv("BOOTSTRAP_CLUSTER")
8792
fetchContainerLogs(&generalContainers, input.ArtifactFolder, input.E2EConfig.MustGetVariable("CONTAINER_RUNTIME"))
88-
if bootstrapCluster == Kind {
93+
if input.BootstrapClusterProxy.GetName() == Kind {
8994
fetchContainerLogs(&ironicContainers, input.ArtifactFolder, input.E2EConfig.MustGetVariable("CONTAINER_RUNTIME"))
9095
}
9196

9297
By("Fetch manifest for bootstrap cluster before pivot")
93-
err = FetchManifests(input.BootstrapClusterProxy, "/tmp/manifests/")
98+
err = FetchManifests(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, "beforePivot", "manifests"))
9499
if err != nil {
95100
Logf("Error fetching manifests for bootstrap cluster before pivot: %v", err)
96101
}
102+
err = FetchManifests(input.TargetCluster, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.TargetCluster.GetName(), "beforePivot", "manifests"))
103+
if err != nil {
104+
Logf("Error fetching manifests for Target cluster before pivot: %v", err)
105+
}
97106
By("Fetch target cluster kubeconfig for target cluster log collection")
98107
kconfigPathWorkload := input.TargetCluster.GetKubeconfigPath()
99108
os.Setenv("KUBECONFIG_WORKLOAD", kconfigPathWorkload)
@@ -109,7 +118,7 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
109118

110119
By("Remove Ironic containers from the source cluster")
111120
ironicDeploymentType := IronicDeploymentTypeBMO
112-
if bootstrapCluster == Kind {
121+
if input.BootstrapClusterProxy.GetName() == Kind {
113122
ironicDeploymentType = IronicDeploymentTypeLocal
114123
} else if GetBoolVariable(input.E2EConfig, "USE_IRSO") {
115124
ironicDeploymentType = IronicDeploymentTypeIrSO
@@ -368,18 +377,25 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
368377
numberOfAllBmh := numberOfWorkers + numberOfControlplane
369378

370379
By("Fetch logs from target cluster after pivot")
371-
err := FetchClusterLogs(input.TargetCluster, filepath.Join(clusterLogCollectionBasePath, "afterPivot"))
380+
err := FetchClusterLogs(input.TargetCluster, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.TargetCluster.GetName(), "afterPivot"))
381+
if err != nil {
382+
Logf("Error: %v", err)
383+
}
384+
By("Fetch logs from bootstrap cluster after pivot")
385+
err = FetchClusterLogs(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, "afterPivot"))
372386
if err != nil {
373387
Logf("Error: %v", err)
374388
}
375389

376390
By("Fetch manifest for workload cluster after pivot")
377-
workloadClusterProxy := framework.NewClusterProxy("workload-cluster-after-pivot", os.Getenv("KUBECONFIG"), runtime.NewScheme())
378-
err = FetchManifests(workloadClusterProxy, "/tmp/manifests/")
391+
err = FetchManifests(input.TargetCluster, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.TargetCluster.GetName(), "afterPivot", "manifests"))
379392
if err != nil {
380393
Logf("Error fetching manifests for workload cluster after pivot: %v", err)
381394
}
382-
os.Unsetenv("KUBECONFIG_WORKLOAD")
395+
err = FetchManifests(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, "afterPivot", "manifests"))
396+
if err != nil {
397+
Logf("Error fetching manifests for bootstrap cluster before pivot: %v", err)
398+
}
383399

384400
By("Remove Ironic deployment from target cluster")
385401
ironicDeploymentType := IronicDeploymentTypeBMO
@@ -497,7 +513,11 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
497513
})
498514

499515
By("Fetch manifest for bootstrap cluster after re-pivot")
500-
err = FetchManifests(input.BootstrapClusterProxy, "/tmp/manifests/")
516+
err = FetchManifests(input.TargetCluster, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.TargetCluster.GetName(), "afterRePivot", "manifests"))
517+
if err != nil {
518+
Logf("Error fetching manifests for workload cluster after pivot: %v", err)
519+
}
520+
err = FetchManifests(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, "afterRePivot", "manifests"))
501521
if err != nil {
502522
Logf("Error fetching manifests for bootstrap cluster before pivot: %v", err)
503523
}

‎test/e2e/upgrade_clusterctl_test.go‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,9 @@ func preInitFunc(clusterProxy framework.ClusterProxy, bmoRelease string, ironicR
269269
Expect(clusterProxy.GetClientSet().CoreV1().Namespaces().Delete(ctx, "test", metav1.DeleteOptions{})).To(Succeed())
270270
}
271271

272-
By("Fetch manifest for bootstrap cluster")
273-
err := FetchManifests(clusterProxy, filepath.Join(artifactFolder, clusterProxy.GetName(), "preInit-manifest"))
272+
err := FetchManifests(clusterProxy, filepath.Join(artifactFolder, bootstrapClusterLogCollectionBasePath, clusterProxy.GetName(), "manifests"))
274273
if err != nil {
275-
Logf("Error fetching manifests for bootstrap cluster: %v", err)
274+
Logf("Error fetching manifests from cluster: %s error: %v", clusterProxy.GetName(), err)
276275
}
277276

278277
By("Fetch target cluster kubeconfig for target cluster log collection")
@@ -305,7 +304,7 @@ func preInitFunc(clusterProxy framework.ClusterProxy, bmoRelease string, ironicR
305304
bmoIronicNamespace := e2eConfig.MustGetVariable(ironicNamespace)
306305
// install ironic
307306
By("Install Ironic in the target cluster")
308-
ironicDeployLogFolder := filepath.Join(clusterLogCollectionBasePath, clusterProxy.GetName(), "ironic-deploy-logs")
307+
ironicDeployLogFolder := filepath.Join(workloadClusterLogCollectionBasePath, clusterProxy.GetName(), "ironic-deploy-logs")
309308
ironicKustomizePath := "IRONIC_RELEASE_" + ironicRelease
310309
initIronicKustomization := e2eConfig.MustGetVariable(ironicKustomizePath)
311310
By(fmt.Sprintf("Installing Ironic from kustomization %s on the upgrade cluster", initIronicKustomization))
@@ -323,7 +322,7 @@ func preInitFunc(clusterProxy framework.ClusterProxy, bmoRelease string, ironicR
323322

324323
// install bmo
325324
By("Install BMO in the target cluster")
326-
bmoDeployLogFolder := filepath.Join(clusterLogCollectionBasePath, clusterProxy.GetName(), "bmo-deploy-logs")
325+
bmoDeployLogFolder := filepath.Join(workloadClusterLogCollectionBasePath, clusterProxy.GetName(), "bmo-deploy-logs")
327326
bmoKustomizePath := "BMO_RELEASE_" + bmoRelease
328327
initBMOKustomization := e2eConfig.MustGetVariable(bmoKustomizePath)
329328
By(fmt.Sprintf("Installing BMO from kustomization %s on the upgrade cluster", initBMOKustomization))
@@ -373,7 +372,7 @@ func preUpgrade(clusterProxy framework.ClusterProxy, bmoUpgradeToRelease string,
373372

374373
bmoIronicNamespace := e2eConfig.MustGetVariable(ironicNamespace)
375374
By("Upgrade Ironic in the target cluster")
376-
ironicDeployLogFolder := filepath.Join(clusterLogCollectionBasePath, clusterProxy.GetName(), "ironic-deploy-logs")
375+
ironicDeployLogFolder := filepath.Join(workloadClusterLogCollectionBasePath, clusterProxy.GetName(), "ironic-deploy-logs")
377376
ironicKustomizePath := "IRONIC_RELEASE_" + ironicTag
378377
initIronicKustomization := e2eConfig.MustGetVariable(ironicKustomizePath)
379378
By(fmt.Sprintf("Upgrading Ironic from kustomization %s on the upgrade cluster", initIronicKustomization))
@@ -391,7 +390,7 @@ func preUpgrade(clusterProxy framework.ClusterProxy, bmoUpgradeToRelease string,
391390

392391
// install bmo
393392
By("Upgrade BMO in the target cluster")
394-
bmoDeployLogFolder := filepath.Join(clusterLogCollectionBasePath, clusterProxy.GetName(), "bmo-deploy-logs")
393+
bmoDeployLogFolder := filepath.Join(workloadClusterLogCollectionBasePath, clusterProxy.GetName(), "bmo-deploy-logs")
395394
bmoKustomizePath := "BMO_RELEASE_" + bmoTag
396395
initBMOKustomization := e2eConfig.MustGetVariable(bmoKustomizePath)
397396
By(fmt.Sprintf("Upgrading BMO from kustomization %s on the upgrade cluster", initBMOKustomization))
@@ -412,7 +411,7 @@ func preUpgrade(clusterProxy framework.ClusterProxy, bmoUpgradeToRelease string,
412411
// it moves back Ironic to the bootstrap cluster.
413412
func preCleanupManagementCluster(clusterProxy framework.ClusterProxy, ironicRelease string) {
414413
By("Fetch logs from target cluster")
415-
err := FetchClusterLogs(clusterProxy, clusterLogCollectionBasePath)
414+
err := FetchClusterLogs(clusterProxy, filepath.Join(artifactFolder, workloadClusterLogCollectionBasePath, clusterProxy.GetName()))
416415
if err != nil {
417416
Logf("Error: %v", err)
418417
}

0 commit comments

Comments
 (0)