Skip to content

Commit d221469

Browse files
adilGhaffarDevadilGhaffarDev
committed
Improve logging in e2e tests
Signed-off-by: Muhammad Adil Ghaffar <[email protected]>
1 parent 838b26d commit d221469

File tree

5 files changed

+137
-148
lines changed

5 files changed

+137
-148
lines changed

‎scripts/fetch_manifests.sh‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎test/e2e/common.go‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ const (
6969
// Out-of-service Taint test actions.
7070
oostAdded = "added"
7171
oostRemoved = "removed"
72+
// Log collection paths.
73+
beforePivotLogCollectionPath = "before-pivot"
74+
afterPivotLogCollectionPath = "after-pivot"
75+
afterRePivotLogCollectionPath = "after-re-pivot"
76+
beforeDeleteLogCollectionPath = "before-delete"
7277
)
7378

7479
func Byf(format string, a ...any) {
@@ -141,11 +146,16 @@ func DumpSpecResourcesAndCleanup(ctx context.Context, specName string, bootstrap
141146

142147
bootstrapClusterProxy.CollectWorkloadClusterLogs(ctx, namespace, clusterName, artifactFolder)
143148

144-
By("Fetch logs from target cluster")
145-
err := FetchClusterLogs(targetClusterProxy, clusterLogCollectionBasePath)
146-
if err != nil {
147-
Logf("Error: %v", err)
148-
}
149+
By("Fetch manifest before deleting clusters")
150+
FetchManifestsAndLogs(func() FetchManifestsAndLogsInput {
151+
return FetchManifestsAndLogsInput{
152+
BootstrapClusterProxy: bootstrapClusterProxy,
153+
WorkloadClusterProxy: targetClusterProxy,
154+
ArtifactFolder: artifactFolder,
155+
LogCollectionPath: beforeDeleteLogCollectionPath,
156+
}
157+
})
158+
149159
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
150160
By(fmt.Sprintf("Dumping all the Cluster API resources in the %q namespace", namespace))
151161
// Dump all Cluster API related resources to artifacts before deleting them.

‎test/e2e/logcollector.go‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ func (Metal3LogCollector) CollectMachinePoolLog(_ context.Context, _ client.Clie
108108
// FetchManifests fetches relevant Metal3, CAPI, and Kubernetes core resources
109109
// and dumps them to a file.
110110
func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) error {
111-
outputPath = filepath.Join(outputPath, clusterProxy.GetName())
111+
outputPath = filepath.Join(outputPath, "manifests")
112112
ctx := context.Background()
113113
restConfig := clusterProxy.GetRESTConfig()
114114
dynamicClient, err := dynamic.NewForConfig(restConfig)
@@ -159,6 +159,7 @@ func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) erro
159159
"m3data",
160160
"m3dataclaim",
161161
"m3datatemplate",
162+
"ironic",
162163
}
163164
client := clusterProxy.GetClient()
164165

@@ -193,10 +194,10 @@ func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) erro
193194
// FetchClusterLogs fetches logs from all pods in the cluster and writes them
194195
// to files.
195196
func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) error {
197+
outputPath = filepath.Join(outputPath, "controller_logs")
196198
ctx := context.Background()
197-
baseDir := filepath.Join(outputPath, clusterProxy.GetName())
198199
// Ensure the base directory exists
199-
if err := os.MkdirAll(baseDir, 0o750); err != nil {
200+
if err := os.MkdirAll(outputPath, 0o750); err != nil {
200201
return fmt.Errorf("couldn't create directory: %w", err)
201202
}
202203

@@ -206,7 +207,7 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
206207
// Print the Pods' information to file
207208
// This does the same thing as:
208209
// kubectl --kubeconfig="${KUBECONFIG_WORKLOAD}" get pods -A
209-
outputFile := filepath.Join(baseDir, "pods.log")
210+
outputFile := filepath.Join(outputPath, "pods.log")
210211
file, err := os.Create(outputFile)
211212
if err != nil {
212213
return fmt.Errorf("failed to create output file: %w", err)
@@ -270,7 +271,7 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
270271
}
271272

272273
machineName := pod.Spec.NodeName
273-
podDir := filepath.Join(baseDir, "machines", machineName, namespace.Name, pod.Name)
274+
podDir := filepath.Join(outputPath, "machines", machineName, namespace.Name, pod.Name)
274275
if err = os.MkdirAll(podDir, 0o750); err != nil {
275276
return fmt.Errorf("couldn't create directory: %w", err)
276277
}
@@ -394,3 +395,37 @@ func DumpGVR(ctx context.Context, dynamicClient *dynamic.DynamicClient, gvr sche
394395
}
395396
return nil
396397
}
398+
399+
type FetchManifestsAndLogsInput struct {
400+
BootstrapClusterProxy framework.ClusterProxy
401+
WorkloadClusterProxy framework.ClusterProxy
402+
ArtifactFolder string
403+
LogCollectionPath string
404+
}
405+
406+
func FetchManifestsAndLogs(inputGetter func() FetchManifestsAndLogsInput) {
407+
input := inputGetter()
408+
By("Fetch logs and manifests for path: " + input.LogCollectionPath)
409+
By(fmt.Sprintf("Fetch manifests from cluster %s", input.WorkloadClusterProxy.GetName()))
410+
err := FetchManifests(input.WorkloadClusterProxy, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.WorkloadClusterProxy.GetName(), input.LogCollectionPath))
411+
if err != nil {
412+
Logf("Error fetching manifests for workload cluster: %v", err)
413+
}
414+
415+
By(fmt.Sprintf("Fetch manifests from cluster %s", input.BootstrapClusterProxy.GetName()))
416+
err = FetchManifests(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, input.LogCollectionPath))
417+
if err != nil {
418+
Logf("Error fetching manifests for bootstrap: %v", err)
419+
}
420+
421+
By(fmt.Sprintf("Fetch logs from cluster %s", input.WorkloadClusterProxy.GetName()))
422+
err = FetchClusterLogs(input.WorkloadClusterProxy, filepath.Join(input.ArtifactFolder, workloadClusterLogCollectionBasePath, input.WorkloadClusterProxy.GetName(), input.LogCollectionPath))
423+
if err != nil {
424+
Logf("Error fetching logs from workload cluster: %v", err)
425+
}
426+
By(fmt.Sprintf("Fetch logs from cluster %s", input.BootstrapClusterProxy.GetName()))
427+
err = FetchClusterLogs(input.BootstrapClusterProxy, filepath.Join(input.ArtifactFolder, bootstrapClusterLogCollectionBasePath, input.LogCollectionPath))
428+
if err != nil {
429+
Logf("Error fetching logs from bootstrap cluster: %v", err)
430+
}
431+
}

‎test/e2e/pivoting.go‎

Lines changed: 43 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
. "github.com/onsi/gomega"
1717
corev1 "k8s.io/api/core/v1"
1818
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19-
"k8s.io/apimachinery/pkg/runtime"
2019
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
2120
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
2221
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
@@ -26,17 +25,18 @@ import (
2625
)
2726

2827
const (
29-
bmoPath = "BMOPATH"
30-
ironicTLSSetup = "IRONIC_TLS_SETUP"
31-
ironicBasicAuth = "IRONIC_BASIC_AUTH"
32-
ironicKeepalived = "IRONIC_KEEPALIVED"
33-
ironicMariadb = "IRONIC_USE_MARIADB"
34-
Kind = "kind"
35-
NamePrefix = "NAMEPREFIX"
36-
restartContainerCertUpdate = "RESTART_CONTAINER_CERTIFICATE_UPDATED"
37-
ironicNamespace = "IRONIC_NAMESPACE"
38-
clusterLogCollectionBasePath = "/tmp/target_cluster_logs"
39-
Metal3ipamProviderName = "metal3"
28+
bmoPath = "BMOPATH"
29+
ironicTLSSetup = "IRONIC_TLS_SETUP"
30+
ironicBasicAuth = "IRONIC_BASIC_AUTH"
31+
ironicKeepalived = "IRONIC_KEEPALIVED"
32+
ironicMariadb = "IRONIC_USE_MARIADB"
33+
Kind = "kind"
34+
NamePrefix = "NAMEPREFIX"
35+
restartContainerCertUpdate = "RESTART_CONTAINER_CERTIFICATE_UPDATED"
36+
ironicNamespace = "IRONIC_NAMESPACE"
37+
workloadClusterLogCollectionBasePath = "workload-cluster-logs"
38+
bootstrapClusterLogCollectionBasePath = "bootstrap-cluster-logs"
39+
Metal3ipamProviderName = "metal3"
4040
)
4141

4242
type PivotingInput struct {
@@ -63,11 +63,14 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
6363
ListMachines(ctx, input.BootstrapClusterProxy.GetClient(), client.InNamespace(input.Namespace))
6464
ListNodes(ctx, input.TargetCluster.GetClient())
6565

66-
By("Fetch logs from target cluster before pivot")
67-
err := FetchClusterLogs(input.TargetCluster, filepath.Join(clusterLogCollectionBasePath, "beforePivot"))
68-
if err != nil {
69-
Logf("Error: %v", err)
70-
}
66+
FetchManifestsAndLogs(func() FetchManifestsAndLogsInput {
67+
return FetchManifestsAndLogsInput{
68+
BootstrapClusterProxy: input.BootstrapClusterProxy,
69+
WorkloadClusterProxy: input.TargetCluster,
70+
ArtifactFolder: input.ArtifactFolder,
71+
LogCollectionPath: beforePivotLogCollectionPath,
72+
}
73+
})
7174

7275
ironicContainers := []string{
7376
"ironic",
@@ -83,17 +86,11 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
8386
}
8487

8588
By("Fetch container logs")
86-
bootstrapCluster := os.Getenv("BOOTSTRAP_CLUSTER")
8789
fetchContainerLogs(&generalContainers, input.ArtifactFolder, input.E2EConfig.MustGetVariable("CONTAINER_RUNTIME"))
88-
if bootstrapCluster == Kind {
90+
if input.BootstrapClusterProxy.GetName() == Kind {
8991
fetchContainerLogs(&ironicContainers, input.ArtifactFolder, input.E2EConfig.MustGetVariable("CONTAINER_RUNTIME"))
9092
}
9193

92-
By("Fetch manifest for bootstrap cluster before pivot")
93-
err = FetchManifests(input.BootstrapClusterProxy, "/tmp/manifests/")
94-
if err != nil {
95-
Logf("Error fetching manifests for bootstrap cluster before pivot: %v", err)
96-
}
9794
By("Fetch target cluster kubeconfig for target cluster log collection")
9895
kconfigPathWorkload := input.TargetCluster.GetKubeconfigPath()
9996
os.Setenv("KUBECONFIG_WORKLOAD", kconfigPathWorkload)
@@ -109,7 +106,7 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
109106

110107
By("Remove Ironic containers from the source cluster")
111108
ironicDeploymentType := IronicDeploymentTypeBMO
112-
if bootstrapCluster == Kind {
109+
if input.BootstrapClusterProxy.GetName() == Kind {
113110
ironicDeploymentType = IronicDeploymentTypeLocal
114111
} else if GetBoolVariable(input.E2EConfig, "USE_IRSO") {
115112
ironicDeploymentType = IronicDeploymentTypeIrSO
@@ -130,7 +127,7 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
130127
Name: input.E2EConfig.MustGetVariable(ironicNamespace),
131128
},
132129
}
133-
_, err = targetClusterClientSet.CoreV1().Namespaces().Create(ctx, ironicNamespaceObj, metav1.CreateOptions{})
130+
_, err := targetClusterClientSet.CoreV1().Namespaces().Create(ctx, ironicNamespaceObj, metav1.CreateOptions{})
134131
Expect(err).ToNot(HaveOccurred(), "Unable to create the Ironic namespace")
135132

136133
By("Initialize Provider component in target cluster")
@@ -367,19 +364,14 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
367364
numberOfControlplane := int(*input.E2EConfig.MustGetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT"))
368365
numberOfAllBmh := numberOfWorkers + numberOfControlplane
369366

370-
By("Fetch logs from target cluster after pivot")
371-
err := FetchClusterLogs(input.TargetCluster, filepath.Join(clusterLogCollectionBasePath, "afterPivot"))
372-
if err != nil {
373-
Logf("Error: %v", err)
374-
}
375-
376-
By("Fetch manifest for workload cluster after pivot")
377-
workloadClusterProxy := framework.NewClusterProxy("workload-cluster-after-pivot", os.Getenv("KUBECONFIG"), runtime.NewScheme())
378-
err = FetchManifests(workloadClusterProxy, "/tmp/manifests/")
379-
if err != nil {
380-
Logf("Error fetching manifests for workload cluster after pivot: %v", err)
381-
}
382-
os.Unsetenv("KUBECONFIG_WORKLOAD")
367+
FetchManifestsAndLogs(func() FetchManifestsAndLogsInput {
368+
return FetchManifestsAndLogsInput{
369+
BootstrapClusterProxy: input.BootstrapClusterProxy,
370+
WorkloadClusterProxy: input.TargetCluster,
371+
ArtifactFolder: input.ArtifactFolder,
372+
LogCollectionPath: afterPivotLogCollectionPath,
373+
}
374+
})
383375

384376
By("Remove Ironic deployment from target cluster")
385377
ironicDeploymentType := IronicDeploymentTypeBMO
@@ -401,14 +393,14 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
401393
//#nosec G204:gosec
402394
cmd := exec.Command("sh", "-c", "export CONTAINER_RUNTIME=docker; "+ironicCommand)
403395
var stdoutStderr []byte
404-
stdoutStderr, err = cmd.CombinedOutput()
396+
stdoutStderr, err := cmd.CombinedOutput()
405397
Logf("Output: %s", stdoutStderr)
406398
Expect(err).ToNot(HaveOccurred(), "Cannot run local ironic")
407399
} else {
408400
By("Install Ironic in the bootstrap cluster")
409401
ironicKustomization := input.E2EConfig.MustGetVariable("IRONIC_RELEASE_PR_TEST")
410402
ironicDeployLogFolder := filepath.Join(os.TempDir(), "source_cluster_logs", "ironic-deploy-logs", input.TargetCluster.GetName())
411-
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
403+
err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
412404
Kustomization: ironicKustomization,
413405
ClusterProxy: input.BootstrapClusterProxy,
414406
WaitForDeployment: true,
@@ -425,7 +417,7 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
425417
bmoKustomization := input.E2EConfig.MustGetVariable("BMO_RELEASE_PR_TEST")
426418
bmoDeployLogFolder := filepath.Join(os.TempDir(), "source_cluster_logs", "bmo-deploy-logs", input.TargetCluster.GetName())
427419
By(fmt.Sprintf("Installing BMO from kustomization %s on the source cluster", bmoKustomization))
428-
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
420+
err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
429421
Kustomization: bmoKustomization,
430422
ClusterProxy: input.BootstrapClusterProxy,
431423
WaitForDeployment: true,
@@ -496,11 +488,15 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
496488
Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-machine-running"),
497489
})
498490

499-
By("Fetch manifest for bootstrap cluster after re-pivot")
500-
err = FetchManifests(input.BootstrapClusterProxy, "/tmp/manifests/")
501-
if err != nil {
502-
Logf("Error fetching manifests for bootstrap cluster before pivot: %v", err)
503-
}
491+
FetchManifestsAndLogs(func() FetchManifestsAndLogsInput {
492+
return FetchManifestsAndLogsInput{
493+
BootstrapClusterProxy: input.BootstrapClusterProxy,
494+
WorkloadClusterProxy: input.TargetCluster,
495+
ArtifactFolder: input.ArtifactFolder,
496+
LogCollectionPath: afterRePivotLogCollectionPath,
497+
}
498+
})
499+
504500
os.Unsetenv("KUBECONFIG_BOOTSTRAP")
505501

506502
By("RE-PIVOTING TEST PASSED!")

0 commit comments

Comments
 (0)