Skip to content

Commit

Permalink
tests: e2e: Add nodeName filter to getCaaPod
Browse files Browse the repository at this point in the history 
As Wainer pointed out in the review, if we ran the e2e tests
on a cluster with multiple worker nodes, then our logic for
only checking for a single CAA pod might fail, so add an
additional nodeName filter, so get the CAA that is running
on the same node as the pod.

Signed-off-by: stevenhorsman <[email protected]>
  • Loading branch information
@stevenhorsman @wainersm
stevenhorsman authored and wainersm committed Oct 29, 2024
1 parent 87ecbab commit 5672559
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 19 deletions.
41 changes: 29 additions & 12 deletions src/cloud-api-adaptor/test/e2e/assessment_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,13 @@ func WatchImagePullTime(ctx context.Context, client klient.Client, caaPod *v1.Po
return pullingtime, nil
}

func getCaaPod(ctx context.Context, client klient.Client, t *testing.T) (*v1.Pod, error) {
func getCaaPod(ctx context.Context, client klient.Client, t *testing.T, nodeName string) (*v1.Pod, error) {
caaPod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "confidential-containers-system",
},
}
pods, err := GetPodNamesByLabel(ctx, client, t, caaPod.GetObjectMeta().GetNamespace(), "app", "cloud-api-adaptor")
pods, err := GetPodNamesByLabel(ctx, client, t, caaPod.GetObjectMeta().GetNamespace(), "app", "cloud-api-adaptor", nodeName)
if err != nil {
return nil, fmt.Errorf("getCaaPod: GetPodNamesByLabel failed: %v", err)
}
Expand All @@ -144,13 +144,13 @@ func getCaaPod(ctx context.Context, client klient.Client, t *testing.T) (*v1.Pod
// <date time> [adaptor/proxy] mount_point:/run/kata-containers/<id>/rootfs source:<image> fstype:overlay driver:image_guest_pull
// <date time> 11:47:42 [adaptor/proxy] CreateContainer: Ignoring PullImage before CreateContainer (cid: "<cid>")
// was output
func IsPulledWithNydusSnapshotter(ctx context.Context, t *testing.T, client klient.Client, containerId string) (bool, error) {
func IsPulledWithNydusSnapshotter(ctx context.Context, t *testing.T, client klient.Client, nodeName string, containerId string) (bool, error) {
nydusSnapshotterPullRegex, err := regexp.Compile(`.*mount_point:/run/kata-containers.*` + containerId + `.*driver:image_guest_pull.*$`)
if err != nil {
return false, err
}

caaPod, err := getCaaPod(ctx, client, t)
caaPod, err := getCaaPod(ctx, client, t, nodeName)
if err != nil {
return false, fmt.Errorf("IsPulledWithNydusSnapshotter: failed to get CAA pod: %v", err)
}
Expand Down Expand Up @@ -277,18 +277,23 @@ func CompareInstanceType(ctx context.Context, t *testing.T, client klient.Client
return fmt.Errorf("no pod matching %v, was found", pod)
}

func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Client, alternateImageName string) error {
func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod, alternateImageName string) error {
nodeName, err := GetNodeNameFromPod(ctx, client, pod)
if err != nil {
return fmt.Errorf("VerifyAlternateImage: GetNodeNameFromPod failed with %v", err)
}

expectedSuccessMessage := "Choosing " + alternateImageName
err := VerifyCaaPodLogContains(ctx, t, client, expectedSuccessMessage)
err = VerifyCaaPodLogContains(ctx, t, client, nodeName, expectedSuccessMessage)
if err != nil {
return fmt.Errorf("VerifyAlternateImage: failed: %v", err)
}
t.Logf("PodVM was brought up using the alternate PodVM image %s", alternateImageName)
return nil
}

func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, expected string) error {
caaPod, err := getCaaPod(ctx, client, t)
func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, nodeName, expected string) error {
caaPod, err := getCaaPod(ctx, client, t, nodeName)
if err != nil {
return fmt.Errorf("VerifyCaaPodLogContains: failed to getCaaPod: %v", err)
}
Expand All @@ -302,13 +307,19 @@ func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Cl
}

func VerifyNydusSnapshotter(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error {
nodeName, err := GetNodeNameFromPod(ctx, client, pod)
if err != nil {
return fmt.Errorf("VerifyNydusSnapshotter: GetNodeNameFromPod failed with %v", err)
}
log.Tracef("Test pod running on node %s", nodeName)

containerId := pod.Status.ContainerStatuses[0].ContainerID
containerId, found := strings.CutPrefix(containerId, "containerd://")
if !found {
return fmt.Errorf("VerifyNydusSnapshotter: unexpected container id format: %s", containerId)
}

usedNydusSnapshotter, err := IsPulledWithNydusSnapshotter(ctx, t, client, containerId)
usedNydusSnapshotter, err := IsPulledWithNydusSnapshotter(ctx, t, client, nodeName, containerId)
if err != nil {
return fmt.Errorf("IsPulledWithNydusSnapshotter: failed with %v", err)
}
Expand All @@ -319,7 +330,12 @@ func VerifyNydusSnapshotter(ctx context.Context, t *testing.T, client klient.Cli
}

func VerifyImagePullTimer(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error {
caaPod, err := getCaaPod(ctx, client, t)
nodeName, err := GetNodeNameFromPod(ctx, client, pod)
if err != nil {
return fmt.Errorf("VerifyImagePullTimer: GetNodeNameFromPod failed with %v", err)
}

caaPod, err := getCaaPod(ctx, client, t, nodeName)
if err != nil {
return fmt.Errorf("VerifyImagePullTimer: failed to getCaaPod: %v", err)
}
Expand Down Expand Up @@ -644,14 +660,15 @@ func AddImagePullSecretToDefaultServiceAccount(ctx context.Context, client klien
return nil
}

func GetPodNamesByLabel(ctx context.Context, client klient.Client, t *testing.T, namespace string, labelName string, labelValue string) (*v1.PodList, error) {
func GetPodNamesByLabel(ctx context.Context, client klient.Client, t *testing.T, namespace string, labelName string, labelValue string, nodeName string) (*v1.PodList, error) {

clientset, err := kubernetes.NewForConfig(client.RESTConfig())
if err != nil {
return nil, fmt.Errorf("GetPodNamesByLabel: get Kubernetes clientSef failed: %v", err)
}

pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelName + "=" + labelValue})
nodeSelector := fmt.Sprintf("spec.nodeName=%s", nodeName)
pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelName + "=" + labelValue, FieldSelector: nodeSelector})
if err != nil {
return nil, fmt.Errorf("GetPodNamesByLabel: get pod list failed: %v", err)
}
Expand Down
14 changes: 7 additions & 7 deletions src/cloud-api-adaptor/test/e2e/assessment_runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,13 @@ func (tc *TestCase) Run() {
t.Errorf("VerifyNydusSnapshotter failed: %v", err)
}
}

if tc.alternateImageName != "" {
err := VerifyAlternateImage(ctx, t, client, tc.pod, tc.alternateImageName)
if err != nil {
t.Errorf("VerifyAlternateImage failed: %v", err)
}
}
}

if tc.extraPods != nil {
Expand Down Expand Up @@ -427,13 +434,6 @@ func (tc *TestCase) Run() {
}
}
}

if tc.alternateImageName != "" {
err := VerifyAlternateImage(ctx, t, client, tc.alternateImageName)
if err != nil {
t.Errorf("VerifyAlternateImage failed: %v", err)
}
}
return ctx
}).
Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
Expand Down

0 comments on commit 5672559

Please sign in to comment.