CommandAndControl failing integration tests #173
Description
Aws Connected Device Framework Affected Module(s):
CommandAndControl
I'm submitting a ...
- bug report
- feature request
Description:
Deploying CommandAndControl module and running integration tests results in errors
Current behavior:
Deploying CommandAndControl, then running integration tests results in errors
Expected behavior:
Integration tests should pass
Steps to reproduce:
Deploy with default settings, IAM auth, and assetLibrary in full mode
"assetLibrary",
"assetLibraryHistory",
"assetLibraryExport",
"certificateActivator",
"certificateVendor",
"commandAndControl",
"deviceMonitoring",
"devicePatcher",
"fleetSimulator",
"greengrass2InstallerConfigGenerators",
"greengrass2Provisioning",
"notifications",
"provisioning"
Run integration tests with
ENVIRONMENT=dev DEPLOY_ARTIFACTS_STORE_BUCKET=your-bucket-name ASSETLIBRARY_MODE='full' ./source/cicd/integrationtestsproject_build.bash
Additional Information:
Test failures are somewhat inconsistent. They are as follows
-
Scenario: Send a message to a thing group # features/commandandcontrol/jobs.feature:76
✔ Before # features/dist/step_definitions/provisioning/things.steps.js:26
✔ Given command-and-control command with operation "cdf-integration-test-ota" exists # features/dist/step_definitions/commandandcontrol/commands.steps.js:16
✔ When I send command-and-control message to last command with attributes: # features/dist/step_definitions/commandandcontrol/messages.steps.js:15
| targets | {"awsIoT": {"thingGroups": [{"name":"cdf-integration-test-cac-jobs-group1", "expand":true}]}} |
| payloadParamValues | {"s3Url":"%property:GREENGRASS_TEMPLATE_S3_LOCATION%"} |
✔ And I wait until last command-and-control message has "awaiting_replies" status # features/dist/step_definitions/commandandcontrol/messages.steps.js:20
✔ Then last command-and-control message exists with attributes: # features/dist/step_definitions/commandandcontrol/messages.steps.js:38
| $.id | world:lastMessageId |
| $.commandId | world:lastCommand.id |
| $.targets.awsIoT.thingGroups.length | 1 |
| $.targets.awsIoT.thingGroups.[0].name | cdf-integration-test-cac-jobs-group1 |
| $.status | awaiting_replies |
| $.createdAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
| $.updatedAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
✔ When I pause for 3000ms # features/dist/step_definitions/common/common.steps.js:73
✖ Then last command-and-control message has recipients: # features/dist/step_definitions/commandandcontrol/messages.steps.js:42
| $.recipients.length | 2 |
| $.recipients.[0].id | cdf-integration-test-cac-jobs-device1 |
| $.recipients.[0].status | success |
| $.recipients.[0].correlationId | regex:^[a-z0-9]{9}$ |
| $.recipients.[1].id | cdf-integration-test-cac-jobs-device2 |
| $.recipients.[1].status | success |
| $.recipients.[1].correlationId | regex:^[a-z0-9]{9}$ |
AssertionError: $.recipients.[0].status: expected 'pending' to equal 'success'
+ expected - actual-pending +success at /workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/common/common.steps.js:124:110 at Array.forEach (<anonymous>) at Object.validateExpectedAttributes (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/common/common.steps.js:83:20) at World.<anonymous> (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/messages.steps.js:44:20) at process.processTicksAndRejections (node:internal/process/task_queues:95:5) -
Scenario: A recipient replies to a message # features/commandandcontrol/jobs.feature:100
✔ Before # features/dist/step_definitions/provisioning/things.steps.js:26
✖ When thing "cdf-integration-test-cac-jobs-device1" replies to last command-and-control message as "IN_PROGRESS" # features/dist/step_definitions/commandandcontrol/replies.steps.js:48
TypeError: Cannot destructure property 'requestId' of 'e.$metadata' as it is undefined.
at reply (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/replies.steps.js:89:25)
at async World. (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/replies.steps.js:49:5) -
Scenario: A recipient replies to a message # features/commandandcontrol/shadows.feature:52
✔ Before # features/dist/step_definitions/provisioning/things.steps.js:26
✔ When thing "cdf-integration-test-cac-shadows-device1" replies to last command-and-control message as "accepted" # features/dist/step_definitions/commandandcontrol/replies.steps.js:48
✔ And thing "cdf-integration-test-cac-shadows-device1" replies to last command-and-control message with payload: # features/dist/step_definitions/commandandcontrol/replies.steps.js:51
| response | first |
✔ And thing "cdf-integration-test-cac-shadows-device1" replies to last command-and-control message with payload: # features/dist/step_definitions/commandandcontrol/replies.steps.js:51
| response | second |
✔ When I pause for 3000ms # features/dist/step_definitions/common/common.steps.js:73
✖ Then last command-and-control message has replies from "cdf-integration-test-cac-shadows-device1": # features/dist/step_definitions/commandandcontrol/replies.steps.js:98
| $.replies.length | 3 |
| $.replies.[0].receivedAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
| $.replies.[0].action | accepted |
| $.replies.[0].payload | undefined |
| $.replies.[1].receivedAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
| $.replies.[1].action | reply |
| $.replies.[1].payload | {"response":"first"} |
| $.replies.[2].receivedAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
| $.replies.[2].action | reply |
| $.replies.[2].payload | {"response":"second"} |
AssertionError: $.replies.length: expected '2' to equal '3'
+ expected - actual-2 +3 at /workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/common/common.steps.js:124:110 at Array.forEach (<anonymous>) at Object.validateExpectedAttributes (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/common/common.steps.js:83:20) at World.<anonymous> (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/replies.steps.js:100:20) at process.processTicksAndRejections (node:internal/process/task_queues:95:5) -
Scenario: Create a command # features/commandandcontrol/topics.feature:8
✔ Before # features/dist/step_definitions/provisioning/things.steps.js:26
✔ Given command-and-control command with operation "cdf-integration-test-reboot" does not exist # features/dist/step_definitions/commandandcontrol/commands.steps.js:24
✔ When I create command-and-control command with attributes: # features/dist/step_definitions/commandandcontrol/commands.steps.js:31
| operation | cdf-integration-test-reboot |
| deliveryMethod | { "type": "TOPIC", "onlineOnly": true, "expectReply": true } |
| payloadTemplate | "{\"category\": ${cat} }" |
| payloadParams | ["cat"] |
| tags | {"cdf-integration-test": true} |
✖ Then last command-and-control command exists with attributes: # features/dist/step_definitions/commandandcontrol/commands.steps.js:43
| $.id | regex:^[a-z0-9]{9}$ |
| $.operation | cdf-integration-test-reboot |
| $.deliveryMethod.type | TOPIC |
| $.deliveryMethod.expectReply | true |
| $.deliveryMethod.onlineOnly | true |
|$.payloadTemplate | "{\"category\": $ {cat} }" |
| $.payloadParams.length | 1 |
| $.payloadParams.[0] | cat |
| $.tags.cdf-integration-test | true |
| $.createdAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
| $.updatedAt | regex:^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d{3})Z$ |
InternalServerError: {"message": "Internal server error"}
at /workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/libraries/clients/commandandcontrol-client/dist/client/commands.apigw.service.js:112:40
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async CommandsApigwService.getCommand (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/libraries/clients/commandandcontrol-client/dist/client/commands.apigw.service.js:104:16)
at async World. (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/commands.steps.js:44:21) -
Scenario: Send a message to multiple things # features/commandandcontrol/topics.feature:29
✔ Before # features/dist/step_definitions/provisioning/things.steps.js:26
✖ Given command-and-control command with operation "cdf-integration-test-reboot" exists # features/dist/step_definitions/commandandcontrol/commands.steps.js:16
InternalServerError: {"message": "Internal server error"}
at /workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/libraries/clients/commandandcontrol-client/dist/client/commands.apigw.service.js:99:40
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async CommandsApigwService.listCommands (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/libraries/clients/commandandcontrol-client/dist/client/commands.apigw.service.js:91:16)
at async listCommands (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/commands.steps.js:52:13)
at async World. (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/commands.steps.js:19:22) -
Scenario: A recipient replies to a message # features/commandandcontrol/topics.feature:54
✔ Before # features/dist/step_definitions/provisioning/things.steps.js:26
✖ When thing "cdf-integration-test-cac-topics-device1" replies to last command-and-control message as "accepted" # features/dist/step_definitions/commandandcontrol/replies.steps.js:48
InternalServerError: {"message": "Internal server error"}
at /workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/libraries/clients/commandandcontrol-client/dist/client/messages.apigw.service.js:82:40
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async MessagesApigwService.getRecipient (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/libraries/clients/commandandcontrol-client/dist/client/messages.apigw.service.js:74:16)
at async reply (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/replies.steps.js:71:23)
at async World. (/workplace/pcozzi/os-cdf/aws-connected-device-framework/source/packages/integration-tests/features/dist/step_definitions/commandandcontrol/replies.steps.js:49:5)