From ac1fae8e4fb5d401eb1b0392e1a0f1b4091daabf Mon Sep 17 00:00:00 2001 From: Jon Lamb Date: Sun, 22 Oct 2023 03:44:06 -0700 Subject: [PATCH] Add firmware update robot test, update renode configs --- .github/workflows/integration_tests.yml | 5 + .gitignore | 7 + agp.robot | 197 +++++++++++++++++------- bootloader/Cargo.toml | 1 + firmware/Cargo.lock | 1 + firmware/Cargo.toml | 1 + renode/agp.resc | 1 + 7 files changed, 157 insertions(+), 56 deletions(-) create mode 100644 .gitignore diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a6029c4..aeacaa9 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -52,6 +52,7 @@ jobs: sleep 2 - name: Run tests + timeout-minutes: 10 run: renode-test agp.robot - name: Archive test results @@ -63,3 +64,7 @@ jobs: report.html log.html robot_output.xml + cmd.stdout + cmd.stderr + agp_uart.txt + agp_renode.log diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..74ffdcd --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# robot framework +agp_logs/ +log.html +report.html +results-agp.robot.xml +robot_output.xml +snapshots diff --git a/agp.robot b/agp.robot index 4206dcf..c40e4b8 100644 --- a/agp.robot +++ b/agp.robot @@ -3,114 +3,199 @@ Documentation Integration tests for air-gradient-pro bootloader, firmware and Default Tags agp Library Process +# These are the defaults provided by renode, automatically set if not supplied +Suite Setup AGP Suite Setup +Suite Teardown Teardown +Test Setup AGP Test Setup +Test Teardown Test Teardown +Resource ${RENODEKEYWORDS} + *** Variables *** # Firmware configs ${AIR_GRADIENT_MAC_ADDRESS} 02:00:04:03:07:04 ${AIR_GRADIENT_IP_ADDRESS} 192.0.2.80 ${AIR_GRADIENT_DEVICE_ID} 255 +${AIR_GRADIENT_DEVICE_PORT} 32101 ${AIR_GRADIENT_LOG} TRACE # Test vars ${CLI} ${CURDIR}/host_tools/air-gradient-cli/target/x86_64-unknown-linux-gnu/release/air-gradient ${AGP_RESC} ${CURDIR}/renode/agp.resc +${FW_IMAGE} ${CURDIR}/firmware/target/agp_images.cpio ${UART} sysbus.usart6 -${UART_TIMEOUT} 10 +${UART_TIMEOUT} 30 ${PING_TIMEOUT} 5 +${VERBOSE_LOGGING_DIR} ${CURDIR}/agp_logs +${VERBOSE_LOGGING_ENABLED} True +${RENODE_LOG_LEVEL} 2 *** Keywords *** +AGP Suite Setup + Setup + Build System + +AGP Test Setup + ${test_name} = Replace String ${TEST NAME} ${SPACE} _ + Set Test Variable \${TEST_NAME} ${test_name} + Test Setup + Prepare Machine + Build Firmware - Set environment variable AIR_GRADIENT_MAC_ADDRESS ${AIR_GRADIENT_MAC_ADDRESS} - Set environment variable AIR_GRADIENT_IP_ADDRESS ${AIR_GRADIENT_IP_ADDRESS} - Set environment variable AIR_GRADIENT_DEVICE_ID ${AIR_GRADIENT_DEVICE_ID} - Set environment variable AIR_GRADIENT_LOG ${AIR_GRADIENT_LOG} - ${result}= Run Process cargo build --release cwd=firmware shell=true - IF ${result.rc} != 0 - Log To Console ${result.stdout} console=yes - Log To Console ${result.stderr} console=yes + Set environment variable AIR_GRADIENT_MAC_ADDRESS ${AIR_GRADIENT_MAC_ADDRESS} + Set environment variable AIR_GRADIENT_IP_ADDRESS ${AIR_GRADIENT_IP_ADDRESS} + Set environment variable AIR_GRADIENT_DEVICE_ID ${AIR_GRADIENT_DEVICE_ID} + Set environment variable AIR_GRADIENT_LOG ${AIR_GRADIENT_LOG} + Set environment variable AIR_GRADIENT_DEVICE_PORT ${AIR_GRADIENT_DEVICE_PORT} + + ${result} = Run Process cargo build --release cwd=firmware shell=true + IF ${result.rc} != 0 + Log To Console ${result.stdout} console=yes + Log To Console ${result.stderr} console=yes END - Should Be Equal As Integers ${result.rc} 0 + Should Be Equal As Integers ${result.rc} 0 Build Bootloader - Set environment variable AIR_GRADIENT_LOG ${AIR_GRADIENT_LOG} - ${result}= Run Process cargo build --release cwd=bootloader shell=true - IF ${result.rc} != 0 - Log To Console ${result.stdout} console=yes - Log To Console ${result.stderr} console=yes + Set environment variable AIR_GRADIENT_LOG ${AIR_GRADIENT_LOG} + ${result} = Run Process cargo build --release cwd=bootloader shell=true + IF ${result.rc} != 0 + Log To Console ${result.stdout} console=yes + Log To Console ${result.stderr} console=yes END - Should Be Equal As Integers ${result.rc} 0 + Should Be Equal As Integers ${result.rc} 0 Build CLI - ${result}= Run Process cargo build --release cwd=host_tools/air-gradient-cli shell=true - IF ${result.rc} != 0 - Log To Console ${result.stdout} console=yes - Log To Console ${result.stderr} console=yes + ${result} = Run Process cargo build --release cwd=host_tools/air-gradient-cli shell=true + IF ${result.rc} != 0 + Log To Console ${result.stdout} console=yes + Log To Console ${result.stderr} console=yes END - Should Be Equal As Integers ${result.rc} 0 + Should Be Equal As Integers ${result.rc} 0 Build System Build Firmware Build Bootloader Build CLI +Enable Verbose Logging + Create Directory ${VERBOSE_LOGGING_DIR} + + Execute Command usart6 CreateFileBackend @${VERBOSE_LOGGING_DIR}/${TEST_NAME}_uart.txt true + Execute Command logFile @${VERBOSE_LOGGING_DIR}/${TEST_NAME}_renode.log true + Execute Command logLevel ${RENODE_LOG_LEVEL} file + Prepare Machine - Execute Command path add @${CURDIR} - Execute Script ${AGP_RESC} + Execute Command path add @${CURDIR} + Execute Script ${AGP_RESC} + Create Terminal Tester ${UART} timeout=${UART_TIMEOUT} + IF ${VERBOSE_LOGGING_ENABLED} == True + Enable Verbose Logging + END Run Command - [Arguments] ${cmd_and_args} - Log To Console ${cmd_and_args} - ${result}= Run Process ${cmd_and_args} shell=true - IF ${result.rc} != 0 - Log To Console ${result.stdout} console=yes - Log To Console ${result.stderr} console=yes + [Arguments] ${cmd_and_args} + IF ${VERBOSE_LOGGING_ENABLED} == True + ${result} = Run Process ${cmd_and_args} shell=true stdout=${VERBOSE_LOGGING_DIR}/${TEST_NAME}_cmd.stdout stderr=${VERBOSE_LOGGING_DIR}/${TEST_NAME}_cmd.stderr + ELSE + ${result} = Run Process ${cmd_and_args} shell=true END - Should Be Equal As Integers ${result.rc} 0 + IF ${result.rc} != 0 + Log To Console ${result.stdout} console=yes + Log To Console ${result.stderr} console=yes + END - RETURN ${result} + Should Be Equal As Integers ${result.rc} 0 -*** Test Cases *** -Boot the System - [Documentation] Boots the system, bootloader and firmware - [Tags] firmware bootloader uart + RETURN ${result} - Build System +Run CLI Command + [Arguments] ${args} ${log_level}=error + ${result} = Run Command RUST_LOG=${log_level} ${CLI} ${args} + RETURN ${result} - Prepare Machine - Create Terminal Tester ${UART} timeout=${UART_TIMEOUT} +Wait For Device Networking + Wait For Line On Uart [D] UM: listening on port ${AIR_GRADIENT_DEVICE_PORT} - Start Emulation +*** Test Cases *** +System Boots + [Documentation] Boots the system, bootloader and firmware + [Tags] firmware bootloader uart - Wait For Line On Uart >>> Initialized <<< + Start Emulation - Provides booted-system + Wait For Line On Uart [W] Invalid boot config, using default + Wait For Line On Uart agp-bootloader + Wait For Line On Uart Reset reason: Power-on reset + Wait For Line On Uart Boot config slot: SLOT0 + Wait For Line On Uart Update pending: false + Wait For Line On Uart Update valid: false + Wait For Line On Uart air-gradient-pro-rs + Wait For Line On Uart Reset reason: Power-on reset + Wait For Line On Uart Update pending: false + Wait For Line On Uart >>> Initialized <<< Responds to Ping - [Documentation] TCP/IP stack should respond to pings - [Tags] firmware bootloader uart network - Requires booted-system + [Documentation] TCP/IP stack should respond to pings + [Tags] firmware bootloader uart network - Set Test Variable ${PING_CMD} ping -w ${PING_TIMEOUT} -W 1 ${AIR_GRADIENT_IP_ADDRESS} + Start Emulation - Wait For Line On Uart [D] UM: listening on port 32101 + Set Test Variable ${PING_CMD} ping -w ${PING_TIMEOUT} -W 1 ${AIR_GRADIENT_IP_ADDRESS} - ${result}= Run Process ${PING_CMD} shell=true - Should Be Equal As Integers ${result.rc} 0 + Wait For Device Networking - Provides running-system + ${result} = Run Process ${PING_CMD} shell=true + Should Be Equal As Integers ${result.rc} 0 Responds with Device Info - [Documentation] Device responds to info request - [Tags] firmware bootloader network device-protocol cli - Requires running-system + [Documentation] Device responds to info request + [Tags] firmware bootloader network device-protocol cli + + Start Emulation + + Wait For Device Networking + + ${result} = Run CLI Command device info -a ${AIR_GRADIENT_IP_ADDRESS} -F device_id + Should Be Equal As Integers ${result.stdout} ${AIR_GRADIENT_DEVICE_ID} + + Wait For Line On Uart [D] UM: processing command Info + +# NOTE: this test flaps sometimes on my local machine but not in CI... +Device Applies Valid Firmware Update + [Documentation] Devices handles firmware updates from the CLI, initially boots from SLOT0, + ... updates and boots to SLOT1 + [Tags] firmware bootloader network device-protocol cli fota + + Start Emulation + + Wait For Device Networking + + ${r0} = Run CLI Command device info -a ${AIR_GRADIENT_IP_ADDRESS} -F active_boot_slot + Should Be Equal ${r0.stdout} "SLOT0" + + # Socket gets reset every time the connection ends + Wait For Device Networking + + Run CLI Command device update -a ${AIR_GRADIENT_IP_ADDRESS} ${FW_IMAGE} log_level=trace + + # firmware msg when upload complete + Wait For Line On Uart [W] Update complete, rebooting now + + # bootloader msg after reboot + Wait For Line On Uart Update pending: true + + # firmware ACKs the pending update and resets + Wait For Line On Uart New application update checks out, marking for BC flash and reseting - Set Test Variable ${CLI_CMD} ${CLI} device info -a ${AIR_GRADIENT_IP_ADDRESS} -F device_id + # bootloader msg after next reboot from ACK + Wait For Line On Uart Update valid: true - ${result} Run Command ${CLI_CMD} + Wait For Device Networking - Should Be Equal As Integers ${result.stdout} ${AIR_GRADIENT_DEVICE_ID} + ${r1} = Run CLI Command device info -a ${AIR_GRADIENT_IP_ADDRESS} -F active_boot_slot + Should Be Equal ${r1.stdout} "SLOT1" # TODO -# - tests for fota related +# - tests for fota related, failover mechanism, etc # - device protocol and CLI ops # - sensors and bcast protocol stuff # - watchdog and panic stuf diff --git a/bootloader/Cargo.toml b/bootloader/Cargo.toml index 7ca74a5..3126421 100644 --- a/bootloader/Cargo.toml +++ b/bootloader/Cargo.toml @@ -78,6 +78,7 @@ pre-start-commands = [ ] reset = ''' sysbus LoadELF $bin +sysbus.cpu VectorTableOffset 0x8000000 sysbus WriteDoubleWord 0x1FFF7A10 $id1 sysbus WriteDoubleWord 0x1FFF7A14 $id2 diff --git a/firmware/Cargo.lock b/firmware/Cargo.lock index 0ddfb55..8f83a42 100644 --- a/firmware/Cargo.lock +++ b/firmware/Cargo.lock @@ -876,6 +876,7 @@ dependencies = [ "byteorder", "cfg-if", "heapless", + "log", "managed", ] diff --git a/firmware/Cargo.toml b/firmware/Cargo.toml index 92dfa22..15a095c 100644 --- a/firmware/Cargo.toml +++ b/firmware/Cargo.toml @@ -253,6 +253,7 @@ pre-start-commands = [ ] reset = ''' sysbus LoadELF $bin +sysbus.cpu VectorTableOffset 0x8000000 # Device ID sysbus WriteDoubleWord 0x1FFF7A10 $id1 diff --git a/renode/agp.resc b/renode/agp.resc index c50ee51..c4e41c9 100644 --- a/renode/agp.resc +++ b/renode/agp.resc @@ -99,6 +99,7 @@ macro reset """ sysbus LoadELF $firmware_bin sysbus LoadELF $bootloader_bin + sysbus.cpu VectorTableOffset 0x8000000 sysbus WriteDoubleWord 0x1FFF7A10 $id1 sysbus WriteDoubleWord 0x1FFF7A14 $id2