Performance test suite

.github/workflows/ci-workflow.yml

@@ -53,6 +53,28 @@ jobs:
           name: bitcoin-testnet-app-${{ matrix.model }}
           path: bitcoin-testnet-bin
 
+  job_build_app_perftest:
+    name: Compile the UX-less version of the app on Nano S+ for performance tests
+
+    runs-on: ubuntu-latest
+
+    container:
+      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Build
+        run: |
+          make DEBUG=0 COIN=bitcoin_testnet BOLOS_SDK="$NANOSP_SDK" AUTOAPPROVE_FOR_PERF_TESTS=1
+
+      - name: Upload Bitcoin Testnet app binary
+        uses: actions/upload-artifact@v2
+        with:
+          name: bitcoin-testnet-perftest-app-nanosp
+          path: bin
+
   job_unit_test:
     name: Unit test
     needs: job_build
@@ -166,6 +188,43 @@ jobs:
           pip install -r requirements.txt
           PYTHONPATH=$PYTHONPATH:/speculos pytest --headless --timeout=300
 
+  job_perftests:
+    name: Performance report
+
+    needs: job_build_app_perftest
+    runs-on: ubuntu-latest
+
+    container:
+      image: ghcr.io/ledgerhq/app-bitcoin-new/speculos-bitcoin:latest
+      ports:
+        - 1234:1234
+        - 9999:9999
+        - 40000:40000
+        - 41000:41000
+        - 42000:42000
+        - 43000:43000
+      options: --entrypoint /bin/bash
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Download Bitcoin app binary for perftests
+        uses: actions/download-artifact@v2
+        with:
+          name: bitcoin-testnet-perftest-app-nanosp
+          path: bin
+
+      - name: Run tests
+        run: |
+          cd tests_perf
+          pip install -r requirements.txt
+          PYTHONPATH=$PYTHONPATH:/speculos pytest --headless --model=nanosp --benchmark-json=benchmarks.json
+      - name: Upload benchmarks summary
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmarks-log
+          path: tests_perf/benchmarks.json
 
   job_test_js_lib:
     name: Tests with the JS library

Makefile

@@ -56,6 +56,13 @@ else
 APPVERSION = "$(APPVERSION_M).$(APPVERSION_N).$(APPVERSION_P)-$(strip $(APPVERSION_SUFFIX))"
 endif
 
+# If set, the app will automatically approve all requests without user interaction. Useful for performance tests.
+# It is critical that no such app is ever deployed in production.
+AUTOAPPROVE_FOR_PERF_TESTS ?= 0
+ifneq ($(AUTOAPPROVE_FOR_PERF_TESTS),0)
+    DEFINES += HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+endif
+
 # Setting to allow building variant applications
 VARIANT_PARAM = COIN
 VARIANT_VALUES = bitcoin_testnet bitcoin
@@ -74,33 +81,36 @@ HAVE_APPLICATION_FLAG_BOLOS_SETTINGS = 1
 HAVE_APPLICATION_FLAG_LIBRARY = 1
 
 ifeq ($(COIN),bitcoin_testnet)
-
-# Bitcoin testnet, no legacy support
-DEFINES   += BIP32_PUBKEY_VERSION=0x043587CF
-DEFINES   += BIP44_COIN_TYPE=1
-DEFINES   += COIN_P2PKH_VERSION=111
-DEFINES   += COIN_P2SH_VERSION=196
-DEFINES   += COIN_NATIVE_SEGWIT_PREFIX=\"tb\"
-DEFINES   += COIN_COINID_SHORT=\"TEST\"
-
-APPNAME = "Bitcoin Test"
-
+    # Bitcoin testnet, no legacy support
+    DEFINES   += BIP32_PUBKEY_VERSION=0x043587CF
+    DEFINES   += BIP44_COIN_TYPE=1
+    DEFINES   += COIN_P2PKH_VERSION=111
+    DEFINES   += COIN_P2SH_VERSION=196
+    DEFINES   += COIN_NATIVE_SEGWIT_PREFIX=\"tb\"
+    DEFINES   += COIN_COINID_SHORT=\"TEST\"
+
+    APPNAME = "Bitcoin Test"
 else ifeq ($(COIN),bitcoin)
-
-# Bitcoin mainnet, no legacy support
-DEFINES   += BIP32_PUBKEY_VERSION=0x0488B21E
-DEFINES   += BIP44_COIN_TYPE=0
-DEFINES   += COIN_P2PKH_VERSION=0
-DEFINES   += COIN_P2SH_VERSION=5
-DEFINES   += COIN_NATIVE_SEGWIT_PREFIX=\"bc\"
-DEFINES   += COIN_COINID_SHORT=\"BTC\"
-
-APPNAME = "Bitcoin"
+    # the version for performance tests automatically approves all requests
+    # there is no reason to ever compile the mainnet app with this flag
+    ifneq ($(AUTOAPPROVE_FOR_PERF_TESTS),0)
+        $(error Use testnet app for performance tests)
+    endif
+
+    # Bitcoin mainnet, no legacy support
+    DEFINES   += BIP32_PUBKEY_VERSION=0x0488B21E
+    DEFINES   += BIP44_COIN_TYPE=0
+    DEFINES   += COIN_P2PKH_VERSION=0
+    DEFINES   += COIN_P2SH_VERSION=5
+    DEFINES   += COIN_NATIVE_SEGWIT_PREFIX=\"bc\"
+    DEFINES   += COIN_COINID_SHORT=\"BTC\"
+
+    APPNAME = "Bitcoin"
 
 else
-ifeq ($(filter clean,$(MAKECMDGOALS)),)
-$(error Unsupported COIN - use bitcoin_testnet, bitcoin)
-endif
+    ifeq ($(filter clean,$(MAKECMDGOALS)),)
+        $(error Unsupported COIN - use bitcoin_testnet, bitcoin)
+    endif
 endif
 
 # Application icons following guidelines:

src/handler/sign_message.c

@@ -174,6 +174,7 @@ void handler_sign_message(dispatcher_context_t* dc, uint8_t protocol_version) {
         snprintf(message_hash_str + 2 * i, 3, "%02X", message_hash[i]);
     }
 
+#ifndef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
     ui_pre_processing_message();
     if (printable) {
         if (!display_message_content_and_confirm(dc,
@@ -189,6 +190,7 @@ void handler_sign_message(dispatcher_context_t* dc, uint8_t protocol_version) {
             return;
         }
     }
+#endif
     uint8_t sig[MAX_DER_SIG_LEN];
 
     uint32_t info;

src/ui/display.c

@@ -93,6 +93,10 @@ bool ui_display_pubkey(dispatcher_context_t *context,
                        const char *bip32_path_str,
                        bool is_path_suspicious,
                        const char *pubkey) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_path_and_pubkey_state_t *state = (ui_path_and_pubkey_state_t *) &g_ui_state;
 
     strncpy(state->bip32_path_str, bip32_path_str, sizeof(state->bip32_path_str));
@@ -110,6 +114,10 @@ bool ui_display_pubkey(dispatcher_context_t *context,
 bool ui_display_path_and_message_content(dispatcher_context_t *context,
                                          const char *path_str,
                                          const char *message_content) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_path_and_message_state_t *state = (ui_path_and_message_state_t *) &g_ui_state;
     strncpy(state->bip32_path_str, path_str, sizeof(state->bip32_path_str));
     strncpy(state->message, message_content, sizeof(state->message));
@@ -122,6 +130,10 @@ bool ui_display_path_and_message_content(dispatcher_context_t *context,
 bool ui_display_message_path_hash_and_confirm(dispatcher_context_t *context,
                                               const char *path_str,
                                               const char *message_hash) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_path_and_message_state_t *state = (ui_path_and_message_state_t *) &g_ui_state;
     strncpy(state->bip32_path_str, path_str, sizeof(state->bip32_path_str));
     strncpy(state->message, message_hash, sizeof(state->message));
@@ -132,6 +144,10 @@ bool ui_display_message_path_hash_and_confirm(dispatcher_context_t *context,
 }
 
 bool ui_display_message_confirm(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     (void) context;
     ui_sign_message_confirm_flow();
 
@@ -141,6 +157,10 @@ bool ui_display_message_confirm(dispatcher_context_t *context) {
 bool ui_display_register_wallet(dispatcher_context_t *context,
                                 const policy_map_wallet_header_t *wallet_header,
                                 const char *policy_descriptor) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_wallet_state_t *state = (ui_wallet_state_t *) &g_ui_state;
 
     strncpy(state->wallet_name, wallet_header->name, sizeof(state->wallet_name));
@@ -158,6 +178,10 @@ bool ui_display_policy_map_cosigner_pubkey(dispatcher_context_t *context,
                                            uint8_t cosigner_index,
                                            uint8_t n_keys,
                                            key_type_e key_type) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     (void) (n_keys);
 
     ui_cosigner_pubkey_and_index_state_t *state =
@@ -190,6 +214,10 @@ bool ui_display_wallet_address(dispatcher_context_t *context,
                                const char *address) {
     ui_wallet_state_t *state = (ui_wallet_state_t *) &g_ui_state;
 
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     strncpy(state->address, address, sizeof(state->address));
 
     if (wallet_name == NULL) {
@@ -205,28 +233,48 @@ bool ui_display_wallet_address(dispatcher_context_t *context,
 bool ui_authorize_wallet_spend(dispatcher_context_t *context, const char *wallet_name) {
     ui_wallet_state_t *state = (ui_wallet_state_t *) &g_ui_state;
 
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     strncpy(state->wallet_name, wallet_name, sizeof(state->wallet_name));
     ui_display_spend_from_wallet_flow();
 
     return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_warn_external_inputs(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_display_warning_external_inputs_flow();
     return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_warn_unverified_segwit_inputs(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_display_unverified_segwit_inputs_flows();
     return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_warn_nondefault_sighash(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_display_nondefault_sighash_flow();
     return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_transaction_prompt(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_display_transaction_prompt();
     return io_ui_process(context, SET_UX_DIRTY);
 }
@@ -237,6 +285,10 @@ bool ui_validate_output(dispatcher_context_t *context,
                         const char *address_or_description,
                         const char *coin_name,
                         uint64_t amount) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_validate_output_state_t *state = (ui_validate_output_state_t *) &g_ui_state;
 
     strncpy(state->address_or_description,
@@ -254,6 +306,10 @@ bool ui_validate_output(dispatcher_context_t *context,
 }
 
 bool ui_warn_high_fee(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_warn_high_fee_flow();
 
     return io_ui_process(context, SET_UX_DIRTY);
@@ -263,6 +319,10 @@ bool ui_validate_transaction(dispatcher_context_t *context,
                              const char *coin_name,
                              uint64_t fee,
                              bool is_self_transfer) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     ui_validate_transaction_state_t *state = (ui_validate_transaction_state_t *) &g_ui_state;
 
     format_sats_amount(coin_name, fee, state->fee);
@@ -298,20 +358,32 @@ void ui_pre_processing_message(void) {
 
 #ifdef HAVE_NBGL
 bool ui_post_processing_confirm_wallet_registration(dispatcher_context_t *context, bool success) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     (void) context;
     ui_display_post_processing_confirm_wallet_registation(success);
 
     return true;
 }
 
 bool ui_post_processing_confirm_transaction(dispatcher_context_t *context, bool success) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     (void) context;
     ui_display_post_processing_confirm_transaction(success);
 
     return true;
 }
 
 bool ui_post_processing_confirm_message(dispatcher_context_t *context, bool success) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+    return true;
+#endif
+
     (void) context;
     ui_display_post_processing_confirm_message(success);