From d1498865e3f2125b7eaa3c4cf1c5fb70cb857aed Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Fri, 10 Oct 2025 00:30:30 -0400 Subject: [PATCH 01/13] Less iOS 26 tests. More shards --- bitrise.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index 85d49b81842c3..ca5f4a5e7a554 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -205,7 +205,7 @@ workflows: run_if: '{{enveq "BITRISE_TEST_BUNDLE_PATH" "" | not}}' inputs: - product_path: $BITRISE_XCTESTRUN_SMOKE_TEST_FILE_PATH - - shard_count: 5 + - shard_count: 6 - destination: $BITRISE_DESTINATION outputs: - BITRISE_TEST_SHARDS_PATH: BITRISE_TEST_SHARDS_PATH_FIREFOX @@ -305,9 +305,7 @@ workflows: timeout: 1200 inputs: - only_testing: | - XCUITests/OnboardingTests XCUITests/BookmarksTests - XCUITests/BrowsingPDFTests XCUITests/FindInPageTests XCUITests/HomePageSettingsUITests XCUITests/NavigationTest From 37276ea7604973875e228fe9450666762bccd07c Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:37:46 -0400 Subject: [PATCH 02/13] Running unit tests in parallel --- bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index ca5f4a5e7a554..b1b32ea65287d 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -198,7 +198,7 @@ workflows: timeout: 1200 inputs: - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO"' + - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled YES" "-parallel-testing-worker-count 2' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_UnitTest_iphonesimulator26.0-arm64.xctestrun - maximum_test_repetitions: 2 - xcode-test-shard-calculation@0: From 7afebbdf9503b31c67ee6784ee423acba5d038ec Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 17:43:53 -0400 Subject: [PATCH 03/13] Fix quote --- bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index b1b32ea65287d..78a23c5f0f0c7 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -198,7 +198,7 @@ workflows: timeout: 1200 inputs: - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled YES" "-parallel-testing-worker-count 2' + - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled YES" "-parallel-testing-worker-count 2"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_UnitTest_iphonesimulator26.0-arm64.xctestrun - maximum_test_repetitions: 2 - xcode-test-shard-calculation@0: From 6b1c550432736f164af7826b1abec3577a6a9b72 Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 18:02:28 -0400 Subject: [PATCH 04/13] Include parallel option in the beginning --- bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index 78a23c5f0f0c7..ae99d2339779a 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -198,7 +198,7 @@ workflows: timeout: 1200 inputs: - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled YES" "-parallel-testing-worker-count 2"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_UnitTest_iphonesimulator26.0-arm64.xctestrun - maximum_test_repetitions: 2 - xcode-test-shard-calculation@0: From 0d2d781617fca5c8c4491dee39ecd0891078718e Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 19:57:15 -0400 Subject: [PATCH 05/13] Parallelize all smoke tests --- bitrise.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index ae99d2339779a..5c8496656a47d 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -313,7 +313,7 @@ workflows: XCUITests/SearchTests XCUITests/SettingsTests - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} @@ -328,7 +328,7 @@ workflows: inputs: - only_testing: $BITRISE_TEST_SHARDS_PATH_FIREFOX/$BITRISE_IO_PARALLEL_INDEX - destination: platform=iOS Simulator,name=iPhone 16,OS=18.6 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} determine_apps_affected: From 57c5903f2935f21f54513ef63487cd44015c8400 Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 21:16:54 -0400 Subject: [PATCH 06/13] Add back ios26 tests. Parallelize unit tests. --- bitrise.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index 5c8496656a47d..3af1d1001b4d6 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -306,9 +306,11 @@ workflows: inputs: - only_testing: | XCUITests/BookmarksTests + XCUITests/BrowsingPDFTests XCUITests/FindInPageTests XCUITests/HomePageSettingsUITests XCUITests/NavigationTest + XCUITests/OnboardingTests XCUITests/PhotonActionSheetTests XCUITests/SearchTests XCUITests/SettingsTests @@ -1618,8 +1620,8 @@ workflows: - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 - test_plan: UnitTest - test_repetition_mode: until_failure - - maximum_test_repetitions: 3 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled" "NO" "-parallel-testing-worker-count" "2"' + - maximum_test_repetitions: 5 + - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2"' - deploy-to-bitrise-io@2.10.0: {} # From b5976019ffdb7f943f3f30ad524ec709175e530c Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 21:45:42 -0400 Subject: [PATCH 07/13] 3 parallel simulators! --- bitrise.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index 3af1d1001b4d6..a959b66e5bdae 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -205,7 +205,7 @@ workflows: run_if: '{{enveq "BITRISE_TEST_BUNDLE_PATH" "" | not}}' inputs: - product_path: $BITRISE_XCTESTRUN_SMOKE_TEST_FILE_PATH - - shard_count: 6 + - shard_count: 5 - destination: $BITRISE_DESTINATION outputs: - BITRISE_TEST_SHARDS_PATH: BITRISE_TEST_SHARDS_PATH_FIREFOX @@ -315,7 +315,7 @@ workflows: XCUITests/SearchTests XCUITests/SettingsTests - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 - - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "3" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} @@ -330,7 +330,7 @@ workflows: inputs: - only_testing: $BITRISE_TEST_SHARDS_PATH_FIREFOX/$BITRISE_IO_PARALLEL_INDEX - destination: platform=iOS Simulator,name=iPhone 16,OS=18.6 - - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "3" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} determine_apps_affected: From 910130e68a152c68ad64c8baa3344b805396182f Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Sun, 12 Oct 2025 22:11:50 -0400 Subject: [PATCH 08/13] Down to 2 simulators --- bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index a959b66e5bdae..cee70b7fa4a95 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -330,7 +330,7 @@ workflows: inputs: - only_testing: $BITRISE_TEST_SHARDS_PATH_FIREFOX/$BITRISE_IO_PARALLEL_INDEX - destination: platform=iOS Simulator,name=iPhone 16,OS=18.6 - - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "3" "COMPILER_INDEX_STORE_ENABLE=NO"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} determine_apps_affected: From cab60404e4e9acacab75de8ee131d6f79e964ffa Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Tue, 14 Oct 2025 16:15:25 -0400 Subject: [PATCH 09/13] Use headless --- bitrise.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index cee70b7fa4a95..c08e2bb1d765a 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -197,7 +197,7 @@ workflows: - xcode-test-without-building@0.4.0: timeout: 1200 inputs: - - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 + - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_UnitTest_iphonesimulator26.0-arm64.xctestrun - maximum_test_repetitions: 2 @@ -306,7 +306,6 @@ workflows: inputs: - only_testing: | XCUITests/BookmarksTests - XCUITests/BrowsingPDFTests XCUITests/FindInPageTests XCUITests/HomePageSettingsUITests XCUITests/NavigationTest @@ -314,7 +313,7 @@ workflows: XCUITests/PhotonActionSheetTests XCUITests/SearchTests XCUITests/SettingsTests - - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 + - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "3" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} @@ -329,7 +328,7 @@ workflows: timeout: 1200 inputs: - only_testing: $BITRISE_TEST_SHARDS_PATH_FIREFOX/$BITRISE_IO_PARALLEL_INDEX - - destination: platform=iOS Simulator,name=iPhone 16,OS=18.6 + - destination: platform=iOS Simulator,name=iPhone 16,OS=18.6,isHeadless=YES - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} From d34f58f84e287af65e83a2ddc90525af5db8eca2 Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Tue, 14 Oct 2025 16:24:53 -0400 Subject: [PATCH 10/13] Remove parallel simulators for unit tests --- bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index c08e2bb1d765a..456bbf1094ef1 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -198,7 +198,7 @@ workflows: timeout: 1200 inputs: - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO"' + - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_UnitTest_iphonesimulator26.0-arm64.xctestrun - maximum_test_repetitions: 2 - xcode-test-shard-calculation@0: From 925c6e4de2d86c0bac0c4c6aa5c396530d14d96f Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Tue, 14 Oct 2025 16:34:59 -0400 Subject: [PATCH 11/13] Use two parallel simulators --- bitrise.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index 456bbf1094ef1..345da225117d5 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -314,7 +314,7 @@ workflows: XCUITests/SearchTests XCUITests/SettingsTests - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "3" "COMPILER_INDEX_STORE_ENABLE=NO"' + - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"' - xctestrun: $BITRISE_TEST_BUNDLE_PATH/Fennec_Smoketest_iphonesimulator26.0-arm64.xctestrun - deploy-to-bitrise-io@2.10.0: {} @@ -1616,11 +1616,11 @@ workflows: - project_path: firefox-ios/Client.xcodeproj - scheme: Fennec - configuration: Fennec_Testing - - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0 + - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - test_plan: UnitTest - test_repetition_mode: until_failure - maximum_test_repetitions: 5 - - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2"' + - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled" "NO" "-parallel-testing-worker-count" "2"' - deploy-to-bitrise-io@2.10.0: {} # From b80be34085b01bdcc33d32cc38aacbbdd87a9fb0 Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Tue, 14 Oct 2025 17:15:32 -0400 Subject: [PATCH 12/13] Revert repetitions for unit tests --- bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index 345da225117d5..52f1734e069ec 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -1619,7 +1619,7 @@ workflows: - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - test_plan: UnitTest - test_repetition_mode: until_failure - - maximum_test_repetitions: 5 + - maximum_test_repetitions: 3 - xcodebuild_options: '"COMPILER_INDEX_STORE_ENABLE=NO" "CODE_SIGN_IDENTITY=" "CODE_SIGNING_REQUIRED=NO" "CODE_SIGNING_ALLOWED=NO" "-parallel-testing-enabled" "NO" "-parallel-testing-worker-count" "2"' - deploy-to-bitrise-io@2.10.0: {} From f738a3f14cf5f61368a812b98ad63f1fc87d8b47 Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Tue, 14 Oct 2025 23:05:34 -0400 Subject: [PATCH 13/13] SearchTests takes too long :( --- bitrise.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index 52f1734e069ec..df7e5dcad4ec8 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -311,7 +311,6 @@ workflows: XCUITests/NavigationTest XCUITests/OnboardingTests XCUITests/PhotonActionSheetTests - XCUITests/SearchTests XCUITests/SettingsTests - destination: platform=iOS Simulator,name=iPhone 17,OS=26.0,isHeadless=YES - xcodebuild_options: '"-parallel-testing-enabled" "YES" "-parallel-testing-worker-count" "2" "COMPILER_INDEX_STORE_ENABLE=NO"'