From effc243f7cbfee75845de59eef3bcf7847ae980b Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 6 Nov 2024 17:13:25 -0300 Subject: [PATCH 01/17] Fire window tests --- DuckDuckGo.xcodeproj/project.pbxproj | 4 + UITests/FireWindowTests.swift | 299 +++++++++++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 UITests/FireWindowTests.swift diff --git a/DuckDuckGo.xcodeproj/project.pbxproj b/DuckDuckGo.xcodeproj/project.pbxproj index c6874ad10a..61ebf93397 100644 --- a/DuckDuckGo.xcodeproj/project.pbxproj +++ b/DuckDuckGo.xcodeproj/project.pbxproj @@ -2797,6 +2797,7 @@ BB470EBC2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB470EBA2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift */; }; BB5789722B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5789712B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift */; }; BB5F46A32C8751F6005F72DF /* BookmarkSortTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5F46A22C8751F6005F72DF /* BookmarkSortTests.swift */; }; + BB731F312CDBA6360023D2E4 /* FireWindowTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */; }; BB7B5F982C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */; }; BB7B5F992C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */; }; BBB881882C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBB881872C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift */; }; @@ -4747,6 +4748,7 @@ BB470EBA2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkManagementDetailViewModel.swift; sourceTree = ""; }; BB5789712B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataBrokerProtectionSubscriptionEventHandler.swift; sourceTree = ""; }; BB5F46A22C8751F6005F72DF /* BookmarkSortTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkSortTests.swift; sourceTree = ""; }; + BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FireWindowTests.swift; sourceTree = ""; }; BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarksSearchAndSortMetrics.swift; sourceTree = ""; }; BBB881872C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkListTreeControllerSearchDataSource.swift; sourceTree = ""; }; BBBB653F2C77BB9400E69AC6 /* BookmarkSearchTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkSearchTests.swift; sourceTree = ""; }; @@ -6953,6 +6955,7 @@ 7B4CE8DB26F02108009134B1 /* UITests */ = { isa = PBXGroup; children = ( + BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */, 376E708D2BD686260082B7EB /* UI Tests.xctestplan */, EEBCE6802BA444FA00B9DF00 /* Common */, EEC7BE2D2BC6C09400F86835 /* AddressBarKeyboardShortcutsTests.swift */, @@ -12604,6 +12607,7 @@ EEC7BE2E2BC6C09500F86835 /* AddressBarKeyboardShortcutsTests.swift in Sources */, EE54F7B32BBFEA49006218DB /* BookmarksAndFavoritesTests.swift in Sources */, EE02D4222BB4611A00DBE6B3 /* TestsURLExtension.swift in Sources */, + BB731F312CDBA6360023D2E4 /* FireWindowTests.swift in Sources */, EE42CBCC2BC8004700AD411C /* PermissionsTests.swift in Sources */, 7B4CE8E726F02135009134B1 /* TabBarTests.swift in Sources */, EEBCE6832BA463DD00B9DF00 /* NSImageExtensions.swift in Sources */, diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift new file mode 100644 index 0000000000..311f26bfcf --- /dev/null +++ b/UITests/FireWindowTests.swift @@ -0,0 +1,299 @@ +// +// FireWindowTests.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest + +class FireWindowTests: XCTestCase { + private var app: XCUIApplication! + private var settingsGeneralButton: XCUIElement! + private var reopenAllWindowsFromLastSessionPreference: XCUIElement! + + override class func setUp() { + UITests.firstRun() + } + + override func setUpWithError() throws { + continueAfterFailure = false + app = XCUIApplication() + app.launchEnvironment["UITEST_MODE"] = "1" + + settingsGeneralButton = app.buttons["PreferencesSidebar.generalButton"] + reopenAllWindowsFromLastSessionPreference = app.radioButtons["PreferencesGeneralView.stateRestorePicker.reopenAllWindowsFromLastSession"] + + app.launch() + app.typeKey("w", modifierFlags: [.command, .option, .shift]) // Let's enforce a single window + } + + func testFireWindowDoesNotStoreHistory() { + openFireWindow() + openSite(pageTitle: "Some site") + openNormalWindow() + assertSiteIsNotShowingInNormalWindowHistory() + } + + func testFireWindowStateIsNotSavedAfterRestart() { + openNormalWindow() + app.typeKey(",", modifierFlags: [.command]) // Open settings + settingsGeneralButton.click(forDuration: 0.5, thenDragTo: settingsGeneralButton) + reopenAllWindowsFromLastSessionPreference.clickAfterExistenceTestSucceeds() + + openThreeSitesOnNormalWindow() + openFireWindow() + openThreeSitesOnFireWindow() + + app.terminate() + app.launch() + + assertSitesOpenedInNormalWindowAreRestored() + assertSitesOpenedOnFireWindowAreNotRestored() + } + + func testFireWindowDoNotShowPinnedTabs() { + openNormalWindow() + openSite(pageTitle: "Page #1") + app.menuItems["Pin Tab"].tap() + + app.openNewTab() + openSite(pageTitle: "Page #2") + app.menuItems["Pin Tab"].tap() + + openFireWindow() + assertFireWindowDoesNotHavePinnedTabs() + } + + func testFireWindowTabsCannotBeDragged() { + openFireWindow() + openSite(pageTitle: "Page #1") + + app.openNewTab() + openSite(pageTitle: "Page #2") + + dragFirstTabOutsideOfFireWindow() + + /// Assert that Page #1 is still on the fire window after the drag + app.typeKey("]", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #2"].exists) + app.typeKey("[", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #1"].exists) + } + + func testFireWindowsSignInDoesNotShowCredentialsPopup() { + openFireWindow() + openSignUpSite() + fillCredentials() + finishSignUp() + assertSavePasswordPopupIsNotShown() + } + + func testCrendentialsAreAutoFilledInFireWindows() { + openNormalWindow() + openLoginSite() + signIn() + saveCredentials() + + /// Here we start the same flow but in the fire window, but we use the autofill credentials saved in the step before. + openFireWindow() + openLoginSite() + signInUsingAutoFill() + } + + // MARK: - Utilities + + private func signInUsingAutoFill() { + let webViewFire = app.webViews.firstMatch + webViewFire.tap() + let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch + emailTextFieldFire.click() + + let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] + let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + coordinate.tap() + + XCTAssertEqual(emailTextFieldFire.value as? String, "test@duck.com") + } + + private func saveCredentials() { + let saveButton = app.buttons["Save"] + saveButton.tap() + } + + private func signIn() { + let webView = app.webViews.firstMatch + webView.tap() + let emailTextField = webView.textFields["Email"].firstMatch + emailTextField.click() + emailTextField.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + + let signInButton = app.webViews.firstMatch.buttons["Sign in"].firstMatch + signInButton.click() + } + + private func openLoginSite() { + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + XCTAssertTrue( + addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), + "The address bar text field didn't become available in a reasonable timeframe." + ) + addressBarTextField.typeURL(URL(string: "https://privacy-test-pages.site/autofill/autoprompt/1-standard-login-form.html")!) + XCTAssertTrue( + app.windows.firstMatch.webViews["Autofill autoprompt for signin forms"].waitForExistence(timeout: UITests.Timeouts.elementExistence), + "Visited site didn't load with the expected title in a reasonable timeframe." + ) + } + + private func assertSavePasswordPopupIsNotShown() { + let credentialsPopup = app.popovers["Save password in DuckDuckGo?"] + XCTAssertFalse(credentialsPopup.exists) + } + + private func finishSignUp() { + let signUpButton = app.webViews.firstMatch.buttons["Sign up"].firstMatch + signUpButton.click() + } + + private func fillCredentials() { + let webView = app.webViews.firstMatch + webView.tap() + let emailTextField = webView.textFields["Email"].firstMatch + emailTextField.click() + emailTextField.typeText("test@duck.com") + + let password = webView.secureTextFields["Password"].firstMatch + password.click() + password.typeText("pa$$word") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } + + private func openSignUpSite() { + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + XCTAssertTrue( + addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), + "The address bar text field didn't become available in a reasonable timeframe." + ) + addressBarTextField.typeURL(URL(string: "https://privacy-test-pages.site/autofill/signup.html")!) + XCTAssertTrue( + app.windows.firstMatch.webViews["Password generation during signup"].waitForExistence(timeout: UITests.Timeouts.elementExistence), + "Visited site didn't load with the expected title in a reasonable timeframe." + ) + } + + private func dragFirstTabOutsideOfFireWindow() { + let toolbar = app.toolbars.firstMatch + let toolbarCoordinate = toolbar.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) + let startPoint = toolbarCoordinate.withOffset(CGVector(dx: 120, dy: 15)) + let endPoint = toolbarCoordinate.withOffset(CGVector(dx: -100, dy: -100)) + startPoint.press(forDuration: 0.5, thenDragTo: endPoint) + } + + private func assertFireWindowDoesNotHavePinnedTabs() { + let existsPredicate = NSPredicate(format: "exists == true") + let staticTextExistsExpectation = expectation(for: existsPredicate, evaluatedWith: app.windows.firstMatch.staticTexts.element(boundBy: 0), handler: nil) + + // Wait up to 10 seconds for the static texts to be available + let result = XCTWaiter().wait(for: [staticTextExistsExpectation], timeout: 10) + XCTAssertEqual(result, .completed, "No static texts were found in the app") + + // After confirming static texts are available, iterate through them + for staticText in app.staticTexts.allElementsBoundByIndex { + if staticText.exists { + XCTAssertFalse(staticText.label.contains("Page #1"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #2"), "Unwanted string found in static text: \(staticText.label)") + } + } + } + + private func assertSitesOpenedInNormalWindowAreRestored() { + XCTAssertTrue(app.staticTexts["Sample text for Page #3"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #3 should exist.") + app.typeKey("[", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #2"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #2 should exist.") + app.typeKey("[", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #1"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #1 should exist.") + } + + private func assertSitesOpenedOnFireWindowAreNotRestored() { + let existsPredicate = NSPredicate(format: "exists == true") + let staticTextExistsExpectation = expectation(for: existsPredicate, evaluatedWith: app.staticTexts.element(boundBy: 0), handler: nil) + + // Wait up to 10 seconds for the static texts to be available + let result = XCTWaiter().wait(for: [staticTextExistsExpectation], timeout: 10) + XCTAssertEqual(result, .completed, "No static texts were found in the app") + + // After confirming static texts are available, iterate through them + for staticText in app.staticTexts.allElementsBoundByIndex { + if staticText.exists { + XCTAssertFalse(staticText.label.contains("Page #4"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #5"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #6"), "Unwanted string found in static text: \(staticText.label)") + } + } + } + + private func openThreeSitesOnNormalWindow() { + app.openNewTab() + openSite(pageTitle: "Page #1") + app.openNewTab() + openSite(pageTitle: "Page #2") + app.openNewTab() + openSite(pageTitle: "Page #3") + } + + private func openThreeSitesOnFireWindow() { + openSite(pageTitle: "Page #4") + app.openNewTab() + openSite(pageTitle: "Page #5") + app.openNewTab() + openSite(pageTitle: "Page #6") + } + + private func cleanAllHistory() { + app.typeKey(.delete, modifierFlags: [.command, .shift]) + let clearButton = app.buttons["Clear"].firstMatch + XCTAssertTrue(clearButton.exists, "Clear button should exist") + clearButton.tap() + } + + private func assertSiteIsNotShowingInNormalWindowHistory() { + let siteMenuItemInHistory = app.menuItems["Some site"] + XCTAssertFalse(siteMenuItemInHistory.exists, "Menu item should not exist because it was not stored in history.") + } + + private func openFireWindow() { + app.typeKey("n", modifierFlags: [.command, .shift]) + } + + private func openNormalWindow() { + app.typeKey("n", modifierFlags: .command) + } + + private func openSite(pageTitle: String) { + let url = UITests.simpleServedPage(titled: pageTitle) + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + XCTAssertTrue( + addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), + "The address bar text field didn't become available in a reasonable timeframe." + ) + addressBarTextField.typeURL(url) + XCTAssertTrue( + app.windows.firstMatch.webViews[pageTitle].waitForExistence(timeout: UITests.Timeouts.elementExistence), + "Visited site didn't load with the expected title in a reasonable timeframe." + ) + } +} From 2562dc6b5b7f1f6e90f57ff71a1007ea5e4078b2 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Mon, 11 Nov 2024 19:18:47 -0300 Subject: [PATCH 02/17] Do not run NTP Search Box experiment --- DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift b/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift index eb49a31193..de1fbea87b 100644 --- a/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift +++ b/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift @@ -75,6 +75,9 @@ protocol NewTabPageSearchBoxExperimentCohortDeciding { struct DefaultNewTabPageSearchBoxExperimentCohortDecider: NewTabPageSearchBoxExperimentCohortDeciding { var cohort: NewTabPageSearchBoxExperiment.Cohort? { + if NSApp.runType == .uiTests { + return nil + } // We enroll all new users if AppDelegate.isNewUser { From f4d0569e3dd9de96f35e06d94da5fd52e2775c42 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 08:47:59 -0300 Subject: [PATCH 03/17] Use first address bar --- UITests/FireWindowTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index 311f26bfcf..a1cf5b799f 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -183,7 +183,7 @@ class FireWindowTests: XCTestCase { } private func openSignUpSite() { - let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch XCTAssertTrue( addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), "The address bar text field didn't become available in a reasonable timeframe." From 9c483ed1df704f7b9da80a1690325d24fea6d930 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 11:04:22 -0300 Subject: [PATCH 04/17] Create default Keychain for UI tests --- .github/workflows/ui_tests.yml | 3 +++ fastlane/Fastfile | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/ui_tests.yml b/.github/workflows/ui_tests.yml index 611b105a5a..1dbe9efd92 100644 --- a/.github/workflows/ui_tests.yml +++ b/.github/workflows/ui_tests.yml @@ -64,6 +64,9 @@ jobs: - name: Set up fastlane run: bundle install + + - name: Create Default Keychain + run: bundle exec fastlane create_keychain - name: Sync code signing assets env: diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 61158943f5..dc347f119d 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -341,6 +341,17 @@ platform :mac do ) end + desc 'Creates a new Kechain to use on UI tests' + lane :create_keychain do |options| + create_keychain( + name: "DefaultKeychain", + default_keychain: true, + unlock: true, + timeout: 54000, + lock_when_sleeps: false + ) + end + ################################################# # Helper functions ################################################# From 15b0f96c9cda737b07b3cdff67b9b26bdd018b75 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 11:04:32 -0300 Subject: [PATCH 05/17] Try to fix tests for macOS 13 --- UITests/FireWindowTests.swift | 41 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index a1cf5b799f..49f5202f27 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -116,14 +116,17 @@ class FireWindowTests: XCTestCase { private func signInUsingAutoFill() { let webViewFire = app.webViews.firstMatch - webViewFire.tap() - let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch - emailTextFieldFire.click() + let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) +// let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch +// emailTextFieldFire.click() let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) coordinate.tap() + let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch XCTAssertEqual(emailTextFieldFire.value as? String, "test@duck.com") } @@ -134,10 +137,13 @@ class FireWindowTests: XCTestCase { private func signIn() { let webView = app.webViews.firstMatch - webView.tap() - let emailTextField = webView.textFields["Email"].firstMatch - emailTextField.click() - emailTextField.typeText("test@duck.com") + let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() +// let emailTextField = webView.textFields["Email"].firstMatch +// emailTextField.click() +// emailTextField.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("test@duck.com") app.typeKey("\t", modifierFlags: []) app.typeText("pa$$word") @@ -170,14 +176,19 @@ class FireWindowTests: XCTestCase { private func fillCredentials() { let webView = app.webViews.firstMatch - webView.tap() - let emailTextField = webView.textFields["Email"].firstMatch - emailTextField.click() - emailTextField.typeText("test@duck.com") - - let password = webView.secureTextFields["Password"].firstMatch - password.click() - password.typeText("pa$$word") + let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() +// webView.tap() +// let emailTextField = webView.textFields["Email"].firstMatch +// emailTextField.click() +// emailTextField.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") +// let password = webView.secureTextFields["Password"].firstMatch +// password.click() +// password.typeText("pa$$word") app.typeKey("\t", modifierFlags: []) app.typeText("pa$$word") } From f8f863ced5f2c1d323e9fc5d1116120a59567581 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 11:26:19 -0300 Subject: [PATCH 06/17] Fix lane name --- .github/workflows/ui_tests.yml | 2 +- fastlane/Fastfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ui_tests.yml b/.github/workflows/ui_tests.yml index 1dbe9efd92..fa86f47a25 100644 --- a/.github/workflows/ui_tests.yml +++ b/.github/workflows/ui_tests.yml @@ -66,7 +66,7 @@ jobs: run: bundle install - name: Create Default Keychain - run: bundle exec fastlane create_keychain + run: bundle exec fastlane create_keychain_ui_tests - name: Sync code signing assets env: diff --git a/fastlane/Fastfile b/fastlane/Fastfile index dc347f119d..9617459d6b 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -342,7 +342,7 @@ platform :mac do end desc 'Creates a new Kechain to use on UI tests' - lane :create_keychain do |options| + lane :create_keychain_ui_tests do |options| create_keychain( name: "DefaultKeychain", default_keychain: true, From 844143d0f4b702eb16134b273c530f7e9eedfdfb Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 11:49:38 -0300 Subject: [PATCH 07/17] Add password to keychain --- fastlane/Fastfile | 1 + 1 file changed, 1 insertion(+) diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 9617459d6b..0b92d927dd 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -345,6 +345,7 @@ platform :mac do lane :create_keychain_ui_tests do |options| create_keychain( name: "DefaultKeychain", + password: "default", default_keychain: true, unlock: true, timeout: 54000, From 51dc3c5be63cce493d971f93a2ca1d6e42f80fff Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 14:38:20 -0300 Subject: [PATCH 08/17] Handle web view elements for macOS 13 and 14 differently --- UITests/FireWindowTests.swift | 102 +++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index 49f5202f27..b4653cb0a4 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -115,19 +115,28 @@ class FireWindowTests: XCTestCase { // MARK: - Utilities private func signInUsingAutoFill() { - let webViewFire = app.webViews.firstMatch - let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) - webViewCoordinate.tap() - app.typeKey("\t", modifierFlags: []) -// let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch -// emailTextFieldFire.click() - - let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] - let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) - coordinate.tap() - - let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch - XCTAssertEqual(emailTextFieldFire.value as? String, "test@duck.com") + if #available(macOS 13.0, *) { + let webViewFire = app.webViews.firstMatch + let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) + let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] + let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + coordinate.tap() + + // TODO: How can I check the value? + XCTAssertEqual(app.textFields["Email"].value as? String, "test@duck.com") + } else { + let webViewFire = app.webViews.firstMatch + webViewFire.tap() + let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch + emailTextFieldFire.click() + let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] + let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + coordinate.tap() + + XCTAssertEqual(emailTextFieldFire.value as? String, "test@duck.com") + } } private func saveCredentials() { @@ -136,16 +145,23 @@ class FireWindowTests: XCTestCase { } private func signIn() { - let webView = app.webViews.firstMatch - let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) - webViewCoordinate.tap() -// let emailTextField = webView.textFields["Email"].firstMatch -// emailTextField.click() -// emailTextField.typeText("test@duck.com") - app.typeKey("\t", modifierFlags: []) - app.typeText("test@duck.com") - app.typeKey("\t", modifierFlags: []) - app.typeText("pa$$word") + if #available(macOS 13.0, *) { + let webView = app.webViews.firstMatch + let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) + app.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } else { + let webView = app.webViews.firstMatch + webView.tap() + let emailTextField = webView.textFields["Email"].firstMatch + emailTextField.click() + emailTextField.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } let signInButton = app.webViews.firstMatch.buttons["Sign in"].firstMatch signInButton.click() @@ -175,22 +191,30 @@ class FireWindowTests: XCTestCase { } private func fillCredentials() { - let webView = app.webViews.firstMatch - let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) - webViewCoordinate.tap() -// webView.tap() -// let emailTextField = webView.textFields["Email"].firstMatch -// emailTextField.click() -// emailTextField.typeText("test@duck.com") - app.typeKey("\t", modifierFlags: []) - app.typeText("test@duck.com") - app.typeKey("\t", modifierFlags: []) - app.typeText("pa$$word") -// let password = webView.secureTextFields["Password"].firstMatch -// password.click() -// password.typeText("pa$$word") - app.typeKey("\t", modifierFlags: []) - app.typeText("pa$$word") + if #available(macOS 13.0, *) { + /// On macOS 13 we tap in the webview coordinate and we use tabs to make it work given that it doesn't find web view elements + let webView = app.webViews.firstMatch + let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) + app.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } else { + let webView = app.webViews.firstMatch + webView.tap() + let emailTextField = webView.textFields["Email"].firstMatch + emailTextField.click() + emailTextField.typeText("test@duck.com") + + let password = webView.secureTextFields["Password"].firstMatch + password.click() + password.typeText("pa$$word") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } } private func openSignUpSite() { From 179eb5530ce71cebf40ad0280bbc8069cae5b5f2 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 14:39:03 -0300 Subject: [PATCH 09/17] Disable other tests only to run Fire Window tests --- UITests/UI Tests.xctestplan | 89 ++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/UITests/UI Tests.xctestplan b/UITests/UI Tests.xctestplan index e1b61add22..a9211cb3c0 100644 --- a/UITests/UI Tests.xctestplan +++ b/UITests/UI Tests.xctestplan @@ -36,9 +36,96 @@ "testTargets" : [ { "skippedTests" : [ + "AddressBarKeyboardShortcutsTests", + "AddressBarKeyboardShortcutsTests\/test_addressBar_allTextToTheLeftShouldBeDeletedByCommandDelete()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_allTextToTheRightShouldBeDeletedByCommandForwardDelete()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_beginning_canBeNavigatedToWithCommandLeftArrow()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_caret_canNavigateThroughWordBoundariesUsingOptionLeftArrow()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_caret_canNavigateThroughWordBoundariesUsingOptionRightArrow()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_commandZShouldUndoLastAction()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_end_canBeNavigatedToWithCommandRightArrow()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestLeftHandCharacterShouldBeDeletedByFnDelete()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestLeftHandWordWithinBoundariesShouldBeDeletedByOptDelete()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestRightHandCharacterShouldBeDeletedByFnForwardDelete()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestRightHandWordWithinBoundariesShouldBeDeletedByOptForwardDelete()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_shiftCommandZShouldRedoLastUndoneAction()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_url_canBeSelected()", + "AddressBarKeyboardShortcutsTests\/test_addressBar_url_word_canBeSelectedByDoubleClick()", + "AutocompleteTests", + "AutocompleteTests\/test_suggestions_showsTypedTitleOfBookmarkedPageAsBookmark()", + "AutocompleteTests\/test_suggestions_showsTypedTitleOfHistoryPageAsHistory()", + "AutocompleteTests\/test_suggestions_showsTypedTitleOfWebsiteNotInBookmarksOrHistoryAsWebsite()", + "BookmarkSearchTests", + "BookmarkSearchTests\/testDragAndDropToReorderIsNotPossibleWhenInSearchOnBookmarksManager()", + "BookmarkSearchTests\/testDragAndDropToReorderIsNotPossibleWhenInSearchOnBookmarksPanel()", + "BookmarkSearchTests\/testEmptyStateWhenSearchingInManager()", + "BookmarkSearchTests\/testEmptyStateWhenSearchingInPanel()", + "BookmarkSearchTests\/testFilteredResultsInManager()", + "BookmarkSearchTests\/testFilteredResultsInPanel()", + "BookmarkSearchTests\/testSearchActionIsDisabledOnBookmarksPanelWhenUserHasNoBookmarks()", + "BookmarkSearchTests\/testShowInFolderFunctionalityOnBookmarksManager()", + "BookmarkSearchTests\/testShowInFolderFunctionalityOnBookmarksPanel()", + "BookmarkSortTests", + "BookmarkSortTests\/testManualSortWorksAsExpectedOnBookmarksManager()", + "BookmarkSortTests\/testManualSortWorksAsExpectedOnBookmarksPanel()", + "BookmarkSortTests\/testNameAscendingSortWorksAsExpectedOnBookmarksManager()", + "BookmarkSortTests\/testNameAscendingSortWorksAsExpectedOnBookmarksPanel()", + "BookmarkSortTests\/testNameDescendingSortWorksAsExpectedOnBookmarksManager()", + "BookmarkSortTests\/testNameDescendingSortWorksAsExpectedOnBookmarksPanel()", + "BookmarkSortTests\/testThatSortIsPersistedThroughBrowserRestarts()", + "BookmarkSortTests\/testWhenChangingSortingInTheManagerIsReflectedInThePanel()", + "BookmarkSortTests\/testWhenChangingSortingInThePanelIsReflectedInTheManager()", + "BookmarkSortTests\/testWhenNoBookmarksThenSortIsDisabledOnTheManager()", + "BookmarkSortTests\/testWhenNoBookmarksThenSortIsDisabledOnThePanel()", + "BookmarksAndFavoritesTests", + "BookmarksAndFavoritesTests\/test_bookmark_canBeRemovedFromBookmarksBarViaRightClick()", + "BookmarksAndFavoritesTests\/test_bookmark_canBeRemovedFromBookmarksTabViaHoverAndContextMenu()", + "BookmarksAndFavoritesTests\/test_bookmark_canBeRemovedViaAddressBarIconClick()", + "BookmarksAndFavoritesTests\/test_bookmarksTab_canBeViewedViaMenuItemManageBookmarks()", "BookmarksAndFavoritesTests\/test_bookmarks_canBeAddedTo_byClickingBookmarksButtonInAddressBar()", + "BookmarksAndFavoritesTests\/test_bookmarks_canBeAddedTo_withContextClickBookmarkThisPage()", + "BookmarksAndFavoritesTests\/test_bookmarks_canBeAddedTo_withSettingsItemBookmarkThisPage()", + "BookmarksAndFavoritesTests\/test_bookmarks_canBeViewedInAddressBarBookmarkDialog()", + "BookmarksAndFavoritesTests\/test_bookmarks_canBeViewedInBookmarkMenuItem()", + "BookmarksAndFavoritesTests\/test_favorites_appearInNewTabFavoritesGrid()", + "BookmarksAndFavoritesTests\/test_favorites_appearWithTheCorrectIndicatorInBookmarksTab()", + "BookmarksAndFavoritesTests\/test_favorites_canBeAddedToFromManageBookmarksView()", "BookmarksAndFavoritesTests\/test_favorites_canBeAddedTo_byClickingAddFavoriteInAddBookmarkPopover()", - "PermissionsTests" + "BookmarksAndFavoritesTests\/test_favorites_canBeAddedTo_byClickingFavoriteThisPageMenuBarItem()", + "BookmarksAndFavoritesTests\/test_favorites_canBeManuallyAddedTo_byClickingAddFavoriteInNewTabPage()", + "BookmarksAndFavoritesTests\/test_favorites_canBeRemovedFromAddressBarBookmarkDialog()", + "BookmarksAndFavoritesTests\/test_favorites_canBeRemovedFromManageBookmarks()", + "BookmarksAndFavoritesTests\/test_favorites_canBeRemovedFromNewTabViaContextClick()", + "BookmarksBarTests", + "BookmarksBarTests\/test_bookmarksBar_whenShowBookmarksBarAlwaysIsSelected_alwaysDynamicallyAppearsOnWindow()", + "BookmarksBarTests\/test_bookmarksBar_whenShowBookmarksBarIsUnchecked_isNeverShownInWindowsAndTabs()", + "BookmarksBarTests\/test_bookmarksBar_whenShowBookmarksNewTabOnlyIsSelected_onlyAppearsOnANewTabUntilASiteIsLoaded()", + "BrowsingHistoryTests", + "BrowsingHistoryTests\/test_history_showsVisitedSiteAfterClosingAndReopeningWindow()", + "BrowsingHistoryTests\/test_recentlyVisited_showsLastVisitedSite()", + "BrowsingHistoryTests\/test_reopenLastClosedWindowMenuItem_canReopenTabsOfLastClosedWindow()", + "FindInPageTests", + "FindInPageTests\/test_findInPage_canBeClosedWithEscape()", + "FindInPageTests\/test_findInPage_canBeClosedWithHideFindMenuItem()", + "FindInPageTests\/test_findInPage_canBeClosedWithShiftCommandF()", + "FindInPageTests\/test_findInPage_canBeOpenedWithKeyCommand()", + "FindInPageTests\/test_findInPage_canBeOpenedWithMenuBarItem()", + "FindInPageTests\/test_findInPage_canBeOpenedWithMoreOptionsMenuItem()", + "FindInPageTests\/test_findInPage_showsCorrectNumberOfOccurrences()", + "FindInPageTests\/test_findInPage_showsFocusAndOccurrenceHighlighting()", + "FindInPageTests\/test_findNext_commandGGoesToNextOccurrence()", + "FindInPageTests\/test_findNext_menuItemGoesToNextOccurrence()", + "FindInPageTests\/test_findNext_nextArrowGoesToNextOccurrence()", + "OnboardingUITests", + "OnboardingUITests\/testOnboardingToBrowsing()", + "PermissionsTests", + "PinnedTabsTests", + "PinnedTabsTests\/testPinnedTabsFunctionality()", + "StateRestorationTests", + "StateRestorationTests\/test_tabStateAtRelaunch_shouldContainNoSitesVisitedInPreviousSession_whenReopenAllWindowsFromLastSessionIsUnset()", + "StateRestorationTests\/test_tabStateAtRelaunch_shouldContainTwoSitesVisitedInPreviousSession_whenReopenAllWindowsFromLastSessionIsSet()", + "TabBarTests", + "TabBarTests\/testWhenClickingAddTab_ThenTabsOpen()" ], "target" : { "containerPath" : "container:DuckDuckGo.xcodeproj", From 599987fa4b66859910b61a9335b10103933cd624 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 15:40:12 -0300 Subject: [PATCH 10/17] Address more feedback on macOS 13 --- UITests/FireWindowTests.swift | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index b4653cb0a4..1e8b3c6986 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -94,6 +94,7 @@ class FireWindowTests: XCTestCase { func testFireWindowsSignInDoesNotShowCredentialsPopup() { openFireWindow() + app.openNewTab() openSignUpSite() fillCredentials() finishSignUp() @@ -102,12 +103,14 @@ class FireWindowTests: XCTestCase { func testCrendentialsAreAutoFilledInFireWindows() { openNormalWindow() + app.openNewTab() openLoginSite() signIn() saveCredentials() /// Here we start the same flow but in the fire window, but we use the autofill credentials saved in the step before. openFireWindow() + app.openNewTab() openLoginSite() signInUsingAutoFill() } @@ -115,17 +118,18 @@ class FireWindowTests: XCTestCase { // MARK: - Utilities private func signInUsingAutoFill() { - if #available(macOS 13.0, *) { + if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { let webViewFire = app.webViews.firstMatch let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) webViewCoordinate.tap() app.typeKey("\t", modifierFlags: []) + sleep(1) let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) coordinate.tap() // TODO: How can I check the value? - XCTAssertEqual(app.textFields["Email"].value as? String, "test@duck.com") + XCTAssertTrue(webViewFire.staticTexts["test@duck.com"].exists) } else { let webViewFire = app.webViews.firstMatch webViewFire.tap() @@ -145,7 +149,7 @@ class FireWindowTests: XCTestCase { } private func signIn() { - if #available(macOS 13.0, *) { + if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { let webView = app.webViews.firstMatch let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) webViewCoordinate.tap() @@ -191,7 +195,7 @@ class FireWindowTests: XCTestCase { } private func fillCredentials() { - if #available(macOS 13.0, *) { + if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { /// On macOS 13 we tap in the webview coordinate and we use tabs to make it work given that it doesn't find web view elements let webView = app.webViews.firstMatch let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) From 5c1ca5dc2213d2fcce261f1b978ad03819aee08b Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Tue, 12 Nov 2024 16:28:09 -0300 Subject: [PATCH 11/17] Hover outside tab to prevent tab preview --- UITests/FireWindowTests.swift | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index 1e8b3c6986..4ab4e232e6 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -94,7 +94,7 @@ class FireWindowTests: XCTestCase { func testFireWindowsSignInDoesNotShowCredentialsPopup() { openFireWindow() - app.openNewTab() + hoverMouseOutsideTabSoPreviewIsNotShown() openSignUpSite() fillCredentials() finishSignUp() @@ -103,20 +103,26 @@ class FireWindowTests: XCTestCase { func testCrendentialsAreAutoFilledInFireWindows() { openNormalWindow() - app.openNewTab() + hoverMouseOutsideTabSoPreviewIsNotShown() openLoginSite() signIn() saveCredentials() /// Here we start the same flow but in the fire window, but we use the autofill credentials saved in the step before. openFireWindow() - app.openNewTab() + hoverMouseOutsideTabSoPreviewIsNotShown() openLoginSite() signInUsingAutoFill() } // MARK: - Utilities + private func hoverMouseOutsideTabSoPreviewIsNotShown() { + let window = app.windows.firstMatch + let coordinate = window.coordinate(withNormalizedOffset: CGVector(dx: -100, dy: -100)) + coordinate.hover() + } + private func signInUsingAutoFill() { if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { let webViewFire = app.webViews.firstMatch @@ -128,8 +134,11 @@ class FireWindowTests: XCTestCase { let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) coordinate.tap() - // TODO: How can I check the value? - XCTAssertTrue(webViewFire.staticTexts["test@duck.com"].exists) + /// On macOS there are some issues when accessing web view elements so we do not check the value of the email text field. + /// If we can access the `test@duck.com privacy-test-pages.site` button means that auto fill is working correctly in the fire window. + /// Checking that the email is being filled correctly is more an autofill test that fire window, so we are okay to skip it. + /// + /// We do run this test on macOS 14 and above. } else { let webViewFire = app.webViews.firstMatch webViewFire.tap() From 4bd8aeaf49f6bb8ab80e25ce86d2b47909cbabce Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 13 Nov 2024 09:51:17 -0300 Subject: [PATCH 12/17] Do some cleaning --- UITests/FireWindowTests.swift | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index 4ab4e232e6..1ce9914eb0 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -124,7 +124,7 @@ class FireWindowTests: XCTestCase { } private func signInUsingAutoFill() { - if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { + if areTestsRunningOnMacos13() { let webViewFire = app.webViews.firstMatch let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) webViewCoordinate.tap() @@ -158,7 +158,7 @@ class FireWindowTests: XCTestCase { } private func signIn() { - if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { + if areTestsRunningOnMacos13() { let webView = app.webViews.firstMatch let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) webViewCoordinate.tap() @@ -204,7 +204,7 @@ class FireWindowTests: XCTestCase { } private func fillCredentials() { - if ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 { + if areTestsRunningOnMacos13() { /// On macOS 13 we tap in the webview coordinate and we use tabs to make it work given that it doesn't find web view elements let webView = app.webViews.firstMatch let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) @@ -311,13 +311,6 @@ class FireWindowTests: XCTestCase { openSite(pageTitle: "Page #6") } - private func cleanAllHistory() { - app.typeKey(.delete, modifierFlags: [.command, .shift]) - let clearButton = app.buttons["Clear"].firstMatch - XCTAssertTrue(clearButton.exists, "Clear button should exist") - clearButton.tap() - } - private func assertSiteIsNotShowingInNormalWindowHistory() { let siteMenuItemInHistory = app.menuItems["Some site"] XCTAssertFalse(siteMenuItemInHistory.exists, "Menu item should not exist because it was not stored in history.") @@ -344,4 +337,8 @@ class FireWindowTests: XCTestCase { "Visited site didn't load with the expected title in a reasonable timeframe." ) } + + private func areTestsRunningOnMacos13() -> Bool { + return ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 + } } From 8ef4794f00d49656f94b208e78e95632fb947087 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 13 Nov 2024 09:51:36 -0300 Subject: [PATCH 13/17] Revert "Disable other tests only to run Fire Window tests" This reverts commit 179eb5530ce71cebf40ad0280bbc8069cae5b5f2. --- UITests/UI Tests.xctestplan | 89 +------------------------------------ 1 file changed, 1 insertion(+), 88 deletions(-) diff --git a/UITests/UI Tests.xctestplan b/UITests/UI Tests.xctestplan index a9211cb3c0..e1b61add22 100644 --- a/UITests/UI Tests.xctestplan +++ b/UITests/UI Tests.xctestplan @@ -36,96 +36,9 @@ "testTargets" : [ { "skippedTests" : [ - "AddressBarKeyboardShortcutsTests", - "AddressBarKeyboardShortcutsTests\/test_addressBar_allTextToTheLeftShouldBeDeletedByCommandDelete()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_allTextToTheRightShouldBeDeletedByCommandForwardDelete()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_beginning_canBeNavigatedToWithCommandLeftArrow()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_caret_canNavigateThroughWordBoundariesUsingOptionLeftArrow()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_caret_canNavigateThroughWordBoundariesUsingOptionRightArrow()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_commandZShouldUndoLastAction()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_end_canBeNavigatedToWithCommandRightArrow()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestLeftHandCharacterShouldBeDeletedByFnDelete()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestLeftHandWordWithinBoundariesShouldBeDeletedByOptDelete()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestRightHandCharacterShouldBeDeletedByFnForwardDelete()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_nearestRightHandWordWithinBoundariesShouldBeDeletedByOptForwardDelete()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_shiftCommandZShouldRedoLastUndoneAction()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_url_canBeSelected()", - "AddressBarKeyboardShortcutsTests\/test_addressBar_url_word_canBeSelectedByDoubleClick()", - "AutocompleteTests", - "AutocompleteTests\/test_suggestions_showsTypedTitleOfBookmarkedPageAsBookmark()", - "AutocompleteTests\/test_suggestions_showsTypedTitleOfHistoryPageAsHistory()", - "AutocompleteTests\/test_suggestions_showsTypedTitleOfWebsiteNotInBookmarksOrHistoryAsWebsite()", - "BookmarkSearchTests", - "BookmarkSearchTests\/testDragAndDropToReorderIsNotPossibleWhenInSearchOnBookmarksManager()", - "BookmarkSearchTests\/testDragAndDropToReorderIsNotPossibleWhenInSearchOnBookmarksPanel()", - "BookmarkSearchTests\/testEmptyStateWhenSearchingInManager()", - "BookmarkSearchTests\/testEmptyStateWhenSearchingInPanel()", - "BookmarkSearchTests\/testFilteredResultsInManager()", - "BookmarkSearchTests\/testFilteredResultsInPanel()", - "BookmarkSearchTests\/testSearchActionIsDisabledOnBookmarksPanelWhenUserHasNoBookmarks()", - "BookmarkSearchTests\/testShowInFolderFunctionalityOnBookmarksManager()", - "BookmarkSearchTests\/testShowInFolderFunctionalityOnBookmarksPanel()", - "BookmarkSortTests", - "BookmarkSortTests\/testManualSortWorksAsExpectedOnBookmarksManager()", - "BookmarkSortTests\/testManualSortWorksAsExpectedOnBookmarksPanel()", - "BookmarkSortTests\/testNameAscendingSortWorksAsExpectedOnBookmarksManager()", - "BookmarkSortTests\/testNameAscendingSortWorksAsExpectedOnBookmarksPanel()", - "BookmarkSortTests\/testNameDescendingSortWorksAsExpectedOnBookmarksManager()", - "BookmarkSortTests\/testNameDescendingSortWorksAsExpectedOnBookmarksPanel()", - "BookmarkSortTests\/testThatSortIsPersistedThroughBrowserRestarts()", - "BookmarkSortTests\/testWhenChangingSortingInTheManagerIsReflectedInThePanel()", - "BookmarkSortTests\/testWhenChangingSortingInThePanelIsReflectedInTheManager()", - "BookmarkSortTests\/testWhenNoBookmarksThenSortIsDisabledOnTheManager()", - "BookmarkSortTests\/testWhenNoBookmarksThenSortIsDisabledOnThePanel()", - "BookmarksAndFavoritesTests", - "BookmarksAndFavoritesTests\/test_bookmark_canBeRemovedFromBookmarksBarViaRightClick()", - "BookmarksAndFavoritesTests\/test_bookmark_canBeRemovedFromBookmarksTabViaHoverAndContextMenu()", - "BookmarksAndFavoritesTests\/test_bookmark_canBeRemovedViaAddressBarIconClick()", - "BookmarksAndFavoritesTests\/test_bookmarksTab_canBeViewedViaMenuItemManageBookmarks()", "BookmarksAndFavoritesTests\/test_bookmarks_canBeAddedTo_byClickingBookmarksButtonInAddressBar()", - "BookmarksAndFavoritesTests\/test_bookmarks_canBeAddedTo_withContextClickBookmarkThisPage()", - "BookmarksAndFavoritesTests\/test_bookmarks_canBeAddedTo_withSettingsItemBookmarkThisPage()", - "BookmarksAndFavoritesTests\/test_bookmarks_canBeViewedInAddressBarBookmarkDialog()", - "BookmarksAndFavoritesTests\/test_bookmarks_canBeViewedInBookmarkMenuItem()", - "BookmarksAndFavoritesTests\/test_favorites_appearInNewTabFavoritesGrid()", - "BookmarksAndFavoritesTests\/test_favorites_appearWithTheCorrectIndicatorInBookmarksTab()", - "BookmarksAndFavoritesTests\/test_favorites_canBeAddedToFromManageBookmarksView()", "BookmarksAndFavoritesTests\/test_favorites_canBeAddedTo_byClickingAddFavoriteInAddBookmarkPopover()", - "BookmarksAndFavoritesTests\/test_favorites_canBeAddedTo_byClickingFavoriteThisPageMenuBarItem()", - "BookmarksAndFavoritesTests\/test_favorites_canBeManuallyAddedTo_byClickingAddFavoriteInNewTabPage()", - "BookmarksAndFavoritesTests\/test_favorites_canBeRemovedFromAddressBarBookmarkDialog()", - "BookmarksAndFavoritesTests\/test_favorites_canBeRemovedFromManageBookmarks()", - "BookmarksAndFavoritesTests\/test_favorites_canBeRemovedFromNewTabViaContextClick()", - "BookmarksBarTests", - "BookmarksBarTests\/test_bookmarksBar_whenShowBookmarksBarAlwaysIsSelected_alwaysDynamicallyAppearsOnWindow()", - "BookmarksBarTests\/test_bookmarksBar_whenShowBookmarksBarIsUnchecked_isNeverShownInWindowsAndTabs()", - "BookmarksBarTests\/test_bookmarksBar_whenShowBookmarksNewTabOnlyIsSelected_onlyAppearsOnANewTabUntilASiteIsLoaded()", - "BrowsingHistoryTests", - "BrowsingHistoryTests\/test_history_showsVisitedSiteAfterClosingAndReopeningWindow()", - "BrowsingHistoryTests\/test_recentlyVisited_showsLastVisitedSite()", - "BrowsingHistoryTests\/test_reopenLastClosedWindowMenuItem_canReopenTabsOfLastClosedWindow()", - "FindInPageTests", - "FindInPageTests\/test_findInPage_canBeClosedWithEscape()", - "FindInPageTests\/test_findInPage_canBeClosedWithHideFindMenuItem()", - "FindInPageTests\/test_findInPage_canBeClosedWithShiftCommandF()", - "FindInPageTests\/test_findInPage_canBeOpenedWithKeyCommand()", - "FindInPageTests\/test_findInPage_canBeOpenedWithMenuBarItem()", - "FindInPageTests\/test_findInPage_canBeOpenedWithMoreOptionsMenuItem()", - "FindInPageTests\/test_findInPage_showsCorrectNumberOfOccurrences()", - "FindInPageTests\/test_findInPage_showsFocusAndOccurrenceHighlighting()", - "FindInPageTests\/test_findNext_commandGGoesToNextOccurrence()", - "FindInPageTests\/test_findNext_menuItemGoesToNextOccurrence()", - "FindInPageTests\/test_findNext_nextArrowGoesToNextOccurrence()", - "OnboardingUITests", - "OnboardingUITests\/testOnboardingToBrowsing()", - "PermissionsTests", - "PinnedTabsTests", - "PinnedTabsTests\/testPinnedTabsFunctionality()", - "StateRestorationTests", - "StateRestorationTests\/test_tabStateAtRelaunch_shouldContainNoSitesVisitedInPreviousSession_whenReopenAllWindowsFromLastSessionIsUnset()", - "StateRestorationTests\/test_tabStateAtRelaunch_shouldContainTwoSitesVisitedInPreviousSession_whenReopenAllWindowsFromLastSessionIsSet()", - "TabBarTests", - "TabBarTests\/testWhenClickingAddTab_ThenTabsOpen()" + "PermissionsTests" ], "target" : { "containerPath" : "container:DuckDuckGo.xcodeproj", From 482cc118f7beff1add3f4f701c64f70b32160959 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 13 Nov 2024 10:04:04 -0300 Subject: [PATCH 14/17] Revert "Do not run NTP Search Box experiment" This reverts commit 2562dc6b5b7f1f6e90f57ff71a1007ea5e4078b2. --- DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift b/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift index de1fbea87b..eb49a31193 100644 --- a/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift +++ b/DuckDuckGo/HomePage/Model/NewTabPageSearchBoxExperiment.swift @@ -75,9 +75,6 @@ protocol NewTabPageSearchBoxExperimentCohortDeciding { struct DefaultNewTabPageSearchBoxExperimentCohortDecider: NewTabPageSearchBoxExperimentCohortDeciding { var cohort: NewTabPageSearchBoxExperiment.Cohort? { - if NSApp.runType == .uiTests { - return nil - } // We enroll all new users if AppDelegate.isNewUser { From 58440ac2d75b47dd8afd7050772755d9ee296dc1 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 13 Nov 2024 10:16:50 -0300 Subject: [PATCH 15/17] Fix SwiftLint --- UITests/FireWindowTests.swift | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index 1ce9914eb0..62659e2734 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -231,7 +231,7 @@ class FireWindowTests: XCTestCase { } private func openSignUpSite() { - let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch XCTAssertTrue( addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), "The address bar text field didn't become available in a reasonable timeframe." @@ -260,11 +260,9 @@ class FireWindowTests: XCTestCase { XCTAssertEqual(result, .completed, "No static texts were found in the app") // After confirming static texts are available, iterate through them - for staticText in app.staticTexts.allElementsBoundByIndex { - if staticText.exists { - XCTAssertFalse(staticText.label.contains("Page #1"), "Unwanted string found in static text: \(staticText.label)") - XCTAssertFalse(staticText.label.contains("Page #2"), "Unwanted string found in static text: \(staticText.label)") - } + for staticText in app.staticTexts.allElementsBoundByIndex where staticText.exists { + XCTAssertFalse(staticText.label.contains("Page #1"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #2"), "Unwanted string found in static text: \(staticText.label)") } } @@ -285,12 +283,10 @@ class FireWindowTests: XCTestCase { XCTAssertEqual(result, .completed, "No static texts were found in the app") // After confirming static texts are available, iterate through them - for staticText in app.staticTexts.allElementsBoundByIndex { - if staticText.exists { - XCTAssertFalse(staticText.label.contains("Page #4"), "Unwanted string found in static text: \(staticText.label)") - XCTAssertFalse(staticText.label.contains("Page #5"), "Unwanted string found in static text: \(staticText.label)") - XCTAssertFalse(staticText.label.contains("Page #6"), "Unwanted string found in static text: \(staticText.label)") - } + for staticText in app.staticTexts.allElementsBoundByIndex where staticText.exists { + XCTAssertFalse(staticText.label.contains("Page #4"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #5"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #6"), "Unwanted string found in static text: \(staticText.label)") } } From a9d0c1c20e306e20c0a7a0dbffe9a14d647d2879 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Wed, 13 Nov 2024 20:19:45 +0100 Subject: [PATCH 16/17] Add check_retries: true to UI tests workflows reports --- .github/workflows/sync_end_to_end.yml | 1 + .github/workflows/ui_tests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/sync_end_to_end.yml b/.github/workflows/sync_end_to_end.yml index 91308789b7..d6820299e7 100644 --- a/.github/workflows/sync_end_to_end.yml +++ b/.github/workflows/sync_end_to_end.yml @@ -153,6 +153,7 @@ jobs: with: check_name: "Test Report ${{ matrix.runner }}" report_paths: ui-tests.xml + check_retries: true - name: Upload logs when workflow failed uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ui_tests.yml b/.github/workflows/ui_tests.yml index fa86f47a25..c041fec09e 100644 --- a/.github/workflows/ui_tests.yml +++ b/.github/workflows/ui_tests.yml @@ -144,6 +144,7 @@ jobs: with: check_name: "Test Report ${{ matrix.runner }}" report_paths: ui-tests.xml + check_retries: true - name: Upload logs when workflow failed uses: actions/upload-artifact@v4 From 7f61ec279c4016715589cf54eadf730e80c95753 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 13 Nov 2024 16:43:15 -0300 Subject: [PATCH 17/17] Fix comment --- UITests/FireWindowTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift index 62659e2734..bedce50dff 100644 --- a/UITests/FireWindowTests.swift +++ b/UITests/FireWindowTests.swift @@ -134,7 +134,7 @@ class FireWindowTests: XCTestCase { let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) coordinate.tap() - /// On macOS there are some issues when accessing web view elements so we do not check the value of the email text field. + /// On macOS 13 there are some issues when accessing web view elements so we do not check the value of the email text field. /// If we can access the `test@duck.com privacy-test-pages.site` button means that auto fill is working correctly in the fire window. /// Checking that the email is being filled correctly is more an autofill test that fire window, so we are okay to skip it. ///