From 7fdf739d346fc201594597030371d850eb950aec Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Wed, 13 Nov 2024 17:30:58 -0300 Subject: [PATCH] Implement Fire Window UI tests (#3544) Task/Issue URL: https://app.asana.com/0/1204006570077678/1205717021705381/f Tech Design URL: CC: **Description**: Implement Fire Window UI tests **Steps to test this PR**: Check that workflow passes on macOS 13 and 14: https://github.com/duckduckgo/macos-browser/actions/runs/11817794843 **Definition of Done**: * [x] Does this PR satisfy our [Definition of Done](https://app.asana.com/0/1202500774821704/1207634633537039/f)? --- ###### Internal references: [Pull Request Review Checklist](https://app.asana.com/0/1202500774821704/1203764234894239/f) [Software Engineering Expectations](https://app.asana.com/0/59792373528535/199064865822552) [Technical Design Template](https://app.asana.com/0/59792373528535/184709971311943) [Pull Request Documentation](https://app.asana.com/0/1202500774821704/1204012835277482/f) --------- Co-authored-by: Dominik Kapusta --- .github/workflows/sync_end_to_end.yml | 1 + .github/workflows/ui_tests.yml | 4 + DuckDuckGo.xcodeproj/project.pbxproj | 4 + UITests/FireWindowTests.swift | 340 ++++++++++++++++++++++++++ fastlane/Fastfile | 12 + 5 files changed, 361 insertions(+) create mode 100644 UITests/FireWindowTests.swift diff --git a/.github/workflows/sync_end_to_end.yml b/.github/workflows/sync_end_to_end.yml index 91308789b7..d6820299e7 100644 --- a/.github/workflows/sync_end_to_end.yml +++ b/.github/workflows/sync_end_to_end.yml @@ -153,6 +153,7 @@ jobs: with: check_name: "Test Report ${{ matrix.runner }}" report_paths: ui-tests.xml + check_retries: true - name: Upload logs when workflow failed uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ui_tests.yml b/.github/workflows/ui_tests.yml index 611b105a5a..c041fec09e 100644 --- a/.github/workflows/ui_tests.yml +++ b/.github/workflows/ui_tests.yml @@ -64,6 +64,9 @@ jobs: - name: Set up fastlane run: bundle install + + - name: Create Default Keychain + run: bundle exec fastlane create_keychain_ui_tests - name: Sync code signing assets env: @@ -141,6 +144,7 @@ jobs: with: check_name: "Test Report ${{ matrix.runner }}" report_paths: ui-tests.xml + check_retries: true - name: Upload logs when workflow failed uses: actions/upload-artifact@v4 diff --git a/DuckDuckGo.xcodeproj/project.pbxproj b/DuckDuckGo.xcodeproj/project.pbxproj index 89b3f30f74..2989d7f8b3 100644 --- a/DuckDuckGo.xcodeproj/project.pbxproj +++ b/DuckDuckGo.xcodeproj/project.pbxproj @@ -2809,6 +2809,7 @@ BB470EBC2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB470EBA2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift */; }; BB5789722B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5789712B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift */; }; BB5F46A32C8751F6005F72DF /* BookmarkSortTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5F46A22C8751F6005F72DF /* BookmarkSortTests.swift */; }; + BB731F312CDBA6360023D2E4 /* FireWindowTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */; }; BB7B5F982C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */; }; BB7B5F992C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */; }; BBB881882C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBB881872C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift */; }; @@ -4760,6 +4761,7 @@ BB470EBA2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkManagementDetailViewModel.swift; sourceTree = ""; }; BB5789712B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataBrokerProtectionSubscriptionEventHandler.swift; sourceTree = ""; }; BB5F46A22C8751F6005F72DF /* BookmarkSortTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkSortTests.swift; sourceTree = ""; }; + BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FireWindowTests.swift; sourceTree = ""; }; BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarksSearchAndSortMetrics.swift; sourceTree = ""; }; BBB881872C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkListTreeControllerSearchDataSource.swift; sourceTree = ""; }; BBBB653F2C77BB9400E69AC6 /* BookmarkSearchTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkSearchTests.swift; sourceTree = ""; }; @@ -6972,6 +6974,7 @@ 7B4CE8DB26F02108009134B1 /* UITests */ = { isa = PBXGroup; children = ( + BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */, 376E708D2BD686260082B7EB /* UI Tests.xctestplan */, EEBCE6802BA444FA00B9DF00 /* Common */, EEC7BE2D2BC6C09400F86835 /* AddressBarKeyboardShortcutsTests.swift */, @@ -12628,6 +12631,7 @@ EEC7BE2E2BC6C09500F86835 /* AddressBarKeyboardShortcutsTests.swift in Sources */, EE54F7B32BBFEA49006218DB /* BookmarksAndFavoritesTests.swift in Sources */, EE02D4222BB4611A00DBE6B3 /* TestsURLExtension.swift in Sources */, + BB731F312CDBA6360023D2E4 /* FireWindowTests.swift in Sources */, EE42CBCC2BC8004700AD411C /* PermissionsTests.swift in Sources */, 7B4CE8E726F02135009134B1 /* TabBarTests.swift in Sources */, EEBCE6832BA463DD00B9DF00 /* NSImageExtensions.swift in Sources */, diff --git a/UITests/FireWindowTests.swift b/UITests/FireWindowTests.swift new file mode 100644 index 0000000000..bedce50dff --- /dev/null +++ b/UITests/FireWindowTests.swift @@ -0,0 +1,340 @@ +// +// FireWindowTests.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest + +class FireWindowTests: XCTestCase { + private var app: XCUIApplication! + private var settingsGeneralButton: XCUIElement! + private var reopenAllWindowsFromLastSessionPreference: XCUIElement! + + override class func setUp() { + UITests.firstRun() + } + + override func setUpWithError() throws { + continueAfterFailure = false + app = XCUIApplication() + app.launchEnvironment["UITEST_MODE"] = "1" + + settingsGeneralButton = app.buttons["PreferencesSidebar.generalButton"] + reopenAllWindowsFromLastSessionPreference = app.radioButtons["PreferencesGeneralView.stateRestorePicker.reopenAllWindowsFromLastSession"] + + app.launch() + app.typeKey("w", modifierFlags: [.command, .option, .shift]) // Let's enforce a single window + } + + func testFireWindowDoesNotStoreHistory() { + openFireWindow() + openSite(pageTitle: "Some site") + openNormalWindow() + assertSiteIsNotShowingInNormalWindowHistory() + } + + func testFireWindowStateIsNotSavedAfterRestart() { + openNormalWindow() + app.typeKey(",", modifierFlags: [.command]) // Open settings + settingsGeneralButton.click(forDuration: 0.5, thenDragTo: settingsGeneralButton) + reopenAllWindowsFromLastSessionPreference.clickAfterExistenceTestSucceeds() + + openThreeSitesOnNormalWindow() + openFireWindow() + openThreeSitesOnFireWindow() + + app.terminate() + app.launch() + + assertSitesOpenedInNormalWindowAreRestored() + assertSitesOpenedOnFireWindowAreNotRestored() + } + + func testFireWindowDoNotShowPinnedTabs() { + openNormalWindow() + openSite(pageTitle: "Page #1") + app.menuItems["Pin Tab"].tap() + + app.openNewTab() + openSite(pageTitle: "Page #2") + app.menuItems["Pin Tab"].tap() + + openFireWindow() + assertFireWindowDoesNotHavePinnedTabs() + } + + func testFireWindowTabsCannotBeDragged() { + openFireWindow() + openSite(pageTitle: "Page #1") + + app.openNewTab() + openSite(pageTitle: "Page #2") + + dragFirstTabOutsideOfFireWindow() + + /// Assert that Page #1 is still on the fire window after the drag + app.typeKey("]", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #2"].exists) + app.typeKey("[", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #1"].exists) + } + + func testFireWindowsSignInDoesNotShowCredentialsPopup() { + openFireWindow() + hoverMouseOutsideTabSoPreviewIsNotShown() + openSignUpSite() + fillCredentials() + finishSignUp() + assertSavePasswordPopupIsNotShown() + } + + func testCrendentialsAreAutoFilledInFireWindows() { + openNormalWindow() + hoverMouseOutsideTabSoPreviewIsNotShown() + openLoginSite() + signIn() + saveCredentials() + + /// Here we start the same flow but in the fire window, but we use the autofill credentials saved in the step before. + openFireWindow() + hoverMouseOutsideTabSoPreviewIsNotShown() + openLoginSite() + signInUsingAutoFill() + } + + // MARK: - Utilities + + private func hoverMouseOutsideTabSoPreviewIsNotShown() { + let window = app.windows.firstMatch + let coordinate = window.coordinate(withNormalizedOffset: CGVector(dx: -100, dy: -100)) + coordinate.hover() + } + + private func signInUsingAutoFill() { + if areTestsRunningOnMacos13() { + let webViewFire = app.webViews.firstMatch + let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) + sleep(1) + let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] + let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + coordinate.tap() + + /// On macOS 13 there are some issues when accessing web view elements so we do not check the value of the email text field. + /// If we can access the `test@duck.com privacy-test-pages.site` button means that auto fill is working correctly in the fire window. + /// Checking that the email is being filled correctly is more an autofill test that fire window, so we are okay to skip it. + /// + /// We do run this test on macOS 14 and above. + } else { + let webViewFire = app.webViews.firstMatch + webViewFire.tap() + let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch + emailTextFieldFire.click() + let autoFillPopup = webViewFire.buttons["test@duck.com privacy-test-pages.site"] + let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + coordinate.tap() + + XCTAssertEqual(emailTextFieldFire.value as? String, "test@duck.com") + } + } + + private func saveCredentials() { + let saveButton = app.buttons["Save"] + saveButton.tap() + } + + private func signIn() { + if areTestsRunningOnMacos13() { + let webView = app.webViews.firstMatch + let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) + app.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } else { + let webView = app.webViews.firstMatch + webView.tap() + let emailTextField = webView.textFields["Email"].firstMatch + emailTextField.click() + emailTextField.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } + + let signInButton = app.webViews.firstMatch.buttons["Sign in"].firstMatch + signInButton.click() + } + + private func openLoginSite() { + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + XCTAssertTrue( + addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), + "The address bar text field didn't become available in a reasonable timeframe." + ) + addressBarTextField.typeURL(URL(string: "https://privacy-test-pages.site/autofill/autoprompt/1-standard-login-form.html")!) + XCTAssertTrue( + app.windows.firstMatch.webViews["Autofill autoprompt for signin forms"].waitForExistence(timeout: UITests.Timeouts.elementExistence), + "Visited site didn't load with the expected title in a reasonable timeframe." + ) + } + + private func assertSavePasswordPopupIsNotShown() { + let credentialsPopup = app.popovers["Save password in DuckDuckGo?"] + XCTAssertFalse(credentialsPopup.exists) + } + + private func finishSignUp() { + let signUpButton = app.webViews.firstMatch.buttons["Sign up"].firstMatch + signUpButton.click() + } + + private func fillCredentials() { + if areTestsRunningOnMacos13() { + /// On macOS 13 we tap in the webview coordinate and we use tabs to make it work given that it doesn't find web view elements + let webView = app.webViews.firstMatch + let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5)) + webViewCoordinate.tap() + app.typeKey("\t", modifierFlags: []) + app.typeText("test@duck.com") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } else { + let webView = app.webViews.firstMatch + webView.tap() + let emailTextField = webView.textFields["Email"].firstMatch + emailTextField.click() + emailTextField.typeText("test@duck.com") + + let password = webView.secureTextFields["Password"].firstMatch + password.click() + password.typeText("pa$$word") + app.typeKey("\t", modifierFlags: []) + app.typeText("pa$$word") + } + } + + private func openSignUpSite() { + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + XCTAssertTrue( + addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), + "The address bar text field didn't become available in a reasonable timeframe." + ) + addressBarTextField.typeURL(URL(string: "https://privacy-test-pages.site/autofill/signup.html")!) + XCTAssertTrue( + app.windows.firstMatch.webViews["Password generation during signup"].waitForExistence(timeout: UITests.Timeouts.elementExistence), + "Visited site didn't load with the expected title in a reasonable timeframe." + ) + } + + private func dragFirstTabOutsideOfFireWindow() { + let toolbar = app.toolbars.firstMatch + let toolbarCoordinate = toolbar.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) + let startPoint = toolbarCoordinate.withOffset(CGVector(dx: 120, dy: 15)) + let endPoint = toolbarCoordinate.withOffset(CGVector(dx: -100, dy: -100)) + startPoint.press(forDuration: 0.5, thenDragTo: endPoint) + } + + private func assertFireWindowDoesNotHavePinnedTabs() { + let existsPredicate = NSPredicate(format: "exists == true") + let staticTextExistsExpectation = expectation(for: existsPredicate, evaluatedWith: app.windows.firstMatch.staticTexts.element(boundBy: 0), handler: nil) + + // Wait up to 10 seconds for the static texts to be available + let result = XCTWaiter().wait(for: [staticTextExistsExpectation], timeout: 10) + XCTAssertEqual(result, .completed, "No static texts were found in the app") + + // After confirming static texts are available, iterate through them + for staticText in app.staticTexts.allElementsBoundByIndex where staticText.exists { + XCTAssertFalse(staticText.label.contains("Page #1"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #2"), "Unwanted string found in static text: \(staticText.label)") + } + } + + private func assertSitesOpenedInNormalWindowAreRestored() { + XCTAssertTrue(app.staticTexts["Sample text for Page #3"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #3 should exist.") + app.typeKey("[", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #2"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #2 should exist.") + app.typeKey("[", modifierFlags: [.command, .shift]) + XCTAssertTrue(app.staticTexts["Sample text for Page #1"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #1 should exist.") + } + + private func assertSitesOpenedOnFireWindowAreNotRestored() { + let existsPredicate = NSPredicate(format: "exists == true") + let staticTextExistsExpectation = expectation(for: existsPredicate, evaluatedWith: app.staticTexts.element(boundBy: 0), handler: nil) + + // Wait up to 10 seconds for the static texts to be available + let result = XCTWaiter().wait(for: [staticTextExistsExpectation], timeout: 10) + XCTAssertEqual(result, .completed, "No static texts were found in the app") + + // After confirming static texts are available, iterate through them + for staticText in app.staticTexts.allElementsBoundByIndex where staticText.exists { + XCTAssertFalse(staticText.label.contains("Page #4"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #5"), "Unwanted string found in static text: \(staticText.label)") + XCTAssertFalse(staticText.label.contains("Page #6"), "Unwanted string found in static text: \(staticText.label)") + } + } + + private func openThreeSitesOnNormalWindow() { + app.openNewTab() + openSite(pageTitle: "Page #1") + app.openNewTab() + openSite(pageTitle: "Page #2") + app.openNewTab() + openSite(pageTitle: "Page #3") + } + + private func openThreeSitesOnFireWindow() { + openSite(pageTitle: "Page #4") + app.openNewTab() + openSite(pageTitle: "Page #5") + app.openNewTab() + openSite(pageTitle: "Page #6") + } + + private func assertSiteIsNotShowingInNormalWindowHistory() { + let siteMenuItemInHistory = app.menuItems["Some site"] + XCTAssertFalse(siteMenuItemInHistory.exists, "Menu item should not exist because it was not stored in history.") + } + + private func openFireWindow() { + app.typeKey("n", modifierFlags: [.command, .shift]) + } + + private func openNormalWindow() { + app.typeKey("n", modifierFlags: .command) + } + + private func openSite(pageTitle: String) { + let url = UITests.simpleServedPage(titled: pageTitle) + let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch + XCTAssertTrue( + addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence), + "The address bar text field didn't become available in a reasonable timeframe." + ) + addressBarTextField.typeURL(url) + XCTAssertTrue( + app.windows.firstMatch.webViews[pageTitle].waitForExistence(timeout: UITests.Timeouts.elementExistence), + "Visited site didn't load with the expected title in a reasonable timeframe." + ) + } + + private func areTestsRunningOnMacos13() -> Bool { + return ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13 + } +} diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 61158943f5..0b92d927dd 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -341,6 +341,18 @@ platform :mac do ) end + desc 'Creates a new Kechain to use on UI tests' + lane :create_keychain_ui_tests do |options| + create_keychain( + name: "DefaultKeychain", + password: "default", + default_keychain: true, + unlock: true, + timeout: 54000, + lock_when_sleeps: false + ) + end + ################################################# # Helper functions #################################################