Implement Fire Window UI tests

.github/workflows/sync_end_to_end.yml

@@ -153,6 +153,7 @@ jobs:
       with:
         check_name: "Test Report ${{ matrix.runner }}"
         report_paths: ui-tests.xml
+        check_retries: true
 
     - name: Upload logs when workflow failed
       uses: actions/upload-artifact@v4

.github/workflows/ui_tests.yml

@@ -64,6 +64,9 @@ jobs:
 
     - name: Set up fastlane
       run: bundle install
+
+    - name: Create Default Keychain
+      run: bundle exec fastlane create_keychain_ui_tests
 
     - name: Sync code signing assets
       env:
@@ -141,6 +144,7 @@ jobs:
       with:
         check_name: "Test Report ${{ matrix.runner }}"
         report_paths: ui-tests.xml
+        check_retries: true
 
     - name: Upload logs when workflow failed
       uses: actions/upload-artifact@v4

DuckDuckGo.xcodeproj/project.pbxproj

@@ -2809,6 +2809,7 @@
 		BB470EBC2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB470EBA2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift */; };
 		BB5789722B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5789712B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift */; };
 		BB5F46A32C8751F6005F72DF /* BookmarkSortTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5F46A22C8751F6005F72DF /* BookmarkSortTests.swift */; };
+		BB731F312CDBA6360023D2E4 /* FireWindowTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */; };
 		BB7B5F982C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */; };
 		BB7B5F992C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */; };
 		BBB881882C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBB881872C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift */; };
@@ -4760,6 +4761,7 @@
 		BB470EBA2C5A66D6002EE91D /* BookmarkManagementDetailViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkManagementDetailViewModel.swift; sourceTree = "<group>"; };
 		BB5789712B2CA70F0009DFE2 /* DataBrokerProtectionSubscriptionEventHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataBrokerProtectionSubscriptionEventHandler.swift; sourceTree = "<group>"; };
 		BB5F46A22C8751F6005F72DF /* BookmarkSortTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkSortTests.swift; sourceTree = "<group>"; };
+		BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FireWindowTests.swift; sourceTree = "<group>"; };
 		BB7B5F972C4ED73800BA4AF8 /* BookmarksSearchAndSortMetrics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarksSearchAndSortMetrics.swift; sourceTree = "<group>"; };
 		BBB881872C4029BA001247C6 /* BookmarkListTreeControllerSearchDataSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkListTreeControllerSearchDataSource.swift; sourceTree = "<group>"; };
 		BBBB653F2C77BB9400E69AC6 /* BookmarkSearchTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarkSearchTests.swift; sourceTree = "<group>"; };
@@ -6972,6 +6974,7 @@
 		7B4CE8DB26F02108009134B1 /* UITests */ = {
 			isa = PBXGroup;
 			children = (
+				BB731F302CDBA6320023D2E4 /* FireWindowTests.swift */,
 				376E708D2BD686260082B7EB /* UI Tests.xctestplan */,
 				EEBCE6802BA444FA00B9DF00 /* Common */,
 				EEC7BE2D2BC6C09400F86835 /* AddressBarKeyboardShortcutsTests.swift */,
@@ -12628,6 +12631,7 @@
 				EEC7BE2E2BC6C09500F86835 /* AddressBarKeyboardShortcutsTests.swift in Sources */,
 				EE54F7B32BBFEA49006218DB /* BookmarksAndFavoritesTests.swift in Sources */,
 				EE02D4222BB4611A00DBE6B3 /* TestsURLExtension.swift in Sources */,
+				BB731F312CDBA6360023D2E4 /* FireWindowTests.swift in Sources */,
 				EE42CBCC2BC8004700AD411C /* PermissionsTests.swift in Sources */,
 				7B4CE8E726F02135009134B1 /* TabBarTests.swift in Sources */,
 				EEBCE6832BA463DD00B9DF00 /* NSImageExtensions.swift in Sources */,

UITests/FireWindowTests.swift

@@ -0,0 +1,340 @@
+//
+//  FireWindowTests.swift
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import XCTest
+
+class FireWindowTests: XCTestCase {
+    private var app: XCUIApplication!
+    private var settingsGeneralButton: XCUIElement!
+    private var reopenAllWindowsFromLastSessionPreference: XCUIElement!
+
+    override class func setUp() {
+        UITests.firstRun()
+    }
+
+    override func setUpWithError() throws {
+        continueAfterFailure = false
+        app = XCUIApplication()
+        app.launchEnvironment["UITEST_MODE"] = "1"
+
+        settingsGeneralButton = app.buttons["PreferencesSidebar.generalButton"]
+        reopenAllWindowsFromLastSessionPreference = app.radioButtons["PreferencesGeneralView.stateRestorePicker.reopenAllWindowsFromLastSession"]
+
+        app.launch()
+        app.typeKey("w", modifierFlags: [.command, .option, .shift]) // Let's enforce a single window
+    }
+
+    func testFireWindowDoesNotStoreHistory() {
+        openFireWindow()
+        openSite(pageTitle: "Some site")
+        openNormalWindow()
+        assertSiteIsNotShowingInNormalWindowHistory()
+    }
+
+    func testFireWindowStateIsNotSavedAfterRestart() {
+        openNormalWindow()
+        app.typeKey(",", modifierFlags: [.command]) // Open settings
+        settingsGeneralButton.click(forDuration: 0.5, thenDragTo: settingsGeneralButton)
+        reopenAllWindowsFromLastSessionPreference.clickAfterExistenceTestSucceeds()
+
+        openThreeSitesOnNormalWindow()
+        openFireWindow()
+        openThreeSitesOnFireWindow()
+
+        app.terminate()
+        app.launch()
+
+        assertSitesOpenedInNormalWindowAreRestored()
+        assertSitesOpenedOnFireWindowAreNotRestored()
+    }
+
+    func testFireWindowDoNotShowPinnedTabs() {
+        openNormalWindow()
+        openSite(pageTitle: "Page #1")
+        app.menuItems["Pin Tab"].tap()
+
+        app.openNewTab()
+        openSite(pageTitle: "Page #2")
+        app.menuItems["Pin Tab"].tap()
+
+        openFireWindow()
+        assertFireWindowDoesNotHavePinnedTabs()
+    }
+
+    func testFireWindowTabsCannotBeDragged() {
+        openFireWindow()
+        openSite(pageTitle: "Page #1")
+
+        app.openNewTab()
+        openSite(pageTitle: "Page #2")
+
+        dragFirstTabOutsideOfFireWindow()
+
+        /// Assert that Page #1 is still on the fire window after the drag
+        app.typeKey("]", modifierFlags: [.command, .shift])
+        XCTAssertTrue(app.staticTexts["Sample text for Page #2"].exists)
+        app.typeKey("[", modifierFlags: [.command, .shift])
+        XCTAssertTrue(app.staticTexts["Sample text for Page #1"].exists)
+    }
+
+    func testFireWindowsSignInDoesNotShowCredentialsPopup() {
+        openFireWindow()
+        hoverMouseOutsideTabSoPreviewIsNotShown()
+        openSignUpSite()
+        fillCredentials()
+        finishSignUp()
+        assertSavePasswordPopupIsNotShown()
+    }
+
+    func testCrendentialsAreAutoFilledInFireWindows() {
+        openNormalWindow()
+        hoverMouseOutsideTabSoPreviewIsNotShown()
+        openLoginSite()
+        signIn()
+        saveCredentials()
+
+        /// Here we start the same flow but in the fire window, but we use the autofill credentials saved in the step before.
+        openFireWindow()
+        hoverMouseOutsideTabSoPreviewIsNotShown()
+        openLoginSite()
+        signInUsingAutoFill()
+    }
+
+    // MARK: - Utilities
+
+    private func hoverMouseOutsideTabSoPreviewIsNotShown() {
+        let window = app.windows.firstMatch
+        let coordinate = window.coordinate(withNormalizedOffset: CGVector(dx: -100, dy: -100))
+        coordinate.hover()
+    }
+
+    private func signInUsingAutoFill() {
+        if areTestsRunningOnMacos13() {
+            let webViewFire = app.webViews.firstMatch
+            let webViewCoordinate = webViewFire.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5))
+            webViewCoordinate.tap()
+            app.typeKey("\t", modifierFlags: [])
+            sleep(1)
+            let autoFillPopup = webViewFire.buttons["[email protected] privacy-test-pages.site"]
+            let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5))
+            coordinate.tap()
+
+            /// On macOS 13 there are some issues when accessing web view elements so we do not check the value of the email text field.
+            /// If we can access the `[email protected] privacy-test-pages.site` button means that auto fill is working correctly in the fire window.
+            /// Checking that the email is being filled correctly is more an autofill test that fire window, so we are okay to skip it.
+            ///
+            /// We do run this test on macOS 14 and above.
+        } else {
+            let webViewFire = app.webViews.firstMatch
+            webViewFire.tap()
+            let emailTextFieldFire = webViewFire.textFields["Email"].firstMatch
+            emailTextFieldFire.click()
+            let autoFillPopup = webViewFire.buttons["[email protected] privacy-test-pages.site"]
+            let coordinate = autoFillPopup.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5))
+            coordinate.tap()
+
+            XCTAssertEqual(emailTextFieldFire.value as? String, "[email protected]")
+        }
+    }
+
+    private func saveCredentials() {
+        let saveButton = app.buttons["Save"]
+        saveButton.tap()
+    }
+
+    private func signIn() {
+        if areTestsRunningOnMacos13() {
+            let webView = app.webViews.firstMatch
+            let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5))
+            webViewCoordinate.tap()
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("[email protected]")
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("pa$$word")
+        } else {
+            let webView = app.webViews.firstMatch
+            webView.tap()
+            let emailTextField = webView.textFields["Email"].firstMatch
+            emailTextField.click()
+            emailTextField.typeText("[email protected]")
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("pa$$word")
+        }
+
+        let signInButton = app.webViews.firstMatch.buttons["Sign in"].firstMatch
+        signInButton.click()
+    }
+
+    private func openLoginSite() {
+        let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch
+        XCTAssertTrue(
+            addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence),
+            "The address bar text field didn't become available in a reasonable timeframe."
+        )
+        addressBarTextField.typeURL(URL(string: "https://privacy-test-pages.site/autofill/autoprompt/1-standard-login-form.html")!)
+        XCTAssertTrue(
+            app.windows.firstMatch.webViews["Autofill autoprompt for signin forms"].waitForExistence(timeout: UITests.Timeouts.elementExistence),
+            "Visited site didn't load with the expected title in a reasonable timeframe."
+        )
+    }
+
+    private func assertSavePasswordPopupIsNotShown() {
+        let credentialsPopup = app.popovers["Save password in DuckDuckGo?"]
+        XCTAssertFalse(credentialsPopup.exists)
+    }
+
+    private func finishSignUp() {
+        let signUpButton = app.webViews.firstMatch.buttons["Sign up"].firstMatch
+        signUpButton.click()
+    }
+
+    private func fillCredentials() {
+        if areTestsRunningOnMacos13() {
+            /// On macOS 13 we tap in the webview coordinate and we use tabs to make it work given that it doesn't find web view elements
+            let webView = app.webViews.firstMatch
+            let webViewCoordinate = webView.coordinate(withNormalizedOffset: CGVector(dx: 5, dy: 5))
+            webViewCoordinate.tap()
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("[email protected]")
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("pa$$word")
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("pa$$word")
+        } else {
+            let webView = app.webViews.firstMatch
+            webView.tap()
+            let emailTextField = webView.textFields["Email"].firstMatch
+            emailTextField.click()
+            emailTextField.typeText("[email protected]")
+
+            let password = webView.secureTextFields["Password"].firstMatch
+            password.click()
+            password.typeText("pa$$word")
+            app.typeKey("\t", modifierFlags: [])
+            app.typeText("pa$$word")
+        }
+    }
+
+    private func openSignUpSite() {
+        let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch
+        XCTAssertTrue(
+            addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence),
+            "The address bar text field didn't become available in a reasonable timeframe."
+        )
+        addressBarTextField.typeURL(URL(string: "https://privacy-test-pages.site/autofill/signup.html")!)
+        XCTAssertTrue(
+            app.windows.firstMatch.webViews["Password generation during signup"].waitForExistence(timeout: UITests.Timeouts.elementExistence),
+            "Visited site didn't load with the expected title in a reasonable timeframe."
+        )
+    }
+
+    private func dragFirstTabOutsideOfFireWindow() {
+        let toolbar = app.toolbars.firstMatch
+        let toolbarCoordinate = toolbar.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
+        let startPoint = toolbarCoordinate.withOffset(CGVector(dx: 120, dy: 15))
+        let endPoint = toolbarCoordinate.withOffset(CGVector(dx: -100, dy: -100))
+        startPoint.press(forDuration: 0.5, thenDragTo: endPoint)
+    }
+
+    private func assertFireWindowDoesNotHavePinnedTabs() {
+        let existsPredicate = NSPredicate(format: "exists == true")
+        let staticTextExistsExpectation = expectation(for: existsPredicate, evaluatedWith: app.windows.firstMatch.staticTexts.element(boundBy: 0), handler: nil)
+
+        // Wait up to 10 seconds for the static texts to be available
+        let result = XCTWaiter().wait(for: [staticTextExistsExpectation], timeout: 10)
+        XCTAssertEqual(result, .completed, "No static texts were found in the app")
+
+        // After confirming static texts are available, iterate through them
+        for staticText in app.staticTexts.allElementsBoundByIndex where staticText.exists {
+            XCTAssertFalse(staticText.label.contains("Page #1"), "Unwanted string found in static text: \(staticText.label)")
+            XCTAssertFalse(staticText.label.contains("Page #2"), "Unwanted string found in static text: \(staticText.label)")
+        }
+    }
+
+    private func assertSitesOpenedInNormalWindowAreRestored() {
+        XCTAssertTrue(app.staticTexts["Sample text for Page #3"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #3 should exist.")
+        app.typeKey("[", modifierFlags: [.command, .shift])
+        XCTAssertTrue(app.staticTexts["Sample text for Page #2"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #2 should exist.")
+        app.typeKey("[", modifierFlags: [.command, .shift])
+        XCTAssertTrue(app.staticTexts["Sample text for Page #1"].waitForExistence(timeout: UITests.Timeouts.elementExistence), "Page #1 should exist.")
+    }
+
+    private func assertSitesOpenedOnFireWindowAreNotRestored() {
+        let existsPredicate = NSPredicate(format: "exists == true")
+        let staticTextExistsExpectation = expectation(for: existsPredicate, evaluatedWith: app.staticTexts.element(boundBy: 0), handler: nil)
+
+        // Wait up to 10 seconds for the static texts to be available
+        let result = XCTWaiter().wait(for: [staticTextExistsExpectation], timeout: 10)
+        XCTAssertEqual(result, .completed, "No static texts were found in the app")
+
+        // After confirming static texts are available, iterate through them
+        for staticText in app.staticTexts.allElementsBoundByIndex where staticText.exists {
+            XCTAssertFalse(staticText.label.contains("Page #4"), "Unwanted string found in static text: \(staticText.label)")
+            XCTAssertFalse(staticText.label.contains("Page #5"), "Unwanted string found in static text: \(staticText.label)")
+            XCTAssertFalse(staticText.label.contains("Page #6"), "Unwanted string found in static text: \(staticText.label)")
+        }
+    }
+
+    private func openThreeSitesOnNormalWindow() {
+        app.openNewTab()
+        openSite(pageTitle: "Page #1")
+        app.openNewTab()
+        openSite(pageTitle: "Page #2")
+        app.openNewTab()
+        openSite(pageTitle: "Page #3")
+    }
+
+    private func openThreeSitesOnFireWindow() {
+        openSite(pageTitle: "Page #4")
+        app.openNewTab()
+        openSite(pageTitle: "Page #5")
+        app.openNewTab()
+        openSite(pageTitle: "Page #6")
+    }
+
+    private func assertSiteIsNotShowingInNormalWindowHistory() {
+        let siteMenuItemInHistory = app.menuItems["Some site"]
+        XCTAssertFalse(siteMenuItemInHistory.exists, "Menu item should not exist because it was not stored in history.")
+    }
+
+    private func openFireWindow() {
+        app.typeKey("n", modifierFlags: [.command, .shift])
+    }
+
+    private func openNormalWindow() {
+        app.typeKey("n", modifierFlags: .command)
+    }
+
+    private func openSite(pageTitle: String) {
+        let url = UITests.simpleServedPage(titled: pageTitle)
+        let addressBarTextField = app.windows.firstMatch.textFields["AddressBarViewController.addressBarTextField"].firstMatch
+        XCTAssertTrue(
+            addressBarTextField.waitForExistence(timeout: UITests.Timeouts.elementExistence),
+            "The address bar text field didn't become available in a reasonable timeframe."
+        )
+        addressBarTextField.typeURL(url)
+        XCTAssertTrue(
+            app.windows.firstMatch.webViews[pageTitle].waitForExistence(timeout: UITests.Timeouts.elementExistence),
+            "Visited site didn't load with the expected title in a reasonable timeframe."
+        )
+    }
+
+    private func areTestsRunningOnMacos13() -> Bool {
+        return ProcessInfo.processInfo.operatingSystemVersion.majorVersion == 13
+    }
+}