Merge branch 'main' into sam/remove-test-rollout-pixel

# By Graeme Arthur (4) and others
# Via GitHub (1) and Graeme Arthur (1)
* main:
  VPN clean-up (#3502)
  add daily pixel for deleting suggestions (#3531)
  Fix VPN memory pressure monitor (#3535)
  Update Ruby to 3.3.4 (#3547)
  Onboarding Add To Dock Pixels (#3543)
  Switch to free runners for tests that run on Maestro (#3546)
  Fix email protection test (#3539)
  Update BSK for PixelKit suffix change (#3534)
  Adding app backgrounded result to rule compilation (#3533)
  Send pixel on sync secure storage failure (#3542)
  Onboarding Add to Dock Refactor for Intro scenario (#3538)
  Update C-S-S to 6.29.0 (#3541)
  Change save password Never for Site button to Not Now (#3471)
  Release 7.144.0-1 (#3540)
  UserDefaults misbehavior monitoring (#3510)
  Send pixel on sync secure storage read failure (#3530)
  Remove NewTabPage retain cycles (#3532)
  Update release notes (#3529)

# Conflicts:
#	DuckDuckGo.xcodeproj/project.pbxproj
#	DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

Author: samsymons

.github/workflows/end-to-end.yml

@@ -68,7 +68,7 @@ jobs:
   end-to-end-tests:
     name: End to end Tests
     needs: build-end-to-end-tests
-    runs-on: macos-14-xlarge
+    runs-on: macos-14
     timeout-minutes: 90
     strategy:
       matrix:

.github/workflows/sync-end-to-end.yml

@@ -68,7 +68,7 @@ jobs:
   sync-end-to-end-tests:
     name: Sync End To End Tests
     needs: build-for-sync-end-to-end-tests
-    runs-on: macos-14-xlarge
+    runs-on: macos-14
     timeout-minutes: 90
     strategy:
       matrix:

.maestro/release_tests/emailprotection.yaml

@@ -27,7 +27,7 @@ tags:
       - scroll
       - assertVisible: Email Protection
       - tapOn: Email Protection
-- assertVisible: Email privacy, simplified.
+- assertVisible: Email privacy, protected.
 - assertVisible:
     id: searchEntry
 - tapOn: 

.ruby-version

@@ -1 +1 @@
-3.0.4
+3.3.4

Core/BoolFileMarker.swift

@@ -0,0 +1,55 @@
+//
+//  BoolFileMarker.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+public struct BoolFileMarker {
+    let fileManager = FileManager.default
+    private let url: URL
+
+    public var isPresent: Bool {
+        fileManager.fileExists(atPath: url.path)
+    }
+
+    public func mark() {
+        if !isPresent {
+            fileManager.createFile(atPath: url.path, contents: nil, attributes: [.protectionKey: FileProtectionType.none])
+        }
+    }
+
+    public func unmark() {
+        if isPresent {
+            try? fileManager.removeItem(at: url)
+        }
+    }
+
+    public init?(name: Name) {
+        guard let applicationSupportDirectory = fileManager.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else {
+            return nil
+        }
+
+        self.url = applicationSupportDirectory.appendingPathComponent(name.rawValue)
+    }
+
+    public struct Name: RawRepresentable {
+        public let rawValue: String
+
+        public init(rawValue: String) {
+            self.rawValue = "\(rawValue).marker"
+        }
+    }
+}

Core/BoolFileMarkerTests.swift

@@ -0,0 +1,58 @@
+//
+//  BoolFileMarkerTests.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import XCTest
+@testable import Core
+
+final class BoolFileMarkerTests: XCTestCase {
+
+    private let marker = BoolFileMarker(name: .init(rawValue: "test"))!
+
+    override func tearDown() {
+        super.tearDown()
+
+        marker.unmark()
+    }
+
+    private var testFileURL: URL? {
+        FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first?.appendingPathComponent("test.marker")
+    }
+
+    func testMarkCreatesCorrectFile() throws {
+
+        marker.mark()
+
+        let fileURL = try XCTUnwrap(testFileURL)
+
+        let attributes = try FileManager.default.attributesOfItem(atPath: fileURL.path)
+        XCTAssertNil(attributes[.protectionKey])
+        XCTAssertTrue(FileManager.default.fileExists(atPath: fileURL.path))
+        XCTAssertEqual(marker.isPresent, true)
+    }
+
+    func testUnmarkRemovesFile() throws {
+        marker.mark()
+        marker.unmark()
+
+        let fileURL = try XCTUnwrap(testFileURL)
+
+        XCTAssertFalse(marker.isPresent)
+        XCTAssertFalse(FileManager.default.fileExists(atPath: fileURL.path))
+    }
+}

Core/PixelEvent.swift

@@ -137,12 +137,15 @@ extension Pixel {
         case autocompleteDisplayedLocalHistory
         case autocompleteDisplayedOpenedTab
         case autocompleteSwipeToDelete
+        case autocompleteSwipeToDeleteDaily
 
         case feedbackPositive
         case feedbackNegativePrefix(category: String)
 
         case brokenSiteReport
 
+        // MARK: - Onboarding
+
         case onboardingIntroShownUnique
         case onboardingIntroComparisonChartShownUnique
         case onboardingIntroChooseBrowserCTAPressed
@@ -173,6 +176,15 @@ extension Pixel {
         case daxDialogsEndOfJourneyNewTabUnique
         case daxDialogsEndOfJourneyDismissed
 
+        // MARK: - Onboarding Add To Dock
+
+        case onboardingAddToDockPromoImpressionsUnique
+        case onboardingAddToDockPromoShowTutorialCTATapped
+        case onboardingAddToDockPromoDismissCTATapped
+        case onboardingAddToDockTutorialDismissCTATapped
+
+        // MARK: - Onboarding Add To Dock
+
         case widgetsOnboardingCTAPressed
         case widgetsOnboardingDeclineOptionPressed
         case widgetsOnboardingMovedToBackground
@@ -400,10 +412,6 @@ extension Pixel {
         case networkProtectionClientFailedToParseRegisteredServersResponse
         case networkProtectionClientFailedToFetchLocations
         case networkProtectionClientFailedToParseLocationsResponse
-        case networkProtectionClientFailedToEncodeRedeemRequest
-        case networkProtectionClientInvalidInviteCode
-        case networkProtectionClientFailedToRedeemInviteCode
-        case networkProtectionClientFailedToParseRedeemResponse
         case networkProtectionClientInvalidAuthToken
 
         case networkProtectionKeychainErrorFailedToCastKeychainValueToData
@@ -623,6 +631,7 @@ extension Pixel {
         case syncRemoveDeviceError
         case syncDeleteAccountError
         case syncLoginExistingAccountError
+        case syncSecureStorageReadError
 
         case syncGetOtherDevices
         case syncGetOtherDevicesCopy
@@ -834,6 +843,11 @@ extension Pixel {
 
         // MARK: WebView Error Page Shown
         case webViewErrorPageShown
+
+        // MARK: UserDefaults incositency monitoring
+        case protectedDataUnavailableWhenBecomeActive
+        case statisticsLoaderATBStateMismatch
+        case adAttributionReportStateMismatch
     }
 
 }
@@ -961,6 +975,7 @@ extension Pixel.Event {
         case .autocompleteDisplayedLocalHistory: return "m_autocomplete_display_local_history"
         case .autocompleteDisplayedOpenedTab: return "m_autocomplete_display_switch_to_tab"
         case .autocompleteSwipeToDelete: return "m_autocomplete_result_deleted"
+        case .autocompleteSwipeToDeleteDaily: return "m_autocomplete_result_deleted_daily"
 
         case .feedbackPositive: return "mfbs_positive_submit"
         case .feedbackNegativePrefix(category: let category): return "mfbs_negative_\(category)"
@@ -997,6 +1012,11 @@ extension Pixel.Event {
         case .daxDialogsEndOfJourneyNewTabUnique: return "m_dx_end_new_tab_unique"
         case .daxDialogsEndOfJourneyDismissed: return "m_dx_end_dialog_dismissed"
 
+        case .onboardingAddToDockPromoImpressionsUnique: return "m_onboarding_add_to_dock_promo_impressions_unique"
+        case .onboardingAddToDockPromoShowTutorialCTATapped: return "m_onboarding_add_to_dock_promo_show_tutorial_button_tapped"
+        case .onboardingAddToDockPromoDismissCTATapped: return "m_onboarding_add_to_dock_promo_dismiss_button_tapped"
+        case .onboardingAddToDockTutorialDismissCTATapped: return "m_onboarding_add_to_dock_tutorial_dismiss_button_tapped"
+
         case .widgetsOnboardingCTAPressed: return "m_o_w_a"
         case .widgetsOnboardingDeclineOptionPressed: return "m_o_w_d"
         case .widgetsOnboardingMovedToBackground: return "m_o_w_b"
@@ -1214,10 +1234,6 @@ extension Pixel.Event {
         case .networkProtectionClientFailedToFetchLocations: return "m_netp_backend_api_error_failed_to_fetch_locations"
         case .networkProtectionClientFailedToParseLocationsResponse:
             return "m_netp_backend_api_error_parsing_locations_response_failed"
-        case .networkProtectionClientFailedToEncodeRedeemRequest: return "m_netp_backend_api_error_encoding_redeem_request_body_failed"
-        case .networkProtectionClientInvalidInviteCode: return "m_netp_backend_api_error_invalid_invite_code"
-        case .networkProtectionClientFailedToRedeemInviteCode: return "m_netp_backend_api_error_failed_to_redeem_invite_code"
-        case .networkProtectionClientFailedToParseRedeemResponse: return "m_netp_backend_api_error_parsing_redeem_response_failed"
         case .networkProtectionClientInvalidAuthToken: return "m_netp_backend_api_error_invalid_auth_token"
         case .networkProtectionKeychainErrorFailedToCastKeychainValueToData: return "m_netp_keychain_error_failed_to_cast_keychain_value_to_data"
         case .networkProtectionKeychainReadError: return "m_netp_keychain_error_read_failed"
@@ -1430,6 +1446,7 @@ extension Pixel.Event {
         case .syncRemoveDeviceError: return "m_d_sync_remove_device_error"
         case .syncDeleteAccountError: return "m_d_sync_delete_account_error"
         case .syncLoginExistingAccountError: return "m_d_sync_login_existing_account_error"
+        case .syncSecureStorageReadError: return "m_d_sync_secure_storage_error"
 
         case .syncGetOtherDevices: return "sync_get_other_devices"
         case .syncGetOtherDevicesCopy: return "sync_get_other_devices_copy"
@@ -1664,6 +1681,11 @@ extension Pixel.Event {
 
         // MARK: - DuckPlayer FE Application Telemetry
         case .duckPlayerLandscapeLayoutImpressions: return "duckplayer_landscape_layout_impressions"
+
+        // MARK: UserDefaults incositency monitoring
+        case .protectedDataUnavailableWhenBecomeActive: return "m_protected_data_unavailable_when_become_active"
+        case .statisticsLoaderATBStateMismatch: return "m_statistics_loader_atb_state_mismatch"
+        case .adAttributionReportStateMismatch: return "m_ad_attribution_report_state_mismatch"
         }
     }
 }
@@ -1715,6 +1737,7 @@ extension Pixel.Event {
 
         case tabClosed = "tab_closed"
         case appQuit = "app_quit"
+        case appBackgrounded = "app_backgrounded"
         case success
 
     }

Core/StatisticsLoader.swift

@@ -33,17 +33,29 @@ public class StatisticsLoader {
     private let returnUserMeasurement: ReturnUserMeasurement
     private let usageSegmentation: UsageSegmenting
     private let parser = AtbParser()
+    private let atbPresenceFileMarker = BoolFileMarker(name: .isATBPresent)
+    private let inconsistencyMonitoring: StatisticsStoreInconsistencyMonitoring
 
     init(statisticsStore: StatisticsStore = StatisticsUserDefaults(),
          returnUserMeasurement: ReturnUserMeasurement = KeychainReturnUserMeasurement(),
-         usageSegmentation: UsageSegmenting = UsageSegmentation()) {
+         usageSegmentation: UsageSegmenting = UsageSegmentation(),
+         inconsistencyMonitoring: StatisticsStoreInconsistencyMonitoring = StorageInconsistencyMonitor()) {
         self.statisticsStore = statisticsStore
         self.returnUserMeasurement = returnUserMeasurement
         self.usageSegmentation = usageSegmentation
+        self.inconsistencyMonitoring = inconsistencyMonitoring
     }
 
     public func load(completion: @escaping Completion = {}) {
-        if statisticsStore.hasInstallStatistics {
+        let hasFileMarker = atbPresenceFileMarker?.isPresent ?? false
+        let hasInstallStatistics = statisticsStore.hasInstallStatistics
+
+        inconsistencyMonitoring.statisticsDidLoad(hasFileMarker: hasFileMarker, hasInstallStatistics: hasInstallStatistics)
+
+        if hasInstallStatistics {
+            // Synchronize file marker with current state
+            createATBFileMarker()
+
             completion()
             return
         }
@@ -85,10 +97,15 @@ public class StatisticsLoader {
             self.statisticsStore.installDate = Date()
             self.statisticsStore.atb = atb.version
             self.returnUserMeasurement.installCompletedWithATB(atb)
+            self.createATBFileMarker()
             completion()
         }
     }
 
+    private func createATBFileMarker() {
+        atbPresenceFileMarker?.mark()
+    }
+
     public func refreshSearchRetentionAtb(completion: @escaping Completion = {}) {
         guard let url = StatisticsDependentURLFactory(statisticsStore: statisticsStore).makeSearchAtbURL() else {
             requestInstallStatistics {
@@ -169,3 +186,7 @@ public class StatisticsLoader {
         processUsageSegmentation(atb: nil, activityType: activityType)
     }
 }
+
+private extension BoolFileMarker.Name {
+    static let isATBPresent = BoolFileMarker.Name(rawValue: "atb-present")
+}

Core/StorageInconsistencyMonitor.swift

@@ -0,0 +1,66 @@
+//
+//  StorageInconsistencyMonitor.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import UIKit
+
+public protocol AppActivationInconsistencyMonitoring {
+    /// See `StorageInconsistencyMonitor` for details
+    func didBecomeActive(isProtectedDataAvailable: Bool)
+}
+
+public protocol StatisticsStoreInconsistencyMonitoring {
+    /// See `StorageInconsistencyMonitor` for details
+    func statisticsDidLoad(hasFileMarker: Bool, hasInstallStatistics: Bool)
+}
+
+public protocol AdAttributionReporterInconsistencyMonitoring {
+    /// See `StorageInconsistencyMonitor` for details
+    func addAttributionReporter(hasFileMarker: Bool, hasCompletedFlag: Bool)
+}
+
+/// Takes care of reporting inconsistency in storage availability and/or state.
+/// See https://app.asana.com/0/481882893211075/1208618515043198/f for details.
+public struct StorageInconsistencyMonitor: AppActivationInconsistencyMonitoring & StatisticsStoreInconsistencyMonitoring & AdAttributionReporterInconsistencyMonitoring {
+
+    public init() { }
+
+    /// Reports a pixel if data is not available while app is active
+    public func didBecomeActive(isProtectedDataAvailable: Bool) {
+        if !isProtectedDataAvailable {
+            Pixel.fire(pixel: .protectedDataUnavailableWhenBecomeActive)
+            assertionFailure("This is unexpected state, debug if possible")
+        }
+    }
+
+    /// Reports a pixel if file marker exists but installStatistics are missing
+    public func statisticsDidLoad(hasFileMarker: Bool, hasInstallStatistics: Bool) {
+        if hasFileMarker == true && hasInstallStatistics == false {
+            Pixel.fire(pixel: .statisticsLoaderATBStateMismatch)
+            assertionFailure("This is unexpected state, debug if possible")
+        }
+    }
+
+    /// Reports a pixel if file marker exists but completion flag is false
+    public func addAttributionReporter(hasFileMarker: Bool, hasCompletedFlag: Bool) {
+        if hasFileMarker == true && hasCompletedFlag == false {
+            Pixel.fire(pixel: .adAttributionReportStateMismatch)
+            assertionFailure("This is unexpected state, debug if possible")
+        }
+    }
+}

Core/SyncErrorHandler.swift

@@ -100,6 +100,8 @@ public class SyncErrorHandler: EventMapping<SyncError> {
                 Pixel.fire(pixel: .syncFailedToLoadAccount, error: error)
             case .failedToSetupEngine:
                 Pixel.fire(pixel: .syncFailedToSetupEngine, error: error)
+            case .failedToReadSecureStore:
+                Pixel.fire(pixel: .syncSecureStorageReadError, error: error)
             default:
                 // Should this be so generic?
                 let domainEvent = Pixel.Event.syncSentUnauthenticatedRequest