Merge branch 'main' into dominik/xcode-16

# By Daniel Bernal (7) and others
# Via Daniel Bernal (3) and others
* main: (29 commits)
  Send pixel on sync secure storage failure (#3542)
  Onboarding Add to Dock Refactor for Intro scenario (#3538)
  Update C-S-S to 6.29.0 (#3541)
  Change save password Never for Site button to Not Now (#3471)
  Release 7.144.0-1 (#3540)
  UserDefaults misbehavior monitoring (#3510)
  Send pixel on sync secure storage read failure (#3530)
  Remove NewTabPage retain cycles (#3532)
  Update release notes (#3529)
  Release 7.144.0-0 (#3528)
  Add Privacy Config feature to control ad attribution reporting (#3506)
  Validate VPN errors before re-throwing them (#3513)
  Allowing users to delete suggestions on macOS (#3465)
  Update build number
  Update build number
  Bump rexml from 3.3.8 to 3.3.9 (#3495)
  Release 7.142.1-1 (#3525)
  Add a debouncer to NavBars animator (#3519)
  Release 7.143.0-1 (#3516)
  Update to subscription cookie (#3512)
  ...

# Conflicts:
#	DuckDuckGo.xcodeproj/project.pbxproj
#	DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

Author: samsymons

Configuration/Version.xcconfig

@@ -1 +1 @@
-MARKETING_VERSION = 7.143.0
+MARKETING_VERSION = 7.144.0

Core/AppPrivacyConfigurationDataProvider.swift

@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"f8b9cfd5f1eb7b77c21d4476f85bd177\""
-        public static let embeddedDataSHA = "c26c97714d73a9e1e99dbd341d5890da42b49d34a296672be3d3cea00bdd37a0"
+        public static let embeddedDataETag = "\"516f95a16f7a556c58e14ee6f193cc30\""
+        public static let embeddedDataSHA = "87314e1ac02784472a722844a27b443b0387a164ac72afaac00d9a70731fc572"
     }
 
     public var embeddedDataEtag: String {

Core/BoolFileMarker.swift

@@ -0,0 +1,55 @@
+//
+//  BoolFileMarker.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+public struct BoolFileMarker {
+    let fileManager = FileManager.default
+    private let url: URL
+
+    public var isPresent: Bool {
+        fileManager.fileExists(atPath: url.path)
+    }
+
+    public func mark() {
+        if !isPresent {
+            fileManager.createFile(atPath: url.path, contents: nil, attributes: [.protectionKey: FileProtectionType.none])
+        }
+    }
+
+    public func unmark() {
+        if isPresent {
+            try? fileManager.removeItem(at: url)
+        }
+    }
+
+    public init?(name: Name) {
+        guard let applicationSupportDirectory = fileManager.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else {
+            return nil
+        }
+
+        self.url = applicationSupportDirectory.appendingPathComponent(name.rawValue)
+    }
+
+    public struct Name: RawRepresentable {
+        public let rawValue: String
+
+        public init(rawValue: String) {
+            self.rawValue = "\(rawValue).marker"
+        }
+    }
+}

Core/BoolFileMarkerTests.swift

@@ -0,0 +1,58 @@
+//
+//  BoolFileMarkerTests.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import XCTest
+@testable import Core
+
+final class BoolFileMarkerTests: XCTestCase {
+
+    private let marker = BoolFileMarker(name: .init(rawValue: "test"))!
+
+    override func tearDown() {
+        super.tearDown()
+
+        marker.unmark()
+    }
+
+    private var testFileURL: URL? {
+        FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first?.appendingPathComponent("test.marker")
+    }
+
+    func testMarkCreatesCorrectFile() throws {
+
+        marker.mark()
+
+        let fileURL = try XCTUnwrap(testFileURL)
+
+        let attributes = try FileManager.default.attributesOfItem(atPath: fileURL.path)
+        XCTAssertNil(attributes[.protectionKey])
+        XCTAssertTrue(FileManager.default.fileExists(atPath: fileURL.path))
+        XCTAssertEqual(marker.isPresent, true)
+    }
+
+    func testUnmarkRemovesFile() throws {
+        marker.mark()
+        marker.unmark()
+
+        let fileURL = try XCTUnwrap(testFileURL)
+
+        XCTAssertFalse(marker.isPresent)
+        XCTAssertFalse(FileManager.default.fileExists(atPath: fileURL.path))
+    }
+}

Core/FeatureFlag.swift

@@ -45,6 +45,7 @@ public enum FeatureFlag: String {
     case onboardingAddToDock
     case autofillSurveys
     case autcompleteTabs
+    case adAttributionReporting
 
     /// https://app.asana.com/0/72649045549333/1208231259093710/f
     case networkProtectionUserTips
@@ -103,6 +104,8 @@ extension FeatureFlag: FeatureFlagSourceProviding {
             return .remoteReleasable(.feature(.autocompleteTabs))
         case .networkProtectionUserTips:
             return .remoteReleasable(.subfeature(NetworkProtectionSubfeature.userTips))
+        case .adAttributionReporting:
+            return .remoteReleasable(.feature(.adAttributionReporting))
         }
     }
 }

Core/HistoryManager.swift

@@ -145,6 +145,10 @@ class NullHistoryCoordinator: HistoryCoordinating {
         completion()
     }
 
+    func removeUrlEntry(_ url: URL, completion: (((any Error)?) -> Void)?) {
+        completion?(nil)
+    }
+
 }
 
 public class HistoryDatabase {

Core/PixelEvent.swift

@@ -460,6 +460,8 @@ extension Pixel {
         case networkProtectionConfigurationInvalidPayload(configuration: Configuration)
         case networkProtectionConfigurationPixelTest
 
+        case networkProtectionMalformedErrorDetected
+
         // MARK: remote messaging pixels
 
         case remoteMessageShown
@@ -622,6 +624,7 @@ extension Pixel {
         case syncRemoveDeviceError
         case syncDeleteAccountError
         case syncLoginExistingAccountError
+        case syncSecureStorageReadError
 
         case syncGetOtherDevices
         case syncGetOtherDevicesCopy
@@ -710,8 +713,8 @@ extension Pixel {
         case privacyProKeychainAccessError
         case privacyProSubscriptionCookieMissingTokenOnSignIn
         case privacyProSubscriptionCookieMissingCookieOnSignOut
-        case privacyProSubscriptionCookieRefreshedWithUpdate
-        case privacyProSubscriptionCookieRefreshedWithDelete
+        case privacyProSubscriptionCookieRefreshedWithAccessToken
+        case privacyProSubscriptionCookieRefreshedWithEmptyValue
         case privacyProSubscriptionCookieFailedToSetSubscriptionCookie
 
         // MARK: Pixel Experiment
@@ -799,6 +802,10 @@ extension Pixel {
         case duckPlayerSettingAlwaysSettings
         case duckPlayerSettingNeverSettings
         case duckPlayerSettingBackToDefault
+        case duckPlayerSettingsAlwaysOverlaySERP
+        case duckPlayerSettingsAlwaysOverlayYoutube
+        case duckPlayerSettingsNeverOverlaySERP
+        case duckPlayerSettingsNeverOverlayYoutube
         case duckPlayerWatchOnYoutube
         case duckPlayerSettingAlwaysOverlayYoutube
         case duckPlayerSettingNeverOverlayYoutube
@@ -826,16 +833,14 @@ extension Pixel {
         case pproFeedbackSubcategoryScreenShow(source: String, reportType: String, category: String)
         case pproFeedbackSubmitScreenShow(source: String, reportType: String, category: String, subcategory: String)
         case pproFeedbackSubmitScreenFAQClick(source: String, reportType: String, category: String, subcategory: String)
-        
-        // MARK: DuckPlayer Pixel Experiment
-        case duckplayerExperimentCohortAssign
-        case duckplayerExperimentSearch
-        case duckplayerExperimentDailySearch
-        case duckplayerExperimentWeeklySearch
-        case duckplayerExperimentYoutubePageView
 
         // MARK: WebView Error Page Shown
         case webViewErrorPageShown
+
+        // MARK: UserDefaults incositency monitoring
+        case protectedDataUnavailableWhenBecomeActive
+        case statisticsLoaderATBStateMismatch
+        case adAttributionReportStateMismatch
     }
 
 }
@@ -1266,6 +1271,8 @@ extension Pixel.Event {
         case .networkProtectionConfigurationInvalidPayload(let config): return "m_netp_vpn_configuration_\(config.rawValue)_invalid_payload"
         case .networkProtectionConfigurationPixelTest: return "m_netp_vpn_configuration_pixel_test"
 
+        case .networkProtectionMalformedErrorDetected: return "m_netp_vpn_malformed_error_detected"
+
             // MARK: remote messaging pixels
 
         case .remoteMessageShown: return "m_remote_message_shown"
@@ -1431,6 +1438,7 @@ extension Pixel.Event {
         case .syncRemoveDeviceError: return "m_d_sync_remove_device_error"
         case .syncDeleteAccountError: return "m_d_sync_delete_account_error"
         case .syncLoginExistingAccountError: return "m_d_sync_login_existing_account_error"
+        case .syncSecureStorageReadError: return "m_d_sync_secure_storage_error"
 
         case .syncGetOtherDevices: return "sync_get_other_devices"
         case .syncGetOtherDevicesCopy: return "sync_get_other_devices_copy"
@@ -1528,8 +1536,8 @@ extension Pixel.Event {
         case .privacyProKeychainAccessError: return "m_privacy-pro_keychain_access_error"
         case .privacyProSubscriptionCookieMissingTokenOnSignIn: return "m_privacy-pro_subscription-cookie-missing_token_on_sign_in"
         case .privacyProSubscriptionCookieMissingCookieOnSignOut: return "m_privacy-pro_subscription-cookie-missing_cookie_on_sign_out"
-        case .privacyProSubscriptionCookieRefreshedWithUpdate: return "m_privacy-pro_subscription-cookie-refreshed_with_update"
-        case .privacyProSubscriptionCookieRefreshedWithDelete: return "m_privacy-pro_subscription-cookie-refreshed_with_delete"
+        case .privacyProSubscriptionCookieRefreshedWithAccessToken: return "m_privacy-pro_subscription-cookie-refreshed_with_access_token"
+        case .privacyProSubscriptionCookieRefreshedWithEmptyValue: return "m_privacy-pro_subscription-cookie-refreshed_with_empty_value"
         case .privacyProSubscriptionCookieFailedToSetSubscriptionCookie: return "m_privacy-pro_subscription-cookie-failed_to_set_subscription_cookie"
 
         // MARK: Pixel Experiment
@@ -1623,6 +1631,10 @@ extension Pixel.Event {
         case .duckPlayerViewFromOther: return "duckplayer_view-from_other"
         case .duckPlayerSettingAlwaysSettings: return "duckplayer_setting_always_settings"
         case .duckPlayerSettingAlwaysDuckPlayer: return "duckplayer_setting_always_duck-player"
+        case .duckPlayerSettingsAlwaysOverlaySERP: return "duckplayer_setting_always_overlay_serp"
+        case .duckPlayerSettingsAlwaysOverlayYoutube: return "duckplayer_setting_always_overlay_youtube"
+        case .duckPlayerSettingsNeverOverlaySERP: return "duckplayer_setting_never_overlay_serp"
+        case .duckPlayerSettingsNeverOverlayYoutube: return "duckplayer_setting_never_overlay_youtube"
         case .duckPlayerOverlayYoutubeImpressions: return "duckplayer_overlay_youtube_impressions"
         case .duckPlayerOverlayYoutubeWatchHere: return "duckplayer_overlay_youtube_watch_here"
         case .duckPlayerSettingNeverSettings: return "duckplayer_setting_never_settings"
@@ -1656,18 +1668,16 @@ extension Pixel.Event {
         case .pproFeedbackSubmitScreenShow: return "m_ppro_feedback_submit-screen_show"
         case .pproFeedbackSubmitScreenFAQClick: return "m_ppro_feedback_submit-screen-faq_click"
 
-        // MARK: Duckplayer experiment
-        case .duckplayerExperimentCohortAssign: return "duckplayer_experiment_cohort_assign_v2"
-        case .duckplayerExperimentSearch: return "duckplayer_experiment_search_v2"
-        case .duckplayerExperimentDailySearch: return "duckplayer_experiment_daily_search_v2"
-        case .duckplayerExperimentWeeklySearch: return "duckplayer_experiment_weekly_search_v2"
-        case .duckplayerExperimentYoutubePageView: return "duckplayer_experiment_youtube_page_view_v2"
-            
         // MARK: - WebView Error Page shown
         case .webViewErrorPageShown: return "m_errorpageshown"
 
         // MARK: - DuckPlayer FE Application Telemetry
         case .duckPlayerLandscapeLayoutImpressions: return "duckplayer_landscape_layout_impressions"
+
+        // MARK: UserDefaults incositency monitoring
+        case .protectedDataUnavailableWhenBecomeActive: return "m_protected_data_unavailable_when_become_active"
+        case .statisticsLoaderATBStateMismatch: return "m_statistics_loader_atb_state_mismatch"
+        case .adAttributionReportStateMismatch: return "m_ad_attribution_report_state_mismatch"
         }
     }
 }

Core/StatisticsLoader.swift

@@ -33,17 +33,29 @@ public class StatisticsLoader {
     private let returnUserMeasurement: ReturnUserMeasurement
     private let usageSegmentation: UsageSegmenting
     private let parser = AtbParser()
+    private let atbPresenceFileMarker = BoolFileMarker(name: .isATBPresent)
+    private let inconsistencyMonitoring: StatisticsStoreInconsistencyMonitoring
 
     init(statisticsStore: StatisticsStore = StatisticsUserDefaults(),
          returnUserMeasurement: ReturnUserMeasurement = KeychainReturnUserMeasurement(),
-         usageSegmentation: UsageSegmenting = UsageSegmentation()) {
+         usageSegmentation: UsageSegmenting = UsageSegmentation(),
+         inconsistencyMonitoring: StatisticsStoreInconsistencyMonitoring = StorageInconsistencyMonitor()) {
         self.statisticsStore = statisticsStore
         self.returnUserMeasurement = returnUserMeasurement
         self.usageSegmentation = usageSegmentation
+        self.inconsistencyMonitoring = inconsistencyMonitoring
     }
 
     public func load(completion: @escaping Completion = {}) {
-        if statisticsStore.hasInstallStatistics {
+        let hasFileMarker = atbPresenceFileMarker?.isPresent ?? false
+        let hasInstallStatistics = statisticsStore.hasInstallStatistics
+
+        inconsistencyMonitoring.statisticsDidLoad(hasFileMarker: hasFileMarker, hasInstallStatistics: hasInstallStatistics)
+
+        if hasInstallStatistics {
+            // Synchronize file marker with current state
+            createATBFileMarker()
+
             completion()
             return
         }
@@ -85,10 +97,15 @@ public class StatisticsLoader {
             self.statisticsStore.installDate = Date()
             self.statisticsStore.atb = atb.version
             self.returnUserMeasurement.installCompletedWithATB(atb)
+            self.createATBFileMarker()
             completion()
         }
     }
 
+    private func createATBFileMarker() {
+        atbPresenceFileMarker?.mark()
+    }
+
     public func refreshSearchRetentionAtb(completion: @escaping Completion = {}) {
         guard let url = StatisticsDependentURLFactory(statisticsStore: statisticsStore).makeSearchAtbURL() else {
             requestInstallStatistics {
@@ -169,3 +186,7 @@ public class StatisticsLoader {
         processUsageSegmentation(atb: nil, activityType: activityType)
     }
 }
+
+private extension BoolFileMarker.Name {
+    static let isATBPresent = BoolFileMarker.Name(rawValue: "atb-present")
+}