DBP: Implement pixels to observe initial scan loading times (#2706)

duckduckgo · Apr 29, 2024 · 79d648d · 79d648d
1 parent f14a2d4
commit 79d648d
Show file tree

Hide file tree

Showing 27 changed files with 204 additions and 51 deletions.
diff --git a/DuckDuckGo/DBP/DataBrokerProtectionAppEvents.swift b/DuckDuckGo/DBP/DataBrokerProtectionAppEvents.swift
@@ -49,8 +49,8 @@ struct DataBrokerProtectionAppEvents {
 
             // If we don't have profileQueries it means there's no user profile saved in our DB
             // In this case, let's disable the agent and delete any left-over data because there's nothing for it to do
-            if let profileQueries = try? DataBrokerProtectionManager.shared.dataManager.fetchBrokerProfileQueryData(ignoresCache: true),
-               profileQueries.count > 0 {
+            if let profileQueriesCount = try? DataBrokerProtectionManager.shared.dataManager.profileQueriesCount(),
+               profileQueriesCount > 0 {
                 restartBackgroundAgent(loginItemsManager: loginItemsManager)
             } else {
                 featureVisibility.disableAndDeleteForWaitlistUsers()

diff --git a/DuckDuckGo/DBP/DataBrokerProtectionDebugMenu.swift b/DuckDuckGo/DBP/DataBrokerProtectionDebugMenu.swift
@@ -227,7 +227,7 @@ final class DataBrokerProtectionDebugMenu: NSMenu {
         os_log("Running scan operations...", log: .dataBrokerProtection)
         let showWebView = sender.representedObject as? Bool ?? false
 
-        DataBrokerProtectionManager.shared.scheduler.startManualScan(showWebView: showWebView) { errors in
+        DataBrokerProtectionManager.shared.scheduler.startManualScan(showWebView: showWebView, startTime: Date()) { errors in
             if let errors = errors {
                 if let oneTimeError = errors.oneTimeError {
                     os_log("scan operations finished, error: %{public}@", log: .dataBrokerProtection, oneTimeError.localizedDescription)

diff --git a/DuckDuckGo/DBP/DataBrokerProtectionLoginItemScheduler.swift b/DuckDuckGo/DBP/DataBrokerProtectionLoginItemScheduler.swift
@@ -56,9 +56,10 @@ extension DataBrokerProtectionLoginItemScheduler: DataBrokerProtectionScheduler
     }
 
     func startManualScan(showWebView: Bool,
+                         startTime: Date,
                          completion: ((DataBrokerProtectionSchedulerErrorCollection?) -> Void)?) {
         enableLoginItem()
-        ipcScheduler.startManualScan(showWebView: showWebView, completion: completion)
+        ipcScheduler.startManualScan(showWebView: showWebView, startTime: startTime, completion: completion)
     }
 
     func startScheduler(showWebView: Bool) {

diff --git a/DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift b/DuckDuckGo/DBP/DataBrokerProtectionPixelsHandler.swift
@@ -99,7 +99,11 @@ public class DataBrokerProtectionPixelsHandler: EventMapping<DataBrokerProtectio
                     .webUILoadingStarted,
                     .webUILoadingSuccess,
                     .emptyAccessTokenDaily,
-                    .generateEmailHTTPErrorDaily:
+                    .generateEmailHTTPErrorDaily,
+                    .initialScanTotalDuration,
+                    .initialScanSiteLoadDuration,
+                    .initialScanPostLoadingDuration,
+                    .initialScanPreStartDuration:
                 PixelKit.fire(event)
 
             case .homeViewShowNoPermissionError,

diff --git a/DuckDuckGoDBPBackgroundAgent/IPCServiceManager.swift b/DuckDuckGoDBPBackgroundAgent/IPCServiceManager.swift
@@ -85,9 +85,10 @@ extension IPCServiceManager: IPCServerInterface {
     }
 
     func startManualScan(showWebView: Bool,
+                         startTime: Date,
                          completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void)) {
         pixelHandler.fire(.ipcServerScanAllBrokersReceivedByAgent)
-        scheduler.startManualScan(showWebView: showWebView) { errors in
+        scheduler.startManualScan(showWebView: showWebView, startTime: startTime) { errors in
             if let error = errors?.oneTimeError {
                 switch error {
                 case DataBrokerProtectionSchedulerError.operationsInterrupted:

diff --git a/...kerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDataManager.swift b/...kerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDataManager.swift
@@ -30,12 +30,7 @@ public protocol DataBrokerProtectionDataManaging {
     func fetchBrokerProfileQueryData(ignoresCache: Bool) throws -> [BrokerProfileQueryData]
     func prepareBrokerProfileQueryDataCache() throws
     func hasMatches() throws -> Bool
-}
-
-extension DataBrokerProtectionDataManaging {
-    func fetchBrokerProfileQueryData() throws -> [BrokerProfileQueryData] {
-        try fetchBrokerProfileQueryData(ignoresCache: false)
-    }
+    func profileQueriesCount() throws -> Int
 }
 
 public protocol DataBrokerProtectionDataManagerDelegate: AnyObject {
@@ -74,6 +69,18 @@ public class DataBrokerProtectionDataManager: DataBrokerProtectionDataManaging {
             return cache.profile
         }
 
+        return try fetchProfileFromDB()
+    }
+
+    public func profileQueriesCount() throws -> Int {
+        guard let profile = try fetchProfileFromDB() else {
+            throw DataBrokerProtectionError.dataNotInDatabase
+        }
+
+        return profile.profileQueries.count
+    }
+
+    private func fetchProfileFromDB() throws -> DataBrokerProtectionProfile? {
         if let profile = try database.fetchProfile() {
             cache.profile = profile
             return profile
@@ -287,7 +294,7 @@ extension InMemoryDataCache: DBPUICommunicationDelegate {
     }
 
     func startScanAndOptOut() -> Bool {
-        return scanDelegate?.startScan() ?? false
+        return scanDelegate?.startScan(startDate: Date()) ?? false
     }
 
     func getInitialScanState() async -> DBPUIInitialScanState {

diff --git a/...kerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift b/...kerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift
@@ -145,6 +145,9 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject {
 
     private let runnerProvider: OperationRunnerProvider
     private let privacyConfigManager: PrivacyConfigurationManaging
+    private let fakePixelHandler: EventMapping<DataBrokerProtectionPixels> = EventMapping { event, _, _, _ in
+        print(event)
+    }
     private let contentScopeProperties: ContentScopeProperties
     private let csvColumns = ["name_input", "age_input", "city_input", "state_input", "name_scraped", "age_scraped", "address_scraped", "relatives_scraped", "url", "broker name", "screenshot_id", "error", "matched_fields", "result_match", "expected_match"]
 
@@ -347,7 +350,7 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject {
 
                     Task {
                         do {
-                            let extractedProfiles = try await runner.scan(query, stageCalculator: FakeStageDurationCalculator(), showWebView: true) { true }
+                            let extractedProfiles = try await runner.scan(query, stageCalculator: FakeStageDurationCalculator(), pixelHandler: fakePixelHandler, showWebView: true) { true }
 
                             DispatchQueue.main.async {
                                 for extractedProfile in extractedProfiles {
@@ -383,7 +386,7 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject {
         )
         Task {
             do {
-                try await runner.optOut(profileQuery: brokerProfileQueryData, extractedProfile: scanResult.extractedProfile, stageCalculator: FakeStageDurationCalculator(), showWebView: true) {
+                try await runner.optOut(profileQuery: brokerProfileQueryData, extractedProfile: scanResult.extractedProfile, stageCalculator: FakeStageDurationCalculator(), pixelHandler: fakePixelHandler, showWebView: true) {
                     true
                 }
 
@@ -473,6 +476,7 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject {
 
 final class FakeStageDurationCalculator: StageDurationCalculator {
     var attemptId: UUID = UUID()
+    var isManualScan: Bool = false
 
     func durationSinceLastStage() -> Double {
         0.0

diff --git a/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift b/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift
@@ -68,6 +68,8 @@ final class DebugScanOperation: DataBrokerOperation {
     var scanURL: String?
     let clickAwaitTime: TimeInterval
     let cookieHandler: CookieHandler
+    let pixelHandler: EventMapping<DataBrokerProtectionPixels>
+    var postLoadingSiteStartTime: Date?
 
     private let fileManager = FileManager.default
     private let debugScanContentPath: String?
@@ -96,6 +98,9 @@ final class DebugScanOperation: DataBrokerOperation {
         }
         self.cookieHandler = EmptyCookieHandler()
         stageCalculator = FakeStageDurationCalculator()
+        pixelHandler =  EventMapping(mapping: { _, _, _, _ in
+            // We do not need the pixel handler for the debug
+        })
     }
 
     func run(inputValue: Void,

diff --git a/...DataBrokerProtection/Sources/DataBrokerProtection/IPC/DataBrokerProtectionIPCClient.swift b/...DataBrokerProtection/Sources/DataBrokerProtection/IPC/DataBrokerProtectionIPCClient.swift
@@ -137,6 +137,7 @@ extension DataBrokerProtectionIPCClient: IPCServerInterface {
     }
 
     public func startManualScan(showWebView: Bool,
+                                startTime: Date,
                                 completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void)) {
         self.pixelHandler.fire(.ipcServerScanAllBrokersCalledByApp)
 
@@ -155,7 +156,7 @@ extension DataBrokerProtectionIPCClient: IPCServerInterface {
         }
 
         xpc.execute(call: { server in
-            server.startManualScan(showWebView: showWebView) { errors in
+            server.startManualScan(showWebView: showWebView, startTime: startTime) { errors in
                 if let error = errors?.oneTimeError {
                     let nsError = error as NSError
                     let interruptedError = DataBrokerProtectionSchedulerError.operationsInterrupted as NSError

diff --git a/...aBrokerProtection/Sources/DataBrokerProtection/IPC/DataBrokerProtectionIPCScheduler.swift b/...aBrokerProtection/Sources/DataBrokerProtection/IPC/DataBrokerProtectionIPCScheduler.swift
@@ -52,9 +52,10 @@ public final class DataBrokerProtectionIPCScheduler: DataBrokerProtectionSchedul
     }
 
     public func startManualScan(showWebView: Bool,
+                                startTime: Date,
                                 completion: ((DataBrokerProtectionSchedulerErrorCollection?) -> Void)?) {
         let completion = completion ?? { _ in }
-        ipcClient.startManualScan(showWebView: showWebView, completion: completion)
+        ipcClient.startManualScan(showWebView: showWebView, startTime: startTime, completion: completion)
     }
 
     public func runQueuedOperations(showWebView: Bool,

diff --git a/...DataBrokerProtection/Sources/DataBrokerProtection/IPC/DataBrokerProtectionIPCServer.swift b/...DataBrokerProtection/Sources/DataBrokerProtection/IPC/DataBrokerProtectionIPCServer.swift
@@ -95,6 +95,7 @@ public protocol IPCServerInterface: AnyObject {
     func optOutAllBrokers(showWebView: Bool,
                           completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void))
     func startManualScan(showWebView: Bool,
+                         startTime: Date,
                          completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void))
     func runQueuedOperations(showWebView: Bool,
                              completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void))
@@ -136,6 +137,7 @@ protocol XPCServerInterface {
     func optOutAllBrokers(showWebView: Bool,
                           completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void))
     func startManualScan(showWebView: Bool,
+                         startTime: Date,
                          completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void))
     func runQueuedOperations(showWebView: Bool,
                              completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void))
@@ -214,8 +216,9 @@ extension DataBrokerProtectionIPCServer: XPCServerInterface {
     }
 
     func startManualScan(showWebView: Bool,
+                         startTime: Date,
                          completion: @escaping ((DataBrokerProtectionSchedulerErrorCollection?) -> Void)) {
-        serverDelegate?.startManualScan(showWebView: showWebView, completion: completion)
+        serverDelegate?.startManualScan(showWebView: showWebView, startTime: startTime, completion: completion)
     }
 
     func runQueuedOperations(showWebView: Bool,

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DBPUIViewModel.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DBPUIViewModel.swift
@@ -23,7 +23,7 @@ import BrowserServicesKit
 import Common
 
 protocol DBPUIScanOps: AnyObject {
-    func startScan() -> Bool
+    func startScan(startDate: Date) -> Bool
     func updateCacheWithCurrentScans() async
     func getBackgroundAgentMetadata() async -> DBPBackgroundAgentMetadata?
 }
@@ -74,8 +74,8 @@ final class DBPUIViewModel {
 }
 
 extension DBPUIViewModel: DBPUIScanOps {
-    func startScan() -> Bool {
-        scheduler.startManualScan()
+    func startScan(startDate: Date) -> Bool {
+        scheduler.startManualScan(startTime: startDate)
         return true
     }
 

diff --git a/...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift
@@ -33,6 +33,7 @@ protocol DataBrokerOperation: CCFCommunicationDelegate {
     var captchaService: CaptchaServiceProtocol { get }
     var cookieHandler: CookieHandler { get }
     var stageCalculator: StageDurationCalculator { get }
+    var pixelHandler: EventMapping<DataBrokerProtectionPixels> { get }
 
     var webViewHandler: WebViewHandler? { get set }
     var actionsHandler: ActionsHandler? { get }
@@ -41,6 +42,7 @@ protocol DataBrokerOperation: CCFCommunicationDelegate {
     var shouldRunNextStep: () -> Bool { get }
     var retriesCountOnError: Int { get set }
     var clickAwaitTime: TimeInterval { get }
+    var postLoadingSiteStartTime: Date? { get set }
 
     func run(inputValue: InputValue,
              webViewHandler: WebViewHandler?,
@@ -151,11 +153,13 @@ extension DataBrokerOperation {
     }
 
     func complete(_ value: ReturnValue) {
+        self.firePostLoadingDurationPixel(hasError: false)
         self.continuation?.resume(returning: value)
         self.continuation = nil
     }
 
     func failed(with error: Error) {
+        self.firePostLoadingDurationPixel(hasError: true)
         self.continuation?.resume(throwing: error)
         self.continuation = nil
     }
@@ -175,6 +179,8 @@ extension DataBrokerOperation {
     // MARK: - CSSCommunicationDelegate
 
     func loadURL(url: URL) async {
+        let webSiteStartLoadingTime = Date()
+
         do {
             // https://app.asana.com/0/1204167627774280/1206912494469284/f
             if query.dataBroker.url == "spokeo.com" {
@@ -183,12 +189,31 @@ extension DataBrokerOperation {
                 }
             }
             try await webViewHandler?.load(url: url)
+            fireSiteLoadingPixel(startTime: webSiteStartLoadingTime, hasError: false)
+            postLoadingSiteStartTime = Date()
             await executeNextStep()
         } catch {
+            fireSiteLoadingPixel(startTime: webSiteStartLoadingTime, hasError: true)
             await onError(error: error)
         }
     }
 
+    private func fireSiteLoadingPixel(startTime: Date, hasError: Bool) {
+        if stageCalculator.isManualScan {
+            let dataBrokerURL = self.query.dataBroker.url
+            let durationInMs = (Date().timeIntervalSince(startTime) * 1000).rounded(.towardZero)
+            pixelHandler.fire(.initialScanSiteLoadDuration(duration: durationInMs, hasError: hasError, brokerURL: dataBrokerURL))
+        }
+    }
+
+    func firePostLoadingDurationPixel(hasError: Bool) {
+        if stageCalculator.isManualScan, let postLoadingSiteStartTime = self.postLoadingSiteStartTime {
+            let dataBrokerURL = self.query.dataBroker.url
+            let durationInMs = (Date().timeIntervalSince(postLoadingSiteStartTime) * 1000).rounded(.towardZero)
+            pixelHandler.fire(.initialScanPostLoadingDuration(duration: durationInMs, hasError: hasError, brokerURL: dataBrokerURL))
+        }
+    }
+
     func success(actionId: String, actionType: ActionType) async {
         switch actionType {
         case .click:

diff --git a/...aBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperationRunner.swift b/...aBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperationRunner.swift
@@ -24,12 +24,14 @@ protocol WebOperationRunner {
 
     func scan(_ profileQuery: BrokerProfileQueryData,
               stageCalculator: StageDurationCalculator,
+              pixelHandler: EventMapping<DataBrokerProtectionPixels>,
               showWebView: Bool,
               shouldRunNextStep: @escaping () -> Bool) async throws -> [ExtractedProfile]
 
     func optOut(profileQuery: BrokerProfileQueryData,
                 extractedProfile: ExtractedProfile,
                 stageCalculator: StageDurationCalculator,
+                pixelHandler: EventMapping<DataBrokerProtectionPixels>,
                 showWebView: Bool,
                 shouldRunNextStep: @escaping () -> Bool) async throws
 }
@@ -38,22 +40,26 @@ extension WebOperationRunner {
 
     func scan(_ profileQuery: BrokerProfileQueryData,
               stageCalculator: StageDurationCalculator,
+              pixelHandler: EventMapping<DataBrokerProtectionPixels>,
               shouldRunNextStep: @escaping () -> Bool) async throws -> [ExtractedProfile] {
 
         try await scan(profileQuery,
                        stageCalculator: stageCalculator,
+                       pixelHandler: pixelHandler,
                        showWebView: false,
                        shouldRunNextStep: shouldRunNextStep)
     }
 
     func optOut(profileQuery: BrokerProfileQueryData,
                 extractedProfile: ExtractedProfile,
                 stageCalculator: StageDurationCalculator,
+                pixelHandler: EventMapping<DataBrokerProtectionPixels>,
                 shouldRunNextStep: @escaping () -> Bool) async throws {
 
         try await optOut(profileQuery: profileQuery,
                          extractedProfile: extractedProfile,
                          stageCalculator: stageCalculator,
+                         pixelHandler: pixelHandler,
                          showWebView: false,
                          shouldRunNextStep: shouldRunNextStep)
     }
@@ -78,6 +84,7 @@ final class DataBrokerOperationRunner: WebOperationRunner {
 
     func scan(_ profileQuery: BrokerProfileQueryData,
               stageCalculator: StageDurationCalculator,
+              pixelHandler: EventMapping<DataBrokerProtectionPixels>,
               showWebView: Bool,
               shouldRunNextStep: @escaping () -> Bool) async throws -> [ExtractedProfile] {
         let scan = ScanOperation(
@@ -87,6 +94,7 @@ final class DataBrokerOperationRunner: WebOperationRunner {
             emailService: emailService,
             captchaService: captchaService,
             stageDurationCalculator: stageCalculator,
+            pixelHandler: pixelHandler,
             shouldRunNextStep: shouldRunNextStep
         )
         return try await scan.run(inputValue: (), showWebView: showWebView)
@@ -95,6 +103,7 @@ final class DataBrokerOperationRunner: WebOperationRunner {
     func optOut(profileQuery: BrokerProfileQueryData,
                 extractedProfile: ExtractedProfile,
                 stageCalculator: StageDurationCalculator,
+                pixelHandler: EventMapping<DataBrokerProtectionPixels>,
                 showWebView: Bool,
                 shouldRunNextStep: @escaping () -> Bool) async throws {
         let optOut = OptOutOperation(
@@ -104,6 +113,7 @@ final class DataBrokerOperationRunner: WebOperationRunner {
             emailService: emailService,
             captchaService: captchaService,
             stageCalculator: stageCalculator,
+            pixelHandler: pixelHandler,
             shouldRunNextStep: shouldRunNextStep
         )
         try await optOut.run(inputValue: extractedProfile, showWebView: showWebView)

diff --git a/...erProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperationsCollection.swift b/...erProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperationsCollection.swift
@@ -181,6 +181,7 @@ final class DataBrokerOperationsCollection: Operation {
                                                                                 runner: runner,
                                                                                 pixelHandler: pixelHandler,
                                                                                 showWebView: showWebView,
+                                                                                isManualScan: operationType == .scan,
                                                                                 userNotificationService: userNotificationService,
                                                                                 shouldRunNextStep: { [weak self] in
                     guard let self = self else { return false }