PIR: Implement pixels for breakage KPIs

PixelDefinitions/pixels/personal_information_removal.json5

@@ -148,5 +148,117 @@
                 "type": "string"
             }
         ]
+    },
+    "dbp_optoutjob_at-7-days_confirmed": {
+        "description": "Pixel that contains if any submitted opt-out has been confirmed 7 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has a confirmed opt-out after 7 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-7-days_unconfirmed": {
+        "description": "Pixel that contains if any submitted opt-out are still unconfirmed 7 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has an unconfirmed opt-out after 7 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-14-days_confirmed": {
+        "description": "Pixel that contains if any submitted opt-out has been confirmed 14 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has a confirmed opt-out after 14 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-14-days_unconfirmed": {
+        "description": "Pixel that contains if any submitted opt-out are still unconfirmed 14 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has an unconfirmed opt-out after 14 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-21-days_confirmed": {
+        "description": "Pixel that contains if any submitted opt-out has been confirmed 21 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has a confirmed opt-out after 21 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-21-days_unconfirmed": {
+        "description": "Pixel that contains if any submitted opt-out are still unconfirmed 21 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has an unconfirmed opt-out after 21 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-42-days_confirmed": {
+        "description": "Pixel that contains if any submitted opt-out has been confirmed 42 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has a confirmed opt-out after 42 days",
+                "type": "string"
+            }
+        ]
+    },
+    "dbp_optoutjob_at-42-days_unconfirmed": {
+        "description": "Pixel that contains if any submitted opt-out are still unconfirmed 42 days after submission.",
+        "owners": ["karlenDimla", "landomen"],
+        "triggers": ["other"],
+        "suffixes": ["form_factor"],
+        "parameters": [
+            "appVersion",
+            {
+                "key": "data_broker",
+                "description": "The URL of the data broker that has an unconfirmed opt-out after 42 days",
+                "type": "string"
+            }
+        ]
     }
 }

pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json

@@ -2,7 +2,7 @@
   "formatVersion": 1,
   "database": {
     "version": 15,
-    "identityHash": "584648b8b3065521786fb33f44214f2e",
+    "identityHash": "a5782d19654dee4cad932b458037bb37",
     "entities": [
       {
         "tableName": "pir_broker_json_etag",
@@ -696,7 +696,7 @@
       },
       {
         "tableName": "pir_optout_job_record",
-        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `reporting_sevenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fourteenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_twentyOneDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fortyTwoDayConfirmationReportSentDateMs` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))",
         "fields": [
           {
             "fieldPath": "extractedProfileId",
@@ -757,6 +757,30 @@
             "columnName": "dateCreatedInMillis",
             "affinity": "INTEGER",
             "notNull": true
+          },
+          {
+            "fieldPath": "reporting.sevenDayConfirmationReportSentDateMs",
+            "columnName": "reporting_sevenDayConfirmationReportSentDateMs",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "reporting.fourteenDayConfirmationReportSentDateMs",
+            "columnName": "reporting_fourteenDayConfirmationReportSentDateMs",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "reporting.twentyOneDayConfirmationReportSentDateMs",
+            "columnName": "reporting_twentyOneDayConfirmationReportSentDateMs",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "reporting.fortyTwoDayConfirmationReportSentDateMs",
+            "columnName": "reporting_fortyTwoDayConfirmationReportSentDateMs",
+            "affinity": "INTEGER",
+            "notNull": true
           }
         ],
         "primaryKey": {
@@ -946,7 +970,7 @@
     "views": [],
     "setupQueries": [
       "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
-      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '584648b8b3065521786fb33f44214f2e')"
+      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'a5782d19654dee4cad932b458037bb37')"
     ]
   }
 }

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt

@@ -22,4 +22,5 @@ object PirJobConstants {
     const val MAX_DETACHED_WEBVIEW_COUNT = 20
     const val SCHEDULED_SCAN_INTERVAL_HOURS = 12L
     const val EMAIL_CONFIRMATION_INTERVAL_HOURS = 8L
+    const val CUSTOM_PIXEL_INTERVAL_HOURS = 5L
 }

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt

@@ -56,6 +56,10 @@ sealed class JobRecord(
         val optOutRemovedDateInMillis: Long = 0L,
         val deprecated: Boolean = false,
         val dateCreatedInMillis: Long = 0L,
+        val confirmation7dayReportSentDateMs: Long = 0L,
+        val confirmation14dayReportSentDateMs: Long = 0L,
+        val confirmation21dayReportSentDateMs: Long = 0L,
+        val confirmation42dayReportSentDateMs: Long = 0L,
     ) : JobRecord(brokerName, userProfileId) {
         enum class OptOutJobStatus {
             /** Opt-out has not been executed yet and should be executed when possible */

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt

@@ -17,9 +17,13 @@
 package com.duckduckgo.pir.impl.pixels
 
 import com.duckduckgo.common.utils.CurrentTimeProvider
+import com.duckduckgo.common.utils.DispatcherProvider
 import com.duckduckgo.di.scopes.AppScope
 import com.duckduckgo.pir.impl.store.PirRepository
+import com.duckduckgo.pir.impl.store.PirSchedulingRepository
 import com.squareup.anvil.annotations.ContributesBinding
+import kotlinx.coroutines.withContext
+import logcat.logcat
 import java.util.concurrent.TimeUnit
 import javax.inject.Inject
 import kotlin.math.abs
@@ -34,33 +38,46 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor(
     private val pirRepository: PirRepository,
     private val currentTimeProvider: CurrentTimeProvider,
     private val pirPixelSender: PirPixelSender,
+    private val pirSchedulingRepository: PirSchedulingRepository,
+    private val dispatcherProvider: DispatcherProvider,
 ) : OptOut24HourSubmissionSuccessRateReporter {
     override suspend fun attemptFirePixel() {
-        val startDate = pirRepository.getCustomStatsPixelsLastSentMs()
-        val now = currentTimeProvider.currentTimeMillis()
+        withContext(dispatcherProvider.io()) {
+            logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" }
+            val startDate = pirRepository.getCustomStatsPixelsLastSentMs()
+            val now = currentTimeProvider.currentTimeMillis()
 
-        if (shouldFirePixel(startDate, now)) {
-            val endDate = now - TimeUnit.HOURS.toMillis(24)
-            val activeBrokers = pirRepository.getAllActiveBrokerObjects()
-            val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty()
+            if (shouldFirePixel(startDate, now)) {
+                logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" }
+                val endDate = now - TimeUnit.HOURS.toMillis(24)
+                val activeBrokers = pirRepository.getAllActiveBrokerObjects()
+                val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty()
+                val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords()
 
-            if (activeBrokers.isNotEmpty() && hasUserProfiles) {
-                activeBrokers.forEach {
-                    val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate(
-                        it.name,
-                        startDate,
-                        endDate,
-                    )
+                if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) {
+                    activeBrokers.forEach { broker ->
+                        val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name }
 
-                    if (successRate != null) {
-                        pirPixelSender.reportBrokerCustomStateOptOutSubmitRate(
-                            brokerUrl = it.url,
-                            optOutSuccessRate = successRate,
+                        if (activeJobRecordsForBroker.isEmpty()) return@forEach
+
+                        val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate(
+                            activeJobRecordsForBroker,
+                            startDate,
+                            endDate,
                         )
+
+                        logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" }
+                        if (successRate != null) {
+                            pirPixelSender.reportBrokerCustomStateOptOutSubmitRate(
+                                brokerUrl = broker.url,
+                                optOutSuccessRate = successRate,
+                            )
+                        }
                     }
-                }
 
-                pirRepository.setCustomStatsPixelsLastSentMs(endDate)
+                    logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" }
+                    pirRepository.setCustomStatsPixelsLastSentMs(endDate)
+                }
             }
         }
     }