Integrate and emit custom stats pixel

Author: karlenDimla

pir/pir-impl/build.gradle

@@ -70,6 +70,7 @@ dependencies {
     testImplementation AndroidX.test.ext.junit
     testImplementation Testing.robolectric
     testImplementation "androidx.lifecycle:lifecycle-runtime-testing:_"
+    testImplementation AndroidX.work.testing
     testImplementation(KotlinX.coroutines.test) {
         // https://github.com/Kotlin/kotlinx.coroutines/issues/2023
         // conflicts with mockito due to direct inclusion of byte buddy

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2025 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.pir.impl.pixels
+
+import com.duckduckgo.common.utils.DispatcherProvider
+import com.duckduckgo.di.scopes.AppScope
+import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED
+import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED
+import com.duckduckgo.pir.impl.store.PirSchedulingRepository
+import com.squareup.anvil.annotations.ContributesBinding
+import kotlinx.coroutines.withContext
+import java.util.concurrent.TimeUnit
+import javax.inject.Inject
+import kotlin.math.round
+
+interface OptOutSubmitRateCalculator {
+    /**
+     * Calculates the opt-out 24h submit rate for a given broker within the specified date range.
+     *
+     * @param brokerName name of the broker to calculate the opt-out submit rate for.
+     * @param startDateMs The opt-out records to include should be created on or after this date. Default is 0L (epoch).
+     * @param endDateMs tThe opt-out records to include should be created on or before this date. Default is 0L (epoch).
+     */
+    suspend fun calculateOptOutSubmitRate(
+        brokerName: String,
+        startDateMs: Long = 0L,
+        endDateMs: Long,
+    ): Double?
+}
+
+@ContributesBinding(AppScope::class)
+class RealOptOutSubmitRateCalculator @Inject constructor(
+    private val dispatcherProvider: DispatcherProvider,
+    private val schedulingRepository: PirSchedulingRepository,
+) : OptOutSubmitRateCalculator {
+    override suspend fun calculateOptOutSubmitRate(
+        brokerName: String,
+        startDateMs: Long,
+        endDateMs: Long,
+    ): Double? = withContext(dispatcherProvider.io()) {
+        // Get all opt out job records created within the given range for the specified broker
+        val recordsCreatedWithinRange = schedulingRepository.getAllValidOptOutJobRecordsForBroker(brokerName).filter {
+            it.brokerName == brokerName && it.dateCreatedInMillis in startDateMs..endDateMs
+        }
+
+        // We don't need to calculate the rate if there are no records
+        if (recordsCreatedWithinRange.isEmpty()) return@withContext null
+
+        // Filter the records to only include those that were requested within 24 hours of creation
+        val requestedRecordsWithinRange = recordsCreatedWithinRange.filter {
+            (it.status == REQUESTED || it.status == REMOVED) && it.optOutRequestedDateInMillis > it.dateCreatedInMillis &&
+                it.optOutRequestedDateInMillis <= it.dateCreatedInMillis + TimeUnit.HOURS.toMillis(
+                    24,
+                )
+        }
+
+        val optOutSuccessRate = requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble()
+        val roundedOptOutSuccessRate = round(optOutSuccessRate * 100) / 100
+        return@withContext roundedOptOutSuccessRate
+    }
+}

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2025 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.pir.impl.pixels
+
+import android.content.Context
+import androidx.work.CoroutineWorker
+import androidx.work.WorkerParameters
+import com.duckduckgo.anvil.annotations.ContributesWorker
+import com.duckduckgo.common.utils.CurrentTimeProvider
+import com.duckduckgo.di.scopes.AppScope
+import com.duckduckgo.pir.impl.store.PirRepository
+import java.util.concurrent.TimeUnit
+import javax.inject.Inject
+import kotlin.math.abs
+
+@ContributesWorker(AppScope::class)
+class PirCustomStatsWorker(
+    context: Context,
+    workerParameters: WorkerParameters,
+) : CoroutineWorker(context, workerParameters) {
+    @Inject
+    lateinit var optOutSubmitRateCalculator: OptOutSubmitRateCalculator
+
+    @Inject
+    lateinit var pirRepository: PirRepository
+
+    @Inject
+    lateinit var currentTimeProvider: CurrentTimeProvider
+
+    @Inject
+    lateinit var pirPixelSender: PirPixelSender
+
+    override suspend fun doWork(): Result {
+        val startDate = pirRepository.getCustomStatsPixelsLastSentMs()
+        val now = currentTimeProvider.currentTimeMillis()
+
+        if (shouldFirePixel(startDate, now)) {
+            val endDate = now - TimeUnit.HOURS.toMillis(24)
+            val activeBrokers = pirRepository.getAllActiveBrokerObjects()
+            val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty()
+
+            if (activeBrokers.isNotEmpty() && hasUserProfiles) {
+                activeBrokers.forEach {
+                    val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate(
+                        it.name,
+                        startDate,
+                        endDate,
+                    )
+
+                    if (successRate != null) {
+                        pirPixelSender.reportBrokerCustomStateOptOutSubmitRate(
+                            brokerUrl = it.url,
+                            optOutSuccessRate = successRate,
+                        )
+                    }
+                }
+
+                pirRepository.setCustomStatsPixelsLastSentMs(endDate)
+            }
+        }
+
+        return Result.success()
+    }
+
+    private fun shouldFirePixel(
+        startDate: Long,
+        now: Long,
+    ): Boolean {
+        return if (startDate == 0L) {
+            // IF first run, we emit the custom stats pixel
+            true
+        } else {
+            // Else we check if at least 24 hours have passed since last emission
+            val nowDiffFromStart = abs(now - startDate)
+            nowDiffFromStart > TimeUnit.HOURS.toMillis(24)
+        }
+    }
+
+    companion object {
+        const val TAG_PIR_RECURRING_CUSTOM_STATS = "TAG_PIR_RECURRING_CUSTOM_STATS"
+    }
+}

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt

@@ -24,6 +24,7 @@ import androidx.work.Data
 import androidx.work.ExistingPeriodicWorkPolicy
 import androidx.work.NetworkType
 import androidx.work.PeriodicWorkRequest
+import androidx.work.PeriodicWorkRequestBuilder
 import androidx.work.WorkManager
 import androidx.work.multiprocess.RemoteListenableWorker
 import com.duckduckgo.app.di.AppCoroutineScope
@@ -34,6 +35,8 @@ import com.duckduckgo.pir.impl.common.PirJobConstants.EMAIL_CONFIRMATION_INTERVA
 import com.duckduckgo.pir.impl.common.PirJobConstants.SCHEDULED_SCAN_INTERVAL_HOURS
 import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker
 import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker.Companion.TAG_EMAIL_CONFIRMATION
+import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker
+import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker.Companion.TAG_PIR_RECURRING_CUSTOM_STATS
 import com.duckduckgo.pir.impl.pixels.PirPixelSender
 import com.duckduckgo.pir.impl.scan.PirScheduledScanRemoteWorker.Companion.TAG_SCHEDULED_SCAN
 import com.duckduckgo.pir.impl.store.PirEventsRepository
@@ -66,6 +69,7 @@ class RealPirScanScheduler @Inject constructor(
 
         schedulePirScans()
         scheduleEmailConfirmation()
+        scheduleRecurringPixelStats()
     }
 
     private fun schedulePirScans() {
@@ -129,9 +133,22 @@ class RealPirScanScheduler @Inject constructor(
         )
     }
 
+    private fun scheduleRecurringPixelStats() {
+        val periodicWorkRequest = PeriodicWorkRequestBuilder<PirCustomStatsWorker>(5, TimeUnit.HOURS)
+            .addTag(TAG_PIR_RECURRING_CUSTOM_STATS)
+            .build()
+
+        workManager.enqueueUniquePeriodicWork(
+            TAG_PIR_RECURRING_CUSTOM_STATS,
+            ExistingPeriodicWorkPolicy.UPDATE,
+            periodicWorkRequest,
+        )
+    }
+
     override fun cancelScheduledScans(context: Context) {
         workManager.cancelUniqueWork(TAG_SCHEDULED_SCAN)
         workManager.cancelUniqueWork(TAG_EMAIL_CONFIRMATION)
+        workManager.cancelUniqueWork(TAG_PIR_RECURRING_CUSTOM_STATS)
         context.stopService(Intent(context, PirRemoteWorkerService::class.java))
     }
 

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt

@@ -22,6 +22,7 @@ import com.duckduckgo.data.store.api.SharedPreferencesProvider
 
 interface PirDataStore {
     var mainConfigEtag: String?
+    var customStatsPixelsLastSentMs: Long
 }
 
 internal class RealPirDataStore(
@@ -42,8 +43,17 @@ internal class RealPirDataStore(
             }
         }
 
+    override var customStatsPixelsLastSentMs: Long
+        get() = preferences.getLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, 0L)
+        set(value) {
+            preferences.edit {
+                putLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, value)
+            }
+        }
+
     companion object {
         private const val FILENAME = "com.duckduckgo.pir.v1"
         private const val KEY_MAIN_ETAG = "KEY_MAIN_ETAG"
+        private const val KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS = "KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS"
     }
 }

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt

@@ -174,6 +174,10 @@ interface PirRepository {
 
     suspend fun deleteEmailData(emailData: List<EmailData>)
 
+    suspend fun getCustomStatsPixelsLastSentMs(): Long
+
+    suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long)
+
     data class GeneratedEmailData(
         val emailAddress: String,
         val pattern: String,
@@ -663,6 +667,14 @@ class RealPirRepository(
             return@withContext
         }
 
+    override suspend fun getCustomStatsPixelsLastSentMs(): Long = withContext(dispatcherProvider.io()) {
+        pirDataStore.customStatsPixelsLastSentMs
+    }
+
+    override suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) = withContext(dispatcherProvider.io()) {
+        pirDataStore.customStatsPixelsLastSentMs = timeMs
+    }
+
     private fun List<EmailData>.toRequest(): PirEmailConfirmationDataRequest =
         PirEmailConfirmationDataRequest(
             items =

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt

@@ -60,6 +60,11 @@ interface PirSchedulingRepository {
      */
     suspend fun getAllValidOptOutJobRecords(): List<OptOutJobRecord>
 
+    /**
+     * Returns all ScanJobRecord whose state is not INVALID for a specific broker
+     */
+    suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List<OptOutJobRecord>
+
     /**
      * Returns a matching [OptOutJobRecord] whose state is not INVALID
      *
@@ -181,6 +186,16 @@ class RealPirSchedulingRepository @Inject constructor(
                 .orEmpty()
         }
 
+    override suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List<OptOutJobRecord> =
+        withContext(dispatcherProvider.io()) {
+            return@withContext jobSchedulingDao()
+                ?.getAllOptOutJobRecordsForBroker(brokerName)
+                ?.map { record -> record.toRecord() }
+                // do not pick-up deprecated jobs as they belong to removed profiles
+                ?.filter { !it.deprecated }
+                .orEmpty()
+        }
+
     override suspend fun saveScanJobRecord(scanJobRecord: ScanJobRecord) {
         withContext(dispatcherProvider.io()) {
             jobSchedulingDao()?.saveScanJobRecord(scanJobRecord.toEntity())

pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt

@@ -40,6 +40,9 @@ interface JobSchedulingDao {
     @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount")
     fun getAllOptOutJobRecords(): List<OptOutJobRecordEntity>
 
+    @Query("SELECT * FROM pir_optout_job_record WHERE brokerName = :brokerName ORDER BY attemptCount")
+    fun getAllOptOutJobRecordsForBroker(brokerName: String): List<OptOutJobRecordEntity>
+
     @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount")
     fun getAllOptOutJobRecordsFlow(): Flow<List<OptOutJobRecordEntity>>