From d15ad2e1cddfa678dd50242761e9859b8097192f Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Tue, 11 Nov 2025 16:44:42 +0100 Subject: [PATCH 01/16] Add pixel definition for 7 days confirmed/unconfirmed --- .../pixels/personal_information_removal.json5 | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index dd0b36814b67..1688ebc781c9 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -148,5 +148,33 @@ "type": "string" } ] + }, + "dbp_optoutjob_at-7-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-7-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 7 days", + "type": "string" + } + ] } } From d270135e6e9b8d36043a0370acf0d145a5ae48ed Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Tue, 11 Nov 2025 17:00:15 +0100 Subject: [PATCH 02/16] Add sender for 7 day confirmed/unconfirmed --- .../duckduckgo/pir/impl/pixels/PirPixel.kt | 10 +++++++ .../pir/impl/pixels/PirPixelSender.kt | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index 1b70031b8118..7639523fa083 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -140,6 +140,16 @@ enum class PirPixel( PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE( baseName = "dbp_databroker_custom_stats_optoutsubmit", type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index 53ff572fabaa..b688940d023f 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -18,6 +18,8 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_FAILED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_START @@ -333,6 +335,16 @@ interface PirPixelSender { brokerUrl: String, optOutSuccessRate: Double, ) + + /** + * Emits a pixel when an opt-out has been confirmed within 7 days. + */ + fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 7 days. + */ + fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -627,6 +639,22 @@ class RealPirPixelSender @Inject constructor( fire(PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE, params) } + override fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), From 755441c74edf2f8927adedf2e9ce3191cb0176f8 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Tue, 11 Nov 2025 17:53:33 +0100 Subject: [PATCH 03/16] Integrate 7 day confirmation pixel --- .../15.json | 30 ++++++- .../pir/impl/models/scheduling/JobRecord.kt | 4 + .../impl/pixels/OptOutConfirmationReporter.kt | 86 +++++++++++++++++++ .../pir/impl/pixels/PirCustomStatsWorker.kt | 5 ++ .../pir/impl/store/PirSchedulingRepository.kt | 25 ++++++ .../pir/impl/store/db/JobSchedulingDao.kt | 12 +++ .../impl/store/db/JobSchedulingEntities.kt | 10 +++ 7 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt diff --git a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json index 1c354877cbb9..a858f45c66f9 100644 --- a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json +++ b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json @@ -2,7 +2,7 @@ "formatVersion": 1, "database": { "version": 15, - "identityHash": "584648b8b3065521786fb33f44214f2e", + "identityHash": "a5782d19654dee4cad932b458037bb37", "entities": [ { "tableName": "pir_broker_json_etag", @@ -696,7 +696,7 @@ }, { "tableName": "pir_optout_job_record", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `reporting_sevenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fourteenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_twentyOneDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fortyTwoDayConfirmationReportSentDateMs` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", "fields": [ { "fieldPath": "extractedProfileId", @@ -757,6 +757,30 @@ "columnName": "dateCreatedInMillis", "affinity": "INTEGER", "notNull": true + }, + { + "fieldPath": "reporting.sevenDayConfirmationReportSentDateMs", + "columnName": "reporting_sevenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fourteenDayConfirmationReportSentDateMs", + "columnName": "reporting_fourteenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.twentyOneDayConfirmationReportSentDateMs", + "columnName": "reporting_twentyOneDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fortyTwoDayConfirmationReportSentDateMs", + "columnName": "reporting_fortyTwoDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true } ], "primaryKey": { @@ -946,7 +970,7 @@ "views": [], "setupQueries": [ "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", - "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '584648b8b3065521786fb33f44214f2e')" + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'a5782d19654dee4cad932b458037bb37')" ] } } \ No newline at end of file diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt index 95f7b6735ed8..965afbf75fe4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt @@ -56,6 +56,10 @@ sealed class JobRecord( val optOutRemovedDateInMillis: Long = 0L, val deprecated: Boolean = false, val dateCreatedInMillis: Long = 0L, + val confirmation7dayReportSentDateMs: Long = 0L, + val confirmation14dayReportSentDateMs: Long = 0L, + val confirmation21dayReportSentDateMs: Long = 0L, + val confirmation42dayReportSentDateMs: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class OptOutJobStatus { /** Opt-out has not been executed yet and should be executed when possible */ diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt new file mode 100644 index 000000000000..0c900e45a4f1 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +interface OptOutConfirmationReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOutConfirmationReporter @Inject constructor( + private val dispatcherProvider: DispatcherProvider, + private val pirSchedulingRepository: PirSchedulingRepository, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pixelSender: PirPixelSender, +) : OptOutConfirmationReporter { + override suspend fun attemptFirePixel() { + withContext(dispatcherProvider.io()) { + val activeBrokers = pirRepository.getAllActiveBrokerObjects().associateBy { it.name } + val allValidRequestedOptOutJobs = pirSchedulingRepository.getAllValidOptOutJobRecords().filter { + it.status == REQUESTED || it.status == REMOVED // TODO: Filter out removed by user + } + + if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext + + attemptFire7dayPixel(allValidRequestedOptOutJobs, activeBrokers) + } + } + + private suspend fun attemptFire7dayPixel( + allValidRequestedOptOutJobs: List, + activeBrokers: Map, + ) { + val now = currentTimeProvider.currentTimeMillis() + val optOutsForSevenDayPixel = allValidRequestedOptOutJobs.filter { + it.daysPassedSinceSubmission(now, 7) && it.confirmation7dayReportSentDateMs == 0L + } + + optOutsForSevenDayPixel.forEach { optOutJobRecord -> + val brokerUrl = activeBrokers[optOutJobRecord.brokerName]?.url ?: return@forEach + + if (optOutJobRecord.status == REMOVED) { + pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) + } else { + pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) + } + + pirSchedulingRepository.markOptOutDay7ConfirmationPixelSent(optOutJobRecord.extractedProfileId, now) + } + } + + private fun OptOutJobRecord.daysPassedSinceSubmission( + now: Long, + interval: Long, + ): Boolean { + return now >= this.optOutRequestedDateInMillis + TimeUnit.DAYS.toMillis(interval) + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt index d7dfe9e13c0f..6481cdb1b07c 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -31,8 +31,13 @@ class PirCustomStatsWorker( @Inject lateinit var optOutSubmissionSuccessRateReporter: OptOut24HourSubmissionSuccessRateReporter + @Inject + lateinit var optOutConfirmationReporter: OptOutConfirmationReporter + override suspend fun doWork(): Result { optOutSubmissionSuccessRateReporter.attemptFirePixel() + optOutConfirmationReporter.attemptFirePixel() + return Result.success() } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index d5461c69f7ed..2ab6a6f8b4ad 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -28,6 +28,7 @@ import com.duckduckgo.pir.impl.models.scheduling.JobRecord.ScanJobRecord.ScanJob import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import com.squareup.anvil.annotations.ContributesBinding @@ -124,6 +125,11 @@ interface PirSchedulingRepository { suspend fun deleteEmailConfirmationJobRecord(extractedProfileId: Long) suspend fun deleteAllEmailConfirmationJobRecords() + + suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -319,6 +325,15 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.updateSevenDayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, @@ -355,6 +370,10 @@ class RealPirSchedulingRepository @Inject constructor( optOutRemovedDateInMillis = this.optOutRemovedDate, deprecated = this.deprecated, dateCreatedInMillis = this.dateCreatedInMillis, + confirmation7dayReportSentDateMs = this.reporting.sevenDayConfirmationReportSentDateMs, + confirmation14dayReportSentDateMs = this.reporting.fourteenDayConfirmationReportSentDateMs, + confirmation21dayReportSentDateMs = this.reporting.twentyOneDayConfirmationReportSentDateMs, + confirmation42dayReportSentDateMs = this.reporting.fortyTwoDayConfirmationReportSentDateMs, ) private fun OptOutJobRecord.toEntity(): OptOutJobRecordEntity = @@ -373,6 +392,12 @@ class RealPirSchedulingRepository @Inject constructor( } else { currentTimeProvider.currentTimeMillis() }, + reporting = ReportingRecord( + sevenDayConfirmationReportSentDateMs = this.confirmation7dayReportSentDateMs, + fourteenDayConfirmationReportSentDateMs = this.confirmation14dayReportSentDateMs, + twentyOneDayConfirmationReportSentDateMs = this.confirmation21dayReportSentDateMs, + fortyTwoDayConfirmationReportSentDateMs = this.confirmation42dayReportSentDateMs, + ), ) private fun EmailConfirmationJobRecord.toEntity(): EmailConfirmationJobRecordEntity = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index e45f1bbad106..b8afe6e22e50 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -118,4 +118,16 @@ interface JobSchedulingDao { deleteOptOutJobRecordsForProfiles(profileQueryIds) deleteEmailConfirmationJobRecordsForProfiles(profileQueryIds) } + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_sevenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun updateSevenDayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt index 62e3bc1092be..69c7836009e1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt @@ -16,6 +16,7 @@ package com.duckduckgo.pir.impl.store.db +import androidx.room.Embedded import androidx.room.Entity import androidx.room.PrimaryKey @@ -44,6 +45,15 @@ data class OptOutJobRecordEntity( val optOutRemovedDate: Long = 0L, val deprecated: Boolean = false, val dateCreatedInMillis: Long, + @Embedded(prefix = "reporting_") + val reporting: ReportingRecord, +) + +data class ReportingRecord( + val sevenDayConfirmationReportSentDateMs: Long = 0L, + val fourteenDayConfirmationReportSentDateMs: Long = 0L, + val twentyOneDayConfirmationReportSentDateMs: Long = 0L, + val fortyTwoDayConfirmationReportSentDateMs: Long = 0L, ) @Entity(tableName = "pir_email_confirmation_job_record") From ca5c87449abb02cab382c470dece37a419da1208 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:33:54 +0100 Subject: [PATCH 04/16] Add pixel definition for 14 days confirmed/unconfirmed --- .../pixels/personal_information_removal.json5 | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index 1688ebc781c9..fac83e9e69cd 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -176,5 +176,33 @@ "type": "string" } ] + }, + "dbp_optoutjob_at-14-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 14 days", + "type": "string" + } + ] } } From 40af82b192ef0140012f8fc73e152594b2187857 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:37:02 +0100 Subject: [PATCH 05/16] Add sender for 14 day confirmed/unconfirmed --- .../duckduckgo/pir/impl/pixels/PirPixel.kt | 9 ++++++ .../pir/impl/pixels/PirPixelSender.kt | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index 7639523fa083..03b88522d25e 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -150,6 +150,15 @@ enum class PirPixel( PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT( baseName = "dbp_optoutjob_at-7-days_unconfirmed", type = Count, + ), + PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index b688940d023f..a62d45a98fd8 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -18,6 +18,8 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE @@ -345,6 +347,16 @@ interface PirPixelSender { * Emits a pixel when an opt-out is unconfirmed within 7 days. */ fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 14 days. + */ + fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 14 days. + */ + fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -655,6 +667,22 @@ class RealPirPixelSender @Inject constructor( fire(PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT, params) } + override fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), From 24bde757fd53aad531430795e7d69f7214fd93b5 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:39:41 +0100 Subject: [PATCH 06/16] Integrate 14 day confirmation pixel --- .../impl/pixels/OptOutConfirmationReporter.kt | 42 +++++++++++++++---- .../pir/impl/store/PirSchedulingRepository.kt | 14 +++++++ .../pir/impl/store/db/JobSchedulingDao.kt | 12 ++++++ 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt index 0c900e45a4f1..23af89bfd83c 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -51,29 +51,55 @@ class RealOptOutConfirmationReporter @Inject constructor( if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext - attemptFire7dayPixel(allValidRequestedOptOutJobs, activeBrokers) + allValidRequestedOptOutJobs.also { + it.attemptFirePixelForConfirmationDay( + activeBrokers, + 7, + { jobRecord -> jobRecord.confirmation7dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay7ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + it.attemptFirePixelForConfirmationDay( + activeBrokers, + 14, + { jobRecord -> jobRecord.confirmation14dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed14Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed14Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay14ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + } } } - private suspend fun attemptFire7dayPixel( - allValidRequestedOptOutJobs: List, + private suspend fun List.attemptFirePixelForConfirmationDay( activeBrokers: Map, + confirmationDay: Long, + jobRecordFilter: (OptOutJobRecord) -> Boolean, + emitConfirmPixel: (String) -> Unit, + emitUnconfirmPixel: (String) -> Unit, + markOptOutJobRecordReporting: suspend (OptOutJobRecord, Long) -> Unit, ) { val now = currentTimeProvider.currentTimeMillis() - val optOutsForSevenDayPixel = allValidRequestedOptOutJobs.filter { - it.daysPassedSinceSubmission(now, 7) && it.confirmation7dayReportSentDateMs == 0L + val optOutsForSevenDayPixel = this.filter { + it.daysPassedSinceSubmission(now, confirmationDay) && jobRecordFilter(it) } optOutsForSevenDayPixel.forEach { optOutJobRecord -> val brokerUrl = activeBrokers[optOutJobRecord.brokerName]?.url ?: return@forEach if (optOutJobRecord.status == REMOVED) { - pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) + emitConfirmPixel(brokerUrl) } else { - pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) + emitUnconfirmPixel(brokerUrl) } - pirSchedulingRepository.markOptOutDay7ConfirmationPixelSent(optOutJobRecord.extractedProfileId, now) + markOptOutJobRecordReporting(optOutJobRecord, now) } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index 2ab6a6f8b4ad..56a32afbca77 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -130,6 +130,11 @@ interface PirSchedulingRepository { extractedProfileId: Long, timestampMs: Long, ) + + suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -334,6 +339,15 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update14DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index b8afe6e22e50..8a50e339db49 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -130,4 +130,16 @@ interface JobSchedulingDao { extractedProfileId: Long, newDate: Long, ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fourteenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update14DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } From ea7efae302459f9be2525afb8dc646860046c810 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:43:17 +0100 Subject: [PATCH 07/16] Add pixel definition for 21 days confirmed/unconfirmed --- .../pixels/personal_information_removal.json5 | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index fac83e9e69cd..514c8e6cbe6d 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -204,5 +204,33 @@ "type": "string" } ] + }, + "dbp_optoutjob_at-21-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 21 days", + "type": "string" + } + ] } } From 4b9ed4955f18e21efc45da92e31781f3f43b5495 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:45:33 +0100 Subject: [PATCH 08/16] Add sender for 21 day confirmed/unconfirmed --- .../duckduckgo/pir/impl/pixels/PirPixel.kt | 10 +++++++ .../pir/impl/pixels/PirPixelSender.kt | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index 03b88522d25e..5f5dd65ca697 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -159,6 +159,16 @@ enum class PirPixel( PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT( baseName = "dbp_optoutjob_at-14-days_unconfirmed", type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index a62d45a98fd8..6b8406029f1a 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -20,6 +20,8 @@ import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE @@ -357,6 +359,16 @@ interface PirPixelSender { * Emits a pixel when an opt-out is unconfirmed within 14 days. */ fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 21 days. + */ + fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 21 days. + */ + fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -683,6 +695,22 @@ class RealPirPixelSender @Inject constructor( fire(PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT, params) } + override fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), From fcbc537ff52689c2cb578951ac8bc78d1e8fa6bc Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:47:58 +0100 Subject: [PATCH 09/16] Integrate 21 day confirmation pixel --- .../pir/impl/pixels/OptOutConfirmationReporter.kt | 11 +++++++++++ .../pir/impl/store/PirSchedulingRepository.kt | 14 ++++++++++++++ .../pir/impl/store/db/JobSchedulingDao.kt | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt index 23af89bfd83c..1e06b68808b2 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -73,6 +73,17 @@ class RealOptOutConfirmationReporter @Inject constructor( pirSchedulingRepository.markOptOutDay14ConfirmationPixelSent(jobRecord.extractedProfileId, now) }, ) + + it.attemptFirePixelForConfirmationDay( + activeBrokers, + 21, + { jobRecord -> jobRecord.confirmation21dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed21Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed21Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay21ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) } } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index 56a32afbca77..ae7ba5992a5a 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -135,6 +135,11 @@ interface PirSchedulingRepository { extractedProfileId: Long, timestampMs: Long, ) + + suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -348,6 +353,15 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update21DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index 8a50e339db49..d8d3ee28ee71 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -142,4 +142,16 @@ interface JobSchedulingDao { extractedProfileId: Long, newDate: Long, ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_twentyOneDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update21DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } From ec9303e8f17eb72a82e4712fa966d17f326f96ea Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:49:44 +0100 Subject: [PATCH 10/16] Add pixel definition for 42 days confirmed/unconfirmed --- .../pixels/personal_information_removal.json5 | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index 514c8e6cbe6d..61b6ae2749f2 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -232,5 +232,33 @@ "type": "string" } ] + }, + "dbp_optoutjob_at-42-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 42 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 42 days", + "type": "string" + } + ] } } From 484d747b3c2d387cb057ddf89b527ae4d1423fda Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 09:52:09 +0100 Subject: [PATCH 11/16] Add sender for 42 day confirmed/unconfirmed --- .../duckduckgo/pir/impl/pixels/PirPixel.kt | 9 ++++++ .../pir/impl/pixels/PirPixelSender.kt | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index 5f5dd65ca697..a676af8f6b66 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -169,6 +169,15 @@ enum class PirPixel( PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT( baseName = "dbp_optoutjob_at-21-days_unconfirmed", type = Count, + ), + PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index 6b8406029f1a..933c7bbe6930 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -22,6 +22,8 @@ import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_CON import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE @@ -369,6 +371,16 @@ interface PirPixelSender { * Emits a pixel when an opt-out is unconfirmed within 21 days. */ fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 42 days. + */ + fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 42 days. + */ + fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -711,6 +723,22 @@ class RealPirPixelSender @Inject constructor( fire(PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT, params) } + override fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), From 885c47b0f057fcfbe5d1d6ebc03d6c13ea07dc0f Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 10:18:13 +0100 Subject: [PATCH 12/16] Integrate 42 day confirmation pixel --- .../impl/pixels/OptOutConfirmationReporter.kt | 28 +++++++++++++++++-- .../pir/impl/store/PirSchedulingRepository.kt | 14 ++++++++++ .../pir/impl/store/db/JobSchedulingDao.kt | 12 ++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt index 1e06b68808b2..710d4cbce257 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -52,9 +52,10 @@ class RealOptOutConfirmationReporter @Inject constructor( if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext allValidRequestedOptOutJobs.also { + // Fire 7 day pixel it.attemptFirePixelForConfirmationDay( activeBrokers, - 7, + INTERVAL_DAY_7, { jobRecord -> jobRecord.confirmation7dayReportSentDateMs == 0L }, { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) }, { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) }, @@ -63,9 +64,10 @@ class RealOptOutConfirmationReporter @Inject constructor( }, ) + // Fire 14 day pixel it.attemptFirePixelForConfirmationDay( activeBrokers, - 14, + INTERVAL_DAY_14, { jobRecord -> jobRecord.confirmation14dayReportSentDateMs == 0L }, { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed14Days(brokerUrl) }, { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed14Days(brokerUrl) }, @@ -74,9 +76,10 @@ class RealOptOutConfirmationReporter @Inject constructor( }, ) + // Fire 21 day pixel it.attemptFirePixelForConfirmationDay( activeBrokers, - 21, + INTERVAL_DAY_21, { jobRecord -> jobRecord.confirmation21dayReportSentDateMs == 0L }, { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed21Days(brokerUrl) }, { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed21Days(brokerUrl) }, @@ -84,6 +87,18 @@ class RealOptOutConfirmationReporter @Inject constructor( pirSchedulingRepository.markOptOutDay21ConfirmationPixelSent(jobRecord.extractedProfileId, now) }, ) + + // Fire 42 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_42, + { jobRecord -> jobRecord.confirmation42dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed42Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed42Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay42ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) } } } @@ -120,4 +135,11 @@ class RealOptOutConfirmationReporter @Inject constructor( ): Boolean { return now >= this.optOutRequestedDateInMillis + TimeUnit.DAYS.toMillis(interval) } + + companion object { + private const val INTERVAL_DAY_7 = 7L + private const val INTERVAL_DAY_14 = 14L + private const val INTERVAL_DAY_21 = 21L + private const val INTERVAL_DAY_42 = 42L + } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index ae7ba5992a5a..676953b49bb1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -140,6 +140,11 @@ interface PirSchedulingRepository { extractedProfileId: Long, timestampMs: Long, ) + + suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -362,6 +367,15 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update42DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index d8d3ee28ee71..8e836a706502 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -154,4 +154,16 @@ interface JobSchedulingDao { extractedProfileId: Long, newDate: Long, ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fortyTwoDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update42DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } From 3718e24d1827dfd052f2ae82fb6dd6aa858d3a8c Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 10:18:37 +0100 Subject: [PATCH 13/16] Fix tests --- .../RealOptOutConfirmationReporterTest.kt | 526 ++++++++++++++++++ .../store/RealPirSchedulingRepositoryTest.kt | 6 + 2 files changed, 532 insertions(+) create mode 100644 pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt new file mode 100644 index 000000000000..f00ff0f098da --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOutConfirmationReporterTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutConfirmationReporter + + private val mockPirSchedulingRepository: PirSchedulingRepository = mock() + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockPixelSender: PirPixelSender = mock() + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val sevenDays = TimeUnit.DAYS.toMillis(7) + private val fourteenDays = TimeUnit.DAYS.toMillis(14) + private val twentyOneDays = TimeUnit.DAYS.toMillis(21) + private val fortyTwoDays = TimeUnit.DAYS.toMillis(42) + + private val testBroker = Broker( + name = "test-broker", + fileName = "test-broker.json", + url = "https://test-broker.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + @Before + fun setUp() { + testee = RealOptOutConfirmationReporter( + dispatcherProvider = coroutineRule.testDispatcherProvider, + pirSchedulingRepository = mockPirSchedulingRepository, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pixelSender = mockPixelSender, + ) + } + + @Test + fun whenNoActiveBrokersThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenNoOptOutJobsThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenOptOutJobNotRequestedOrRemovedThenDoesNotFirePixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.NOT_EXECUTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedAndStatusIsRemovedThenFiresConfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPassedAndStatusIsRequestedThenFiresUnconfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation7dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenLessThan7DaysPassedThenDoesNotFire7DayPixel() = runTest { + val now = baseTime + sevenDays - TimeUnit.HOURS.toMillis(1) // 1 hour before 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenExactly7DaysPassedThenFires7DayPixel() = runTest { + val now = baseTime + sevenDays // Exactly 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when14DaysPassedThenFires14DayPixel() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay14ConfirmationPixelSent(1L, now) + } + + @Test + fun when21DaysPassedThenFires21DayPixel() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay21ConfirmationPixelSent(1L, now) + } + + @Test + fun when42DaysPassedThenFires42DayPixel() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay42ConfirmationPixelSent(1L, now) + } + + @Test + fun whenMultipleIntervalsPassedThenFiresAllApplicablePixels() = runTest { + val now = baseTime + fortyTwoDays // 42 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + } + + @Test + fun whenBrokerNotFoundThenSkipsJobRecord() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = "unknown-broker", + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenMultipleJobRecordsThenFiresPixelsForEach() = runTest { + val now = baseTime + sevenDays + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(2L, now) + } + + @Test + fun when14DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation14dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay14ConfirmationPixelSent(any(), any()) + } + + @Test + fun when21DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation21dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay21ConfirmationPixelSent(any(), any()) + } + + @Test + fun when42DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation42dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed42Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay42ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedBut14DaysNotPassedThenOnlyFires7DayPixel() = runTest { + val now = baseTime + sevenDays + oneDay // 8 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + } + + @Test + fun when14DaysPassedBut21DaysNotPassedThenFires7And14DayPixels() = runTest { + val now = baseTime + fourteenDays + oneDay // 15 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + } + + @Test + fun whenRequestedStatusThenFiresUnconfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + } + + @Test + fun whenRemovedStatusThenFiresConfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutUnconfirmed7Days(any()) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis: Long = baseTime, + optOutRemovedDateInMillis: Long = 0L, + confirmation7dayReportSentDateMs: Long = 0L, + confirmation14dayReportSentDateMs: Long = 0L, + confirmation21dayReportSentDateMs: Long = 0L, + confirmation42dayReportSentDateMs: Long = 0L, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = 0, + lastOptOutAttemptDateInMillis = 0L, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = false, + dateCreatedInMillis = baseTime, + confirmation7dayReportSentDateMs = confirmation7dayReportSentDateMs, + confirmation14dayReportSentDateMs = confirmation14dayReportSentDateMs, + confirmation21dayReportSentDateMs = confirmation21dayReportSentDateMs, + confirmation42dayReportSentDateMs = confirmation42dayReportSentDateMs, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt index 301541d36c7b..770d39f032fe 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt @@ -27,6 +27,7 @@ import com.duckduckgo.pir.impl.store.db.BrokerJsonDao import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import kotlinx.coroutines.runBlocking @@ -103,6 +104,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 2000L, optOutRemovedDate = 0L, dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val deprecatedOptOutJobEntity = @@ -117,6 +119,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 4000L, optOutRemovedDate = 0L, dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val scanJobRecord = @@ -405,6 +408,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 2000L, optOutRemovedDate = 0L, dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ) } @@ -439,6 +443,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 2000L, optOutRemovedDate = 0L, dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), OptOutJobRecordEntity( extractedProfileId = 999L, @@ -450,6 +455,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 4000L, optOutRemovedDate = 5000L, dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ), ) From 474a878fdeb2e124454d2e55c81f40affa5114f6 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 10:38:37 +0100 Subject: [PATCH 14/16] Add logs --- .../pir/impl/common/PirJobConstants.kt | 1 + ...OptOut24HourSubmissionSuccessRateReporter.kt | 5 +++++ .../impl/pixels/OptOutConfirmationReporter.kt | 17 ++++++++++++----- .../pir/impl/pixels/PirCustomStatsWorker.kt | 2 ++ .../pir/impl/scan/PirScanScheduler.kt | 4 +++- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt index 9c3f0ac50ba8..53674d6ddbd5 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt @@ -22,4 +22,5 @@ object PirJobConstants { const val MAX_DETACHED_WEBVIEW_COUNT = 20 const val SCHEDULED_SCAN_INTERVAL_HOURS = 12L const val EMAIL_CONFIRMATION_INTERVAL_HOURS = 8L + const val CUSTOM_PIXEL_INTERVAL_HOURS = 5L } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt index 774a8df2551a..edf821d5cf07 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -20,6 +20,7 @@ import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.pir.impl.store.PirRepository import com.squareup.anvil.annotations.ContributesBinding +import logcat.logcat import java.util.concurrent.TimeUnit import javax.inject.Inject import kotlin.math.abs @@ -36,10 +37,12 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( private val pirPixelSender: PirPixelSender, ) : OptOut24HourSubmissionSuccessRateReporter { override suspend fun attemptFirePixel() { + logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" } val startDate = pirRepository.getCustomStatsPixelsLastSentMs() val now = currentTimeProvider.currentTimeMillis() if (shouldFirePixel(startDate, now)) { + logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } val endDate = now - TimeUnit.HOURS.toMillis(24) val activeBrokers = pirRepository.getAllActiveBrokerObjects() val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() @@ -52,6 +55,7 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( endDate, ) + logcat { "PIR-CUSTOM-STATS: 24hr submission ${it.name} : $successRate" } if (successRate != null) { pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( brokerUrl = it.url, @@ -60,6 +64,7 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( } } + logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } pirRepository.setCustomStatsPixelsLastSentMs(endDate) } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt index 710d4cbce257..ea0ebe694060 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -27,6 +27,7 @@ import com.duckduckgo.pir.impl.store.PirRepository import com.duckduckgo.pir.impl.store.PirSchedulingRepository import com.squareup.anvil.annotations.ContributesBinding import kotlinx.coroutines.withContext +import logcat.logcat import java.util.concurrent.TimeUnit import javax.inject.Inject @@ -43,6 +44,8 @@ class RealOptOutConfirmationReporter @Inject constructor( private val pixelSender: PirPixelSender, ) : OptOutConfirmationReporter { override suspend fun attemptFirePixel() { + logcat { "PIR-CUSTOM-STATS: Attempt to fire confirmation pixels" } + withContext(dispatcherProvider.io()) { val activeBrokers = pirRepository.getAllActiveBrokerObjects().associateBy { it.name } val allValidRequestedOptOutJobs = pirSchedulingRepository.getAllValidOptOutJobRecords().filter { @@ -51,6 +54,7 @@ class RealOptOutConfirmationReporter @Inject constructor( if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext + logcat { "PIR-CUSTOM-STATS: Will fire confirmation pixels for ${allValidRequestedOptOutJobs.size} jobs" } allValidRequestedOptOutJobs.also { // Fire 7 day pixel it.attemptFirePixelForConfirmationDay( @@ -112,17 +116,20 @@ class RealOptOutConfirmationReporter @Inject constructor( markOptOutJobRecordReporting: suspend (OptOutJobRecord, Long) -> Unit, ) { val now = currentTimeProvider.currentTimeMillis() - val optOutsForSevenDayPixel = this.filter { + val optOutsForPixel = this.filter { it.daysPassedSinceSubmission(now, confirmationDay) && jobRecordFilter(it) } - optOutsForSevenDayPixel.forEach { optOutJobRecord -> - val brokerUrl = activeBrokers[optOutJobRecord.brokerName]?.url ?: return@forEach + logcat { "PIR-CUSTOM-STATS: Firing 7day confirmation pixels for ${optOutsForPixel.size} jobs" } + optOutsForPixel.forEach { optOutJobRecord -> + val broker = activeBrokers[optOutJobRecord.brokerName] ?: return@forEach if (optOutJobRecord.status == REMOVED) { - emitConfirmPixel(brokerUrl) + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${broker.name}" } + emitConfirmPixel(broker.url) } else { - emitUnconfirmPixel(brokerUrl) + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day unconfirmation pixels for ${broker.name}" } + emitUnconfirmPixel(broker.url) } markOptOutJobRecordReporting(optOutJobRecord, now) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt index 6481cdb1b07c..b0bc5e2733ca 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -21,6 +21,7 @@ import androidx.work.CoroutineWorker import androidx.work.WorkerParameters import com.duckduckgo.anvil.annotations.ContributesWorker import com.duckduckgo.di.scopes.AppScope +import logcat.logcat import javax.inject.Inject @ContributesWorker(AppScope::class) @@ -35,6 +36,7 @@ class PirCustomStatsWorker( lateinit var optOutConfirmationReporter: OptOutConfirmationReporter override suspend fun doWork(): Result { + logcat { "PIR-CUSTOM-STATS: Attempt to fire custom pixels" } optOutSubmissionSuccessRateReporter.attemptFirePixel() optOutConfirmationReporter.attemptFirePixel() diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt index 28f3a414ce38..50fe401d48c9 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt @@ -31,6 +31,7 @@ import com.duckduckgo.app.di.AppCoroutineScope import com.duckduckgo.appbuildconfig.api.AppBuildConfig import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.common.PirJobConstants.CUSTOM_PIXEL_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.EMAIL_CONFIRMATION_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.SCHEDULED_SCAN_INTERVAL_HOURS import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker @@ -134,8 +135,9 @@ class RealPirScanScheduler @Inject constructor( } private fun scheduleRecurringPixelStats() { - val periodicWorkRequest = PeriodicWorkRequestBuilder(5, TimeUnit.HOURS) + val periodicWorkRequest = PeriodicWorkRequestBuilder(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) .addTag(TAG_PIR_RECURRING_CUSTOM_STATS) + .setInitialDelay(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) .build() workManager.enqueueUniquePeriodicWork( From b20e19626c266e0824a224b441618df5bfb68c65 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Wed, 12 Nov 2025 13:10:39 +0100 Subject: [PATCH 15/16] Minor refactor --- ...tOut24HourSubmissionSuccessRateReporter.kt | 58 +++-- .../impl/pixels/OptOutConfirmationReporter.kt | 2 +- .../impl/pixels/OptOutSubmitRateCalculator.kt | 16 +- ...24HourSubmissionSuccessRateReporterTest.kt | 170 ++++++++++++-- .../RealOptOutSubmitRateCalculatorTest.kt | 216 +++--------------- 5 files changed, 223 insertions(+), 239 deletions(-) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt index edf821d5cf07..e83c5cb367f9 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -17,9 +17,12 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext import logcat.logcat import java.util.concurrent.TimeUnit import javax.inject.Inject @@ -35,37 +38,46 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( private val pirRepository: PirRepository, private val currentTimeProvider: CurrentTimeProvider, private val pirPixelSender: PirPixelSender, + private val pirSchedulingRepository: PirSchedulingRepository, + private val dispatcherProvider: DispatcherProvider, ) : OptOut24HourSubmissionSuccessRateReporter { override suspend fun attemptFirePixel() { - logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" } - val startDate = pirRepository.getCustomStatsPixelsLastSentMs() - val now = currentTimeProvider.currentTimeMillis() + withContext(dispatcherProvider.io()) { + logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" } + val startDate = pirRepository.getCustomStatsPixelsLastSentMs() + val now = currentTimeProvider.currentTimeMillis() - if (shouldFirePixel(startDate, now)) { - logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } - val endDate = now - TimeUnit.HOURS.toMillis(24) - val activeBrokers = pirRepository.getAllActiveBrokerObjects() - val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + if (shouldFirePixel(startDate, now)) { + logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } + val endDate = now - TimeUnit.HOURS.toMillis(24) + val activeBrokers = pirRepository.getAllActiveBrokerObjects() + val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords() - if (activeBrokers.isNotEmpty() && hasUserProfiles) { - activeBrokers.forEach { - val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( - it.name, - startDate, - endDate, - ) + if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { broker -> + val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name } - logcat { "PIR-CUSTOM-STATS: 24hr submission ${it.name} : $successRate" } - if (successRate != null) { - pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( - brokerUrl = it.url, - optOutSuccessRate = successRate, + if (activeJobRecordsForBroker.isEmpty()) return@forEach + + val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( + activeJobRecordsForBroker, + startDate, + endDate, ) + + logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" } + if (successRate != null) { + pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = broker.url, + optOutSuccessRate = successRate, + ) + } } - } - logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } - pirRepository.setCustomStatsPixelsLastSentMs(endDate) + logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } + pirRepository.setCustomStatsPixelsLastSentMs(endDate) + } } } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt index ea0ebe694060..5e7a36029d28 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -120,7 +120,7 @@ class RealOptOutConfirmationReporter @Inject constructor( it.daysPassedSinceSubmission(now, confirmationDay) && jobRecordFilter(it) } - logcat { "PIR-CUSTOM-STATS: Firing 7day confirmation pixels for ${optOutsForPixel.size} jobs" } + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${optOutsForPixel.size} jobs" } optOutsForPixel.forEach { optOutJobRecord -> val broker = activeBrokers[optOutJobRecord.brokerName] ?: return@forEach diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt index 23840aacd614..688fa2578b64 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt @@ -18,9 +18,9 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.common.utils.DispatcherProvider import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.scheduling.JobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED -import com.duckduckgo.pir.impl.store.PirSchedulingRepository import com.squareup.anvil.annotations.ContributesBinding import kotlinx.coroutines.withContext import java.util.concurrent.TimeUnit @@ -31,12 +31,12 @@ interface OptOutSubmitRateCalculator { /** * Calculates the opt-out 24h submit rate for a given broker within the specified date range. * - * @param brokerName name of the broker to calculate the opt-out submit rate for. + * @param allActiveOptOutJobsForBroker all active opt-out job records for the broker. * @param startDateMs The opt-out records to include should be created on or after this date. Default is 0L (epoch). * @param endDateMs tThe opt-out records to include should be created on or before this date. Default is 0L (epoch). */ suspend fun calculateOptOutSubmitRate( - brokerName: String, + allActiveOptOutJobsForBroker: List, startDateMs: Long = 0L, endDateMs: Long, ): Double? @@ -45,16 +45,15 @@ interface OptOutSubmitRateCalculator { @ContributesBinding(AppScope::class) class RealOptOutSubmitRateCalculator @Inject constructor( private val dispatcherProvider: DispatcherProvider, - private val schedulingRepository: PirSchedulingRepository, ) : OptOutSubmitRateCalculator { override suspend fun calculateOptOutSubmitRate( - brokerName: String, + allActiveOptOutJobsForBroker: List, startDateMs: Long, endDateMs: Long, ): Double? = withContext(dispatcherProvider.io()) { // Get all opt out job records created within the given range for the specified broker - val recordsCreatedWithinRange = schedulingRepository.getAllValidOptOutJobRecordsForBroker(brokerName).filter { - it.brokerName == brokerName && it.dateCreatedInMillis in startDateMs..endDateMs + val recordsCreatedWithinRange = allActiveOptOutJobsForBroker.filter { + it.dateCreatedInMillis in startDateMs..endDateMs } // We don't need to calculate the rate if there are no records @@ -68,7 +67,8 @@ class RealOptOutSubmitRateCalculator @Inject constructor( ) } - val optOutSuccessRate = requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() + val optOutSuccessRate = + requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() val roundedOptOutSuccessRate = round(optOutSuccessRate * 100) / 100 return@withContext roundedOptOutSuccessRate } diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt index e0dd3f314159..274404334961 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt @@ -16,12 +16,14 @@ package com.duckduckgo.pir.impl.pixels -import android.content.Context import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.pir.impl.models.Broker import com.duckduckgo.pir.impl.models.ProfileQuery +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository import kotlinx.coroutines.test.runTest import org.junit.Before import org.junit.Rule @@ -43,7 +45,7 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { private val mockCurrentTimeProvider: CurrentTimeProvider = mock() private val mockOptOutSubmitRateCalculator: OptOutSubmitRateCalculator = mock() private val mockPirPixelSender: PirPixelSender = mock() - private val context: Context = mock() + private val mockSchedulingRepository: PirSchedulingRepository = mock() private lateinit var toTest: RealOptOut24HourSubmissionSuccessRateReporter @@ -93,16 +95,24 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { pirRepository = mockPirRepository, currentTimeProvider = mockCurrentTimeProvider, pirPixelSender = mockPirPixelSender, + pirSchedulingRepository = mockSchedulingRepository, + dispatcherProvider = coroutineRule.testDispatcherProvider, ) } @Test fun whenFirstRunThenShouldFirePixel() = runTest { val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -139,11 +149,16 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { fun whenMoreThan24HoursPassedThenShouldFirePixel() = runTest { val startDate = baseTime val now = baseTime + twentyFourHours + oneHour // 25 hours passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -201,6 +216,7 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) toTest.attemptFirePixel() @@ -216,6 +232,15 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenMultipleBrokersThenShouldFirePixelForEach() = runTest { val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( @@ -225,19 +250,20 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { ), ) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker1.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.5) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker2.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.8) @@ -258,6 +284,15 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenSuccessRateIsNullThenShouldNotFirePixelForThatBroker() = runTest { val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( @@ -267,19 +302,20 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { ), ) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker1.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.5) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker2.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(null) @@ -300,10 +336,16 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenAllSuccessRatesAreNullThenShouldNotFireAnyPixels() = runTest { val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -323,11 +365,16 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { val startDate = baseTime val now = baseTime + twentyFourHours + oneHour val expectedEndDate = now - twentyFourHours + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -339,9 +386,9 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { toTest.attemptFirePixel() verify(mockOptOutSubmitRateCalculator).calculateOptOutSubmitRate( - brokerName = testBroker1.name, - startDateMs = startDate, - endDateMs = expectedEndDate, + allActiveOptOutJobsForBroker = eq(listOf(jobRecord)), + startDateMs = eq(startDate), + endDateMs = eq(expectedEndDate), ) verify(mockPirRepository).setCustomStatsPixelsLastSentMs(expectedEndDate) } @@ -349,6 +396,15 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenMultipleBrokersWithMixedSuccessRatesThenFiresPixelsForNonNullRates() = runTest { val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( @@ -358,19 +414,20 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { ), ) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker1.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(null) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker2.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.9) @@ -433,6 +490,7 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) toTest.attemptFirePixel() @@ -444,4 +502,70 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) } + + @Test + fun whenBrokerHasNoJobRecordsThenSkipsThatBroker() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, // This broker has no job records + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) // Only for broker1 + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord)), + eq(0L), + eq(now - twentyFourHours), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + // Only broker1 should fire pixel, broker2 should be skipped + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker1.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } } diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt index f51f6fa8523b..6f3cc7407121 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt @@ -19,15 +19,12 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus -import com.duckduckgo.pir.impl.store.PirSchedulingRepository import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Assert.assertNull import org.junit.Before import org.junit.Rule import org.junit.Test -import org.mockito.kotlin.mock -import org.mockito.kotlin.whenever import java.util.concurrent.TimeUnit class RealOptOutSubmitRateCalculatorTest { @@ -37,19 +34,15 @@ class RealOptOutSubmitRateCalculatorTest { private lateinit var testee: RealOptOutSubmitRateCalculator - private val mockSchedulingRepository: PirSchedulingRepository = mock() - @Before fun setUp() { testee = RealOptOutSubmitRateCalculator( dispatcherProvider = coroutineRule.testDispatcherProvider, - schedulingRepository = mockSchedulingRepository, ) } // Test data private val testBrokerName = "test-broker" - private val testBrokerName2 = "test-broker-2" // January 15, 2024 10:00:00 UTC private val baseTime = 1705309200000L @@ -62,10 +55,7 @@ class RealOptOutSubmitRateCalculatorTest { val startDate = baseTime val endDate = baseTime + oneDay - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(emptyList()) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(emptyList(), startDate, endDate) assertNull(result) } @@ -86,10 +76,11 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = baseTime + oneDay + oneHour, ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordBeforeRange, recordAfterRange)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate( + listOf(recordBeforeRange, recordAfterRange), + startDate, + endDate, + ) assertNull(result) } @@ -111,11 +102,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.ERROR, dateCreatedInMillis = baseTime + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record1, record2)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) assertEquals(0.0, result!!, 0.0) } @@ -140,11 +127,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // Within 24 hours ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record1, record2)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -168,11 +151,11 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedRecord, notExecutedRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate( + listOf(requestedRecord, notExecutedRecord), + startDate, + endDate, + ) assertEquals(0.5, result!!, 0.0) } @@ -197,11 +180,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24 hours ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedWithinWindow, requestedOutsideWindow)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithinWindow, requestedOutsideWindow), startDate, endDate) assertEquals(0.5, result!!, 0.0) } @@ -225,46 +204,11 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedAt24Hours, notExecutedRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAt24Hours, notExecutedRecord), startDate, endDate) assertEquals(0.5, result!!, 0.0) } - @Test - fun whenRecordsFromDifferentBrokersThenOnlyCountMatchingBroker() = runTest { - val startDate = baseTime - val endDate = baseTime + oneDay - val dateCreated = baseTime + oneHour - - val recordForTestBroker = createOptOutJobRecord( - extractedProfileId = 1L, - brokerName = testBrokerName, - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated, - optOutRequestedDateInMillis = dateCreated + oneHour, - ) - val recordForOtherBroker = createOptOutJobRecord( - extractedProfileId = 2L, - brokerName = testBrokerName2, - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated + 2 * oneHour, - optOutRequestedDateInMillis = dateCreated + 3 * oneHour, - ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordForTestBroker)) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName2)) - .thenReturn(listOf(recordForOtherBroker)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) - - assertEquals(1.0, result!!, 0.0) - } - @Test fun whenStartDateIsZeroThenUseDefaultStartDate() = runTest { val endDate = baseTime + oneDay @@ -277,11 +221,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated, optOutRequestedDateInMillis = dateCreated + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, endDateMs = endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record), endDateMs = endDate) assertEquals(1.0, result!!, 0.0) } @@ -298,11 +238,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = startDate, // Exactly at start optOutRequestedDateInMillis = startDate + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordAtStart)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtStart), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -319,11 +255,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = endDate, // Exactly at end optOutRequestedDateInMillis = endDate + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordAtEnd)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtEnd), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -359,11 +291,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.PENDING_EMAIL_CONFIRMATION, dateCreatedInMillis = dateCreated + 4 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested, removed, error, pendingEmail)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested, removed, error, pendingEmail), startDate, endDate) assertEquals(0.25, result!!, 0.0) } @@ -408,11 +336,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.ERROR, dateCreatedInMillis = dateCreated + 5 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested1, requested2, requested3, notExecuted, error)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, requested3, notExecuted, error), startDate, endDate) // Only requested1 and requested3 count (2 out of 5) assertEquals(0.4, result!!, 0.0) @@ -438,11 +362,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 2 * oneHour + oneHour, // After creation ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedAtSameTime, validRequested)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAtSameTime, validRequested), startDate, endDate) // Only validRequested counts (1 out of 2) assertEquals(0.5, result!!, 0.0) @@ -468,11 +388,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // After creation ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedBeforeCreation, validRequested)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedBeforeCreation, validRequested), startDate, endDate) // Only validRequested counts (1 out of 2) assertEquals(0.5, result!!, 0.0) @@ -493,10 +409,7 @@ class RealOptOutSubmitRateCalculatorTest { optOutRequestedDateInMillis = dateCreated + oneMillisecond, // Just 1ms after (should be counted) ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedJustAfter)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -515,11 +428,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated, optOutRequestedDateInMillis = dateCreated + twentyFourHours - oneMillisecond, // Just before 24h limit ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedJustBefore24h)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustBefore24h), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -544,11 +453,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedJustAfter24h, notExecuted)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter24h, notExecuted), startDate, endDate) // requestedJustAfter24h doesn't count, so 0 out of 2 assertEquals(0.0, result!!, 0.0) @@ -580,11 +485,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 3 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested, notExecuted1, notExecuted2)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested, notExecuted1, notExecuted2), startDate, endDate) // 1/3 = 0.333... rounded to 0.33 assertEquals(0.33, result!!, 0.0) @@ -617,48 +518,12 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 4 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested1, requested2, notExecuted)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, notExecuted), startDate, endDate) // 2/3 = 0.666... rounded to 0.67 assertEquals(0.67, result!!, 0.0) } - @Test - fun whenRepositoryReturnsWrongBrokerNameThenExcluded() = runTest { - val startDate = baseTime - val endDate = baseTime + oneDay - val dateCreated = baseTime + oneHour - - // Repository might return records with wrong broker name (should be filtered out) - val recordWithWrongBroker = createOptOutJobRecord( - extractedProfileId = 1L, - brokerName = testBrokerName2, // Wrong broker - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated, - optOutRequestedDateInMillis = dateCreated + oneHour, - ) - val recordWithCorrectBroker = createOptOutJobRecord( - extractedProfileId = 2L, - brokerName = testBrokerName, // Correct broker - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated + 2 * oneHour, - optOutRequestedDateInMillis = dateCreated + 3 * oneHour, - ) - - // Repository returns both, but only correct broker should be counted - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordWithWrongBroker, recordWithCorrectBroker)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) - - // Only recordWithCorrectBroker counts (1 out of 1 after filtering) - assertEquals(1.0, result!!, 0.0) - } - @Test fun whenSingleRecordRequestedThenReturnOne() = runTest { val startDate = baseTime @@ -672,11 +537,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated, optOutRequestedDateInMillis = dateCreated + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(singleRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -693,11 +554,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(singleRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) assertEquals(0.0, result!!, 0.0) } @@ -729,10 +586,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(300), // 10 months later ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record1, record2, record3)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2, record3), startDate, endDate) // 2 out of 3 assertEquals(0.67, result!!, 0.0) @@ -759,10 +613,7 @@ class RealOptOutSubmitRateCalculatorTest { optOutRequestedDateInMillis = dateCreated + 3 * oneHour, ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedWithZeroDate, validRequested)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithZeroDate, validRequested), startDate, endDate) // Only validRequested counts (1 out of 2) assertEquals(0.5, result!!, 0.0) @@ -791,10 +642,7 @@ class RealOptOutSubmitRateCalculatorTest { ) } - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested) + notExecutedRecords) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested) + notExecutedRecords, startDate, endDate) // 1/7 = 0.142857... rounded to 0.14 assertEquals(0.14, result!!, 0.0) From 731a10a0b3042748cad3662ce6ddf28a6fa19a38 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Mon, 17 Nov 2025 11:27:34 +0100 Subject: [PATCH 16/16] Address comments --- ...tOut24HourSubmissionSuccessRateReporter.kt | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt index e83c5cb367f9..a81adfda76f7 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -47,37 +47,36 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( val startDate = pirRepository.getCustomStatsPixelsLastSentMs() val now = currentTimeProvider.currentTimeMillis() - if (shouldFirePixel(startDate, now)) { - logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } - val endDate = now - TimeUnit.HOURS.toMillis(24) - val activeBrokers = pirRepository.getAllActiveBrokerObjects() - val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() - val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords() + if (!shouldFirePixel(startDate, now)) return@withContext + logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } + val endDate = now - TimeUnit.HOURS.toMillis(24) + val activeBrokers = pirRepository.getAllActiveBrokerObjects() + val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords() - if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) { - activeBrokers.forEach { broker -> - val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name } + if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { broker -> + val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name } - if (activeJobRecordsForBroker.isEmpty()) return@forEach + if (activeJobRecordsForBroker.isEmpty()) return@forEach - val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( - activeJobRecordsForBroker, - startDate, - endDate, - ) + val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( + activeJobRecordsForBroker, + startDate, + endDate, + ) - logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" } - if (successRate != null) { - pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( - brokerUrl = broker.url, - optOutSuccessRate = successRate, - ) - } + logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" } + if (successRate != null) { + pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = broker.url, + optOutSuccessRate = successRate, + ) } - - logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } - pirRepository.setCustomStatsPixelsLastSentMs(endDate) } + + logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } + pirRepository.setCustomStatsPixelsLastSentMs(endDate) } } }