diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index c5f4f5611e6c..61b6ae2749f2 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -129,5 +129,136 @@ "enum": ["on", "off", "unsupported"] } ] + }, + "dbp_databroker_custom_stats_optoutsubmit": { + "description": "Pixel that contains a broker's opt-out 24h submission success rate.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that this opt-out attempt targets", + "type": "string" + }, + { + "key": "optout_submit_success_rate", + "description": "The success rate of how many opt-out jobs successfully requested within 24 hours of creation", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-7-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-7-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 42 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 42 days", + "type": "string" + } + ] } } diff --git a/pir/pir-impl/build.gradle b/pir/pir-impl/build.gradle index 58c4926233f8..3a9dbc204b75 100644 --- a/pir/pir-impl/build.gradle +++ b/pir/pir-impl/build.gradle @@ -70,6 +70,7 @@ dependencies { testImplementation AndroidX.test.ext.junit testImplementation Testing.robolectric testImplementation "androidx.lifecycle:lifecycle-runtime-testing:_" + testImplementation AndroidX.work.testing testImplementation(KotlinX.coroutines.test) { // https://github.com/Kotlin/kotlinx.coroutines/issues/2023 // conflicts with mockito due to direct inclusion of byte buddy diff --git a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json new file mode 100644 index 000000000000..a858f45c66f9 --- /dev/null +++ b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json @@ -0,0 +1,976 @@ +{ + "formatVersion": 1, + "database": { + "version": 15, + "identityHash": "a5782d19654dee4cad932b458037bb37", + "entities": [ + { + "tableName": "pir_broker_json_etag", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`fileName` TEXT NOT NULL, `etag` TEXT NOT NULL, PRIMARY KEY(`fileName`))", + "fields": [ + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "etag", + "columnName": "etag", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "fileName" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_details", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `fileName` TEXT NOT NULL, `url` TEXT NOT NULL, `version` TEXT NOT NULL, `parent` TEXT, `addedDatetime` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parent", + "columnName": "parent", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addedDatetime", + "columnName": "addedDatetime", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_opt_out", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT NOT NULL, `optOutUrl` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scan", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scheduling_config", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `retryError` INTEGER NOT NULL, `confirmOptOutScan` INTEGER NOT NULL, `maintenanceScan` INTEGER NOT NULL, `maxAttempts` INTEGER, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "retryError", + "columnName": "retryError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "confirmOptOutScan", + "columnName": "confirmOptOutScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maintenanceScan", + "columnName": "maintenanceScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maxAttempts", + "columnName": "maxAttempts", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_user_profile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `birthYear` INTEGER NOT NULL, `phone` TEXT, `deprecated` INTEGER NOT NULL, `user_firstName` TEXT NOT NULL, `user_lastName` TEXT NOT NULL, `user_middleName` TEXT, `user_suffix` TEXT, `address_city` TEXT NOT NULL, `address_state` TEXT NOT NULL, `address_street` TEXT, `address_zip` TEXT)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "birthYear", + "columnName": "birthYear", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "phone", + "columnName": "phone", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "userName.firstName", + "columnName": "user_firstName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.lastName", + "columnName": "user_lastName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.middleName", + "columnName": "user_middleName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "userName.suffix", + "columnName": "user_suffix", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addresses.city", + "columnName": "address_city", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.state", + "columnName": "address_state", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.street", + "columnName": "address_street", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addresses.zip", + "columnName": "address_zip", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_events_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_scan_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_scan_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `profileQueryId` INTEGER NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSuccess` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `profileQueryId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSuccess", + "columnName": "isSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "profileQueryId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_opt_out_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSubmitSuccess` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSubmitSuccess", + "columnName": "isSubmitSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_opt_out_action_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `completionTimeInMillis` INTEGER NOT NULL, `actionType` TEXT NOT NULL, `isError` INTEGER NOT NULL, `result` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "completionTimeInMillis", + "columnName": "completionTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionType", + "columnName": "actionType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isError", + "columnName": "isError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "result", + "columnName": "result", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_extracted_profiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `profileQueryId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `name` TEXT NOT NULL, `alternativeNames` TEXT NOT NULL, `age` TEXT NOT NULL, `addresses` TEXT NOT NULL, `phoneNumbers` TEXT NOT NULL, `relatives` TEXT NOT NULL, `profileUrl` TEXT NOT NULL, `identifier` TEXT NOT NULL, `reportId` TEXT NOT NULL, `email` TEXT NOT NULL, `fullName` TEXT NOT NULL, `dateAddedInMillis` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "alternativeNames", + "columnName": "alternativeNames", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "age", + "columnName": "age", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses", + "columnName": "addresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "phoneNumbers", + "columnName": "phoneNumbers", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "relatives", + "columnName": "relatives", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileUrl", + "columnName": "profileUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "identifier", + "columnName": "identifier", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reportId", + "columnName": "reportId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "fullName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateAddedInMillis", + "columnName": "dateAddedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier", + "unique": true, + "columnNames": [ + "profileQueryId", + "brokerName", + "name", + "profileUrl", + "identifier" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier` ON `${TABLE_NAME}` (`profileQueryId`, `brokerName`, `name`, `profileUrl`, `identifier`)" + } + ], + "foreignKeys": [] + }, + { + "tableName": "pir_scan_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `lastScanDateInMillis` INTEGER, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `userProfileId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastScanDateInMillis", + "columnName": "lastScanDateInMillis", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "userProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_optout_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `reporting_sevenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fourteenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_twentyOneDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fortyTwoDayConfirmationReportSentDateMs` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptCount", + "columnName": "attemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastOptOutAttemptDate", + "columnName": "lastOptOutAttemptDate", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "optOutRequestedDate", + "columnName": "optOutRequestedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutRemovedDate", + "columnName": "optOutRemovedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.sevenDayConfirmationReportSentDateMs", + "columnName": "reporting_sevenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fourteenDayConfirmationReportSentDateMs", + "columnName": "reporting_fourteenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.twentyOneDayConfirmationReportSentDateMs", + "columnName": "reporting_twentyOneDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fortyTwoDayConfirmationReportSentDateMs", + "columnName": "reporting_fortyTwoDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_mirror_sites", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `url` TEXT NOT NULL, `addedAt` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, `optOutUrl` TEXT NOT NULL, `parentSite` TEXT NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addedAt", + "columnName": "addedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parentSite", + "columnName": "parentSite", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_email_confirmation_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `email` TEXT NOT NULL, `attemptId` TEXT NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `emailConfirmationLink` TEXT NOT NULL, `linkFetchAttemptCount` INTEGER NOT NULL, `lastLinkFetchDateInMillis` INTEGER NOT NULL, `jobAttemptCount` INTEGER NOT NULL, `lastJobAttemptDateInMillis` INTEGER NOT NULL, `lastJobAttemptActionId` TEXT NOT NULL, `deprecated` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptId", + "columnName": "attemptId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "emailConfirmationLink", + "columnName": "emailConfirmationLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "linkFetchAttemptCount", + "columnName": "linkFetchAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastLinkFetchDateInMillis", + "columnName": "lastLinkFetchDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "jobAttemptCount", + "columnName": "jobAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptDateInMillis", + "columnName": "lastJobAttemptDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptActionId", + "columnName": "lastJobAttemptActionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_email_confirmation_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, `value` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'a5782d19654dee4cad932b458037bb37')" + ] + } +} \ No newline at end of file diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt index 9c3f0ac50ba8..53674d6ddbd5 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt @@ -22,4 +22,5 @@ object PirJobConstants { const val MAX_DETACHED_WEBVIEW_COUNT = 20 const val SCHEDULED_SCAN_INTERVAL_HOURS = 12L const val EMAIL_CONFIRMATION_INTERVAL_HOURS = 8L + const val CUSTOM_PIXEL_INTERVAL_HOURS = 5L } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt index ae996deedd8e..965afbf75fe4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt @@ -55,6 +55,11 @@ sealed class JobRecord( val optOutRequestedDateInMillis: Long = 0L, val optOutRemovedDateInMillis: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long = 0L, + val confirmation7dayReportSentDateMs: Long = 0L, + val confirmation14dayReportSentDateMs: Long = 0L, + val confirmation21dayReportSentDateMs: Long = 0L, + val confirmation42dayReportSentDateMs: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class OptOutJobStatus { /** Opt-out has not been executed yet and should be executed when possible */ @@ -90,6 +95,7 @@ sealed class JobRecord( val status: ScanJobStatus = ScanJobStatus.NOT_EXECUTED, val lastScanDateInMillis: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class ScanJobStatus { /** Scan has not been executed yet and should be executed when possible */ diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt new file mode 100644 index 000000000000..a81adfda76f7 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import logcat.logcat +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.abs + +interface OptOut24HourSubmissionSuccessRateReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( + private val optOutSubmitRateCalculator: OptOutSubmitRateCalculator, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pirPixelSender: PirPixelSender, + private val pirSchedulingRepository: PirSchedulingRepository, + private val dispatcherProvider: DispatcherProvider, +) : OptOut24HourSubmissionSuccessRateReporter { + override suspend fun attemptFirePixel() { + withContext(dispatcherProvider.io()) { + logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" } + val startDate = pirRepository.getCustomStatsPixelsLastSentMs() + val now = currentTimeProvider.currentTimeMillis() + + if (!shouldFirePixel(startDate, now)) return@withContext + logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } + val endDate = now - TimeUnit.HOURS.toMillis(24) + val activeBrokers = pirRepository.getAllActiveBrokerObjects() + val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords() + + if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { broker -> + val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name } + + if (activeJobRecordsForBroker.isEmpty()) return@forEach + + val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( + activeJobRecordsForBroker, + startDate, + endDate, + ) + + logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" } + if (successRate != null) { + pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = broker.url, + optOutSuccessRate = successRate, + ) + } + } + + logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } + pirRepository.setCustomStatsPixelsLastSentMs(endDate) + } + } + } + + private fun shouldFirePixel( + startDate: Long, + now: Long, + ): Boolean { + return if (startDate == 0L) { + // IF first run, we emit the custom stats pixel + true + } else { + // Else we check if at least 24 hours have passed since last emission + val nowDiffFromStart = abs(now - startDate) + nowDiffFromStart > TimeUnit.HOURS.toMillis(24) + } + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt new file mode 100644 index 000000000000..5e7a36029d28 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import logcat.logcat +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +interface OptOutConfirmationReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOutConfirmationReporter @Inject constructor( + private val dispatcherProvider: DispatcherProvider, + private val pirSchedulingRepository: PirSchedulingRepository, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pixelSender: PirPixelSender, +) : OptOutConfirmationReporter { + override suspend fun attemptFirePixel() { + logcat { "PIR-CUSTOM-STATS: Attempt to fire confirmation pixels" } + + withContext(dispatcherProvider.io()) { + val activeBrokers = pirRepository.getAllActiveBrokerObjects().associateBy { it.name } + val allValidRequestedOptOutJobs = pirSchedulingRepository.getAllValidOptOutJobRecords().filter { + it.status == REQUESTED || it.status == REMOVED // TODO: Filter out removed by user + } + + if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext + + logcat { "PIR-CUSTOM-STATS: Will fire confirmation pixels for ${allValidRequestedOptOutJobs.size} jobs" } + allValidRequestedOptOutJobs.also { + // Fire 7 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_7, + { jobRecord -> jobRecord.confirmation7dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay7ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 14 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_14, + { jobRecord -> jobRecord.confirmation14dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed14Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed14Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay14ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 21 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_21, + { jobRecord -> jobRecord.confirmation21dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed21Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed21Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay21ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 42 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_42, + { jobRecord -> jobRecord.confirmation42dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed42Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed42Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay42ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + } + } + } + + private suspend fun List.attemptFirePixelForConfirmationDay( + activeBrokers: Map, + confirmationDay: Long, + jobRecordFilter: (OptOutJobRecord) -> Boolean, + emitConfirmPixel: (String) -> Unit, + emitUnconfirmPixel: (String) -> Unit, + markOptOutJobRecordReporting: suspend (OptOutJobRecord, Long) -> Unit, + ) { + val now = currentTimeProvider.currentTimeMillis() + val optOutsForPixel = this.filter { + it.daysPassedSinceSubmission(now, confirmationDay) && jobRecordFilter(it) + } + + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${optOutsForPixel.size} jobs" } + optOutsForPixel.forEach { optOutJobRecord -> + val broker = activeBrokers[optOutJobRecord.brokerName] ?: return@forEach + + if (optOutJobRecord.status == REMOVED) { + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${broker.name}" } + emitConfirmPixel(broker.url) + } else { + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day unconfirmation pixels for ${broker.name}" } + emitUnconfirmPixel(broker.url) + } + + markOptOutJobRecordReporting(optOutJobRecord, now) + } + } + + private fun OptOutJobRecord.daysPassedSinceSubmission( + now: Long, + interval: Long, + ): Boolean { + return now >= this.optOutRequestedDateInMillis + TimeUnit.DAYS.toMillis(interval) + } + + companion object { + private const val INTERVAL_DAY_7 = 7L + private const val INTERVAL_DAY_14 = 14L + private const val INTERVAL_DAY_21 = 21L + private const val INTERVAL_DAY_42 = 42L + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt new file mode 100644 index 000000000000..688fa2578b64 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.scheduling.JobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.round + +interface OptOutSubmitRateCalculator { + /** + * Calculates the opt-out 24h submit rate for a given broker within the specified date range. + * + * @param allActiveOptOutJobsForBroker all active opt-out job records for the broker. + * @param startDateMs The opt-out records to include should be created on or after this date. Default is 0L (epoch). + * @param endDateMs tThe opt-out records to include should be created on or before this date. Default is 0L (epoch). + */ + suspend fun calculateOptOutSubmitRate( + allActiveOptOutJobsForBroker: List, + startDateMs: Long = 0L, + endDateMs: Long, + ): Double? +} + +@ContributesBinding(AppScope::class) +class RealOptOutSubmitRateCalculator @Inject constructor( + private val dispatcherProvider: DispatcherProvider, +) : OptOutSubmitRateCalculator { + override suspend fun calculateOptOutSubmitRate( + allActiveOptOutJobsForBroker: List, + startDateMs: Long, + endDateMs: Long, + ): Double? = withContext(dispatcherProvider.io()) { + // Get all opt out job records created within the given range for the specified broker + val recordsCreatedWithinRange = allActiveOptOutJobsForBroker.filter { + it.dateCreatedInMillis in startDateMs..endDateMs + } + + // We don't need to calculate the rate if there are no records + if (recordsCreatedWithinRange.isEmpty()) return@withContext null + + // Filter the records to only include those that were requested within 24 hours of creation + val requestedRecordsWithinRange = recordsCreatedWithinRange.filter { + (it.status == REQUESTED || it.status == REMOVED) && it.optOutRequestedDateInMillis > it.dateCreatedInMillis && + it.optOutRequestedDateInMillis <= it.dateCreatedInMillis + TimeUnit.HOURS.toMillis( + 24, + ) + } + + val optOutSuccessRate = + requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() + val roundedOptOutSuccessRate = round(optOutSuccessRate * 100) / 100 + return@withContext roundedOptOutSuccessRate + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt new file mode 100644 index 000000000000..b0bc5e2733ca --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.duckduckgo.anvil.annotations.ContributesWorker +import com.duckduckgo.di.scopes.AppScope +import logcat.logcat +import javax.inject.Inject + +@ContributesWorker(AppScope::class) +class PirCustomStatsWorker( + context: Context, + workerParameters: WorkerParameters, +) : CoroutineWorker(context, workerParameters) { + @Inject + lateinit var optOutSubmissionSuccessRateReporter: OptOut24HourSubmissionSuccessRateReporter + + @Inject + lateinit var optOutConfirmationReporter: OptOutConfirmationReporter + + override suspend fun doWork(): Result { + logcat { "PIR-CUSTOM-STATS: Attempt to fire custom pixels" } + optOutSubmissionSuccessRateReporter.attemptFirePixel() + optOutConfirmationReporter.attemptFirePixel() + + return Result.success() + } + + companion object { + const val TAG_PIR_RECURRING_CUSTOM_STATS = "TAG_PIR_RECURRING_CUSTOM_STATS" + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index c849c89f07a4..a676af8f6b66 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -135,6 +135,49 @@ enum class PirPixel( PIR_OPTOUT_SUBMIT_FAILURE( baseName = "dbp_optout_process_failure", types = setOf(Count), + ), + + PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE( + baseName = "dbp_databroker_custom_stats_optoutsubmit", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_unconfirmed", + type = Count, + ), + PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_unconfirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_unconfirmed", + type = Count, + ), + PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index 85ac7b2be51d..933c7bbe6930 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -18,6 +18,15 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_FAILED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_START import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_SUCCESS @@ -321,6 +330,57 @@ interface PirPixelSender { * Emits a pixel to signal that PIR encrypted database is unavailable. */ fun reportSecureStorageUnavailable() + + /** + * Emits a pixel containing the opt-out submit success rate for a broker for the last 24 hours + * + * @param brokerUrl url of the Broker for which the opt-out submit rate is for + * @param optOutSuccessRate opt out submit success rate for the past 24 hours + */ + fun reportBrokerCustomStateOptOutSubmitRate( + brokerUrl: String, + optOutSuccessRate: Double, + ) + + /** + * Emits a pixel when an opt-out has been confirmed within 7 days. + */ + fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 7 days. + */ + fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 14 days. + */ + fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 14 days. + */ + fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 21 days. + */ + fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 21 days. + */ + fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 42 days. + */ + fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 42 days. + */ + fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -603,6 +663,82 @@ class RealPirPixelSender @Inject constructor( fire(PIR_INTERNAL_SECURE_STORAGE_UNAVAILABLE) } + override fun reportBrokerCustomStateOptOutSubmitRate( + brokerUrl: String, + optOutSuccessRate: Double, + ) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + PARAM_KEY_OPTOUT_SUBMIT_SUCCESS_RATE to optOutSuccessRate.toString(), + ) + + fire(PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE, params) + } + + override fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), @@ -638,5 +774,6 @@ class RealPirPixelSender @Inject constructor( private const val PARAM_KEY_STAGE = "stage" private const val PARAM_KEY_PATTERN = "pattern" private const val PARAM_KEY_ACTION_TYPE = "action_type" + private const val PARAM_KEY_OPTOUT_SUBMIT_SUCCESS_RATE = "optout_submit_success_rate" } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt index ee14153d07a7..50fe401d48c9 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt @@ -24,16 +24,20 @@ import androidx.work.Data import androidx.work.ExistingPeriodicWorkPolicy import androidx.work.NetworkType import androidx.work.PeriodicWorkRequest +import androidx.work.PeriodicWorkRequestBuilder import androidx.work.WorkManager import androidx.work.multiprocess.RemoteListenableWorker import com.duckduckgo.app.di.AppCoroutineScope import com.duckduckgo.appbuildconfig.api.AppBuildConfig import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.common.PirJobConstants.CUSTOM_PIXEL_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.EMAIL_CONFIRMATION_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.SCHEDULED_SCAN_INTERVAL_HOURS import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker.Companion.TAG_EMAIL_CONFIRMATION +import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker +import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker.Companion.TAG_PIR_RECURRING_CUSTOM_STATS import com.duckduckgo.pir.impl.pixels.PirPixelSender import com.duckduckgo.pir.impl.scan.PirScheduledScanRemoteWorker.Companion.TAG_SCHEDULED_SCAN import com.duckduckgo.pir.impl.store.PirEventsRepository @@ -66,6 +70,7 @@ class RealPirScanScheduler @Inject constructor( schedulePirScans() scheduleEmailConfirmation() + scheduleRecurringPixelStats() } private fun schedulePirScans() { @@ -129,9 +134,23 @@ class RealPirScanScheduler @Inject constructor( ) } + private fun scheduleRecurringPixelStats() { + val periodicWorkRequest = PeriodicWorkRequestBuilder(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) + .addTag(TAG_PIR_RECURRING_CUSTOM_STATS) + .setInitialDelay(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) + .build() + + workManager.enqueueUniquePeriodicWork( + TAG_PIR_RECURRING_CUSTOM_STATS, + ExistingPeriodicWorkPolicy.UPDATE, + periodicWorkRequest, + ) + } + override fun cancelScheduledScans(context: Context) { workManager.cancelUniqueWork(TAG_SCHEDULED_SCAN) workManager.cancelUniqueWork(TAG_EMAIL_CONFIRMATION) + workManager.cancelUniqueWork(TAG_PIR_RECURRING_CUSTOM_STATS) context.stopService(Intent(context, PirRemoteWorkerService::class.java)) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt index 38e81fb7b60f..c2db7aec25f4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt @@ -22,6 +22,7 @@ import com.duckduckgo.data.store.api.SharedPreferencesProvider interface PirDataStore { var mainConfigEtag: String? + var customStatsPixelsLastSentMs: Long } internal class RealPirDataStore( @@ -42,8 +43,17 @@ internal class RealPirDataStore( } } + override var customStatsPixelsLastSentMs: Long + get() = preferences.getLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, 0L) + set(value) { + preferences.edit { + putLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, value) + } + } + companion object { private const val FILENAME = "com.duckduckgo.pir.v1" private const val KEY_MAIN_ETAG = "KEY_MAIN_ETAG" + private const val KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS = "KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS" } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt index ef026e0b66a5..071fbcc70255 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt @@ -53,7 +53,7 @@ import com.squareup.moshi.Types @Database( exportSchema = true, - version = 14, + version = 15, entities = [ BrokerJsonEtag::class, BrokerEntity::class, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt index 248919fe9abc..9843d2724f6c 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt @@ -174,6 +174,10 @@ interface PirRepository { suspend fun deleteEmailData(emailData: List) + suspend fun getCustomStatsPixelsLastSentMs(): Long + + suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) + data class GeneratedEmailData( val emailAddress: String, val pattern: String, @@ -663,6 +667,14 @@ class RealPirRepository( return@withContext } + override suspend fun getCustomStatsPixelsLastSentMs(): Long = withContext(dispatcherProvider.io()) { + pirDataStore.customStatsPixelsLastSentMs + } + + override suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) = withContext(dispatcherProvider.io()) { + pirDataStore.customStatsPixelsLastSentMs = timeMs + } + private fun List.toRequest(): PirEmailConfirmationDataRequest = PirEmailConfirmationDataRequest( items = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index 311db495677e..676953b49bb1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -28,6 +28,7 @@ import com.duckduckgo.pir.impl.models.scheduling.JobRecord.ScanJobRecord.ScanJob import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import com.squareup.anvil.annotations.ContributesBinding @@ -60,6 +61,11 @@ interface PirSchedulingRepository { */ suspend fun getAllValidOptOutJobRecords(): List + /** + * Returns all ScanJobRecord whose state is not INVALID for a specific broker + */ + suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List + /** * Returns a matching [OptOutJobRecord] whose state is not INVALID * @@ -119,6 +125,26 @@ interface PirSchedulingRepository { suspend fun deleteEmailConfirmationJobRecord(extractedProfileId: Long) suspend fun deleteAllEmailConfirmationJobRecords() + + suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -181,6 +207,16 @@ class RealPirSchedulingRepository @Inject constructor( .orEmpty() } + override suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List = + withContext(dispatcherProvider.io()) { + return@withContext jobSchedulingDao() + ?.getAllOptOutJobRecordsForBroker(brokerName) + ?.map { record -> record.toRecord() } + // do not pick-up deprecated jobs as they belong to removed profiles + ?.filter { !it.deprecated } + .orEmpty() + } + override suspend fun saveScanJobRecord(scanJobRecord: ScanJobRecord) { withContext(dispatcherProvider.io()) { jobSchedulingDao()?.saveScanJobRecord(scanJobRecord.toEntity()) @@ -304,6 +340,42 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.updateSevenDayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update14DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update21DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update42DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, @@ -311,6 +383,7 @@ class RealPirSchedulingRepository @Inject constructor( status = ScanJobStatus.entries.find { it.name == this.status } ?: ScanJobStatus.ERROR, lastScanDateInMillis = this.lastScanDateInMillis ?: 0L, deprecated = this.deprecated, + dateCreatedInMillis = this.dateCreatedInMillis, ) private fun ScanJobRecord.toEntity(): ScanJobRecordEntity = @@ -320,6 +393,11 @@ class RealPirSchedulingRepository @Inject constructor( status = this.status.name, lastScanDateInMillis = this.lastScanDateInMillis, deprecated = this.deprecated, + dateCreatedInMillis = if (this.dateCreatedInMillis != 0L) { + this.dateCreatedInMillis + } else { + currentTimeProvider.currentTimeMillis() + }, ) private fun OptOutJobRecordEntity.toRecord(): OptOutJobRecord = @@ -333,6 +411,11 @@ class RealPirSchedulingRepository @Inject constructor( optOutRequestedDateInMillis = this.optOutRequestedDate, optOutRemovedDateInMillis = this.optOutRemovedDate, deprecated = this.deprecated, + dateCreatedInMillis = this.dateCreatedInMillis, + confirmation7dayReportSentDateMs = this.reporting.sevenDayConfirmationReportSentDateMs, + confirmation14dayReportSentDateMs = this.reporting.fourteenDayConfirmationReportSentDateMs, + confirmation21dayReportSentDateMs = this.reporting.twentyOneDayConfirmationReportSentDateMs, + confirmation42dayReportSentDateMs = this.reporting.fortyTwoDayConfirmationReportSentDateMs, ) private fun OptOutJobRecord.toEntity(): OptOutJobRecordEntity = @@ -346,6 +429,17 @@ class RealPirSchedulingRepository @Inject constructor( optOutRequestedDate = this.optOutRequestedDateInMillis, optOutRemovedDate = this.optOutRemovedDateInMillis, deprecated = this.deprecated, + dateCreatedInMillis = if (this.dateCreatedInMillis != 0L) { + this.dateCreatedInMillis + } else { + currentTimeProvider.currentTimeMillis() + }, + reporting = ReportingRecord( + sevenDayConfirmationReportSentDateMs = this.confirmation7dayReportSentDateMs, + fourteenDayConfirmationReportSentDateMs = this.confirmation14dayReportSentDateMs, + twentyOneDayConfirmationReportSentDateMs = this.confirmation21dayReportSentDateMs, + fortyTwoDayConfirmationReportSentDateMs = this.confirmation42dayReportSentDateMs, + ), ) private fun EmailConfirmationJobRecord.toEntity(): EmailConfirmationJobRecordEntity = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index 30d040df1fde..8e836a706502 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -40,6 +40,9 @@ interface JobSchedulingDao { @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount") fun getAllOptOutJobRecords(): List + @Query("SELECT * FROM pir_optout_job_record WHERE brokerName = :brokerName ORDER BY attemptCount") + fun getAllOptOutJobRecordsForBroker(brokerName: String): List + @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount") fun getAllOptOutJobRecordsFlow(): Flow> @@ -115,4 +118,52 @@ interface JobSchedulingDao { deleteOptOutJobRecordsForProfiles(profileQueryIds) deleteEmailConfirmationJobRecordsForProfiles(profileQueryIds) } + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_sevenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun updateSevenDayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fourteenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update14DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_twentyOneDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update21DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fortyTwoDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update42DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt index adacba5232a0..69c7836009e1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt @@ -16,6 +16,7 @@ package com.duckduckgo.pir.impl.store.db +import androidx.room.Embedded import androidx.room.Entity import androidx.room.PrimaryKey @@ -29,6 +30,7 @@ data class ScanJobRecordEntity( val status: String, val lastScanDateInMillis: Long? = null, val deprecated: Boolean = false, + val dateCreatedInMillis: Long, ) @Entity(tableName = "pir_optout_job_record") @@ -42,6 +44,16 @@ data class OptOutJobRecordEntity( val optOutRequestedDate: Long = 0L, val optOutRemovedDate: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long, + @Embedded(prefix = "reporting_") + val reporting: ReportingRecord, +) + +data class ReportingRecord( + val sevenDayConfirmationReportSentDateMs: Long = 0L, + val fourteenDayConfirmationReportSentDateMs: Long = 0L, + val twentyOneDayConfirmationReportSentDateMs: Long = 0L, + val fortyTwoDayConfirmationReportSentDateMs: Long = 0L, ) @Entity(tableName = "pir_email_confirmation_job_record") diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt new file mode 100644 index 000000000000..274404334961 --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt @@ -0,0 +1,571 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.ProfileQuery +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOut24HourSubmissionSuccessRateReporterTest { + + @get:Rule + var coroutineRule = CoroutineTestRule() + + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockOptOutSubmitRateCalculator: OptOutSubmitRateCalculator = mock() + private val mockPirPixelSender: PirPixelSender = mock() + private val mockSchedulingRepository: PirSchedulingRepository = mock() + + private lateinit var toTest: RealOptOut24HourSubmissionSuccessRateReporter + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneHour = TimeUnit.HOURS.toMillis(1) + private val twentyFourHours = TimeUnit.HOURS.toMillis(24) + + private val testBroker1 = Broker( + name = "test-broker-1", + fileName = "test-broker-1.json", + url = "https://test-broker-1.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + private val testBroker2 = Broker( + name = "test-broker-2", + fileName = "test-broker-2.json", + url = "https://test-broker-2.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + private val testProfileQuery = ProfileQuery( + id = 1L, + firstName = "John", + lastName = "Doe", + city = "New York", + state = "NY", + addresses = emptyList(), + birthYear = 1990, + fullName = "John Doe", + age = 33, + deprecated = false, + ) + + @Before + fun setUp() { + toTest = RealOptOut24HourSubmissionSuccessRateReporter( + optOutSubmitRateCalculator = mockOptOutSubmitRateCalculator, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pirPixelSender = mockPirPixelSender, + pirSchedulingRepository = mockSchedulingRepository, + dispatcherProvider = coroutineRule.testDispatcherProvider, + ) + } + + @Test + fun whenFirstRunThenShouldFirePixel() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenLessThan24HoursPassedThenShouldNotFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + oneHour // Only 1 hour passed + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + toTest.attemptFirePixel() + + verify(mockPirRepository, never()).getAllActiveBrokerObjects() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenMoreThan24HoursPassedThenShouldFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours + oneHour // 25 hours passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.75) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.75, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenExactly24HoursPassedThenShouldNotFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours // Exactly 24 hours + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + toTest.attemptFirePixel() + + verify(mockPirRepository, never()).getAllActiveBrokerObjects() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + } + + @Test + fun whenNoActiveBrokersThenShouldNotFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenNoUserProfilesThenShouldNotFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenMultipleBrokersThenShouldFirePixelForEach() = runTest { + val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.5) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.8) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker2.url, + optOutSuccessRate = 0.8, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenSuccessRateIsNullThenShouldNotFirePixelForThatBroker() = runTest { + val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.5) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(null) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenAllSuccessRatesAreNullThenShouldNotFireAnyPixels() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(null) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenShouldFirePixelThenUsesCorrectDateRange() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours + oneHour + val expectedEndDate = now - twentyFourHours + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator).calculateOptOutSubmitRate( + allActiveOptOutJobsForBroker = eq(listOf(jobRecord)), + startDateMs = eq(startDate), + endDateMs = eq(expectedEndDate), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(expectedEndDate) + } + + @Test + fun whenMultipleBrokersWithMixedSuccessRatesThenFiresPixelsForNonNullRates() = runTest { + val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(null) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.9) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker1.url), + optOutSuccessRate = any(), + ) + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker2.url, + optOutSuccessRate = 0.9, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenShouldFirePixelButNoBrokersAndNoProfilesThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenShouldFirePixelButNoBrokersWithProfilesThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenShouldFirePixelButNoProfilesWithBrokersThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenBrokerHasNoJobRecordsThenSkipsThatBroker() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, // This broker has no job records + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) // Only for broker1 + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord)), + eq(0L), + eq(now - twentyFourHours), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + // Only broker1 should fire pixel, broker2 should be skipped + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker1.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt new file mode 100644 index 000000000000..f00ff0f098da --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOutConfirmationReporterTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutConfirmationReporter + + private val mockPirSchedulingRepository: PirSchedulingRepository = mock() + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockPixelSender: PirPixelSender = mock() + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val sevenDays = TimeUnit.DAYS.toMillis(7) + private val fourteenDays = TimeUnit.DAYS.toMillis(14) + private val twentyOneDays = TimeUnit.DAYS.toMillis(21) + private val fortyTwoDays = TimeUnit.DAYS.toMillis(42) + + private val testBroker = Broker( + name = "test-broker", + fileName = "test-broker.json", + url = "https://test-broker.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + @Before + fun setUp() { + testee = RealOptOutConfirmationReporter( + dispatcherProvider = coroutineRule.testDispatcherProvider, + pirSchedulingRepository = mockPirSchedulingRepository, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pixelSender = mockPixelSender, + ) + } + + @Test + fun whenNoActiveBrokersThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenNoOptOutJobsThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenOptOutJobNotRequestedOrRemovedThenDoesNotFirePixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.NOT_EXECUTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedAndStatusIsRemovedThenFiresConfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPassedAndStatusIsRequestedThenFiresUnconfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation7dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenLessThan7DaysPassedThenDoesNotFire7DayPixel() = runTest { + val now = baseTime + sevenDays - TimeUnit.HOURS.toMillis(1) // 1 hour before 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenExactly7DaysPassedThenFires7DayPixel() = runTest { + val now = baseTime + sevenDays // Exactly 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when14DaysPassedThenFires14DayPixel() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay14ConfirmationPixelSent(1L, now) + } + + @Test + fun when21DaysPassedThenFires21DayPixel() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay21ConfirmationPixelSent(1L, now) + } + + @Test + fun when42DaysPassedThenFires42DayPixel() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay42ConfirmationPixelSent(1L, now) + } + + @Test + fun whenMultipleIntervalsPassedThenFiresAllApplicablePixels() = runTest { + val now = baseTime + fortyTwoDays // 42 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + } + + @Test + fun whenBrokerNotFoundThenSkipsJobRecord() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = "unknown-broker", + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenMultipleJobRecordsThenFiresPixelsForEach() = runTest { + val now = baseTime + sevenDays + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(2L, now) + } + + @Test + fun when14DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation14dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay14ConfirmationPixelSent(any(), any()) + } + + @Test + fun when21DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation21dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay21ConfirmationPixelSent(any(), any()) + } + + @Test + fun when42DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation42dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed42Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay42ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedBut14DaysNotPassedThenOnlyFires7DayPixel() = runTest { + val now = baseTime + sevenDays + oneDay // 8 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + } + + @Test + fun when14DaysPassedBut21DaysNotPassedThenFires7And14DayPixels() = runTest { + val now = baseTime + fourteenDays + oneDay // 15 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + } + + @Test + fun whenRequestedStatusThenFiresUnconfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + } + + @Test + fun whenRemovedStatusThenFiresConfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutUnconfirmed7Days(any()) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis: Long = baseTime, + optOutRemovedDateInMillis: Long = 0L, + confirmation7dayReportSentDateMs: Long = 0L, + confirmation14dayReportSentDateMs: Long = 0L, + confirmation21dayReportSentDateMs: Long = 0L, + confirmation42dayReportSentDateMs: Long = 0L, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = 0, + lastOptOutAttemptDateInMillis = 0L, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = false, + dateCreatedInMillis = baseTime, + confirmation7dayReportSentDateMs = confirmation7dayReportSentDateMs, + confirmation14dayReportSentDateMs = confirmation14dayReportSentDateMs, + confirmation21dayReportSentDateMs = confirmation21dayReportSentDateMs, + confirmation42dayReportSentDateMs = confirmation42dayReportSentDateMs, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt new file mode 100644 index 000000000000..6f3cc7407121 --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt @@ -0,0 +1,676 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import java.util.concurrent.TimeUnit + +class RealOptOutSubmitRateCalculatorTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutSubmitRateCalculator + + @Before + fun setUp() { + testee = RealOptOutSubmitRateCalculator( + dispatcherProvider = coroutineRule.testDispatcherProvider, + ) + } + + // Test data + private val testBrokerName = "test-broker" + + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneHour = TimeUnit.HOURS.toMillis(1) + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val twentyFourHours = TimeUnit.HOURS.toMillis(24) + + @Test + fun whenNoRecordsInDateRangeThenReturnNull() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val result = testee.calculateOptOutSubmitRate(emptyList(), startDate, endDate) + + assertNull(result) + } + + @Test + fun whenRecordsExistButNoneInDateRangeThenReturnNull() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordBeforeRange = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + dateCreatedInMillis = baseTime - oneDay, + ) + val recordAfterRange = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + dateCreatedInMillis = baseTime + oneDay + oneHour, + ) + + val result = testee.calculateOptOutSubmitRate( + listOf(recordBeforeRange, recordAfterRange), + startDate, + endDate, + ) + + assertNull(result) + } + + @Test + fun whenRecordsInDateRangeButNoneRequestedThenReturnZero() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = baseTime + oneHour, + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = baseTime + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) + + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenAllRecordsInDateRangeAreRequestedThenReturnOne() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // Within 24 hours + ) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenHalfRecordsInDateRangeAreRequestedThenReturnHalf() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val notExecutedRecord = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate( + listOf(requestedRecord, notExecutedRecord), + startDate, + endDate, + ) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRequestedRecordOutside24HourWindowThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedWithinWindow = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val requestedOutsideWindow = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24 hours + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithinWindow, requestedOutsideWindow), startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRequestedRecordExactlyAt24HourWindowThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedAt24Hours = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours, // Exactly 24 hours + ) + val notExecutedRecord = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAt24Hours, notExecutedRecord), startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenStartDateIsZeroThenUseDefaultStartDate() = runTest { + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val record = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(record), endDateMs = endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenRecordAtStartDateBoundaryThenIncluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordAtStart = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = startDate, // Exactly at start + optOutRequestedDateInMillis = startDate + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtStart), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenRecordAtEndDateBoundaryThenIncluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordAtEnd = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = endDate, // Exactly at end + optOutRequestedDateInMillis = endDate + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtEnd), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenMultipleStatusesThenOnlyRequestedCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val removed = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REMOVED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val error = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = dateCreated + 3 * oneHour, + ) + val pendingEmail = createOptOutJobRecord( + extractedProfileId = 4L, + brokerName = testBrokerName, + status = OptOutJobStatus.PENDING_EMAIL_CONFIRMATION, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested, removed, error, pendingEmail), startDate, endDate) + + assertEquals(0.25, result!!, 0.0) + } + + @Test + fun whenComplexScenarioThenCalculateCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 5 records in range + val requested1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24h + ) + val requested2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24h + ) + val requested3 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 3 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour + twentyFourHours, // Exactly 24h + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 4L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val error = createOptOutJobRecord( + extractedProfileId = 5L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = dateCreated + 5 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, requested3, notExecuted, error), startDate, endDate) + + // Only requested1 and requested3 count (2 out of 5) + assertEquals(0.4, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateEqualsDateCreatedThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedAtSameTime = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated, // Exactly equal (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + oneHour, // After creation + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAtSameTime, validRequested), startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateBeforeDateCreatedThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedBeforeCreation = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated - oneHour, // Before creation (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // After creation + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedBeforeCreation, validRequested), startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustAfterDateCreatedThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustAfter = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneMillisecond, // Just 1ms after (should be counted) + ) + + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustBefore24HourLimitThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustBefore24h = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours - oneMillisecond, // Just before 24h limit + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustBefore24h), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustAfter24HourLimitThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustAfter24h = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours + oneMillisecond, // Just after 24h limit + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter24h, notExecuted), startDate, endDate) + + // requestedJustAfter24h doesn't count, so 0 out of 2 + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenResultNeedsRoundingThenRoundsToTwoDecimalPlaces() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 1 out of 3 = 0.333... should round to 0.33 + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val notExecuted1 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val notExecuted2 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 3 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested, notExecuted1, notExecuted2), startDate, endDate) + + // 1/3 = 0.333... rounded to 0.33 + assertEquals(0.33, result!!, 0.0) + } + + @Test + fun whenResultNeedsRoundingUpThenRoundsCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 2 out of 3 = 0.666... should round to 0.67 + val requested1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val requested2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, notExecuted), startDate, endDate) + + // 2/3 = 0.666... rounded to 0.67 + assertEquals(0.67, result!!, 0.0) + } + + @Test + fun whenSingleRecordRequestedThenReturnOne() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val singleRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenSingleRecordNotRequestedThenReturnZero() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val singleRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated, + ) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) + + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenLargeDateRangeThenFiltersCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + TimeUnit.DAYS.toMillis(365) // 1 year + val dateCreated = baseTime + oneDay + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(180), // 6 months later + optOutRequestedDateInMillis = dateCreated + TimeUnit.DAYS.toMillis(180) + oneHour, + ) + val record3 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(300), // 10 months later + ) + + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2, record3), startDate, endDate) + + // 2 out of 3 + assertEquals(0.67, result!!, 0.0) + } + + @Test + fun whenRequestedRecordHasZeroOptOutRequestedDateThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedWithZeroDate = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = 0L, // Zero (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithZeroDate, validRequested), startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenFractionalResultRoundsDownThenRoundsCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 1 out of 7 = 0.142857... should round to 0.14 + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val notExecutedRecords = (2L..7L).map { id -> + createOptOutJobRecord( + extractedProfileId = id, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + id * oneHour, + ) + } + + val result = testee.calculateOptOutSubmitRate(listOf(requested) + notExecutedRecords, startDate, endDate) + + // 1/7 = 0.142857... rounded to 0.14 + assertEquals(0.14, result!!, 0.0) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBrokerName, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt index ad93c1383d0e..770d39f032fe 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt @@ -27,6 +27,7 @@ import com.duckduckgo.pir.impl.store.db.BrokerJsonDao import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import kotlinx.coroutines.runBlocking @@ -79,6 +80,7 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = ScanJobStatus.NOT_EXECUTED.name, lastScanDateInMillis = 1000L, + dateCreatedInMillis = 100L, ) private val deprecatedScanJobEntity = @@ -88,6 +90,7 @@ class RealPirSchedulingRepositoryTest { status = ScanJobStatus.MATCHES_FOUND.name, deprecated = true, lastScanDateInMillis = 2000L, + dateCreatedInMillis = 100L, ) private val validOptOutJobEntity = @@ -100,6 +103,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val deprecatedOptOutJobEntity = @@ -113,6 +118,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 3000L, optOutRequestedDate = 4000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val scanJobRecord = @@ -243,6 +250,7 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = "NOT_EXECUTED", lastScanDateInMillis = 1000L, + dateCreatedInMillis = 9000L, ), ) } @@ -268,12 +276,14 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = "NOT_EXECUTED", lastScanDateInMillis = 1000L, + dateCreatedInMillis = 9000L, ), ScanJobRecordEntity( brokerName = "another-broker", userProfileId = 456L, status = "MATCHES_FOUND", lastScanDateInMillis = 5000L, + dateCreatedInMillis = 9000L, ), ), ) @@ -397,6 +407,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ) } @@ -430,6 +442,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), OptOutJobRecordEntity( extractedProfileId = 999L, @@ -440,6 +454,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 3000L, optOutRequestedDate = 4000L, optOutRemovedDate = 5000L, + dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ), )