diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 new file mode 100644 index 000000000000..61b6ae2749f2 --- /dev/null +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -0,0 +1,264 @@ +{ + "dbp_optout_process_submit-success": { + "description": "Fired when an opt-out submission succeeds.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that this opt-out attempt targets", + "type": "string" + }, + { + "key": "parent", + "description": "The parent data broker of the one this opt-out attempt targets", + "type": "string" + }, + { + "key": "attempt_id", + "description": "Client-generated ID of the opt-out attempt", + "type": "string" + }, + { + "key": "duration", + "description": "Total duration of the opt-out attempt in milliseconds", + "type": "string" + }, + { + "key": "tries", + "description": "The number of tries it took to submit successfully", + "type": "string" + }, + { + "key": "pattern", + "description": "Email pattern used during submission, when available", + "type": "string" + }, + { + "key": "vpn_connection_state", + "description": "Reported VPN connection state when the submission succeeded", + "type": "string" + }, + { + "key": "vpn_bypass", + "description": "VPN bypass status when the submission succeeded", + "type": "string", + "enum": ["on", "off", "unsupported"] + } + ] + }, + "dbp_optout_process_failure": { + "description": "Fired when an opt-out attempt fails.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that this opt-out attempt targets", + "type": "string" + }, + { + "key": "parent", + "description": "The parent data broker of the one this opt-out attempt targets", + "type": "string" + }, + { + "key": "broker_version", + "description": "The version of the broker JSON file", + "type": "string" + }, + { + "key": "attempt_id", + "description": "Client-generated ID of the opt-out attempt", + "type": "string" + }, + { + "key": "duration", + "description": "Total duration of the opt-out attempt in milliseconds", + "type": "string" + }, + { + "key": "stage", + "description": "The stage where the failure occurred", + "type": "string" + }, + { + "key": "tries", + "description": "The number of tries recorded when the failure happened", + "type": "string" + }, + { + "key": "pattern", + "description": "Email pattern used during the attempt, when available", + "type": "string" + }, + { + "key": "action_id", + "description": "Predefined identifier of the broker action that failed", + "type": "string" + }, + { + "key": "action_type", + "description": "Type of action that failed", + "type": "string", + "enum": [ + "extract", + "navigate", + "fillForm", + "click", + "expectation", + "emailConfirmation", + "getCaptchaInfo", + "solveCaptcha", + "condition" + ] + }, + { + "key": "vpn_connection_state", + "description": "Reported VPN connection state when the failure happened", + "type": "string" + }, + { + "key": "vpn_bypass", + "description": "VPN bypass status when the failure happened", + "type": "string", + "enum": ["on", "off", "unsupported"] + } + ] + }, + "dbp_databroker_custom_stats_optoutsubmit": { + "description": "Pixel that contains a broker's opt-out 24h submission success rate.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that this opt-out attempt targets", + "type": "string" + }, + { + "key": "optout_submit_success_rate", + "description": "The success rate of how many opt-out jobs successfully requested within 24 hours of creation", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-7-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-7-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 42 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 42 days", + "type": "string" + } + ] + } +} diff --git a/pir/pir-impl/build.gradle b/pir/pir-impl/build.gradle index 58c4926233f8..3a9dbc204b75 100644 --- a/pir/pir-impl/build.gradle +++ b/pir/pir-impl/build.gradle @@ -70,6 +70,7 @@ dependencies { testImplementation AndroidX.test.ext.junit testImplementation Testing.robolectric testImplementation "androidx.lifecycle:lifecycle-runtime-testing:_" + testImplementation AndroidX.work.testing testImplementation(KotlinX.coroutines.test) { // https://github.com/Kotlin/kotlinx.coroutines/issues/2023 // conflicts with mockito due to direct inclusion of byte buddy diff --git a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json new file mode 100644 index 000000000000..a858f45c66f9 --- /dev/null +++ b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json @@ -0,0 +1,976 @@ +{ + "formatVersion": 1, + "database": { + "version": 15, + "identityHash": "a5782d19654dee4cad932b458037bb37", + "entities": [ + { + "tableName": "pir_broker_json_etag", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`fileName` TEXT NOT NULL, `etag` TEXT NOT NULL, PRIMARY KEY(`fileName`))", + "fields": [ + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "etag", + "columnName": "etag", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "fileName" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_details", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `fileName` TEXT NOT NULL, `url` TEXT NOT NULL, `version` TEXT NOT NULL, `parent` TEXT, `addedDatetime` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parent", + "columnName": "parent", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addedDatetime", + "columnName": "addedDatetime", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_opt_out", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT NOT NULL, `optOutUrl` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scan", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scheduling_config", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `retryError` INTEGER NOT NULL, `confirmOptOutScan` INTEGER NOT NULL, `maintenanceScan` INTEGER NOT NULL, `maxAttempts` INTEGER, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "retryError", + "columnName": "retryError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "confirmOptOutScan", + "columnName": "confirmOptOutScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maintenanceScan", + "columnName": "maintenanceScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maxAttempts", + "columnName": "maxAttempts", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_user_profile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `birthYear` INTEGER NOT NULL, `phone` TEXT, `deprecated` INTEGER NOT NULL, `user_firstName` TEXT NOT NULL, `user_lastName` TEXT NOT NULL, `user_middleName` TEXT, `user_suffix` TEXT, `address_city` TEXT NOT NULL, `address_state` TEXT NOT NULL, `address_street` TEXT, `address_zip` TEXT)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "birthYear", + "columnName": "birthYear", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "phone", + "columnName": "phone", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "userName.firstName", + "columnName": "user_firstName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.lastName", + "columnName": "user_lastName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.middleName", + "columnName": "user_middleName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "userName.suffix", + "columnName": "user_suffix", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addresses.city", + "columnName": "address_city", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.state", + "columnName": "address_state", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.street", + "columnName": "address_street", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addresses.zip", + "columnName": "address_zip", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_events_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_scan_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_scan_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `profileQueryId` INTEGER NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSuccess` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `profileQueryId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSuccess", + "columnName": "isSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "profileQueryId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_opt_out_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSubmitSuccess` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSubmitSuccess", + "columnName": "isSubmitSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_opt_out_action_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `completionTimeInMillis` INTEGER NOT NULL, `actionType` TEXT NOT NULL, `isError` INTEGER NOT NULL, `result` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "completionTimeInMillis", + "columnName": "completionTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionType", + "columnName": "actionType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isError", + "columnName": "isError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "result", + "columnName": "result", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_extracted_profiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `profileQueryId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `name` TEXT NOT NULL, `alternativeNames` TEXT NOT NULL, `age` TEXT NOT NULL, `addresses` TEXT NOT NULL, `phoneNumbers` TEXT NOT NULL, `relatives` TEXT NOT NULL, `profileUrl` TEXT NOT NULL, `identifier` TEXT NOT NULL, `reportId` TEXT NOT NULL, `email` TEXT NOT NULL, `fullName` TEXT NOT NULL, `dateAddedInMillis` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "alternativeNames", + "columnName": "alternativeNames", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "age", + "columnName": "age", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses", + "columnName": "addresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "phoneNumbers", + "columnName": "phoneNumbers", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "relatives", + "columnName": "relatives", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileUrl", + "columnName": "profileUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "identifier", + "columnName": "identifier", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reportId", + "columnName": "reportId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "fullName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateAddedInMillis", + "columnName": "dateAddedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier", + "unique": true, + "columnNames": [ + "profileQueryId", + "brokerName", + "name", + "profileUrl", + "identifier" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier` ON `${TABLE_NAME}` (`profileQueryId`, `brokerName`, `name`, `profileUrl`, `identifier`)" + } + ], + "foreignKeys": [] + }, + { + "tableName": "pir_scan_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `lastScanDateInMillis` INTEGER, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `userProfileId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastScanDateInMillis", + "columnName": "lastScanDateInMillis", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "userProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_optout_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `reporting_sevenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fourteenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_twentyOneDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fortyTwoDayConfirmationReportSentDateMs` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptCount", + "columnName": "attemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastOptOutAttemptDate", + "columnName": "lastOptOutAttemptDate", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "optOutRequestedDate", + "columnName": "optOutRequestedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutRemovedDate", + "columnName": "optOutRemovedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.sevenDayConfirmationReportSentDateMs", + "columnName": "reporting_sevenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fourteenDayConfirmationReportSentDateMs", + "columnName": "reporting_fourteenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.twentyOneDayConfirmationReportSentDateMs", + "columnName": "reporting_twentyOneDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fortyTwoDayConfirmationReportSentDateMs", + "columnName": "reporting_fortyTwoDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_mirror_sites", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `url` TEXT NOT NULL, `addedAt` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, `optOutUrl` TEXT NOT NULL, `parentSite` TEXT NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addedAt", + "columnName": "addedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parentSite", + "columnName": "parentSite", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_email_confirmation_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `email` TEXT NOT NULL, `attemptId` TEXT NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `emailConfirmationLink` TEXT NOT NULL, `linkFetchAttemptCount` INTEGER NOT NULL, `lastLinkFetchDateInMillis` INTEGER NOT NULL, `jobAttemptCount` INTEGER NOT NULL, `lastJobAttemptDateInMillis` INTEGER NOT NULL, `lastJobAttemptActionId` TEXT NOT NULL, `deprecated` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptId", + "columnName": "attemptId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "emailConfirmationLink", + "columnName": "emailConfirmationLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "linkFetchAttemptCount", + "columnName": "linkFetchAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastLinkFetchDateInMillis", + "columnName": "lastLinkFetchDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "jobAttemptCount", + "columnName": "jobAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptDateInMillis", + "columnName": "lastJobAttemptDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptActionId", + "columnName": "lastJobAttemptActionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_email_confirmation_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, `value` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'a5782d19654dee4cad932b458037bb37')" + ] + } +} \ No newline at end of file diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/NativeBrokerActionHandler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/NativeBrokerActionHandler.kt index 660232580599..3d82a0c8e87e 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/NativeBrokerActionHandler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/NativeBrokerActionHandler.kt @@ -30,6 +30,7 @@ import com.duckduckgo.pir.impl.common.NativeBrokerActionHandler.NativeActionResu import com.duckduckgo.pir.impl.common.NativeBrokerActionHandler.NativeActionResult.Success.NativeSuccessData.CaptchaTransactionIdReceived import com.duckduckgo.pir.impl.service.DbpService.CaptchaSolutionMeta import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirRepository.GeneratedEmailData import kotlinx.coroutines.withContext interface NativeBrokerActionHandler { @@ -62,7 +63,7 @@ interface NativeBrokerActionHandler { ) : NativeActionResult() { sealed class NativeSuccessData { data class Email( - val email: String, + val generatedEmailData: GeneratedEmailData, ) : NativeSuccessData() data class CaptchaTransactionIdReceived( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirActionsRunner.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirActionsRunner.kt index 8f271bebbc71..c3e7cb2d4397 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirActionsRunner.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirActionsRunner.kt @@ -399,7 +399,7 @@ class RealPirActionsRunner @AssistedInject constructor( if (it is Success) { engine?.dispatch( EmailReceived( - email = (it.data as Email).email, + generatedEmailData = (it.data as Email).generatedEmailData, ), ) } else { diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt index 9c3f0ac50ba8..53674d6ddbd5 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt @@ -22,4 +22,5 @@ object PirJobConstants { const val MAX_DETACHED_WEBVIEW_COUNT = 20 const val SCHEDULED_SCAN_INTERVAL_HOURS = 12L const val EMAIL_CONFIRMATION_INTERVAL_HOURS = 8L + const val CUSTOM_PIXEL_INTERVAL_HOURS = 5L } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt index 8ead44d0261c..813dbf42f9e4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirRunStateHandler.kt @@ -27,18 +27,23 @@ import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerOptOu import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationCompleted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationNeeded import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationStarted -import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutCompleted +import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutFailed import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutStarted +import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutSubmitted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScanActionFailed import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScanActionSucceeded import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScheduledScanCompleted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScheduledScanStarted import com.duckduckgo.pir.impl.models.AddressCityState import com.duckduckgo.pir.impl.models.ExtractedProfile +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord import com.duckduckgo.pir.impl.pixels.PirPixelSender +import com.duckduckgo.pir.impl.pixels.PirStage import com.duckduckgo.pir.impl.scheduling.JobRecordUpdater +import com.duckduckgo.pir.impl.scripts.models.BrokerAction import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse.ExtractedResponse +import com.duckduckgo.pir.impl.scripts.models.asActionType import com.duckduckgo.pir.impl.store.PirEventsRepository import com.duckduckgo.pir.impl.store.PirRepository import com.duckduckgo.pir.impl.store.PirSchedulingRepository @@ -127,12 +132,24 @@ interface PirRunStateHandler { val extractedProfile: ExtractedProfile, ) : PirRunState(brokerName) - data class BrokerRecordOptOutCompleted( + data class BrokerRecordOptOutSubmitted( override val brokerName: String, val extractedProfile: ExtractedProfile, + val attemptId: String, val startTimeInMillis: Long, val endTimeInMillis: Long, - val isSubmitSuccess: Boolean, + val emailPattern: String?, + ) : PirRunState(brokerName) + + data class BrokerRecordOptOutFailed( + override val brokerName: String, + val extractedProfile: ExtractedProfile, + val attemptId: String, + val startTimeInMillis: Long, + val endTimeInMillis: Long, + val failedAction: BrokerAction, + val stage: PirStage, + val emailPattern: String?, ) : PirRunState(brokerName) data class BrokerOptOutActionSucceeded( @@ -177,7 +194,8 @@ class RealPirRunStateHandler @Inject constructor( is BrokerScanActionSucceeded -> handleBrokerScanActionSucceeded(pirRunState) is BrokerScanActionFailed -> handleBrokerScanActionFailed(pirRunState) is BrokerRecordOptOutStarted -> handleRecordOptOutStarted(pirRunState) - is BrokerRecordOptOutCompleted -> handleRecordOptOutCompleted(pirRunState) + is BrokerRecordOptOutSubmitted -> handleBrokerRecordOptOutSubmitted(pirRunState) + is BrokerRecordOptOutFailed -> handleBrokerRecordOptOutFailed(pirRunState) is BrokerOptOutActionSucceeded -> handleBrokerOptOutActionSucceeded(pirRunState) is BrokerOptOutActionFailed -> handleBrokerOptOutActionFailed(pirRunState) is BrokerRecordEmailConfirmationNeeded -> handleBrokerRecordEmailConfirmationNeeded(pirRunState) @@ -406,19 +424,55 @@ class RealPirRunStateHandler @Inject constructor( ) } - private suspend fun handleRecordOptOutCompleted(state: BrokerRecordOptOutCompleted) { - updateOptOutRecord(state.isSubmitSuccess, state.extractedProfile.dbId) - pixelSender.reportOptOutCompleted( + private suspend fun handleBrokerRecordOptOutSubmitted(state: BrokerRecordOptOutSubmitted) { + val broker = repository.getBrokerForName(state.brokerName) + val optOutJobRecord = updateOptOutRecord(true, state.extractedProfile.dbId) + + if (broker == null || optOutJobRecord == null) return + + pixelSender.reportOptOutSubmitted( + brokerUrl = broker.url, + parent = broker.parent ?: "", + attemptId = state.attemptId, + durationMs = state.endTimeInMillis - state.startTimeInMillis, + tries = optOutJobRecord.attemptCount, + emailPattern = state.emailPattern, + ) + + eventsRepository.saveOptOutCompleted( brokerName = state.brokerName, - totalTimeInMillis = state.endTimeInMillis - state.startTimeInMillis, - isSuccess = state.isSubmitSuccess, + extractedProfile = state.extractedProfile, + startTimeInMillis = state.startTimeInMillis, + endTimeInMillis = state.endTimeInMillis, + isSubmitSuccess = true, ) + } + + private suspend fun handleBrokerRecordOptOutFailed(state: BrokerRecordOptOutFailed) { + val broker = repository.getBrokerForName(state.brokerName) + val optOutJobRecord = updateOptOutRecord(false, state.extractedProfile.dbId) + + if (broker == null || optOutJobRecord == null) return + + pixelSender.reportOptOutFailed( + brokerUrl = broker.url, + parent = broker.parent ?: "", + brokerJsonVersion = broker.version, + attemptId = state.attemptId, + durationMs = state.endTimeInMillis - state.startTimeInMillis, + tries = optOutJobRecord.attemptCount, + emailPattern = state.emailPattern, + stage = state.stage, + actionId = state.failedAction.id, + actionType = state.failedAction.asActionType(), + ) + eventsRepository.saveOptOutCompleted( brokerName = state.brokerName, extractedProfile = state.extractedProfile, startTimeInMillis = state.startTimeInMillis, endTimeInMillis = state.endTimeInMillis, - isSubmitSuccess = state.isSubmitSuccess, + isSubmitSuccess = false, ) } @@ -457,8 +511,8 @@ class RealPirRunStateHandler @Inject constructor( private suspend fun updateOptOutRecord( isSubmitted: Boolean, extractedProfileId: Long, - ) { - if (isSubmitted) { + ): OptOutJobRecord? { + return if (isSubmitted) { jobRecordUpdater.updateOptOutRequested(extractedProfileId) } else { jobRecordUpdater.updateOptOutError(extractedProfileId) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/BrokerStepCompletedEventHandler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/BrokerStepCompletedEventHandler.kt index 67b1b1339fc8..16242046e18a 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/BrokerStepCompletedEventHandler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/BrokerStepCompletedEventHandler.kt @@ -25,13 +25,15 @@ import com.duckduckgo.pir.impl.common.PirJob.RunType.EMAIL_CONFIRMATION import com.duckduckgo.pir.impl.common.PirRunStateHandler import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerManualScanCompleted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationCompleted -import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutCompleted +import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutFailed +import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutSubmitted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScheduledScanCompleted import com.duckduckgo.pir.impl.common.actions.EventHandler.Next import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.Event import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.Event.BrokerStepCompleted import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.Event.ExecuteNextBrokerStep import com.duckduckgo.pir.impl.common.actions.PirActionsRunnerStateEngine.State +import com.duckduckgo.pir.impl.pixels.PirStage import com.squareup.anvil.annotations.ContributesMultibinding import javax.inject.Inject import kotlin.reflect.KClass @@ -76,6 +78,7 @@ class BrokerStepCompletedEventHandler @Inject constructor( state.copy( currentBrokerStepIndex = state.currentBrokerStepIndex + 1, actionRetryCount = 0, + generatedEmailData = null, ), nextEvent = ExecuteNextBrokerStep, ) @@ -115,15 +118,32 @@ class BrokerStepCompletedEventHandler @Inject constructor( RunType.OPTOUT -> { val currentOptOutStep = currentBrokerStep as OptOutStep - pirRunStateHandler.handleState( - BrokerRecordOptOutCompleted( + if (isSuccess) { + BrokerRecordOptOutSubmitted( brokerName = currentOptOutStep.brokerName, extractedProfile = currentOptOutStep.profileToOptOut, + attemptId = state.attemptId ?: "no-attempt-id", startTimeInMillis = state.brokerStepStartTime, endTimeInMillis = currentTimeProvider.currentTimeMillis(), - isSubmitSuccess = isSuccess, - ), - ) + emailPattern = state.generatedEmailData?.pattern, + ) + } else { + // Whatever last action that was executed is the last action that failed. + val lastAction = currentBrokerStep.actions[state.currentBrokerStepIndex] + + BrokerRecordOptOutFailed( + brokerName = currentOptOutStep.brokerName, + extractedProfile = currentOptOutStep.profileToOptOut, + startTimeInMillis = state.brokerStepStartTime, + endTimeInMillis = currentTimeProvider.currentTimeMillis(), + attemptId = state.attemptId ?: "no-attempt-id", + failedAction = lastAction, + stage = PirStage.OTHER, // TODO: Integrate stages properly later on + emailPattern = state.generatedEmailData?.pattern, + ) + }.also { + pirRunStateHandler.handleState(it) + } } EMAIL_CONFIRMATION -> { diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/EmailReceivedEventHandler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/EmailReceivedEventHandler.kt index 24c9ae823bd7..4ed2faead5b6 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/EmailReceivedEventHandler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/EmailReceivedEventHandler.kt @@ -48,7 +48,7 @@ class EmailReceivedEventHandler @Inject constructor() : EventHandler { val currentBrokerStep = state.brokerStepsToExecute[state.currentBrokerStepIndex] as OptOutStep val updatedProfileWithEmail = currentBrokerStep.profileToOptOut.copy( - email = (event as EmailReceived).email, + email = (event as EmailReceived).generatedEmailData.emailAddress, ) val updatedBrokerSteps = state.brokerStepsToExecute.toMutableList().apply { @@ -60,6 +60,7 @@ class EmailReceivedEventHandler @Inject constructor() : EventHandler { return Next( nextState = state.copy( brokerStepsToExecute = updatedBrokerSteps, + generatedEmailData = event.generatedEmailData, ), nextEvent = ExecuteBrokerStepAction( actionRequestData = UserProfile( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/PirActionsRunnerStateEngine.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/PirActionsRunnerStateEngine.kt index 20c58983680d..b8ee1dd659e1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/PirActionsRunnerStateEngine.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/actions/PirActionsRunnerStateEngine.kt @@ -25,6 +25,7 @@ import com.duckduckgo.pir.impl.scripts.models.PirError import com.duckduckgo.pir.impl.scripts.models.PirScriptRequestData import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse.GetCaptchaInfoResponse.ResponseData +import com.duckduckgo.pir.impl.store.PirRepository.GeneratedEmailData import kotlinx.coroutines.flow.Flow interface PirActionsRunnerStateEngine { @@ -53,6 +54,7 @@ interface PirActionsRunnerStateEngine { val transactionID: String = "", val pendingUrl: String? = null, val actionRetryCount: Int = 0, + val generatedEmailData: GeneratedEmailData? = null, ) /** @@ -76,7 +78,7 @@ interface PirActionsRunnerStateEngine { ) : Event() data class EmailReceived( - val email: String, + val generatedEmailData: GeneratedEmailData, ) : Event() data object ExecuteNextBrokerStep : Event() diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt index ae996deedd8e..965afbf75fe4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt @@ -55,6 +55,11 @@ sealed class JobRecord( val optOutRequestedDateInMillis: Long = 0L, val optOutRemovedDateInMillis: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long = 0L, + val confirmation7dayReportSentDateMs: Long = 0L, + val confirmation14dayReportSentDateMs: Long = 0L, + val confirmation21dayReportSentDateMs: Long = 0L, + val confirmation42dayReportSentDateMs: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class OptOutJobStatus { /** Opt-out has not been executed yet and should be executed when possible */ @@ -90,6 +95,7 @@ sealed class JobRecord( val status: ScanJobStatus = ScanJobStatus.NOT_EXECUTED, val lastScanDateInMillis: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class ScanJobStatus { /** Scan has not been executed yet and should be executed when possible */ diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt new file mode 100644 index 000000000000..a81adfda76f7 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import logcat.logcat +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.abs + +interface OptOut24HourSubmissionSuccessRateReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( + private val optOutSubmitRateCalculator: OptOutSubmitRateCalculator, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pirPixelSender: PirPixelSender, + private val pirSchedulingRepository: PirSchedulingRepository, + private val dispatcherProvider: DispatcherProvider, +) : OptOut24HourSubmissionSuccessRateReporter { + override suspend fun attemptFirePixel() { + withContext(dispatcherProvider.io()) { + logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" } + val startDate = pirRepository.getCustomStatsPixelsLastSentMs() + val now = currentTimeProvider.currentTimeMillis() + + if (!shouldFirePixel(startDate, now)) return@withContext + logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } + val endDate = now - TimeUnit.HOURS.toMillis(24) + val activeBrokers = pirRepository.getAllActiveBrokerObjects() + val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords() + + if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { broker -> + val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name } + + if (activeJobRecordsForBroker.isEmpty()) return@forEach + + val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( + activeJobRecordsForBroker, + startDate, + endDate, + ) + + logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" } + if (successRate != null) { + pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = broker.url, + optOutSuccessRate = successRate, + ) + } + } + + logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } + pirRepository.setCustomStatsPixelsLastSentMs(endDate) + } + } + } + + private fun shouldFirePixel( + startDate: Long, + now: Long, + ): Boolean { + return if (startDate == 0L) { + // IF first run, we emit the custom stats pixel + true + } else { + // Else we check if at least 24 hours have passed since last emission + val nowDiffFromStart = abs(now - startDate) + nowDiffFromStart > TimeUnit.HOURS.toMillis(24) + } + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt new file mode 100644 index 000000000000..5e7a36029d28 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import logcat.logcat +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +interface OptOutConfirmationReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOutConfirmationReporter @Inject constructor( + private val dispatcherProvider: DispatcherProvider, + private val pirSchedulingRepository: PirSchedulingRepository, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pixelSender: PirPixelSender, +) : OptOutConfirmationReporter { + override suspend fun attemptFirePixel() { + logcat { "PIR-CUSTOM-STATS: Attempt to fire confirmation pixels" } + + withContext(dispatcherProvider.io()) { + val activeBrokers = pirRepository.getAllActiveBrokerObjects().associateBy { it.name } + val allValidRequestedOptOutJobs = pirSchedulingRepository.getAllValidOptOutJobRecords().filter { + it.status == REQUESTED || it.status == REMOVED // TODO: Filter out removed by user + } + + if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext + + logcat { "PIR-CUSTOM-STATS: Will fire confirmation pixels for ${allValidRequestedOptOutJobs.size} jobs" } + allValidRequestedOptOutJobs.also { + // Fire 7 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_7, + { jobRecord -> jobRecord.confirmation7dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay7ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 14 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_14, + { jobRecord -> jobRecord.confirmation14dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed14Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed14Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay14ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 21 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_21, + { jobRecord -> jobRecord.confirmation21dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed21Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed21Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay21ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 42 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_42, + { jobRecord -> jobRecord.confirmation42dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed42Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed42Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay42ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + } + } + } + + private suspend fun List.attemptFirePixelForConfirmationDay( + activeBrokers: Map, + confirmationDay: Long, + jobRecordFilter: (OptOutJobRecord) -> Boolean, + emitConfirmPixel: (String) -> Unit, + emitUnconfirmPixel: (String) -> Unit, + markOptOutJobRecordReporting: suspend (OptOutJobRecord, Long) -> Unit, + ) { + val now = currentTimeProvider.currentTimeMillis() + val optOutsForPixel = this.filter { + it.daysPassedSinceSubmission(now, confirmationDay) && jobRecordFilter(it) + } + + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${optOutsForPixel.size} jobs" } + optOutsForPixel.forEach { optOutJobRecord -> + val broker = activeBrokers[optOutJobRecord.brokerName] ?: return@forEach + + if (optOutJobRecord.status == REMOVED) { + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${broker.name}" } + emitConfirmPixel(broker.url) + } else { + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day unconfirmation pixels for ${broker.name}" } + emitUnconfirmPixel(broker.url) + } + + markOptOutJobRecordReporting(optOutJobRecord, now) + } + } + + private fun OptOutJobRecord.daysPassedSinceSubmission( + now: Long, + interval: Long, + ): Boolean { + return now >= this.optOutRequestedDateInMillis + TimeUnit.DAYS.toMillis(interval) + } + + companion object { + private const val INTERVAL_DAY_7 = 7L + private const val INTERVAL_DAY_14 = 14L + private const val INTERVAL_DAY_21 = 21L + private const val INTERVAL_DAY_42 = 42L + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt new file mode 100644 index 000000000000..688fa2578b64 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.scheduling.JobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.round + +interface OptOutSubmitRateCalculator { + /** + * Calculates the opt-out 24h submit rate for a given broker within the specified date range. + * + * @param allActiveOptOutJobsForBroker all active opt-out job records for the broker. + * @param startDateMs The opt-out records to include should be created on or after this date. Default is 0L (epoch). + * @param endDateMs tThe opt-out records to include should be created on or before this date. Default is 0L (epoch). + */ + suspend fun calculateOptOutSubmitRate( + allActiveOptOutJobsForBroker: List, + startDateMs: Long = 0L, + endDateMs: Long, + ): Double? +} + +@ContributesBinding(AppScope::class) +class RealOptOutSubmitRateCalculator @Inject constructor( + private val dispatcherProvider: DispatcherProvider, +) : OptOutSubmitRateCalculator { + override suspend fun calculateOptOutSubmitRate( + allActiveOptOutJobsForBroker: List, + startDateMs: Long, + endDateMs: Long, + ): Double? = withContext(dispatcherProvider.io()) { + // Get all opt out job records created within the given range for the specified broker + val recordsCreatedWithinRange = allActiveOptOutJobsForBroker.filter { + it.dateCreatedInMillis in startDateMs..endDateMs + } + + // We don't need to calculate the rate if there are no records + if (recordsCreatedWithinRange.isEmpty()) return@withContext null + + // Filter the records to only include those that were requested within 24 hours of creation + val requestedRecordsWithinRange = recordsCreatedWithinRange.filter { + (it.status == REQUESTED || it.status == REMOVED) && it.optOutRequestedDateInMillis > it.dateCreatedInMillis && + it.optOutRequestedDateInMillis <= it.dateCreatedInMillis + TimeUnit.HOURS.toMillis( + 24, + ) + } + + val optOutSuccessRate = + requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() + val roundedOptOutSuccessRate = round(optOutSuccessRate * 100) / 100 + return@withContext roundedOptOutSuccessRate + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt new file mode 100644 index 000000000000..b0bc5e2733ca --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.duckduckgo.anvil.annotations.ContributesWorker +import com.duckduckgo.di.scopes.AppScope +import logcat.logcat +import javax.inject.Inject + +@ContributesWorker(AppScope::class) +class PirCustomStatsWorker( + context: Context, + workerParameters: WorkerParameters, +) : CoroutineWorker(context, workerParameters) { + @Inject + lateinit var optOutSubmissionSuccessRateReporter: OptOut24HourSubmissionSuccessRateReporter + + @Inject + lateinit var optOutConfirmationReporter: OptOutConfirmationReporter + + override suspend fun doWork(): Result { + logcat { "PIR-CUSTOM-STATS: Attempt to fire custom pixels" } + optOutSubmissionSuccessRateReporter.attemptFirePixel() + optOutConfirmationReporter.attemptFirePixel() + + return Result.success() + } + + companion object { + const val TAG_PIR_RECURRING_CUSTOM_STATS = "TAG_PIR_RECURRING_CUSTOM_STATS" + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index 4c9f96301f08..a676af8f6b66 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -76,11 +76,6 @@ enum class PirPixel( type = Count, ), - PIR_INTERNAL_BROKER_OPT_OUT_COMPLETED( - baseName = "pir_internal_opt-out_completed", - type = Count, - ), - PIR_INTERNAL_CPU_USAGE( baseName = "pir_internal_cpu_usage", type = Count, @@ -130,6 +125,59 @@ enum class PirPixel( PIR_INTERNAL_SECURE_STORAGE_UNAVAILABLE( baseName = "pir_internal_secure-storage_unavailable", types = setOf(Count, Daily()), + ), + + PIR_OPTOUT_SUBMIT_SUCCESS( + baseName = "dbp_optout_process_submit-success", + types = setOf(Count), + ), + + PIR_OPTOUT_SUBMIT_FAILURE( + baseName = "dbp_optout_process_failure", + types = setOf(Count), + ), + + PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE( + baseName = "dbp_databroker_custom_stats_optoutsubmit", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_unconfirmed", + type = Count, + ), + PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_unconfirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_unconfirmed", + type = Count, + ), + PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index 406270938c69..933c7bbe6930 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -18,6 +18,15 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_FAILED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_START import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_SUCCESS @@ -27,7 +36,6 @@ import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_LINK_RECEI import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_MAX_RETRIES_EXCEEDED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_RUN_COMPLETED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_RUN_STARTED -import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_BROKER_OPT_OUT_COMPLETED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_BROKER_OPT_OUT_STARTED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_BROKER_SCAN_COMPLETED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_BROKER_SCAN_STARTED @@ -41,6 +49,8 @@ import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_SCHEDULED_SCAN_SCHED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_SCHEDULED_SCAN_STARTED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_INTERNAL_SECURE_STORAGE_UNAVAILABLE import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_OPTOUT_STAGE_PENDING_EMAIL_CONFIRMATION +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_OPTOUT_SUBMIT_FAILURE +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_OPTOUT_SUBMIT_SUCCESS import com.squareup.anvil.annotations.ContributesBinding import logcat.logcat import javax.inject.Inject @@ -109,17 +119,49 @@ interface PirPixelSender { ) /** - * Emits a pixel to signal that an opt-out job for a specific extractedProfile has been completed. - * It could mean that the opt-out for the record was successful or failed. + * Emits a pixel to signal that an opt-out job for a specific extractedProfile has been successfully submitted. * - * @param brokerName for which the opt-out is for - * @param totalTimeInMillis How long it took to complete the opt-out for the record. - * @param isSuccess - if result was not an error, it is a success. + * @param brokerUrl url of the Broker for which the opt-out is for + * @param parent The parent data broker of the one this opt-out attempt targets + * @param attemptId - Client-generated ID of the opt-out attempt + * @param durationMs - Total duration of the opt-out attempt in milliseconds + * @param tries - The number of tries it took to submit successfully. + * @param emailPattern - Email pattern used during submission, when available else null. */ - fun reportOptOutCompleted( - brokerName: String, - totalTimeInMillis: Long, - isSuccess: Boolean, + fun reportOptOutSubmitted( + brokerUrl: String, + parent: String, + attemptId: String, + durationMs: Long, + tries: Int, + emailPattern: String?, + ) + + /** + * Emits a pixel to signal that an opt-out job for a specific extractedProfile has been failed. + * + * @param brokerUrl url of the Broker for which the opt-out is for + * @param parent The parent data broker of the one this opt-out attempt targets + * @param brokerJsonVersion The version of the broker JSON file + * @param attemptId - Client-generated ID of the opt-out attempt + * @param durationMs - Total duration of the opt-out attempt in milliseconds + * @param stage - The stage where the failure occurred + * @param tries - The number of tries it took to submit successfully. + * @param emailPattern - Email pattern used during submission, when available else null. + * @param actionId - Predefined identifier of the broker action that failed + * @param actionType - Type of action that failed + */ + fun reportOptOutFailed( + brokerUrl: String, + parent: String, + brokerJsonVersion: String, + attemptId: String, + durationMs: Long, + stage: PirStage, + tries: Int, + emailPattern: String?, + actionId: String, + actionType: String, ) /** @@ -288,6 +330,57 @@ interface PirPixelSender { * Emits a pixel to signal that PIR encrypted database is unavailable. */ fun reportSecureStorageUnavailable() + + /** + * Emits a pixel containing the opt-out submit success rate for a broker for the last 24 hours + * + * @param brokerUrl url of the Broker for which the opt-out submit rate is for + * @param optOutSuccessRate opt out submit success rate for the past 24 hours + */ + fun reportBrokerCustomStateOptOutSubmitRate( + brokerUrl: String, + optOutSuccessRate: Double, + ) + + /** + * Emits a pixel when an opt-out has been confirmed within 7 days. + */ + fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 7 days. + */ + fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 14 days. + */ + fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 14 days. + */ + fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 21 days. + */ + fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 21 days. + */ + fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 42 days. + */ + fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 42 days. + */ + fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -353,17 +446,51 @@ class RealPirPixelSender @Inject constructor( fire(PIR_INTERNAL_BROKER_OPT_OUT_STARTED, params) } - override fun reportOptOutCompleted( - brokerName: String, - totalTimeInMillis: Long, - isSuccess: Boolean, + override fun reportOptOutSubmitted( + brokerUrl: String, + parent: String, + attemptId: String, + durationMs: Long, + tries: Int, + emailPattern: String?, ) { val params = mapOf( - PARAM_KEY_BROKER_NAME to brokerName, - PARAM_KEY_TOTAL_TIME to totalTimeInMillis.toString(), - PARAM_KEY_SUCCESS to isSuccess.toString(), + PARAM_KEY_BROKER to brokerUrl, + PARAM_KEY_PARENT to parent, + PARAM_ATTEMPT_ID to attemptId, + PARAM_DURATION to durationMs.toString(), + PARAM_TRIES to tries.toString(), + PARAM_KEY_PATTERN to (emailPattern ?: ""), ) - fire(PIR_INTERNAL_BROKER_OPT_OUT_COMPLETED, params) + fire(PIR_OPTOUT_SUBMIT_SUCCESS, params) + } + + override fun reportOptOutFailed( + brokerUrl: String, + parent: String, + brokerJsonVersion: String, + attemptId: String, + durationMs: Long, + stage: PirStage, + tries: Int, + emailPattern: String?, + actionId: String, + actionType: String, + ) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + PARAM_KEY_PARENT to parent, + PARAM_BROKER_VERSION to brokerJsonVersion, + PARAM_ATTEMPT_ID to attemptId, + PARAM_DURATION to durationMs.toString(), + PARAM_KEY_STAGE to stage.stageName, + PARAM_TRIES to tries.toString(), + PARAM_KEY_PATTERN to (emailPattern ?: ""), + PARAM_ACTION_ID to actionId, + PARAM_KEY_ACTION_TYPE to actionType, + ) + + fire(PIR_OPTOUT_SUBMIT_FAILURE, params) } override fun reportScanStats(totalScanToRun: Int) { @@ -536,6 +663,82 @@ class RealPirPixelSender @Inject constructor( fire(PIR_INTERNAL_SECURE_STORAGE_UNAVAILABLE) } + override fun reportBrokerCustomStateOptOutSubmitRate( + brokerUrl: String, + optOutSuccessRate: Double, + ) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + PARAM_KEY_OPTOUT_SUBMIT_SUCCESS_RATE to optOutSuccessRate.toString(), + ) + + fire(PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE, params) + } + + override fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), @@ -565,5 +768,12 @@ class RealPirPixelSender @Inject constructor( private const val PARAM_ATTEMPT_NUMBER = "attempt_number" private const val PARAM_TOTAL_FETCH = "totalFetchAttempts" private const val PARAM_TOTAL_EMAIL_CONFIRMATION = "totalEmailConfirmationJobs" + + private const val PARAM_KEY_BROKER = "data_broker" + private const val PARAM_KEY_PARENT = "parent" + private const val PARAM_KEY_STAGE = "stage" + private const val PARAM_KEY_PATTERN = "pattern" + private const val PARAM_KEY_ACTION_TYPE = "action_type" + private const val PARAM_KEY_OPTOUT_SUBMIT_SUCCESS_RATE = "optout_submit_success_rate" } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirStage.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirStage.kt new file mode 100644 index 000000000000..2ed7fb3cbd92 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirStage.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +enum class PirStage(val stageName: String) { + /** + * + * Start of an opt-out job execution, after verifying all dependencies + */ + START("start"), + + /** + * Stage before querying dbp-api v0 for a generated email. + */ + EMAIL_GENERATE("email-generate"), + + /** + * Stage where we start the getCaptchaInfo action + */ + CAPTCHA_PARSE("captcha-parse"), + + /** + * Stage after captcha info is parsed and before it's submitted to the captcha service + */ + CAPTCHA_SEND("captcha-send"), + + /** + * Stage during the solveCaptcha action and just before submitting the captcha to be solved + */ + CAPTCHA_SOLVE("captcha-solve"), + + /** + * Stage can be set in two places: + * (1) at the start of the expectation action + * (2) right before executing the emailConfirmation action (prior to email decoupling flow changes) + */ + SUBMIT("submit"), + + /** + * Stage after halting opt-out jobs at email confirmation step. + */ + EMAIL_CONFIRM_HALTED("email-confirm-halted"), + + /** + * Stage when email confirmation job resumes, after verifying all dependencies. + */ + EMAIL_CONFIRM_DECOUPLED("email-confirm-decoupled"), + + /** + * Stage when finalizing an opt-out job, after it has been submitted and right before submitting a dbp_optout_process_submit-success pixel. + */ + VALIDATE("validate"), + + /** + * Catch-all stage + */ + OTHER("other"), + + /** + * Stage at the start of the click and fillForm actions. + */ + FILL_FORM("fill-form"), + + /** + * Stage when the opt-out's condition action meets its expectation. + */ + CONDITION_FOUND("condition-found"), + + /** + * + * Stage when either: + * (1) the opt-out's condition action completes with no follow-up actions or + * (2) when it doesn't meet its expectation, continuing with regular action execution. + */ + CONDITION_NOT_FOUND("condition-not-found"), +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt index ee14153d07a7..50fe401d48c9 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt @@ -24,16 +24,20 @@ import androidx.work.Data import androidx.work.ExistingPeriodicWorkPolicy import androidx.work.NetworkType import androidx.work.PeriodicWorkRequest +import androidx.work.PeriodicWorkRequestBuilder import androidx.work.WorkManager import androidx.work.multiprocess.RemoteListenableWorker import com.duckduckgo.app.di.AppCoroutineScope import com.duckduckgo.appbuildconfig.api.AppBuildConfig import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.common.PirJobConstants.CUSTOM_PIXEL_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.EMAIL_CONFIRMATION_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.SCHEDULED_SCAN_INTERVAL_HOURS import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker.Companion.TAG_EMAIL_CONFIRMATION +import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker +import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker.Companion.TAG_PIR_RECURRING_CUSTOM_STATS import com.duckduckgo.pir.impl.pixels.PirPixelSender import com.duckduckgo.pir.impl.scan.PirScheduledScanRemoteWorker.Companion.TAG_SCHEDULED_SCAN import com.duckduckgo.pir.impl.store.PirEventsRepository @@ -66,6 +70,7 @@ class RealPirScanScheduler @Inject constructor( schedulePirScans() scheduleEmailConfirmation() + scheduleRecurringPixelStats() } private fun schedulePirScans() { @@ -129,9 +134,23 @@ class RealPirScanScheduler @Inject constructor( ) } + private fun scheduleRecurringPixelStats() { + val periodicWorkRequest = PeriodicWorkRequestBuilder(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) + .addTag(TAG_PIR_RECURRING_CUSTOM_STATS) + .setInitialDelay(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) + .build() + + workManager.enqueueUniquePeriodicWork( + TAG_PIR_RECURRING_CUSTOM_STATS, + ExistingPeriodicWorkPolicy.UPDATE, + periodicWorkRequest, + ) + } + override fun cancelScheduledScans(context: Context) { workManager.cancelUniqueWork(TAG_SCHEDULED_SCAN) workManager.cancelUniqueWork(TAG_EMAIL_CONFIRMATION) + workManager.cancelUniqueWork(TAG_PIR_RECURRING_CUSTOM_STATS) context.stopService(Intent(context, PirRemoteWorkerService::class.java)) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt index 6677937748e8..9b61388be716 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scheduling/JobRecordUpdater.kt @@ -129,7 +129,7 @@ interface JobRecordUpdater { * * @param extractedProfileId The id stored in our database for the [ExtractedProfile] */ - suspend fun updateOptOutRequested(extractedProfileId: Long) + suspend fun updateOptOutRequested(extractedProfileId: Long): OptOutJobRecord? /** * Updates the [OptOutJobRecord] associated with a given [extractedProfileId]. @@ -140,7 +140,7 @@ interface JobRecordUpdater { * * @param extractedProfileId The id stored in our database for the [ExtractedProfile] */ - suspend fun updateOptOutError(extractedProfileId: Long) + suspend fun updateOptOutError(extractedProfileId: Long): OptOutJobRecord? /** * Updates the [OptOutJobRecord] associated with the given [extractedProfileId]. @@ -403,33 +403,32 @@ class RealJobRecordUpdater @Inject constructor( } } - override suspend fun updateOptOutRequested(extractedProfileId: Long) { - withContext(dispatcherProvider.io()) { + override suspend fun updateOptOutRequested(extractedProfileId: Long): OptOutJobRecord? { + return withContext(dispatcherProvider.io()) { schedulingRepository.getValidOptOutJobRecord(extractedProfileId)?.also { - schedulingRepository.saveOptOutJobRecord( - it - .copy( - status = OptOutJobStatus.REQUESTED, - optOutRequestedDateInMillis = currentTimeProvider.currentTimeMillis(), - ).also { - logcat { "PIR-JOB-RECORD: Updating OptOutRecord for $extractedProfileId to $it" } - }, + val updatedRecord = it.copy( + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = currentTimeProvider.currentTimeMillis(), ) + schedulingRepository.saveOptOutJobRecord(updatedRecord) + + logcat { "PIR-JOB-RECORD: Updating OptOutRecord for $extractedProfileId to $it" } + + return@withContext updatedRecord } } } - override suspend fun updateOptOutError(extractedProfileId: Long) { - withContext(dispatcherProvider.io()) { + override suspend fun updateOptOutError(extractedProfileId: Long): OptOutJobRecord? { + return withContext(dispatcherProvider.io()) { schedulingRepository.getValidOptOutJobRecord(extractedProfileId)?.also { - schedulingRepository.saveOptOutJobRecord( - it - .copy( - status = OptOutJobStatus.ERROR, - ).also { - logcat { "PIR-JOB-RECORD: Updating OptOutRecord for $extractedProfileId to $it" } - }, + val updatedRecord = it.copy( + status = OptOutJobStatus.ERROR, ) + schedulingRepository.saveOptOutJobRecord(updatedRecord) + logcat { "PIR-JOB-RECORD: Updating OptOutRecord for $extractedProfileId to $it" } + + return@withContext updatedRecord } } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt index 38e81fb7b60f..c2db7aec25f4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt @@ -22,6 +22,7 @@ import com.duckduckgo.data.store.api.SharedPreferencesProvider interface PirDataStore { var mainConfigEtag: String? + var customStatsPixelsLastSentMs: Long } internal class RealPirDataStore( @@ -42,8 +43,17 @@ internal class RealPirDataStore( } } + override var customStatsPixelsLastSentMs: Long + get() = preferences.getLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, 0L) + set(value) { + preferences.edit { + putLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, value) + } + } + companion object { private const val FILENAME = "com.duckduckgo.pir.v1" private const val KEY_MAIN_ETAG = "KEY_MAIN_ETAG" + private const val KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS = "KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS" } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt index ef026e0b66a5..071fbcc70255 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt @@ -53,7 +53,7 @@ import com.squareup.moshi.Types @Database( exportSchema = true, - version = 14, + version = 15, entities = [ BrokerJsonEtag::class, BrokerEntity::class, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt index 1399d26996ba..9843d2724f6c 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt @@ -32,6 +32,7 @@ import com.duckduckgo.pir.impl.service.DbpService.PirEmailConfirmationDataReques import com.duckduckgo.pir.impl.service.DbpService.PirJsonBroker import com.duckduckgo.pir.impl.store.PirRepository.BrokerJson import com.duckduckgo.pir.impl.store.PirRepository.EmailConfirmationLinkFetchStatus +import com.duckduckgo.pir.impl.store.PirRepository.GeneratedEmailData import com.duckduckgo.pir.impl.store.db.BrokerDao import com.duckduckgo.pir.impl.store.db.BrokerEntity import com.duckduckgo.pir.impl.store.db.BrokerJsonDao @@ -167,12 +168,21 @@ interface PirRepository { profileQueryIdsToDelete: List, ): Boolean - suspend fun getEmailForBroker(dataBroker: String): String + suspend fun getEmailForBroker(dataBroker: String): GeneratedEmailData suspend fun getEmailConfirmationLinkStatus(emailData: List): Map suspend fun deleteEmailData(emailData: List) + suspend fun getCustomStatsPixelsLastSentMs(): Long + + suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) + + data class GeneratedEmailData( + val emailAddress: String, + val pattern: String, + ) + data class BrokerJson( val fileName: String, val etag: String, @@ -606,9 +616,14 @@ class RealPirRepository( } } - override suspend fun getEmailForBroker(dataBroker: String): String = + override suspend fun getEmailForBroker(dataBroker: String): GeneratedEmailData = withContext(dispatcherProvider.io()) { - return@withContext dbpService.getEmail(brokerDao()?.getBrokerDetails(dataBroker)!!.url).emailAddress + return@withContext dbpService.getEmail(brokerDao()?.getBrokerDetails(dataBroker)!!.url).run { + GeneratedEmailData( + emailAddress, + pattern, + ) + } } override suspend fun getEmailConfirmationLinkStatus(emailData: List): Map = @@ -652,6 +667,14 @@ class RealPirRepository( return@withContext } + override suspend fun getCustomStatsPixelsLastSentMs(): Long = withContext(dispatcherProvider.io()) { + pirDataStore.customStatsPixelsLastSentMs + } + + override suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) = withContext(dispatcherProvider.io()) { + pirDataStore.customStatsPixelsLastSentMs = timeMs + } + private fun List.toRequest(): PirEmailConfirmationDataRequest = PirEmailConfirmationDataRequest( items = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index 311db495677e..676953b49bb1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -28,6 +28,7 @@ import com.duckduckgo.pir.impl.models.scheduling.JobRecord.ScanJobRecord.ScanJob import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import com.squareup.anvil.annotations.ContributesBinding @@ -60,6 +61,11 @@ interface PirSchedulingRepository { */ suspend fun getAllValidOptOutJobRecords(): List + /** + * Returns all ScanJobRecord whose state is not INVALID for a specific broker + */ + suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List + /** * Returns a matching [OptOutJobRecord] whose state is not INVALID * @@ -119,6 +125,26 @@ interface PirSchedulingRepository { suspend fun deleteEmailConfirmationJobRecord(extractedProfileId: Long) suspend fun deleteAllEmailConfirmationJobRecords() + + suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -181,6 +207,16 @@ class RealPirSchedulingRepository @Inject constructor( .orEmpty() } + override suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List = + withContext(dispatcherProvider.io()) { + return@withContext jobSchedulingDao() + ?.getAllOptOutJobRecordsForBroker(brokerName) + ?.map { record -> record.toRecord() } + // do not pick-up deprecated jobs as they belong to removed profiles + ?.filter { !it.deprecated } + .orEmpty() + } + override suspend fun saveScanJobRecord(scanJobRecord: ScanJobRecord) { withContext(dispatcherProvider.io()) { jobSchedulingDao()?.saveScanJobRecord(scanJobRecord.toEntity()) @@ -304,6 +340,42 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.updateSevenDayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update14DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update21DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update42DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, @@ -311,6 +383,7 @@ class RealPirSchedulingRepository @Inject constructor( status = ScanJobStatus.entries.find { it.name == this.status } ?: ScanJobStatus.ERROR, lastScanDateInMillis = this.lastScanDateInMillis ?: 0L, deprecated = this.deprecated, + dateCreatedInMillis = this.dateCreatedInMillis, ) private fun ScanJobRecord.toEntity(): ScanJobRecordEntity = @@ -320,6 +393,11 @@ class RealPirSchedulingRepository @Inject constructor( status = this.status.name, lastScanDateInMillis = this.lastScanDateInMillis, deprecated = this.deprecated, + dateCreatedInMillis = if (this.dateCreatedInMillis != 0L) { + this.dateCreatedInMillis + } else { + currentTimeProvider.currentTimeMillis() + }, ) private fun OptOutJobRecordEntity.toRecord(): OptOutJobRecord = @@ -333,6 +411,11 @@ class RealPirSchedulingRepository @Inject constructor( optOutRequestedDateInMillis = this.optOutRequestedDate, optOutRemovedDateInMillis = this.optOutRemovedDate, deprecated = this.deprecated, + dateCreatedInMillis = this.dateCreatedInMillis, + confirmation7dayReportSentDateMs = this.reporting.sevenDayConfirmationReportSentDateMs, + confirmation14dayReportSentDateMs = this.reporting.fourteenDayConfirmationReportSentDateMs, + confirmation21dayReportSentDateMs = this.reporting.twentyOneDayConfirmationReportSentDateMs, + confirmation42dayReportSentDateMs = this.reporting.fortyTwoDayConfirmationReportSentDateMs, ) private fun OptOutJobRecord.toEntity(): OptOutJobRecordEntity = @@ -346,6 +429,17 @@ class RealPirSchedulingRepository @Inject constructor( optOutRequestedDate = this.optOutRequestedDateInMillis, optOutRemovedDate = this.optOutRemovedDateInMillis, deprecated = this.deprecated, + dateCreatedInMillis = if (this.dateCreatedInMillis != 0L) { + this.dateCreatedInMillis + } else { + currentTimeProvider.currentTimeMillis() + }, + reporting = ReportingRecord( + sevenDayConfirmationReportSentDateMs = this.confirmation7dayReportSentDateMs, + fourteenDayConfirmationReportSentDateMs = this.confirmation14dayReportSentDateMs, + twentyOneDayConfirmationReportSentDateMs = this.confirmation21dayReportSentDateMs, + fortyTwoDayConfirmationReportSentDateMs = this.confirmation42dayReportSentDateMs, + ), ) private fun EmailConfirmationJobRecord.toEntity(): EmailConfirmationJobRecordEntity = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index 30d040df1fde..8e836a706502 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -40,6 +40,9 @@ interface JobSchedulingDao { @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount") fun getAllOptOutJobRecords(): List + @Query("SELECT * FROM pir_optout_job_record WHERE brokerName = :brokerName ORDER BY attemptCount") + fun getAllOptOutJobRecordsForBroker(brokerName: String): List + @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount") fun getAllOptOutJobRecordsFlow(): Flow> @@ -115,4 +118,52 @@ interface JobSchedulingDao { deleteOptOutJobRecordsForProfiles(profileQueryIds) deleteEmailConfirmationJobRecordsForProfiles(profileQueryIds) } + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_sevenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun updateSevenDayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fourteenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update14DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_twentyOneDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update21DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fortyTwoDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update42DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt index adacba5232a0..69c7836009e1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt @@ -16,6 +16,7 @@ package com.duckduckgo.pir.impl.store.db +import androidx.room.Embedded import androidx.room.Entity import androidx.room.PrimaryKey @@ -29,6 +30,7 @@ data class ScanJobRecordEntity( val status: String, val lastScanDateInMillis: Long? = null, val deprecated: Boolean = false, + val dateCreatedInMillis: Long, ) @Entity(tableName = "pir_optout_job_record") @@ -42,6 +44,16 @@ data class OptOutJobRecordEntity( val optOutRequestedDate: Long = 0L, val optOutRemovedDate: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long, + @Embedded(prefix = "reporting_") + val reporting: ReportingRecord, +) + +data class ReportingRecord( + val sevenDayConfirmationReportSentDateMs: Long = 0L, + val fourteenDayConfirmationReportSentDateMs: Long = 0L, + val twentyOneDayConfirmationReportSentDateMs: Long = 0L, + val fortyTwoDayConfirmationReportSentDateMs: Long = 0L, ) @Entity(tableName = "pir_email_confirmation_job_record") diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt index 6c6d653b4690..3e9ea33e7769 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/common/RealPirRunStateHandlerTest.kt @@ -22,8 +22,9 @@ import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerManua import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationCompleted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationNeeded import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordEmailConfirmationStarted -import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutCompleted +import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutFailed import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutStarted +import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerRecordOptOutSubmitted import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScanActionSucceeded import com.duckduckgo.pir.impl.common.PirRunStateHandler.PirRunState.BrokerScheduledScanCompleted import com.duckduckgo.pir.impl.models.AddressCityState @@ -33,12 +34,17 @@ import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobR import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord.EmailData import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord.JobAttemptData import com.duckduckgo.pir.impl.models.scheduling.JobRecord.EmailConfirmationJobRecord.LinkFetchData +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus import com.duckduckgo.pir.impl.pixels.PirPixelSender +import com.duckduckgo.pir.impl.pixels.PirStage.OTHER import com.duckduckgo.pir.impl.scheduling.JobRecordUpdater +import com.duckduckgo.pir.impl.scripts.models.BrokerAction import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse.ExtractedResponse import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse.ExtractedResponse.ScriptAddressCityState import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse.ExtractedResponse.ScriptExtractedProfile import com.duckduckgo.pir.impl.scripts.models.PirSuccessResponse.NavigateResponse +import com.duckduckgo.pir.impl.scripts.models.asActionType import com.duckduckgo.pir.impl.store.PirEventsRepository import com.duckduckgo.pir.impl.store.PirRepository import com.duckduckgo.pir.impl.store.PirSchedulingRepository @@ -449,13 +455,27 @@ class RealPirRunStateHandlerTest { fun whenHandleBrokerRecordOptOutCompletedWithSuccessThenUpdatesRecordAndReportsPixel() = runTest { val state = - BrokerRecordOptOutCompleted( + BrokerRecordOptOutSubmitted( brokerName = testBrokerName, extractedProfile = testExtractedProfile, startTimeInMillis = testStartTimeInMillis, endTimeInMillis = testEventTimeInMillis, - isSubmitSuccess = true, + attemptId = "c9982ded-021a-4251-9e03-2c58b130410f", + emailPattern = "ep15", ) + whenever(mockRepository.getBrokerForName(testBrokerName)).thenReturn(testBroker) + whenever(mockJobRecordUpdater.updateOptOutRequested(any())).thenReturn( + OptOutJobRecord( + extractedProfileId = 789L, + brokerName = testBrokerName, + userProfileId = 123L, + status = OptOutJobStatus.REQUESTED, + attemptCount = 2, + lastOptOutAttemptDateInMillis = 1000L, + optOutRequestedDateInMillis = 2000L, + optOutRemovedDateInMillis = 0L, + ), + ) testee.handleState(state) @@ -467,10 +487,13 @@ class RealPirRunStateHandlerTest { endTimeInMillis = testEventTimeInMillis, isSubmitSuccess = true, ) - verify(mockPixelSender).reportOptOutCompleted( - brokerName = testBrokerName, - totalTimeInMillis = testEventTimeInMillis - testStartTimeInMillis, - isSuccess = true, + verify(mockPixelSender).reportOptOutSubmitted( + brokerUrl = testBroker.url, + parent = "", + attemptId = state.attemptId, + durationMs = testEventTimeInMillis - testStartTimeInMillis, + tries = 2, + emailPattern = state.emailPattern, ) } @@ -478,13 +501,32 @@ class RealPirRunStateHandlerTest { fun whenHandleBrokerRecordOptOutCompletedWithFailureThenUpdatesRecordAndReportsPixel() = runTest { val state = - BrokerRecordOptOutCompleted( + BrokerRecordOptOutFailed( brokerName = testBrokerName, extractedProfile = testExtractedProfile, startTimeInMillis = testStartTimeInMillis, endTimeInMillis = testEventTimeInMillis, - isSubmitSuccess = false, + attemptId = "c9982ded-021a-4251-9e03-2c58b130410f", + failedAction = BrokerAction.Navigate( + id = "fail82ded-021a-4251-9e03-2c58b130410f", + url = "https://example.com/fail", + ), + stage = OTHER, + emailPattern = "ep15", ) + whenever(mockRepository.getBrokerForName(testBrokerName)).thenReturn(testBroker) + whenever(mockJobRecordUpdater.updateOptOutError(any())).thenReturn( + OptOutJobRecord( + extractedProfileId = 789L, + brokerName = testBrokerName, + userProfileId = 123L, + status = OptOutJobStatus.ERROR, + attemptCount = 2, + lastOptOutAttemptDateInMillis = 1000L, + optOutRequestedDateInMillis = 2000L, + optOutRemovedDateInMillis = 0L, + ), + ) testee.handleState(state) @@ -496,10 +538,17 @@ class RealPirRunStateHandlerTest { endTimeInMillis = testEventTimeInMillis, isSubmitSuccess = false, ) - verify(mockPixelSender).reportOptOutCompleted( - brokerName = testBrokerName, - totalTimeInMillis = testEventTimeInMillis - testStartTimeInMillis, - isSuccess = false, + verify(mockPixelSender).reportOptOutFailed( + brokerUrl = testBroker.url, + parent = "", + brokerJsonVersion = testBroker.version, + attemptId = "c9982ded-021a-4251-9e03-2c58b130410f", + durationMs = testEventTimeInMillis - testStartTimeInMillis, + stage = state.stage, + tries = 2, + emailPattern = state.emailPattern, + actionId = state.failedAction.id, + actionType = state.failedAction.asActionType(), ) } diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt new file mode 100644 index 000000000000..274404334961 --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt @@ -0,0 +1,571 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.ProfileQuery +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOut24HourSubmissionSuccessRateReporterTest { + + @get:Rule + var coroutineRule = CoroutineTestRule() + + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockOptOutSubmitRateCalculator: OptOutSubmitRateCalculator = mock() + private val mockPirPixelSender: PirPixelSender = mock() + private val mockSchedulingRepository: PirSchedulingRepository = mock() + + private lateinit var toTest: RealOptOut24HourSubmissionSuccessRateReporter + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneHour = TimeUnit.HOURS.toMillis(1) + private val twentyFourHours = TimeUnit.HOURS.toMillis(24) + + private val testBroker1 = Broker( + name = "test-broker-1", + fileName = "test-broker-1.json", + url = "https://test-broker-1.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + private val testBroker2 = Broker( + name = "test-broker-2", + fileName = "test-broker-2.json", + url = "https://test-broker-2.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + private val testProfileQuery = ProfileQuery( + id = 1L, + firstName = "John", + lastName = "Doe", + city = "New York", + state = "NY", + addresses = emptyList(), + birthYear = 1990, + fullName = "John Doe", + age = 33, + deprecated = false, + ) + + @Before + fun setUp() { + toTest = RealOptOut24HourSubmissionSuccessRateReporter( + optOutSubmitRateCalculator = mockOptOutSubmitRateCalculator, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pirPixelSender = mockPirPixelSender, + pirSchedulingRepository = mockSchedulingRepository, + dispatcherProvider = coroutineRule.testDispatcherProvider, + ) + } + + @Test + fun whenFirstRunThenShouldFirePixel() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenLessThan24HoursPassedThenShouldNotFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + oneHour // Only 1 hour passed + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + toTest.attemptFirePixel() + + verify(mockPirRepository, never()).getAllActiveBrokerObjects() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenMoreThan24HoursPassedThenShouldFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours + oneHour // 25 hours passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.75) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.75, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenExactly24HoursPassedThenShouldNotFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours // Exactly 24 hours + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + toTest.attemptFirePixel() + + verify(mockPirRepository, never()).getAllActiveBrokerObjects() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + } + + @Test + fun whenNoActiveBrokersThenShouldNotFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenNoUserProfilesThenShouldNotFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenMultipleBrokersThenShouldFirePixelForEach() = runTest { + val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.5) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.8) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker2.url, + optOutSuccessRate = 0.8, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenSuccessRateIsNullThenShouldNotFirePixelForThatBroker() = runTest { + val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.5) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(null) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenAllSuccessRatesAreNullThenShouldNotFireAnyPixels() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(null) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenShouldFirePixelThenUsesCorrectDateRange() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours + oneHour + val expectedEndDate = now - twentyFourHours + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator).calculateOptOutSubmitRate( + allActiveOptOutJobsForBroker = eq(listOf(jobRecord)), + startDateMs = eq(startDate), + endDateMs = eq(expectedEndDate), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(expectedEndDate) + } + + @Test + fun whenMultipleBrokersWithMixedSuccessRatesThenFiresPixelsForNonNullRates() = runTest { + val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(null) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), + ), + ) + .thenReturn(0.9) + + toTest.attemptFirePixel() + + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker1.url), + optOutSuccessRate = any(), + ) + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker2.url, + optOutSuccessRate = 0.9, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenShouldFirePixelButNoBrokersAndNoProfilesThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenShouldFirePixelButNoBrokersWithProfilesThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenShouldFirePixelButNoProfilesWithBrokersThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + toTest.attemptFirePixel() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenBrokerHasNoJobRecordsThenSkipsThatBroker() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, // This broker has no job records + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) // Only for broker1 + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord)), + eq(0L), + eq(now - twentyFourHours), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + // Only broker1 should fire pixel, broker2 should be skipped + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker1.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt new file mode 100644 index 000000000000..f00ff0f098da --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOutConfirmationReporterTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutConfirmationReporter + + private val mockPirSchedulingRepository: PirSchedulingRepository = mock() + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockPixelSender: PirPixelSender = mock() + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val sevenDays = TimeUnit.DAYS.toMillis(7) + private val fourteenDays = TimeUnit.DAYS.toMillis(14) + private val twentyOneDays = TimeUnit.DAYS.toMillis(21) + private val fortyTwoDays = TimeUnit.DAYS.toMillis(42) + + private val testBroker = Broker( + name = "test-broker", + fileName = "test-broker.json", + url = "https://test-broker.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + @Before + fun setUp() { + testee = RealOptOutConfirmationReporter( + dispatcherProvider = coroutineRule.testDispatcherProvider, + pirSchedulingRepository = mockPirSchedulingRepository, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pixelSender = mockPixelSender, + ) + } + + @Test + fun whenNoActiveBrokersThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenNoOptOutJobsThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenOptOutJobNotRequestedOrRemovedThenDoesNotFirePixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.NOT_EXECUTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedAndStatusIsRemovedThenFiresConfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPassedAndStatusIsRequestedThenFiresUnconfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation7dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenLessThan7DaysPassedThenDoesNotFire7DayPixel() = runTest { + val now = baseTime + sevenDays - TimeUnit.HOURS.toMillis(1) // 1 hour before 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenExactly7DaysPassedThenFires7DayPixel() = runTest { + val now = baseTime + sevenDays // Exactly 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when14DaysPassedThenFires14DayPixel() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay14ConfirmationPixelSent(1L, now) + } + + @Test + fun when21DaysPassedThenFires21DayPixel() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay21ConfirmationPixelSent(1L, now) + } + + @Test + fun when42DaysPassedThenFires42DayPixel() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay42ConfirmationPixelSent(1L, now) + } + + @Test + fun whenMultipleIntervalsPassedThenFiresAllApplicablePixels() = runTest { + val now = baseTime + fortyTwoDays // 42 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + } + + @Test + fun whenBrokerNotFoundThenSkipsJobRecord() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = "unknown-broker", + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenMultipleJobRecordsThenFiresPixelsForEach() = runTest { + val now = baseTime + sevenDays + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(2L, now) + } + + @Test + fun when14DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation14dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay14ConfirmationPixelSent(any(), any()) + } + + @Test + fun when21DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation21dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay21ConfirmationPixelSent(any(), any()) + } + + @Test + fun when42DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation42dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed42Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay42ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedBut14DaysNotPassedThenOnlyFires7DayPixel() = runTest { + val now = baseTime + sevenDays + oneDay // 8 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + } + + @Test + fun when14DaysPassedBut21DaysNotPassedThenFires7And14DayPixels() = runTest { + val now = baseTime + fourteenDays + oneDay // 15 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + } + + @Test + fun whenRequestedStatusThenFiresUnconfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + } + + @Test + fun whenRemovedStatusThenFiresConfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutUnconfirmed7Days(any()) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis: Long = baseTime, + optOutRemovedDateInMillis: Long = 0L, + confirmation7dayReportSentDateMs: Long = 0L, + confirmation14dayReportSentDateMs: Long = 0L, + confirmation21dayReportSentDateMs: Long = 0L, + confirmation42dayReportSentDateMs: Long = 0L, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = 0, + lastOptOutAttemptDateInMillis = 0L, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = false, + dateCreatedInMillis = baseTime, + confirmation7dayReportSentDateMs = confirmation7dayReportSentDateMs, + confirmation14dayReportSentDateMs = confirmation14dayReportSentDateMs, + confirmation21dayReportSentDateMs = confirmation21dayReportSentDateMs, + confirmation42dayReportSentDateMs = confirmation42dayReportSentDateMs, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt new file mode 100644 index 000000000000..6f3cc7407121 --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt @@ -0,0 +1,676 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import java.util.concurrent.TimeUnit + +class RealOptOutSubmitRateCalculatorTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutSubmitRateCalculator + + @Before + fun setUp() { + testee = RealOptOutSubmitRateCalculator( + dispatcherProvider = coroutineRule.testDispatcherProvider, + ) + } + + // Test data + private val testBrokerName = "test-broker" + + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneHour = TimeUnit.HOURS.toMillis(1) + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val twentyFourHours = TimeUnit.HOURS.toMillis(24) + + @Test + fun whenNoRecordsInDateRangeThenReturnNull() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val result = testee.calculateOptOutSubmitRate(emptyList(), startDate, endDate) + + assertNull(result) + } + + @Test + fun whenRecordsExistButNoneInDateRangeThenReturnNull() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordBeforeRange = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + dateCreatedInMillis = baseTime - oneDay, + ) + val recordAfterRange = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + dateCreatedInMillis = baseTime + oneDay + oneHour, + ) + + val result = testee.calculateOptOutSubmitRate( + listOf(recordBeforeRange, recordAfterRange), + startDate, + endDate, + ) + + assertNull(result) + } + + @Test + fun whenRecordsInDateRangeButNoneRequestedThenReturnZero() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = baseTime + oneHour, + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = baseTime + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) + + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenAllRecordsInDateRangeAreRequestedThenReturnOne() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // Within 24 hours + ) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenHalfRecordsInDateRangeAreRequestedThenReturnHalf() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val notExecutedRecord = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate( + listOf(requestedRecord, notExecutedRecord), + startDate, + endDate, + ) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRequestedRecordOutside24HourWindowThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedWithinWindow = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val requestedOutsideWindow = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24 hours + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithinWindow, requestedOutsideWindow), startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRequestedRecordExactlyAt24HourWindowThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedAt24Hours = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours, // Exactly 24 hours + ) + val notExecutedRecord = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAt24Hours, notExecutedRecord), startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenStartDateIsZeroThenUseDefaultStartDate() = runTest { + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val record = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(record), endDateMs = endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenRecordAtStartDateBoundaryThenIncluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordAtStart = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = startDate, // Exactly at start + optOutRequestedDateInMillis = startDate + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtStart), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenRecordAtEndDateBoundaryThenIncluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordAtEnd = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = endDate, // Exactly at end + optOutRequestedDateInMillis = endDate + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtEnd), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenMultipleStatusesThenOnlyRequestedCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val removed = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REMOVED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val error = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = dateCreated + 3 * oneHour, + ) + val pendingEmail = createOptOutJobRecord( + extractedProfileId = 4L, + brokerName = testBrokerName, + status = OptOutJobStatus.PENDING_EMAIL_CONFIRMATION, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested, removed, error, pendingEmail), startDate, endDate) + + assertEquals(0.25, result!!, 0.0) + } + + @Test + fun whenComplexScenarioThenCalculateCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 5 records in range + val requested1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24h + ) + val requested2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24h + ) + val requested3 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 3 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour + twentyFourHours, // Exactly 24h + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 4L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val error = createOptOutJobRecord( + extractedProfileId = 5L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = dateCreated + 5 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, requested3, notExecuted, error), startDate, endDate) + + // Only requested1 and requested3 count (2 out of 5) + assertEquals(0.4, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateEqualsDateCreatedThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedAtSameTime = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated, // Exactly equal (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + oneHour, // After creation + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAtSameTime, validRequested), startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateBeforeDateCreatedThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedBeforeCreation = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated - oneHour, // Before creation (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // After creation + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedBeforeCreation, validRequested), startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustAfterDateCreatedThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustAfter = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneMillisecond, // Just 1ms after (should be counted) + ) + + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustBefore24HourLimitThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustBefore24h = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours - oneMillisecond, // Just before 24h limit + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustBefore24h), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustAfter24HourLimitThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustAfter24h = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours + oneMillisecond, // Just after 24h limit + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter24h, notExecuted), startDate, endDate) + + // requestedJustAfter24h doesn't count, so 0 out of 2 + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenResultNeedsRoundingThenRoundsToTwoDecimalPlaces() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 1 out of 3 = 0.333... should round to 0.33 + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val notExecuted1 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val notExecuted2 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 3 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested, notExecuted1, notExecuted2), startDate, endDate) + + // 1/3 = 0.333... rounded to 0.33 + assertEquals(0.33, result!!, 0.0) + } + + @Test + fun whenResultNeedsRoundingUpThenRoundsCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 2 out of 3 = 0.666... should round to 0.67 + val requested1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val requested2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, notExecuted), startDate, endDate) + + // 2/3 = 0.666... rounded to 0.67 + assertEquals(0.67, result!!, 0.0) + } + + @Test + fun whenSingleRecordRequestedThenReturnOne() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val singleRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenSingleRecordNotRequestedThenReturnZero() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val singleRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated, + ) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) + + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenLargeDateRangeThenFiltersCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + TimeUnit.DAYS.toMillis(365) // 1 year + val dateCreated = baseTime + oneDay + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(180), // 6 months later + optOutRequestedDateInMillis = dateCreated + TimeUnit.DAYS.toMillis(180) + oneHour, + ) + val record3 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(300), // 10 months later + ) + + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2, record3), startDate, endDate) + + // 2 out of 3 + assertEquals(0.67, result!!, 0.0) + } + + @Test + fun whenRequestedRecordHasZeroOptOutRequestedDateThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedWithZeroDate = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = 0L, // Zero (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithZeroDate, validRequested), startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenFractionalResultRoundsDownThenRoundsCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 1 out of 7 = 0.142857... should round to 0.14 + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val notExecutedRecords = (2L..7L).map { id -> + createOptOutJobRecord( + extractedProfileId = id, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + id * oneHour, + ) + } + + val result = testee.calculateOptOutSubmitRate(listOf(requested) + notExecutedRecords, startDate, endDate) + + // 1/7 = 0.142857... rounded to 0.14 + assertEquals(0.14, result!!, 0.0) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBrokerName, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt index ad93c1383d0e..770d39f032fe 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt @@ -27,6 +27,7 @@ import com.duckduckgo.pir.impl.store.db.BrokerJsonDao import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import kotlinx.coroutines.runBlocking @@ -79,6 +80,7 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = ScanJobStatus.NOT_EXECUTED.name, lastScanDateInMillis = 1000L, + dateCreatedInMillis = 100L, ) private val deprecatedScanJobEntity = @@ -88,6 +90,7 @@ class RealPirSchedulingRepositoryTest { status = ScanJobStatus.MATCHES_FOUND.name, deprecated = true, lastScanDateInMillis = 2000L, + dateCreatedInMillis = 100L, ) private val validOptOutJobEntity = @@ -100,6 +103,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val deprecatedOptOutJobEntity = @@ -113,6 +118,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 3000L, optOutRequestedDate = 4000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val scanJobRecord = @@ -243,6 +250,7 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = "NOT_EXECUTED", lastScanDateInMillis = 1000L, + dateCreatedInMillis = 9000L, ), ) } @@ -268,12 +276,14 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = "NOT_EXECUTED", lastScanDateInMillis = 1000L, + dateCreatedInMillis = 9000L, ), ScanJobRecordEntity( brokerName = "another-broker", userProfileId = 456L, status = "MATCHES_FOUND", lastScanDateInMillis = 5000L, + dateCreatedInMillis = 9000L, ), ), ) @@ -397,6 +407,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ) } @@ -430,6 +442,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), OptOutJobRecordEntity( extractedProfileId = 999L, @@ -440,6 +454,8 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 3000L, optOutRequestedDate = 4000L, optOutRemovedDate = 5000L, + dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ), )