diff --git a/catalyst-toolbox/src/bin/cli/rewards/veterans.rs b/catalyst-toolbox/src/bin/cli/rewards/veterans.rs index 8c2c157e..6077aaa0 100644 --- a/catalyst-toolbox/src/bin/cli/rewards/veterans.rs +++ b/catalyst-toolbox/src/bin/cli/rewards/veterans.rs @@ -62,6 +62,14 @@ pub struct VeteransRewards { /// if the first cutoff is selected then the first modifier is used. #[structopt(long, required = true)] reputation_agreement_rate_modifiers: Vec, + + /// Value in range [0.5, 1] + /// The minimum confidence for a vCA ranking to be excluded from eligible rankings when in disagreement from simple majority. + /// Confidence is either `#FO / #Rankings` or `(#Excellent + #Good) / #Rankings` depending on the final ranking. + /// Simple majority is 50%. Qualified majority is 70%. Using 70% avoids punishing vCAs where this confidence is not clear. + /// 70% is because when #vCA == 3 confidence is only 66% and thus in this case, where there is just 1 vote in disagreement, all 3 vCAs get rewarded. + #[structopt(long)] + minimum_confidence: Decimal, } impl VeteransRewards { @@ -77,6 +85,7 @@ impl VeteransRewards { rewards_agreement_rate_modifiers, reputation_agreement_rate_cutoffs, reputation_agreement_rate_modifiers, + minimum_confidence, } = self; let reviews: Vec = csv::load_data_from_csv::<_, b','>(&from)?; @@ -100,6 +109,10 @@ impl VeteransRewards { bail!("Expected rewards_agreement_rate_cutoffs to be descending"); } + if minimum_confidence < Decimal::new(5, 1) || minimum_confidence > Decimal::ONE { + bail!("Expected minimum_confidence to range between .5 and 1"); + } + let results = veterans::calculate_veteran_advisors_incentives( &reviews, total_rewards, @@ -113,6 +126,7 @@ impl VeteransRewards { .into_iter() .zip(reputation_agreement_rate_modifiers.into_iter()) .collect(), + minimum_confidence, ); csv::dump_data_to_csv(rewards_to_csv_data(results).iter(), &to).unwrap(); diff --git a/catalyst-toolbox/src/rewards/veterans.rs b/catalyst-toolbox/src/rewards/veterans.rs index 76cc9d3a..817640d7 100644 --- a/catalyst-toolbox/src/rewards/veterans.rs +++ b/catalyst-toolbox/src/rewards/veterans.rs @@ -16,6 +16,15 @@ pub struct VeteranAdvisorIncentive { pub reputation: u64, } +struct FinalRankingWithConfidence { + review_ranking: ReviewRanking, + + /// This is to be used in conjunction with `ReviewRanking::is_positive()` to assess the + /// confidence of the boolean reply. It is either `#FO / #Rankings` or `(#Excellent + #Good) / #Rankings` depending on the final ranking. + /// For now we do not discriminate between Good and Excellent but this might change in the future. + confidence: Decimal, +} + pub type VcaRewards = HashMap; pub type EligibilityThresholds = std::ops::RangeInclusive; @@ -25,18 +34,38 @@ pub type EligibilityThresholds = std::ops::RangeInclusive; // e.g. something like an expanded version of a AdvisorReviewRow // [proposal_id, advisor, ratings, ..(other fields from AdvisorReviewRow).., ranking (good/excellent/filtered out), vca] -fn calc_final_ranking_per_review(rankings: &[impl Borrow]) -> ReviewRanking { +fn calc_final_ranking_with_confidence_per_review( + rankings: &[impl Borrow], +) -> FinalRankingWithConfidence { let rankings_majority = Decimal::from(rankings.len()) / Decimal::from(2); let ranks = rankings.iter().counts_by(|r| r.borrow().score()); - match (ranks.get(&FilteredOut), ranks.get(&Excellent)) { - (Some(filtered_out), _) if Decimal::from(*filtered_out) >= rankings_majority => { - ReviewRanking::FilteredOut + match ( + ranks.get(&Excellent), + ranks.get(&Good), + ranks.get(&FilteredOut), + ) { + (_, _, Some(filtered_out)) if Decimal::from(*filtered_out) >= rankings_majority => { + FinalRankingWithConfidence { + review_ranking: FilteredOut, + confidence: Decimal::from(*filtered_out) / Decimal::from(rankings.len()), + } } - (_, Some(excellent)) if Decimal::from(*excellent) > rankings_majority => { - ReviewRanking::Excellent + (Some(excellent), maybe_good, _) if Decimal::from(*excellent) > rankings_majority => { + FinalRankingWithConfidence { + review_ranking: Excellent, + confidence: (Decimal::from(maybe_good.copied().unwrap_or_default()) + + Decimal::from(*excellent)) + / Decimal::from(rankings.len()), + } } - _ => ReviewRanking::Good, + (maybe_excellent, Some(good), _) => FinalRankingWithConfidence { + review_ranking: Good, + confidence: (Decimal::from(maybe_excellent.copied().unwrap_or_default()) + + Decimal::from(*good)) + / Decimal::from(rankings.len()), + }, + _ => unreachable!(), } } @@ -84,12 +113,18 @@ pub fn calculate_veteran_advisors_incentives( reputation_thresholds: EligibilityThresholds, rewards_mod_args: Vec<(Decimal, Decimal)>, reputation_mod_args: Vec<(Decimal, Decimal)>, + minimum_confidence: Decimal, ) -> HashMap { - let final_rankings_per_review = veteran_rankings + let final_rankings_with_confidence_per_review = veteran_rankings .iter() .into_group_map_by(|ranking| ranking.review_id()) .into_iter() - .map(|(review, rankings)| (review, calc_final_ranking_per_review(&rankings))) + .map(|(review, rankings)| { + ( + review, + calc_final_ranking_with_confidence_per_review(&rankings), + ) + }) .collect::>(); let rankings_per_vca = veteran_rankings @@ -99,11 +134,13 @@ pub fn calculate_veteran_advisors_incentives( let eligible_rankings_per_vca = veteran_rankings .iter() .filter(|ranking| { - final_rankings_per_review + let final_ranking_with_confidence = final_rankings_with_confidence_per_review .get(&ranking.review_id()) - .unwrap() - .is_positive() + .unwrap(); + + final_ranking_with_confidence.review_ranking.is_positive() == ranking.score().is_positive() + || final_ranking_with_confidence.confidence < minimum_confidence }) .counts_by(|ranking| ranking.vca.clone()); @@ -156,6 +193,8 @@ mod tests { const VCA_1: &str = "vca1"; const VCA_2: &str = "vca2"; const VCA_3: &str = "vca3"; + const SIMPLE_MAJORITY_CONFIDENCE: Decimal = dec!(.5); + const QUALIFIED_MAJORITY_CONFIDENCE: Decimal = dec!(.7); struct RandomIterator; impl Iterator for RandomIterator { @@ -188,23 +227,36 @@ mod tests { #[test] fn final_ranking_is_correct() { assert!(matches!( - calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 5, 5, 5, RandomIterator),), - ReviewRanking::Good + calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 5, 5, 5, RandomIterator)), + FinalRankingWithConfidence { + review_ranking: Good, + confidence + } if confidence == (dec!(10) / dec!(15)) )); assert!(matches!( - calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 4, 2, 5, RandomIterator)), - ReviewRanking::Good + calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 4, 2, 5, RandomIterator)), + FinalRankingWithConfidence { + review_ranking: Good, + confidence + } if confidence == (dec!(6) / dec!(11)) + )); assert!(matches!( - calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 4, 1, 5, RandomIterator)), - ReviewRanking::FilteredOut + calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 4, 1, 5, RandomIterator)), + FinalRankingWithConfidence { + review_ranking: FilteredOut, + confidence, + } if confidence == (dec!(5) / dec!(10)) )); assert!(matches!( - calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator)), - ReviewRanking::Excellent + calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator)), + FinalRankingWithConfidence { + review_ranking: Excellent, + confidence, + } if confidence == (dec!(4) / dec!(5)) )); } @@ -231,6 +283,7 @@ mod tests { .into_iter() .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter()) .collect(), + SIMPLE_MAJORITY_CONFIDENCE, ); assert!(results.get(VCA_1).is_none()); let res = results.get(VCA_2).unwrap(); @@ -260,6 +313,7 @@ mod tests { .into_iter() .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter()) .collect(), + SIMPLE_MAJORITY_CONFIDENCE, ); let res1 = results.get(VCA_1).unwrap(); assert_eq!(res1.reputation, 1); @@ -283,21 +337,55 @@ mod tests { (Rewards::new(8, 1), Rewards::ONE, Rewards::ONE), (Rewards::new(9, 1), Rewards::new(125, 2), Rewards::ONE), ]; - for (agreement, reward_modifier, reputation_modifier) in inputs { + for (vca3_agreement, reward_modifier, reputation_modifier) in inputs { let rankings = (0..100) .flat_map(|i| { let vcas = vec![VCA_1.to_owned(), VCA_2.to_owned(), VCA_3.to_owned()].into_iter(); - let (good, filtered_out) = if Rewards::from(i) < agreement * Rewards::from(100) - { - (3, 0) - } else { - (2, 1) - }; + let (good, filtered_out) = + if Rewards::from(i) < vca3_agreement * Rewards::from(100) { + (3, 0) + } else { + (2, 1) + }; gen_dummy_rankings(i.to_string(), 0, good, filtered_out, vcas).into_iter() }) .collect::>(); - let results = calculate_veteran_advisors_incentives( + let results_simple_confidence = calculate_veteran_advisors_incentives( + &rankings, + total_rewards, + 1..=200, + 1..=200, + THRESHOLDS + .into_iter() + .zip(REWARDS_DISAGREEMENT_MODIFIERS.into_iter()) + .collect(), + THRESHOLDS + .into_iter() + .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter()) + .collect(), + SIMPLE_MAJORITY_CONFIDENCE, + ); + let vca3_expected_reward_portion_simple_confidence = + vca3_agreement * Rewards::from(100) * reward_modifier; + dbg!(vca3_expected_reward_portion_simple_confidence); + dbg!(vca3_agreement, reward_modifier, reputation_modifier); + let vca3_expected_rewards_simple_confidence = total_rewards + / (Rewards::from(125 * 2) + vca3_expected_reward_portion_simple_confidence) + * vca3_expected_reward_portion_simple_confidence; + let res_vca3_simple_confidence = results_simple_confidence.get(VCA_3).unwrap(); + assert_eq!( + res_vca3_simple_confidence.reputation, + (Rewards::from(100) * vca3_agreement * reputation_modifier) + .to_u64() + .unwrap() + ); + assert!(are_close( + res_vca3_simple_confidence.rewards, + vca3_expected_rewards_simple_confidence + )); + + let results_qualified_confidence = calculate_veteran_advisors_incentives( &rankings, total_rewards, 1..=200, @@ -310,21 +398,29 @@ mod tests { .into_iter() .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter()) .collect(), + QUALIFIED_MAJORITY_CONFIDENCE, ); - let expected_reward_portion = agreement * Rewards::from(100) * reward_modifier; - dbg!(expected_reward_portion); - dbg!(agreement, reward_modifier, reputation_modifier); - let expected_rewards = total_rewards - / (Rewards::from(125 * 2) + expected_reward_portion) - * expected_reward_portion; - let res = results.get(VCA_3).unwrap(); + + let vca3_expected_reward_portion_qualified_confidence = Rewards::from(100) * dec!(1.25); // low confidence so max reward modifier, agreement ratio doesn't count as all and rankings are all eligible + dbg!(vca3_expected_reward_portion_qualified_confidence); + dbg!(vca3_agreement, reward_modifier, reputation_modifier); + + let vca3_expected_rewards_qualified_confidence = total_rewards + / (Rewards::from(125 * 2) + vca3_expected_reward_portion_qualified_confidence) + * vca3_expected_reward_portion_qualified_confidence; // 1/3 of the reward + + let res_vca3_qualified_confidence = results_qualified_confidence.get(VCA_3).unwrap(); + assert_eq!( - res.reputation, - (Rewards::from(100) * agreement * reputation_modifier) + res_vca3_qualified_confidence.reputation, + (Rewards::from(100)) // all assessment are valid since confidence is low (2/3 < 0.7) .to_u64() .unwrap() ); - assert!(are_close(res.rewards, expected_rewards)); + assert!(are_close( + res_vca3_qualified_confidence.rewards, + vca3_expected_rewards_qualified_confidence + )); } } }