initial addition of filtering mechanism

Author: cameron1024

catalyst-toolbox/src/bin/cli/ideascale/mod.rs

@@ -1,10 +1,16 @@
-use catalyst_toolbox::ideascale::{
-    build_challenges, build_fund, build_proposals, fetch_all, CustomFieldTags,
-    Error as IdeascaleError, Scores, Sponsors,
+use catalyst_toolbox::{
+    community_advisors::models::{ReviewRanking, VeteranRankingRow},
+    ideascale::{
+        build_challenges, build_fund, build_proposals, fetch_all, CustomFieldTags,
+        Error as IdeascaleError, Scores, Sponsors,
+    },
+    utils::csv::{dump_data_to_csv, load_data_from_csv},
 };
+use core::cmp::max;
+use itertools::Itertools;
 use jcli_lib::utils::io as io_utils;
 use jormungandr_lib::interfaces::VotePrivacy;
-use std::collections::HashSet;
+use std::{collections::HashSet, ffi::OsStr};
 
 use structopt::StructOpt;
 
@@ -30,6 +36,7 @@ pub enum Error {
 #[derive(Debug, StructOpt)]
 pub enum Ideascale {
     Import(Import),
+    Filter(Filter),
 }
 
 // We need this type because structopt uses Vec<String> as a special type, so it is not compatible
@@ -88,10 +95,21 @@ pub struct Import {
     stages_filters: Filters,
 }
 
+#[derive(Debug, StructOpt)]
+#[structopt(rename_all = "kebab")]
+pub struct Filter {
+    #[structopt(long)]
+    input: PathBuf,
+
+    #[structopt(long)]
+    output: Option<PathBuf>,
+}
+
 impl Ideascale {
     pub fn exec(&self) -> Result<(), Error> {
         match self {
             Ideascale::Import(import) => import.exec(),
+            Ideascale::Filter(filter) => filter.exec(),
         }
     }
 }
@@ -182,6 +200,70 @@ impl Import {
     }
 }
 
+impl Filter {
+    fn output_file(input: &Path, output: Option<&Path>) -> PathBuf {
+        if let Some(output) = output {
+            output.to_path_buf()
+        } else {
+            let name = input.file_name().and_then(OsStr::to_str).unwrap_or("");
+            let name = format!("{name}.output");
+            let temp = input.with_file_name(name);
+            println!("no output specified, writing to {}", temp.to_string_lossy());
+            temp
+        }
+    }
+
+    fn filter_rows(rows: &[VeteranRankingRow]) -> Vec<VeteranRankingRow> {
+        let groups = rows
+            .iter()
+            .group_by(|row| (&row.assessor, &row.proposal_id));
+        groups
+            .into_iter()
+            .flat_map(|(_, group)| {
+                let group = group.collect_vec();
+                let excellent = group
+                    .iter()
+                    .filter(|row| row.score() == ReviewRanking::Excellent)
+                    .count();
+                let good = group
+                    .iter()
+                    .filter(|row| row.score() == ReviewRanking::Good)
+                    .count();
+                let filtered = group
+                    .iter()
+                    .filter(|row| row.score() == ReviewRanking::FilteredOut)
+                    .count();
+
+                let max_count = max(excellent, max(good, filtered));
+
+                let include_excellent = excellent == max_count;
+                let include_good = good == max_count;
+                let include_filtered = filtered == max_count;
+
+                group.into_iter().filter(move |row| match row.score() {
+                    ReviewRanking::Excellent => include_excellent,
+                    ReviewRanking::Good => include_good,
+                    ReviewRanking::FilteredOut => include_filtered,
+                    ReviewRanking::NA => true, // if unknown, ignore
+                })
+            })
+            .cloned()
+            .collect()
+    }
+
+    fn exec(&self) -> Result<(), Error> {
+        let Self { input, output } = self;
+        let output = Self::output_file(input, output.as_deref());
+
+        let rows = load_data_from_csv::<_, b','>(input)?;
+        let rows = Self::filter_rows(&rows);
+
+        dump_data_to_csv(&rows, &output)?;
+
+        Ok(())
+    }
+}
+
 fn dump_content_to_file(content: impl Serialize, file_path: &Path) -> Result<(), Error> {
     let writer = jcli_lib::utils::io::open_file_write(&Some(file_path))?;
     serde_json::to_writer_pretty(writer, &content).map_err(Error::Serde)
@@ -238,3 +320,36 @@ fn read_sponsors_file(path: &Option<PathBuf>) -> Result<Sponsors, Error> {
     }
     Ok(sponsors)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn correctly_formats_output_file_for_filter() {
+        let input = PathBuf::from("/foo/bar/file.txt");
+        let output = PathBuf::from("/baz/qux/output.txt");
+
+        let result = Filter::output_file(&input, Some(&output));
+        assert_eq!(result, output);
+
+        let result = Filter::output_file(&input, None);
+        assert_eq!(result, PathBuf::from("/foo/bar/file.txt.output"));
+    }
+
+    #[test]
+    fn filters_rows_correctly() {
+        use ReviewRanking::*;
+
+        let pid = String::from("pid");
+        let assessor = String::from("assessor");
+        let first = VeteranRankingRow::new(pid.clone(), assessor.clone(), "1".into(), Excellent);
+        let second = VeteranRankingRow::new(pid.clone(), assessor.clone(), "2".into(), Excellent);
+        let third = VeteranRankingRow::new(pid.clone(), assessor.clone(), "3".into(), Good);
+
+        let rows = vec![first.clone(), second.clone(), third];
+        let expected_rows = vec![first, second];
+
+        assert_eq!(Filter::filter_rows(&rows), expected_rows);
+    }
+}

catalyst-toolbox/src/community_advisors/models/de.rs

@@ -1,5 +1,5 @@
 use crate::utils::serde::deserialize_truthy_falsy;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use vit_servicing_station_lib::db::models::community_advisors_reviews::ReviewRanking as VitReviewRanking;
 
 /// (Proposal Id, Assessor Id), an assessor cannot assess the same proposal more than once
@@ -37,7 +37,7 @@ pub struct AdvisorReviewRow {
     filtered_out: bool,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
 pub struct VeteranRankingRow {
     pub proposal_id: String,
     #[serde(alias = "Assessor")]
@@ -82,6 +82,26 @@ impl ReviewRanking {
 }
 
 impl VeteranRankingRow {
+    pub fn new(
+        proposal_id: String,
+        assessor: String,
+        vca: VeteranAdvisorId,
+        ranking: ReviewRanking,
+    ) -> Self {
+        let excellent = ranking == ReviewRanking::Excellent;
+        let good = ranking == ReviewRanking::Good;
+        let filtered_out = ranking == ReviewRanking::FilteredOut;
+
+        Self {
+            proposal_id,
+            assessor,
+            vca,
+            excellent,
+            good,
+            filtered_out,
+        }
+    }
+
     pub fn score(&self) -> ReviewRanking {
         ranking_mux(self.excellent, self.good, self.filtered_out)
     }