added download-or-update-course-exercises and fixed related issues

Author: Heliozoa

tmc-langs-cli/src/app.rs

@@ -357,6 +357,16 @@ fn create_core_app() -> App<'static, 'static> {
                 .long("submission-url")
                 .takes_value(true)))
 
+        .subcommand(SubCommand::with_name("download-or-update-course-exercises")
+            .about("Downloads exercises. If downloading an exercise that has been downloaded before, the student file policy will be used to avoid overwriting student files, effectively just updating the exercise files")
+            .long_about(SCHEMA_NULL)
+            .arg(Arg::with_name("exercise-id")
+                .help("Exercise id of an exercise that should be downloaded. Multiple ids can be given.")
+                .long("exercise-id")
+                .required(true)
+                .takes_value(true)
+                .multiple(true)))
+
         .subcommand(SubCommand::with_name("download-or-update-exercises")
             .about("Downloads exercises. If downloading an exercise on top of an existing one, the student file policy will be used to avoid overwriting student files, effectively just updating the exercise files")
             .long_about(SCHEMA_NULL)

tmc-langs-cli/src/config.rs

@@ -5,7 +5,7 @@ mod projects_config;
 mod tmc_config;
 
 pub use credentials::Credentials;
-pub use projects_config::ProjectsConfig;
+pub use projects_config::{CourseConfig, Exercise, ProjectsConfig};
 pub use tmc_config::{ConfigValue, TmcConfig};
 
 use anyhow::{Context, Error};

tmc-langs-cli/src/config/projects_config.rs

@@ -60,8 +60,17 @@ pub struct CourseConfig {
 }
 
 impl CourseConfig {
-    pub fn save_to_projects_dir(self, projects_dir: &Path) -> Result<()> {
-        let target = projects_dir.join(&self.course);
+    pub fn save_to_projects_dir(&self, projects_dir: &Path) -> Result<()> {
+        let course_dir = projects_dir.join(&self.course);
+        if !course_dir.exists() {
+            fs::create_dir_all(&course_dir).with_context(|| {
+                format!(
+                    "Failed to create course directory at {}",
+                    course_dir.display()
+                )
+            })?;
+        }
+        let target = course_dir.join("course_config.toml");
         let s = toml::to_string_pretty(&self)?;
         fs::write(&target, s.as_bytes())
             .with_context(|| format!("Failed to write course config to {}", target.display()))?;

tmc-langs-cli/src/main.rs

@@ -7,7 +7,8 @@ mod output;
 
 use anyhow::{Context, Result};
 use clap::{ArgMatches, Error, ErrorKind};
-use config::{Credentials, TmcConfig};
+use config::ProjectsConfig;
+use config::{CourseConfig, Credentials, Exercise, TmcConfig};
 use error::{InvalidTokenError, SandboxTestError};
 use output::{
     CombinedCourseData, ErrorData, Kind, Output, OutputData, OutputResult, Status, Warnings,
@@ -789,6 +790,84 @@ fn run_core(
             });
             print_output(&output, pretty, &warnings)?
         }
+        ("download-or-update-course-exercises", Some(matches)) => {
+            let exercise_ids = matches.values_of("exercise-id").unwrap();
+
+            // collect exercise into (id, path) pairs
+            let exercises = exercise_ids
+                .into_iter()
+                .map(into_usize)
+                .collect::<Result<_>>()?;
+            let exercises_details = core.get_exercises_details(exercises)?;
+
+            let tmc_config = TmcConfig::load(client_name)?;
+
+            let projects_dir = tmc_config.projects_dir;
+            let mut projects_config = ProjectsConfig::load(&projects_dir)?;
+
+            let mut course_data = HashMap::<String, Vec<(String, String)>>::new();
+            let mut exercises_and_paths = vec![];
+            for exercise_detail in exercises_details {
+                // get course and exercise name from server
+                let ex_details = core.get_exercise_details(exercise_detail.id)?;
+                // check if the checksum is different from what's already on disk
+                if let Some(course_config) = projects_config.courses.get(&ex_details.course_name) {
+                    if let Some(exercise) = course_config.exercises.get(&ex_details.exercise_name) {
+                        if exercise_detail.checksum == exercise.checksum {
+                            // skip this exercise
+                            log::info!(
+                                "Skipping exercise {} ({} in {}) due to identical checksum",
+                                exercise_detail.id,
+                                ex_details.course_name,
+                                ex_details.exercise_name
+                            );
+                            continue;
+                        }
+                    }
+                }
+
+                let target = ProjectsConfig::get_exercise_download_target(
+                    &projects_dir,
+                    &ex_details.course_name,
+                    &ex_details.exercise_name,
+                );
+
+                let entry = course_data.entry(ex_details.course_name);
+                let course_exercises = entry.or_default();
+                course_exercises.push((ex_details.exercise_name, exercise_detail.checksum));
+
+                exercises_and_paths.push((exercise_detail.id, target));
+            }
+            core.download_or_update_exercises(exercises_and_paths)
+                .context("Failed to download exercises")?;
+
+            for (course_name, exercise_names) in course_data {
+                let mut exercises = HashMap::new();
+                for (exercise_name, checksum) in exercise_names {
+                    exercises.insert(exercise_name, Exercise { checksum });
+                }
+
+                if let Some(course_config) = projects_config.courses.get_mut(&course_name) {
+                    course_config.exercises.extend(exercises);
+                    course_config.save_to_projects_dir(&projects_dir)?;
+                } else {
+                    let course_config = CourseConfig {
+                        course: course_name,
+                        exercises,
+                    };
+                    course_config.save_to_projects_dir(&projects_dir)?;
+                };
+            }
+
+            let output = Output::OutputData::<()>(OutputData {
+                status: Status::Finished,
+                message: None,
+                result: OutputResult::RetrievedData,
+                percent_done: 1.0,
+                data: None,
+            });
+            print_output(&output, pretty, &warnings)?
+        }
         ("download-or-update-exercises", Some(matches)) => {
             let mut exercise_args = matches.values_of("exercise").unwrap();
 

tmc-langs-core/src/error.rs

@@ -44,6 +44,8 @@ pub enum CoreError {
     NotLoggedIn,
     #[error("Failed to find cache directory")]
     CacheDir,
+    #[error("No values found in exercise details map returned by server")]
+    MissingDetailsValue,
 
     #[error(transparent)]
     SystemTime(#[from] std::time::SystemTimeError),

tmc-langs-core/src/lib.rs

@@ -19,7 +19,7 @@ pub use oauth2;
 pub use request::FeedbackAnswer;
 pub use response::{
     Course, CourseData, CourseDataExercise, CourseDataExercisePoint, CourseDetails, CourseExercise,
-    Exercise, ExerciseDetails, NewSubmission, Organization, Review, Submission,
+    Exercise, ExerciseChecksums, ExerciseDetails, NewSubmission, Organization, Review, Submission,
     SubmissionFeedbackResponse, SubmissionFinished, SubmissionProcessingStatus, SubmissionStatus,
     UpdateResult, User,
 };

tmc-langs-core/src/response.rs

@@ -210,6 +210,12 @@ pub struct ExerciseDetails {
     pub submissions: Vec<ExerciseSubmission>,
 }
 
+#[derive(Debug, Deserialize, Serialize, JsonSchema)]
+pub struct ExerciseChecksums {
+    pub id: usize,
+    pub checksum: String,
+}
+
 #[derive(Debug, Deserialize, Serialize, JsonSchema)]
 pub struct Submission {
     pub id: usize,

tmc-langs-core/src/tmc_core.rs

@@ -287,7 +287,7 @@ impl TmcCore {
                     "Downloaded exercise {} to '{}'. ({} out of {})",
                     exercise_id,
                     target.display(),
-                    n,
+                    n + 1,
                     exercises_len
                 ),
                 progress,
@@ -329,7 +329,7 @@ impl TmcCore {
     pub fn get_exercises_details(
         &self,
         exercise_ids: Vec<usize>,
-    ) -> Result<Vec<ExerciseDetails>, CoreError> {
+    ) -> Result<Vec<ExerciseChecksums>, CoreError> {
         self.core_exercise_details(exercise_ids)
     }
 

tmc-langs-core/src/tmc_core/api.rs

@@ -4,8 +4,8 @@ use crate::error::CoreError;
 use crate::response::ErrorResponse;
 use crate::{
     Course, CourseData, CourseDataExercise, CourseDataExercisePoint, CourseDetails, CourseExercise,
-    ExerciseDetails, FeedbackAnswer, NewSubmission, Organization, Review, Submission,
-    SubmissionFeedbackResponse, TmcCore, User,
+    ExerciseChecksums, ExerciseDetails, FeedbackAnswer, NewSubmission, Organization, Review,
+    Submission, SubmissionFeedbackResponse, TmcCore, User,
 };
 use oauth2::TokenResponse;
 use reqwest::{
@@ -610,7 +610,7 @@ impl TmcCore {
     pub(super) fn core_exercise_details(
         &self,
         exercise_ids: Vec<usize>,
-    ) -> Result<Vec<ExerciseDetails>, CoreError> {
+    ) -> Result<Vec<ExerciseChecksums>, CoreError> {
         if self.token.is_none() {
             return Err(CoreError::NotLoggedIn);
         }
@@ -623,7 +623,15 @@ impl TmcCore {
                 .collect::<Vec<_>>()
                 .join(","),
         );
-        self.get_json_with_params(&url_tail, &[exercise_ids])
+
+        // returns map with result in key "exercises"
+        let res: HashMap<String, Vec<ExerciseChecksums>> =
+            self.get_json_with_params(&url_tail, &[exercise_ids])?;
+        if let Some((_, val)) = res.into_iter().next() {
+            // just return whatever value is found first
+            return Ok(val);
+        }
+        Err(CoreError::MissingDetailsValue)
     }
 
     pub(super) fn download_solution(

tmc-langs-core/tests/compare_server_results.rs

@@ -3,7 +3,7 @@
 
 use dotenv::dotenv;
 use std::env;
-use std::path::Path;
+use std::path::PathBuf;
 use std::thread;
 use std::time::Duration;
 use tmc_langs_core::{CoreError, Exercise, SubmissionProcessingStatus, SubmissionStatus, TmcCore};
@@ -60,7 +60,7 @@ fn dl_test_submit_course_solutions(course_id: usize) {
             .join("solution/download")
             .unwrap();
         dl_test_submit_exercise(&core, exercise, |target| {
-            core.download_model_solution(solution_url, target)
+            core.download_model_solution(solution_url, &target)
         });
     }
 
@@ -106,13 +106,13 @@ where
 // downloader should download the submission target to the path arg
 fn dl_test_submit_exercise<F>(core: &TmcCore, exercise: Exercise, downloader: F)
 where
-    F: FnOnce(&Path) -> Result<(), CoreError>,
+    F: FnOnce(PathBuf) -> Result<(), CoreError>,
 {
     log::debug!("submitting exercise {:#?}", exercise);
     let temp = tempfile::tempdir().unwrap();
     let submission_path = temp.path().join(exercise.id.to_string());
     log::debug!("downloading to {}", submission_path.display());
-    downloader(&submission_path).unwrap();
+    downloader(submission_path.clone()).unwrap();
 
     log::debug!("testing locally {}", submission_path.display());
     let test_results = core.run_tests(&submission_path, &mut vec![]).unwrap();