Improve publisher tool (#263)

* Fix issues with publish tool

* Add `--continue-from` argument to publish

* Fix batch sorting issue

* Check if a package is published before publishing

* Add ability to yank package versions in bulk

* Disable incremental compilation to reduce disk space used

* Don't wait for correct `aws-sigv4` version

* Correct crate ownership during publish

* Always correct crate ownership

* replace cargo search with crates.io API

* Publisher updates

* Fix clippy lints against Rust 1.56.1

* Incorporate CR feedback

Co-authored-by: Russell Cohen <rcoh@amazon.com>

Author: jdisanti

tools/publisher/Cargo.toml

@@ -9,13 +9,17 @@ publish = false
 
 [dependencies]
 anyhow = "1.0"
+async-trait = "0.1.51"
 cargo_toml = "0.10.1"
 clap = "2.33"
+crates_io_api = "0.7.3"
+lazy_static = "1"
 dialoguer = "0.8"
 num_cpus = "1.13"
+regex = "1.5.4"
 semver = "1.0"
 thiserror = "1.0"
 tokio = { version = "1.12", features = ["full"] }
 toml = { version = "0.5.8", features = ["preserve_order"] }
 tracing = "0.1.29"
-tracing-subscriber = "0.2.25"
+tracing-subscriber = "0.2.25"

tools/publisher/fake_cargo/cargo_fails

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "some stdout failure message"
+>&2 echo "some stderr failure message"
+exit 1

tools/publisher/fake_cargo/cargo_owner_list

@@ -0,0 +1,4 @@
+#!/bin/bash
+>&2 echo "     Updating crates.io index"
+echo "rcoh (Russell Cohen)"
+echo "github:awslabs:rust-sdk-owners (rust-sdk-owners)"

tools/publisher/fake_cargo/cargo_publish_already_published

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo 'error: crate version `0.0.22-alpha` is already uploaded'
+exit 1

tools/publisher/fake_cargo/cargo_search_success

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Fake `cargo search aws-sdk-dynamodb` for unit testing
+echo 'aws-sdk-dynamodb = "0.0.22-alpha"          # AWS SDK for Amazon DynamoDB'
+echo 'aws-sdk-dynamodbstreams = "0.0.22-alpha"   # AWS SDK for Amazon DynamoDB Streams'

tools/publisher/fake_cargo/cargo_success

@@ -0,0 +1,2 @@
+#!/bin/bash
+exit 0

tools/publisher/fake_cargo/cargo_yank_not_found

@@ -0,0 +1,10 @@
+#!/bin/bash
+{
+    echo "     Updating crates.io index"
+    echo "        Yank aws-sigv4:0.0.0"
+    echo "error: failed to yank from the registry at https://crates.io"
+    echo
+    echo "Caused by:"
+    echo '  the remote server responded with an error: crate `aws-sigv4` does not have a version `0.0.0`'
+} >&2
+exit 101

tools/publisher/src/cargo.rs

@@ -5,82 +5,70 @@
 
 //! Module for interacting with Cargo.
 
+mod add_owner;
+mod get_owners;
+mod publish;
+mod yank;
+
+pub use add_owner::AddOwner;
+pub use get_owners::GetOwners;
+pub use publish::Publish;
+pub use yank::Yank;
+
 use anyhow::{Context, Result};
-use std::path::{Path, PathBuf};
+use async_trait::async_trait;
+use std::borrow::Cow;
 use std::process::{Command, Output};
 
-macro_rules! cmd {
-    [ $( $x:expr ),* ] => {
-        {
-            let mut cmd = Cmd::new();
-            $(cmd.push($x);)*
-            cmd
-        }
-    };
-}
+#[async_trait]
+pub trait CargoOperation {
+    type Output;
 
-/// Confirms that cargo exists on the path.
-pub async fn confirm_installed_on_path() -> Result<()> {
-    cmd!["cargo", "--version"]
-        .spawn()
-        .await
-        .context("cargo is not installed on the PATH")?;
-    Ok(())
-}
+    /// Runs the command asynchronously.
+    async fn spawn(&self) -> Result<Self::Output>;
 
-/// Returns a `Cmd` that, when spawned, will asynchronously run `cargo publish` in the given crate path.
-pub fn publish_task(crate_path: &Path) -> Cmd {
-    cmd!["cargo", "publish"].working_dir(crate_path)
+    /// Returns a plan string that can be output to the user to describe the command.
+    fn plan(&self) -> Option<Cow<'static, str>>;
 }
 
-#[derive(Default)]
-pub struct Cmd {
-    parts: Vec<String>,
-    working_dir: Option<PathBuf>,
+/// Confirms that cargo exists on the path.
+pub fn confirm_installed_on_path() -> Result<()> {
+    handle_failure(
+        "discover cargo version",
+        &Command::new("cargo")
+            .arg("version")
+            .output()
+            .context("cargo is not installed on the PATH")?,
+    )
+    .context("cargo is not installed on the PATH")
 }
 
-impl Cmd {
-    fn new() -> Cmd {
-        Default::default()
-    }
-
-    fn push(&mut self, part: impl Into<String>) {
-        self.parts.push(part.into());
-    }
-
-    fn working_dir(mut self, working_dir: impl AsRef<Path>) -> Self {
-        self.working_dir = Some(working_dir.as_ref().into());
-        self
-    }
-
-    /// Returns a plan string that can be output to the user to describe the command.
-    pub fn plan(&self) -> String {
-        let mut plan = String::new();
-        if let Some(working_dir) = &self.working_dir {
-            plan.push_str(&format!("[in {:?}]: ", working_dir));
-        }
-        plan.push_str(&self.parts.join(" "));
-        plan
-    }
+/// Returns (stdout, stderr)
+fn output_text(output: &Output) -> (String, String) {
+    (
+        String::from_utf8_lossy(&output.stdout).to_string(),
+        String::from_utf8_lossy(&output.stderr).to_string(),
+    )
+}
 
-    /// Runs the command asynchronously.
-    pub async fn spawn(mut self) -> Result<Output> {
-        let working_dir = self
-            .working_dir
-            .take()
-            .unwrap_or_else(|| std::env::current_dir().unwrap());
-        let mut command: Command = self.into();
-        tokio::task::spawn_blocking(move || Ok(command.current_dir(working_dir).output()?)).await?
+fn handle_failure(operation_name: &str, output: &Output) -> Result<(), anyhow::Error> {
+    if !output.status.success() {
+        return Err(capture_error(operation_name, output));
     }
+    Ok(())
 }
 
-impl From<Cmd> for Command {
-    fn from(cmd: Cmd) -> Self {
-        assert!(!cmd.parts.is_empty());
-        let mut command = Command::new(&cmd.parts[0]);
-        for i in 1..cmd.parts.len() {
-            command.arg(&cmd.parts[i]);
-        }
-        command
-    }
+fn capture_error(operation_name: &str, output: &Output) -> anyhow::Error {
+    let message = format!(
+        "Failed to {name}:\nStatus: {status}\nStdout: {stdout}\nStderr: {stderr}\n",
+        name = operation_name,
+        status = if let Some(code) = output.status.code() {
+            format!("{}", code)
+        } else {
+            "Killed by signal".to_string()
+        },
+        stdout = String::from_utf8_lossy(&output.stdout),
+        stderr = String::from_utf8_lossy(&output.stderr)
+    );
+    anyhow::Error::msg(message)
 }

tools/publisher/src/cargo/add_owner.rs

@@ -0,0 +1,84 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+use crate::cargo::{handle_failure, CargoOperation};
+use anyhow::Result;
+use async_trait::async_trait;
+use std::borrow::Cow;
+use std::process::Command;
+
+pub struct AddOwner<'a> {
+    program: &'static str,
+    package_name: &'a str,
+    owner: &'a str,
+}
+
+impl<'a> AddOwner<'a> {
+    pub fn new(package_name: &'a str, owner: &'a str) -> AddOwner<'a> {
+        AddOwner {
+            program: "cargo",
+            package_name,
+            owner,
+        }
+    }
+}
+
+#[async_trait]
+impl<'a> CargoOperation for AddOwner<'a> {
+    type Output = ();
+
+    async fn spawn(&self) -> Result<()> {
+        let mut command = Command::new(self.program);
+        command
+            .arg("owner")
+            .arg("--add")
+            .arg(self.owner)
+            .arg(self.package_name);
+        let output = tokio::task::spawn_blocking(move || command.output()).await??;
+        handle_failure("add owner", &output)?;
+        Ok(())
+    }
+
+    fn plan(&self) -> Option<Cow<'static, str>> {
+        None
+    }
+}
+
+#[cfg(all(test, not(target_os = "windows")))]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn add_owner_success() {
+        AddOwner {
+            program: "./fake_cargo/cargo_success",
+            package_name: "aws-sdk-s3",
+            owner: "github:awslabs:rust-sdk-owners",
+        }
+        .spawn()
+        .await
+        .unwrap();
+    }
+
+    #[tokio::test]
+    async fn get_owners_failed() {
+        let result = AddOwner {
+            program: "./fake_cargo/cargo_fails",
+            package_name: "aws-sdk-s3",
+            owner: "github:awslabs:rust-sdk-owners",
+        }
+        .spawn()
+        .await;
+
+        assert!(result.is_err(), "expected error, got {:?}", result);
+        assert_eq!(
+            "Failed to add owner:\n\
+            Status: 1\n\
+            Stdout: some stdout failure message\n\n\
+            Stderr: some stderr failure message\n\n",
+            format!("{}", result.err().unwrap())
+        );
+    }
+}

tools/publisher/src/cargo/get_owners.rs

@@ -0,0 +1,99 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+use crate::cargo::{handle_failure, output_text, CargoOperation};
+use anyhow::Result;
+use async_trait::async_trait;
+use regex::Regex;
+use std::borrow::Cow;
+use std::process::Command;
+
+pub struct GetOwners<'a> {
+    program: &'static str,
+    package_name: &'a str,
+}
+
+impl<'a> GetOwners<'a> {
+    pub fn new(package_name: &'a str) -> GetOwners<'a> {
+        GetOwners {
+            program: "cargo",
+            package_name,
+        }
+    }
+}
+
+#[async_trait]
+impl<'a> CargoOperation for GetOwners<'a> {
+    type Output = Vec<String>;
+
+    async fn spawn(&self) -> Result<Vec<String>> {
+        let mut command = Command::new(self.program);
+        command.arg("owner").arg("--list").arg(self.package_name);
+        let output = tokio::task::spawn_blocking(move || command.output()).await??;
+        handle_failure("get crate owners", &output)?;
+
+        let mut result = Vec::new();
+        let (stdout, _) = output_text(&output);
+        let line_re = Regex::new(r#"^([\w\d\-_:]+)\s+\([\w\d\s\-_]+\)$"#).unwrap();
+        for line in stdout.lines() {
+            if let Some(captures) = line_re.captures(line) {
+                let user_id = captures.get(1).unwrap().as_str();
+                result.push(user_id.to_string());
+            } else {
+                return Err(anyhow::Error::msg(format!(
+                    "unrecognized line in `cargo owner` output: {}",
+                    line
+                )));
+            }
+        }
+        Ok(result)
+    }
+
+    fn plan(&self) -> Option<Cow<'static, str>> {
+        None
+    }
+}
+
+#[cfg(all(test, not(target_os = "windows")))]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn get_owners_success() {
+        let owners = GetOwners {
+            program: "./fake_cargo/cargo_owner_list",
+            package_name: "aws-sdk-s3",
+        }
+        .spawn()
+        .await
+        .unwrap();
+        assert_eq!(
+            vec![
+                "rcoh".to_string(),
+                "github:awslabs:rust-sdk-owners".to_string()
+            ],
+            owners
+        );
+    }
+
+    #[tokio::test]
+    async fn get_owners_failed() {
+        let result = GetOwners {
+            program: "./fake_cargo/cargo_fails",
+            package_name: "aws-sdk-s3",
+        }
+        .spawn()
+        .await;
+
+        assert!(result.is_err(), "expected error, got {:?}", result);
+        assert_eq!(
+            "Failed to get crate owners:\n\
+            Status: 1\n\
+            Stdout: some stdout failure message\n\n\
+            Stderr: some stderr failure message\n\n",
+            format!("{}", result.err().unwrap())
+        );
+    }
+}