Feat reattach to manual destroy run

Author: baptman21

internal/controller/workspace_controller_deletion_policy.go

@@ -101,6 +101,30 @@ func (r *WorkspaceReconciler) deleteWorkspace(ctx context.Context, w *workspaceI
 
 		if _, ok := runStatusUnsuccessful[run.Status]; ok {
 			w.log.Info("Destroy Run", "msg", fmt.Sprintf("destroy run %s is unsuccessful: %s", run.ID, run.Status))
+
+			workspace, err := w.tfClient.Client.Workspaces.ReadByID(ctx, w.instance.Status.WorkspaceID)
+			if err != nil {
+				return r.handleWorkspaceErrorNotFound(ctx, w, err)
+			}
+
+			w.log.Info("Destroy Run", "msg", fmt.Sprintf("CurrentRun: %s %s %v", workspace.CurrentRun.ID, workspace.CurrentRun.Status, workspace.CurrentRun.IsDestroy))
+
+			if workspace.CurrentRun != nil && workspace.CurrentRun.ID != w.instance.Status.DestroyRunID {
+
+				run, err := w.tfClient.Client.Runs.Read(ctx, w.instance.Status.DestroyRunID)
+				if err != nil {
+					// ignore this run id, and let the next reconcile loop handle the error
+					return nil
+				}
+				if run.IsDestroy {
+					w.log.Info("Destroy Run", "msg", fmt.Sprintf("found more recent destroy run %s, updating DestroyRunID", workspace.CurrentRun.ID))
+
+					w.instance.Status.DestroyRunID = workspace.CurrentRun.ID
+					w.updateWorkspaceStatusRun(run)
+					return r.Status().Update(ctx, &w.instance)
+				}
+			}
+
 			return nil
 		}
 		w.log.Info("Destroy Run", "msg", fmt.Sprintf("destroy run %s is not finished", run.ID))

internal/controller/workspace_controller_deletion_policy_test.go

@@ -195,6 +195,97 @@ var _ = Describe("Workspace controller", Ordered, func() {
 				return err == tfc.ErrResourceNotFound
 			}).Should(BeTrue())
 		})
+		It("can destroy delete a workspace when the destroy was retried manually after failing", func() {
+			if cloudEndpoint != tfcDefaultAddress {
+				Skip("Does not run against TFC, skip this test")
+			}
+			instance.Spec.AllowDestroyPlan = true
+			instance.Spec.DeletionPolicy = appv1alpha2.DeletionPolicyDestroy
+			createWorkspace(instance)
+			workspaceID := instance.Status.WorkspaceID
+
+			cv := createAndUploadConfigurationVersion(instance.Status.WorkspaceID, "hoi")
+			Eventually(func() bool {
+				listOpts := tfc.ListOptions{
+					PageNumber: 1,
+					PageSize:   maxPageSize,
+				}
+				for listOpts.PageNumber != 0 {
+					runs, err := tfClient.Runs.List(ctx, workspaceID, &tfc.RunListOptions{
+						ListOptions: listOpts,
+					})
+					Expect(err).To(Succeed())
+					for _, r := range runs.Items {
+						if r.ConfigurationVersion.ID == cv.ID {
+							return r.Status == tfc.RunApplied
+						}
+					}
+					listOpts.PageNumber = runs.NextPage
+				}
+				return false
+			}).Should(BeTrue())
+
+			// create an errored ConfigurationVersion for the delete to fail
+			cv = createAndUploadErroredConfigurationVersion(instance.Status.WorkspaceID, false)
+
+			Expect(k8sClient.Delete(ctx, instance)).To(Succeed())
+
+			var destroyRunID string
+			Eventually(func() bool {
+				ws, err := tfClient.Workspaces.ReadByID(ctx, workspaceID)
+				Expect(err).To(Succeed())
+				Expect(ws).ToNot(BeNil())
+				Expect(ws.CurrentRun).ToNot(BeNil())
+				run, err := tfClient.Runs.Read(ctx, ws.CurrentRun.ID)
+				Expect(err).To(Succeed())
+				Expect(run).ToNot(BeNil())
+				destroyRunID = run.ID
+
+				return run.IsDestroy
+			}).Should(BeTrue())
+
+			Eventually(func() bool {
+				run, _ := tfClient.Runs.Read(ctx, destroyRunID)
+				if run.Status == tfc.RunErrored {
+					return true
+				}
+
+				return false
+			}).Should(BeTrue())
+
+			// put back a working configuration
+			cv = createAndUploadConfigurationVersion(instance.Status.WorkspaceID, "hoi")
+
+			// start a new destroy run manually
+			run, err := tfClient.Runs.Create(ctx, tfc.RunCreateOptions{
+				IsDestroy: tfc.Bool(true),
+				Message:   tfc.String(runMessage),
+				Workspace: &tfc.Workspace{
+					ID: workspaceID,
+				},
+			})
+			Expect(err).To(Succeed())
+			Expect(run).ToNot(BeNil())
+
+			var newDestroyRunID string
+			Eventually(func() bool {
+				ws, err := tfClient.Workspaces.ReadByID(ctx, workspaceID)
+				Expect(err).To(Succeed())
+				Expect(ws).ToNot(BeNil())
+				Expect(ws.CurrentRun).ToNot(BeNil())
+				run, err := tfClient.Runs.Read(ctx, ws.CurrentRun.ID)
+				Expect(err).To(Succeed())
+				Expect(run).ToNot(BeNil())
+				newDestroyRunID = run.ID
+
+				return run.IsDestroy && newDestroyRunID != destroyRunID
+			}).Should(BeTrue())
+
+			Eventually(func() bool {
+				_, err := tfClient.Workspaces.ReadByID(ctx, workspaceID)
+				return err == tfc.ErrResourceNotFound
+			}).Should(BeTrue())
+		})
 		It("can force delete a workspace", func() {
 			instance.Spec.DeletionPolicy = appv1alpha2.DeletionPolicyForce
 			createWorkspace(instance)

internal/controller/workspace_controller_outputs_test.go

@@ -180,3 +180,46 @@ func createAndUploadConfigurationVersion(workspaceID string, outputValue string)
 
 	return cv
 }
+
+func createAndUploadErroredConfigurationVersion(workspaceID string, autoQueueRuns bool) *tfc.ConfigurationVersion {
+	GinkgoHelper()
+	// Create a temporary dir in the current one
+	cd, err := os.Getwd()
+	Expect(err).Should(Succeed())
+	td, err := os.MkdirTemp(cd, "tf-*")
+	Expect(err).Should(Succeed())
+	defer os.RemoveAll(td)
+	// Create a temporary file in the temporary dir
+	f, err := os.CreateTemp(td, "*.tf")
+	Expect(err).Should(Succeed())
+	defer os.Remove(f.Name())
+	// Terraform code to upload
+	tf := fmt.Sprint(`
+				resource "test_non_existent_resource" "this" {}
+				`)
+	// Save the Terraform code to the temporary file
+	_, err = f.WriteString(tf)
+	Expect(err).Should(Succeed())
+
+	cv, err := tfClient.ConfigurationVersions.Create(ctx, workspaceID, tfc.ConfigurationVersionCreateOptions{
+		AutoQueueRuns: tfc.Bool(autoQueueRuns),
+		Speculative:   tfc.Bool(false),
+	})
+	Expect(err).Should(Succeed())
+	Expect(cv).ShouldNot(BeNil())
+
+	Expect(tfClient.ConfigurationVersions.Upload(ctx, cv.UploadURL, td)).Should(Succeed())
+
+	Eventually(func() bool {
+		c, err := tfClient.ConfigurationVersions.Read(ctx, cv.ID)
+		if err != nil {
+			return false
+		}
+		if c.Status == tfc.ConfigurationUploaded {
+			return true
+		}
+		return false
+	}).Should(BeTrue())
+
+	return cv
+}