send workflow url from cli (#2048)

* send workflow url from cli

Author: motatoes

backend/controllers/projects.go

@@ -621,6 +621,7 @@ type SetJobStatusRequest struct {
 	PrCommentUrl    string                      `json:"pr_comment_url"`
 	PrCommentId     string                      `json:"pr_comment_id"`
 	TerraformOutput string                      `json:"terraform_output"`
+	WorkflowUrl     string                      `json:"workflow_url,omitempty"`
 }
 
 func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
@@ -664,6 +665,10 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 	switch request.Status {
 	case "started":
 		job.Status = orchestrator_scheduler.DiggerJobStarted
+		if request.WorkflowUrl != "" {
+			slog.Debug("Adding workflow url to job", "jobId", jobId, "workflowUrl", request.WorkflowUrl)
+			job.WorkflowRunUrl = &request.WorkflowUrl
+		}
 		err := models.DB.UpdateDiggerJob(job)
 		if err != nil {
 			slog.Error("Error updating job status",
@@ -862,44 +867,13 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 		return
 	}
 
-	// attempt to update workflow run url
+	// attempt to update workflow run url, note we only have this for backwards compatibility with
+	// older digger cli versions, newer cli versions after v0.6.110 will send the workflow url so
+	// we don't need to pull API, saving us rate limit exceeded errors
 	slog.Debug("Attempting to update workflow run URL", "jobId", jobId)
-
-	client, _, err := utils.GetGithubClient(d.GithubClientProvider, job.Batch.GithubInstallationId, job.Batch.RepoFullName)
+	err = updateWorkflowUrlForJob(d.GithubClientProvider, job)
 	if err != nil {
-		slog.Warn("Error creating GitHub client for workflow URL update",
-			"jobId", jobId,
-			"repoFullName", job.Batch.RepoFullName,
-			"error", err,
-		)
-	} else {
-		_, workflowRunUrl, err := utils.GetWorkflowIdAndUrlFromDiggerJobId(
-			client,
-			job.Batch.RepoOwner,
-			job.Batch.RepoName,
-			job.DiggerJobID,
-		)
-		if err != nil {
-			slog.Warn("Error getting workflow ID from job",
-				"jobId", jobId,
-				"error", err,
-			)
-		} else if workflowRunUrl != "#" && workflowRunUrl != "" {
-			job.WorkflowRunUrl = &workflowRunUrl
-			err = models.DB.UpdateDiggerJob(job)
-			if err != nil {
-				slog.Error("Error updating job with workflow URL",
-					"jobId", jobId,
-					"workflowUrl", workflowRunUrl,
-					"error", err,
-				)
-			} else {
-				slog.Debug("Updated job with workflow URL",
-					"jobId", jobId,
-					"workflowUrl", workflowRunUrl,
-				)
-			}
-		}
+		slog.Warn("Failed to update workflow run URL", "jobId", jobId, "error", err)
 	}
 
 	job.StatusUpdatedAt = request.Timestamp
@@ -991,6 +965,63 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 	c.JSON(http.StatusOK, res)
 }
 
+func updateWorkflowUrlForJob(githubClientProvider utils.GithubClientProvider, job *models.DiggerJob) error {
+	if job == nil {
+		return fmt.Errorf("job is nil")
+	}
+	if job.WorkflowRunUrl != nil && *job.WorkflowRunUrl != "#" && *job.WorkflowRunUrl != "" {
+		slog.Debug("Workflow URL already exists", "jobId", job.DiggerJobID)
+		return nil
+	}
+	jobId := job.DiggerJobID
+	client, _, err := utils.GetGithubClient(githubClientProvider, job.Batch.GithubInstallationId, job.Batch.RepoFullName)
+	if err != nil {
+		slog.Warn("Error creating GitHub client for workflow URL update",
+			"jobId", jobId,
+			"repoFullName", job.Batch.RepoFullName,
+			"error", err,
+		)
+		return fmt.Errorf("error creating GitHub client for workflow URL update: %v", err)
+	}
+
+	_, workflowRunUrl, err := utils.GetWorkflowIdAndUrlFromDiggerJobId(
+		client,
+		job.Batch.RepoOwner,
+		job.Batch.RepoName,
+		job.DiggerJobID,
+	)
+	if err != nil {
+		slog.Warn("Error getting workflow ID from job",
+			"jobId", jobId,
+			"error", err,
+		)
+		return fmt.Errorf("error getting workflow ID from job: %v", err)
+	}
+
+	if workflowRunUrl != "#" && workflowRunUrl != "" {
+		job.WorkflowRunUrl = &workflowRunUrl
+		err = models.DB.UpdateDiggerJob(job)
+		if err != nil {
+			slog.Error("Error updating job with workflow URL",
+				"jobId", jobId,
+				"workflowUrl", workflowRunUrl,
+				"error", err,
+			)
+			return fmt.Errorf("error updating job with workflow URL: %v", err)
+		} else {
+			slog.Debug("Updated job with workflow URL",
+				"jobId", jobId,
+				"workflowUrl", workflowRunUrl,
+			)
+		}
+	} else {
+		slog.Debug("Workflow URL not found for job",
+			"jobId", jobId)
+		return fmt.Errorf("workflow URL not found for job (workflowUrl returned: %v)", workflowRunUrl)
+	}
+	return nil
+}
+
 type CreateProjectRunRequest struct {
 	StartedAt time.Time `json:"startedAt"`
 	EndedAt   time.Time `json:"endedAt"`

backend/services/scheduler.go

@@ -2,18 +2,14 @@ package services
 
 import (
 	"fmt"
-	"log/slog"
-	"runtime/debug"
-	"time"
-
 	"github.com/diggerhq/digger/backend/ci_backends"
 	"github.com/diggerhq/digger/backend/config"
 	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/backend/utils"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
-	"github.com/diggerhq/digger/libs/spec"
 	"github.com/google/go-github/v61/github"
 	"github.com/google/uuid"
+	"log/slog"
 )
 
 func DiggerJobCompleted(client *github.Client, batchId *uuid.UUID, parentJob *models.DiggerJob, repoFullName string, repoOwner string, repoName string, workflowFileName string, gh utils.GithubClientProvider) error {
@@ -168,63 +164,6 @@ func TriggerJob(gh utils.GithubClientProvider, ciBackend ci_backends.CiBackend,
 	}
 
 	slog.Info("Job successfully triggered", "jobId", job.DiggerJobID, "status", job.Status)
-	go UpdateWorkflowUrlForJob(job, ciBackend, spec)
 
 	return nil
 }
-
-// This is meant to run asynchronously since it queries for job url
-func UpdateWorkflowUrlForJob(job *models.DiggerJob, ciBackend ci_backends.CiBackend, spec *spec.Spec) {
-	defer func() {
-		if r := recover(); r != nil {
-			stack := string(debug.Stack())
-			slog.Error("Recovered from panic in UpdateWorkflowUrlForJob",
-				"jobId", job.DiggerJobID,
-				"error", r,
-				"stackTrace", stack,
-			)
-		}
-	}()
-
-	batch := job.Batch
-	// for now we only perform this update for github
-	if batch.VCS != models.DiggerVCSGithub {
-		slog.Debug("Skipping workflow URL update for non-GitHub VCS", "jobId", job.DiggerJobID, "vcs", batch.VCS)
-		return
-	}
-
-	slog.Info("Starting workflow URL update", "jobId", job.DiggerJobID)
-
-	for n := 0; n < 30; n++ {
-		time.Sleep(1 * time.Second)
-		workflowUrl, err := ciBackend.GetWorkflowUrl(*spec)
-		if err != nil {
-			slog.Debug("Error fetching workflow URL",
-				"jobId", job.DiggerJobID,
-				"attempt", n+1,
-				"error", err)
-		} else {
-			if workflowUrl == "#" || workflowUrl == "" {
-				slog.Debug("Received blank workflow URL", "jobId", job.DiggerJobID, "attempt", n+1)
-			} else {
-				job.WorkflowRunUrl = &workflowUrl
-				err = models.DB.UpdateDiggerJob(job)
-				if err != nil {
-					slog.Error("Failed to update job with workflow URL",
-						"jobId", job.DiggerJobID,
-						"url", workflowUrl,
-						"error", err)
-					continue
-				} else {
-					slog.Info("Successfully updated workflow URL",
-						"jobId", job.DiggerJobID,
-						"url", workflowUrl)
-				}
-				return
-			}
-		}
-	}
-
-	slog.Warn("Failed to obtain workflow URL after multiple attempts", "jobId", job.DiggerJobID)
-	// if we get to here its highly likely that the workflow job entirely failed to start for some reason
-}

libs/backendapi/diggerapi.go

@@ -135,6 +135,22 @@ func (d DiggerApi) ReportProjectRun(namespace string, projectName string, starte
 	return nil
 }
 
+func isGitHubActions() bool {
+	return os.Getenv("GITHUB_ACTIONS") == "true"
+}
+
+func getWorkflowUrl() string {
+	if isGitHubActions() {
+		githubServerURL := os.Getenv("GITHUB_SERVER_URL")  // e.g., https://github.com
+		githubRepository := os.Getenv("GITHUB_REPOSITORY") // e.g., diggerhq/demo-opentofu
+		githubRunID := os.Getenv("GITHUB_RUN_ID")          // numeric run ID
+		workflowURL := fmt.Sprintf("%s/%s/actions/runs/%s", githubServerURL, githubRepository, githubRunID)
+		return workflowURL
+	} else {
+		return "#"
+	}
+}
+
 func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId string, status string, timestamp time.Time, summary *iac_utils.IacSummary, planJson string, PrCommentUrl string, PrCommentId string, terraformOutput string, iacUtils iac_utils.IacUtils) (*scheduler.SerializedBatch, error) {
 	repoNameForBackendReporting := strings.ReplaceAll(repo, "/", "-")
 	u, err := url.Parse(d.DiggerHost)
@@ -160,6 +176,7 @@ func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId
 		}
 	}
 
+	workflowUrl := getWorkflowUrl()
 	u.Path = filepath.Join(u.Path, "repos", repoNameForBackendReporting, "projects", projectName, "jobs", jobId, "set-status")
 	request := map[string]interface{}{
 		"status":             status,
@@ -169,6 +186,7 @@ func (d DiggerApi) ReportProjectJobStatus(repo string, projectName string, jobId
 		"pr_comment_url":     PrCommentUrl,
 		"pr_comment_id":      PrCommentId,
 		"terraform_output":   terraformOutput,
+		"workflow_url":       workflowUrl,
 	}
 
 	jsonData, err := json.Marshal(request)