adding serviceBusy tag for transient-poller-failure counter metric. (#1212)

adding serviceBusy tag for transient-poller-failure counter metric.

Author: mindaugasbarcauskas

internal/internal_logging_tags.go

@@ -41,4 +41,5 @@ const (
 	tagVisibilityQuery   = "VisibilityQuery"
 	tagPanicError        = "PanicError"
 	tagPanicStack        = "PanicStack"
+	causeTag             = "pollerrorcause"
 )

internal/internal_task_pollers.go

@@ -25,6 +25,7 @@ package internal
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -47,6 +48,7 @@ const (
 	stickyDecisionScheduleToStartTimeoutSeconds = 5
 
 	ratioToForceCompleteDecisionTaskComplete = 0.8
+	serviceBusy                              = "serviceBusy"
 )
 
 type (
@@ -770,8 +772,13 @@ func (wtp *workflowTaskPoller) poll(ctx context.Context) (interface{}, error) {
 	response, err := wtp.service.PollForDecisionTask(ctx, request, getYarpcCallOptions(wtp.featureFlags)...)
 	if err != nil {
 		retryable := isServiceTransientError(err)
+
 		if retryable {
-			wtp.metricsScope.Counter(metrics.DecisionPollTransientFailedCounter).Inc(1)
+			if target := (*s.ServiceBusyError)(nil); errors.As(err, &target) {
+				wtp.metricsScope.Tagged(map[string]string{causeTag: serviceBusy}).Counter(metrics.DecisionPollTransientFailedCounter).Inc(1)
+			} else {
+				wtp.metricsScope.Counter(metrics.DecisionPollTransientFailedCounter).Inc(1)
+			}
 		} else {
 			wtp.metricsScope.Counter(metrics.DecisionPollFailedCounter).Inc(1)
 		}
@@ -1011,7 +1018,12 @@ func (atp *activityTaskPoller) poll(ctx context.Context) (*s.PollForActivityTask
 	if err != nil {
 		retryable := isServiceTransientError(err)
 		if retryable {
-			atp.metricsScope.Counter(metrics.ActivityPollTransientFailedCounter).Inc(1)
+
+			if target := (*s.ServiceBusyError)(nil); errors.As(err, &target) {
+				atp.metricsScope.Tagged(map[string]string{causeTag: serviceBusy}).Counter(metrics.ActivityPollTransientFailedCounter).Inc(1)
+			} else {
+				atp.metricsScope.Counter(metrics.ActivityPollTransientFailedCounter).Inc(1)
+			}
 		} else {
 			atp.metricsScope.Counter(metrics.ActivityPollFailedCounter).Inc(1)
 		}