Create new poller thread pool usage metrics (#1275)

Create new poller thread pool usage metrics. ConcurrentTaskQuota is the number of buffer in poller thread channel and ConcurrentTaskRunning is current utilization of that channel.
If ConcurrentTaskRunning is exceeding ConcurrentTaskQuota, polling will halt and result in under utilization and backlog as a result.

Author: timl3136

internal/common/metrics/constants.go

@@ -114,4 +114,7 @@ const (
 	MemoryUsedHeap  = CadenceMetricsPrefix + "memory-used-heap"
 	MemoryUsedStack = CadenceMetricsPrefix + "memory-used-stack"
 	NumGoRoutines   = CadenceMetricsPrefix + "num-go-routines"
+
+	ConcurrentTaskQuota   = CadenceMetricsPrefix + "concurrent-task-quota"
+	ConcurrentTaskRunning = CadenceMetricsPrefix + "concurrent-task-running"
 )

internal/internal_worker_base.go

@@ -252,6 +252,8 @@ func (bw *baseWorker) runPoller() {
 		case <-bw.shutdownCh:
 			return
 		case <-bw.pollerRequestCh:
+			bw.metricsScope.Gauge(metrics.ConcurrentTaskQuota).Update(float64(cap(bw.pollerRequestCh)))
+			bw.metricsScope.Gauge(metrics.ConcurrentTaskRunning).Update(float64(cap(bw.pollerRequestCh) - len(bw.pollerRequestCh)))
 			if bw.sessionTokenBucket != nil {
 				bw.sessionTokenBucket.waitForAvailableToken()
 			}