Do not retry on unrecoverable errors.

For error caused by unexpected metric name suffix or by unexpected
metric type, it should return as unrecoverable error because retrying
doesn't help the situation.

Author: Yen-Cheng Chou

retrieval/series_cache.go

@@ -43,6 +43,10 @@ var (
 	keyReason, _ = tag.NewKey("reason")
 )
 
+type unrecoverableError struct {
+	error
+}
+
 func init() {
 	if err := view.Register(&view.View{
 		Name:        "prometheus_sidecar/dropped_series",
@@ -447,14 +451,14 @@ func (c *seriesCache) refresh(ctx context.Context, ref uint64) error {
 			ts.MetricKind = metric_pb.MetricDescriptor_GAUGE
 			ts.ValueType = metric_pb.MetricDescriptor_DOUBLE
 		default:
-			return errors.Errorf("unexpected metric name suffix %q", suffix)
+			return unrecoverableError{errors.Errorf("unexpected metric name suffix %q", suffix)}
 		}
 	case textparse.MetricTypeHistogram:
 		ts.Metric.Type = c.getMetricType(c.metricsPrefix, baseMetricName)
 		ts.MetricKind = metric_pb.MetricDescriptor_CUMULATIVE
 		ts.ValueType = metric_pb.MetricDescriptor_DISTRIBUTION
 	default:
-		return errors.Errorf("unexpected metric type %s", metadata.Type)
+		return unrecoverableError{errors.Errorf("unexpected metric type %s", metadata.Type)}
 	}
 
 	entry.proto = ts

retrieval/transform.go

@@ -143,6 +143,9 @@ func (b *sampleBuilder) getSeriesWithRetry(ctx context.Context, sample tsdb.RefS
 		if err == nil {
 			break
 		}
+		if _, ok := err.(unrecoverableError); ok {
+			return nil, false, err
+		}
 		level.Warn(b.logger).Log("msg", "failed to get seriesCacheEntry", "err", err)
 		backoff = exponential(backoff)
 		if backoff > 0 {