llm-proxy: annotate spans of sync jobs with whether they were skipped (#52433)

Makes it easier to tell whether a span is _supposed_ to have no child
spans, or if GCP is just a bit slow.

## Test plan

n/a

Author: bobheadxi

enterprise/cmd/llm-proxy/internal/actor/source.go

@@ -159,7 +159,9 @@ func (s *sourcesSyncHandler) Handle(ctx context.Context) (err error) {
 		if err := s.rmux.LockContext(ctx); errors.HasType(err, &redsync.ErrTaken{}) {
 			msg := fmt.Sprintf("did not acquire lock, another worker is likely active: %s", err.Error())
 			handleLogger.Debug(msg)
-			trace.SpanFromContext(ctx).AddEvent(msg)
+			trace.SpanFromContext(ctx).SetAttributes(
+				attribute.Bool("skipped", true),
+				attribute.String("reason", msg))
 
 			return nil // ignore lock contention errors
 		} else if err != nil {
@@ -171,6 +173,9 @@ func (s *sourcesSyncHandler) Handle(ctx context.Context) (err error) {
 		_, _ = s.rmux.ExtendContext(ctx)
 	}
 
+	// Annotate span to indicate we're actually doing work today!
+	trace.SpanFromContext(ctx).SetAttributes(attribute.Bool("skipped", false))
+
 	p := pool.New().WithErrors().WithContext(ctx)
 	for _, src := range s.sources {
 		if src, ok := src.(SourceSyncer); ok {