[jobmanager] Recover from job panics

Signed-off-by: Dmitrii Okunev <xaionaro@meta.com>

Author: xaionaro

pkg/jobmanager/jobmanager.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/facebookincubator/go-belt/beltctx"
+	"github.com/facebookincubator/go-belt/tool/experimental/errmon"
 	"github.com/insomniacslk/xjson"
 
 	"github.com/linuxboot/contest/pkg/api"
@@ -183,6 +184,9 @@ loop:
 			logging.Debugf(ev.Context, "Handling event %+v", ev)
 			handlerWg.Add(1)
 			go func() {
+				defer func() {
+					errmon.ObserveRecoverCtx(ev.Context, recover())
+				}()
 				defer handlerWg.Done()
 				jm.handleEvent(ev)
 			}()

pkg/jobmanager/start.go

@@ -102,6 +102,9 @@ func (jm *JobManager) startJob(ctx context.Context, j *job.Job, resumeState *job
 }
 
 func (jm *JobManager) runJob(ctx context.Context, j *job.Job, resumeState *job.PauseEventPayload) {
+	defer func() {
+		errmon.ObserveRecoverCtx(ctx, recover())
+	}()
 	defer func() {
 		jm.jobsMu.Lock()
 		delete(jm.jobs, j.ID)