feat(hatchery): timeout of 10 minutes to start a worker (#3161)

fsamin · yesnault · commit 0a8e85bada97 · 2018-08-07T17:00:51.000+02:00
diff --git a/sdk/hatchery/hatchery.go b/sdk/hatchery/hatchery.go
@@ -214,7 +214,8 @@ func Create(h Interface) error {
 				continue
 			}
 
-			currentCtx := context.WithValue(ctx, tracing.TagWorkflowNodeJobRun, j.ID)
+			var traceEnded *struct{}
+			currentCtx, currentCancel := context.WithTimeout(ctx, 10*time.Minute)
 			if val, has := j.Header.Get(tracingutils.SampledHeader); has && val == "1" {
 				currentCtx, _ = tracing.New(currentCtx, h.ServiceName(), "hatchery.JobReceive", trace.AlwaysSample(), trace.SpanKindServer)
 
@@ -236,8 +237,16 @@ func Create(h Interface) error {
 					)
 				}
 				tracing.End(currentCtx, nil, nil) // nolint
+				var T struct{}
+				traceEnded = &T
+				currentCancel()
 			}
-
+			go func() {
+				<-currentCtx.Done()
+				if traceEnded == nil {
+					endTrace(currentCtx.Err().Error())
+				}
+			}()
 			//Check if the jobs is concerned by a pending worker creation
 			if _, exist := spawnIDs.Get(strconv.FormatInt(j.ID, 10)); exist {
 				log.Debug("job %d already spawned in previous routine", j.ID)
@@ -297,7 +306,7 @@ func Create(h Interface) error {
 					isRun:        false,
 					temptToSpawn: true,
 				}
-				endTrace("no model")
+				endTrace("no model or service ratio reached")
 
 				continue
 			}

Original file line number	Diff line number	Diff line change
`@@ -214,7 +214,8 @@ func Create(h Interface) error {`
`214`	`214`	`continue`
`215`	`215`	`}`
`216`	`216`
`217`		`- currentCtx := context.WithValue(ctx, tracing.TagWorkflowNodeJobRun, j.ID)`
	`217`	`+ var traceEnded *struct{}`
	`218`	`+ currentCtx, currentCancel := context.WithTimeout(ctx, 10*time.Minute)`
`218`	`219`	`if val, has := j.Header.Get(tracingutils.SampledHeader); has && val == "1" {`
`219`	`220`	`currentCtx, _ = tracing.New(currentCtx, h.ServiceName(), "hatchery.JobReceive", trace.AlwaysSample(), trace.SpanKindServer)`
`220`	`221`
`@@ -236,8 +237,16 @@ func Create(h Interface) error {`
`236`	`237`	`)`
`237`	`238`	`}`
`238`	`239`	`tracing.End(currentCtx, nil, nil) // nolint`
	`240`	`+ var T struct{}`
	`241`	`+ traceEnded = &T`
	`242`	`+ currentCancel()`
`239`	`243`	`}`
`240`		`-`
	`244`	`+ go func() {`
	`245`	`+ <-currentCtx.Done()`
	`246`	`+ if traceEnded == nil {`
	`247`	`+ endTrace(currentCtx.Err().Error())`
	`248`	`+ }`
	`249`	`+ }()`
`241`	`250`	`//Check if the jobs is concerned by a pending worker creation`
`242`	`251`	`if _, exist := spawnIDs.Get(strconv.FormatInt(j.ID, 10)); exist {`
`243`	`252`	`log.Debug("job %d already spawned in previous routine", j.ID)`
`@@ -297,7 +306,7 @@ func Create(h Interface) error {`
`297`	`306`	`isRun: false,`
`298`	`307`	`temptToSpawn: true,`
`299`	`308`	`}`
`300`		`- endTrace("no model")`
	`309`	`+ endTrace("no model or service ratio reached")`
`301`	`310`
`302`	`311`	`continue`
`303`	`312`	`}`