studio/flamenco
studio/flamenco
12
30
Fork 38
Code Issues 63 Pull Requests 5 Releases Wiki Activity

Check for number of workers before soft failing the task. #104195

Merged
Sybren A. Stüvel merged 15 commits from Nitin-Rawat-1/flamenco:104190-job-stuck into main 2023-04-20 11:53:43 +02:00
Conversation 2 Commits 15 Files Changed 2 +18 -10
1 changed files with 18 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 6e24e0be3b - Show all commits

28
internal/manager/api_impl/worker_task_updates.go
View File

@ -187,17 +187,25 @@ func (f *Flamenco) onTaskFailed(
Int("failedByWorkerCount", numFailed). Int("failedByWorkerCount", numFailed).
Int("threshold", threshold). Int("threshold", threshold).
Logger() Logger()
if numFailed < threshold {
numWorkers, err := f.numWorkersCapableOfRunningTask(ctx, task) if numFailed > threshold {
if err != nil { return f.hardFailTask(ctx, logger, worker, task, numFailed)
return err
}
if numWorkers == 1 {
return f.failJobAfterCatastroficTaskFailure(ctx, logger, worker, task)
}
return f.softFailTask(ctx, logger, worker, task, numFailed)
} }
return f.hardFailTask(ctx, logger, worker, task, numFailed)
numWorkers, err := f.numWorkersCapableOfRunningTask(ctx, task)
if err != nil {
return err
}
// If number of workers capable of running the failed task again is "1",
// that means we have no worker besides the one that actually failed the task.
// Because at this point in code the worker hasn't been registered as failing this task yet,
// and thus it is still counted.
// In such condition we should just fail the job itself.
if numWorkers <= 1 {
return f.failJobAfterCatastroficTaskFailure(ctx, logger, worker, task)
}
return f.softFailTask(ctx, logger, worker, task, numFailed)
} }
// maybeBlocklistWorker potentially block-lists the Worker, and checks whether // maybeBlocklistWorker potentially block-lists the Worker, and checks whether