Check for number of workers before soft failing the task. #104195
@ -318,3 +318,68 @@ func TestBlockingAfterFailure(t *testing.T) {
|
||||
assertResponseNoContent(t, echoCtx)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobFailureAfterWorkerTaskFailure(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
mf := newMockedFlamenco(mockCtrl)
|
||||
worker := testWorker()
|
||||
|
||||
// Contruct the JSON request object
|
||||
taskUpdate := api.TaskUpdateJSONRequestBody{
|
||||
TaskStatus: ptr(api.TaskStatusFailed),
|
||||
}
|
||||
|
||||
// Construct the task that's supposed to be updated.
|
||||
taskID := "181eab68-1123-4790-93b1-94309a899411"
|
||||
jobID := "e4719398-7cfa-4877-9bab-97c2d6c158b5"
|
||||
mockJob := persistence.Job{UUID: jobID}
|
||||
mockTask := persistence.Task{
|
||||
UUID: taskID,
|
||||
Worker: &worker,
|
||||
WorkerID: &worker.ID,
|
||||
Job: &mockJob,
|
||||
Activity: "pre-update activity",
|
||||
Type: "misc",
|
||||
}
|
||||
|
||||
conf := config.Conf{
|
||||
Base: config.Base{
|
||||
TaskFailAfterSoftFailCount: 3,
|
||||
BlocklistThreshold: 65535, // This test doesn't cover blocklisting.
|
||||
},
|
||||
}
|
||||
|
||||
mf.config.EXPECT().Get().Return(&conf).AnyTimes()
|
||||
|
||||
mf.persistence.EXPECT().FetchTask(gomock.Any(), taskID).Return(&mockTask, nil)
|
||||
|
||||
mf.persistence.EXPECT().TaskTouchedByWorker(gomock.Any(), &mockTask)
|
||||
mf.persistence.EXPECT().WorkerSeen(gomock.Any(), &worker)
|
||||
|
||||
mf.persistence.EXPECT().CountTaskFailuresOfWorker(gomock.Any(), &mockJob, &worker, "misc").Return(0, nil)
|
||||
|
||||
mf.persistence.EXPECT().AddWorkerToTaskFailedList(gomock.Any(), &mockTask, &worker).Return(1, nil)
|
||||
|
||||
mf.persistence.EXPECT().WorkersLeftToRun(gomock.Any(), &mockJob, "misc").
|
||||
Return(map[string]bool{"60453eec-5a26-43e9-9da2-d00506d492cc": true}, nil)
|
||||
mf.persistence.EXPECT().FetchTaskFailureList(gomock.Any(), &mockTask).
|
||||
Return([]*persistence.Worker{ /* It shouldn't matter whether the failing worker is here or not. */ }, nil)
|
||||
|
||||
// Expect hard failure of the task, because there are no workers left to perfom it.
|
||||
mf.stateMachine.EXPECT().TaskStatusChange(gomock.Any(), &mockTask, api.TaskStatusFailed)
|
||||
mf.logStorage.EXPECT().WriteTimestamped(gomock.Any(), jobID, taskID,
|
||||
"Task failed by worker дрон (e7632d62-c3b8-4af0-9e78-01752928952c), Manager will fail the entire job "+
|
||||
"as there are no more workers left for tasks of type \"misc\".")
|
||||
|
||||
// Expect failure of the job.
|
||||
mf.stateMachine.EXPECT().
|
||||
JobStatusChange(gomock.Any(), &mockJob, api.JobStatusFailed, "no more workers left to run tasks of type \"misc\"")
|
||||
|
||||
echoCtx := mf.prepareMockedJSONRequest(taskUpdate)
|
||||
requestWorkerStore(echoCtx, &worker)
|
||||
err := mf.flamenco.TaskUpdate(echoCtx, taskID)
|
||||
assert.NoError(t, err)
|
||||
assertResponseNoContent(t, echoCtx)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user