From ec540e47a7095541beddf9ee735cfb12b2c565c6 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Thu, 30 May 2024 21:17:26 +0800 Subject: [PATCH 01/25] Manager: draft task and job status transition; regenerated openapi with pause-requested status; start frontend work --- addon/flamenco/manager/docs/JobStatus.md | 2 +- addon/flamenco/manager/model/job_status.py | 9 +- go.mod | 1 + go.sum | 2 + .../task_state_machine/task_state_machine.go | 73 ++- pkg/api/flamenco-openapi.yaml | 1 + pkg/api/openapi_spec.gen.go | 454 +++++++++--------- pkg/api/openapi_types.gen.go | 2 + web/app/src/components/jobs/JobActionsBar.vue | 3 + web/app/src/manager-api/model/JobStatus.js | 7 + web/app/src/stores/tasks.js | 3 + 11 files changed, 323 insertions(+), 234 deletions(-) diff --git a/addon/flamenco/manager/docs/JobStatus.md b/addon/flamenco/manager/docs/JobStatus.md index c7e1da07..2c1d142b 100644 --- a/addon/flamenco/manager/docs/JobStatus.md +++ b/addon/flamenco/manager/docs/JobStatus.md @@ -4,7 +4,7 @@ ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**value** | **str** | | must be one of ["active", "canceled", "completed", "failed", "paused", "queued", "cancel-requested", "requeueing", "under-construction", ] +**value** | **str** | | must be one of ["active", "canceled", "completed", "failed", "paused", "pause-requested", "queued", "cancel-requested", "requeueing", "under-construction", ] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/addon/flamenco/manager/model/job_status.py b/addon/flamenco/manager/model/job_status.py index f1e535fe..bcde8127 100644 --- a/addon/flamenco/manager/model/job_status.py +++ b/addon/flamenco/manager/model/job_status.py @@ -57,6 +57,7 @@ class JobStatus(ModelSimple): 'COMPLETED': "completed", 'FAILED': "failed", 'PAUSED': "paused", + 'PAUSE-REQUESTED': "pause-requested", 'QUEUED': "queued", 'CANCEL-REQUESTED': "cancel-requested", 'REQUEUEING': "requeueing", @@ -112,10 +113,10 @@ class JobStatus(ModelSimple): Note that value can be passed either in args or in kwargs, but not in both. Args: - args[0] (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 + args[0] (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "pause-requested", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 Keyword Args: - value (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 + value (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "pause-requested", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 _check_type (bool): if True, values for parameters in openapi_types will be type checked and a TypeError will be raised if the wrong type is input. @@ -202,10 +203,10 @@ class JobStatus(ModelSimple): Note that value can be passed either in args or in kwargs, but not in both. Args: - args[0] (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 + args[0] (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "pause-requested", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 Keyword Args: - value (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 + value (str):, must be one of ["active", "canceled", "completed", "failed", "paused", "pause-requested", "queued", "cancel-requested", "requeueing", "under-construction", ] # noqa: E501 _check_type (bool): if True, values for parameters in openapi_types will be type checked and a TypeError will be raised if the wrong type is input. diff --git a/go.mod b/go.mod index b621f8cc..f6335774 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect + golang.org/toolchain v0.0.1-go1.9rc2.windows-amd64 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 17976c6d..e7ce247b 100644 --- a/go.sum +++ b/go.sum @@ -190,6 +190,8 @@ github.com/ziflex/lecho/v3 v3.1.0 h1:65bSzSc0yw7EEhi44lMnkOI877ZzbE7tGDWfYCQXZwI github.com/ziflex/lecho/v3 v3.1.0/go.mod h1:dwQ6xCAKmSBHhwZ6XmiAiDptD7iklVkW7xQYGUncX0Q= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +golang.org/toolchain v0.0.1-go1.9rc2.windows-amd64 h1:1f9RozPx9d/MkNM8NMgJDmTj6WNwWPixB1qIWVz5ORc= +golang.org/toolchain v0.0.1-go1.9rc2.windows-amd64/go.mod h1:8wlg68NqwW7eMnI1aABk/C2pDYXj8mrMY4TyRfiLeS0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 2bdb673e..085837aa 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -119,8 +119,7 @@ func (sm *StateMachine) updateJobAfterTaskStatusChange( return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusCompleted, api.JobStatusRequeueing, "task was queued") case api.TaskStatusPaused: - // Pausing a task has no impact on the job. - return nil + return sm.updateJobOnTaskStatusPaused(ctx, logger, job) case api.TaskStatusCanceled: return sm.updateJobOnTaskStatusCanceled(ctx, logger, job) @@ -183,6 +182,38 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge return nil } +// updateJobOnTaskStatusPaused conditionally escalates the pausing of a task to pause the job. +func (sm *StateMachine) updateJobOnTaskStatusPaused(ctx context.Context, logger zerolog.Logger, job *persistence.Job) error { + // If no more tasks can run, pause the job. + numRunnable, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed) + if err != nil { + return err + } + if numRunnable == 0 { + logger.Info().Msg("paused task was last runnable task of job, pausing job") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "paused task was last runnable task of job, pausing job") + } + + if job.Status == api.JobStatusPauseRequested { + // if the job is in pause-requested state, and all other tasks are paused, + // then the job can be paused. + numPaused, numTotal, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusPaused) + if err != nil { + return err + } + if numPaused == numTotal { + logger.Info().Msg("all tasks of job are paused, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks paused") + } + } else { + // if the job is not in pause-requested state, then some error occurred and the job should be failed. + logger.Info().Msg("task cannot be changed to paused when job is not in pause-requested state") + } + + return nil +} + // updateJobOnTaskStatusFailed conditionally escalates the failure of a task to fail the entire job. func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger zerolog.Logger, job *persistence.Job) error { // Count the number of failed tasks. If it is over the threshold, fail the job. @@ -385,6 +416,13 @@ func (sm *StateMachine) updateTasksAfterJobStatusChange( massTaskUpdate: true, }, err + case api.JobStatusPauseRequested: + jobStatus, err := sm.pauseTasks(ctx, logger, job) + return tasksUpdateResult{ + followingJobStatus: jobStatus, + massTaskUpdate: true, + }, err + case api.JobStatusRequeueing: jobStatus, err := sm.requeueTasks(ctx, logger, job, oldJobStatus) return tasksUpdateResult{ @@ -438,6 +476,37 @@ func (sm *StateMachine) cancelTasks( return "", nil } +func (sm *StateMachine) pauseTasks( + ctx context.Context, logger zerolog.Logger, job *persistence.Job, +) (api.JobStatus, error) { + logger.Info().Msg("pausing tasks of job") + + // Any task that might run in the future should get paused. + // Active jobs should remain active until finished + taskStatusesToPause := []api.TaskStatus{ + api.TaskStatusActive, + api.TaskStatusQueued, + api.TaskStatusCanceled, + } + err := sm.persist.UpdateJobsTaskStatusesConditional( + ctx, job, taskStatusesToPause, api.TaskStatusPaused, + fmt.Sprintf("Manager paused this task because the job got status %q.", job.Status), + ) + if err != nil { + return "", fmt.Errorf("pausing tasks of job %s: %w", job.UUID, err) + } + + // If pause was requested, it has now happened, so the job can transition. + if job.Status == api.JobStatusPauseRequested { + logger.Info().Msg("all tasks of job paused, job can go to 'paused' status") + return api.JobStatusPaused, nil + } + + // This could mean pause was triggered by failure of the job, in which case the + // job is already in the correct status. + return "", nil +} + // requeueTasks re-queues all tasks of the job. // // This function assumes that the current job status is "requeueing". diff --git a/pkg/api/flamenco-openapi.yaml b/pkg/api/flamenco-openapi.yaml index 8f19c6bd..491dbd25 100644 --- a/pkg/api/flamenco-openapi.yaml +++ b/pkg/api/flamenco-openapi.yaml @@ -1680,6 +1680,7 @@ components: - completed - failed - paused + - pause-requested - queued - cancel-requested - requeueing diff --git a/pkg/api/openapi_spec.gen.go b/pkg/api/openapi_spec.gen.go index 671be5ea..2de53300 100644 --- a/pkg/api/openapi_spec.gen.go +++ b/pkg/api/openapi_spec.gen.go @@ -19,233 +19,233 @@ import ( var swaggerSpec = []string{ "H4sIAAAAAAAC/+y923LcOJYo+iuInBPhqpjMlCz5Ula/HLcvVaq2yxpL7jonWhVKJInMhEUCbAJUOtvh", - "iPmI8ydnT8R+2PO0f6Dmj3ZgLQAESTAvsiWr3NMP1RaTxGVhYd0vHweJzAspmNBqcPRxoJIFyyn886lS", - "fC5YekbVpfk7ZSopeaG5FIOjxq+EK0KJNv+iinBt/i5ZwvgVS8l0RfSCkV9lecnK8WA4KEpZsFJzBrMk", - "Ms+pSOHfXLMc/vF/lWw2OBr8y169uD27sr1n+MHg03CgVwUbHA1oWdKV+fu9nJqv7WOlSy7m9vlFUXJZ", - "cr0KXuBCszkr3Rv4NPK5oHn8h/VjKk11tXE7Bn6n+KbZEVWX/QupKp6aH2ayzKkeHOGDYfvFT8NByf5e", - "8ZKlg6O/uZcMcOxe/NqCLbSgFIAkXNWwPq/f/Lxy+p4l2izw6RXlGZ1m7Gc5PWVam+V0MOeUi3nGiMLf", - "iZwRSn6WU2JGUxEEWUie4D+b4/y6YILM+RUTQ5LxnGvAsyua8dT8t2KKaGmeKUbsIGPyRmQrUimzRrLk", - "ekEQaDC5mdujYAf4bWRL2YxWme6u62zBiP0R10HUQi6FXQypFCvJ0qw9ZZqVORcw/4IrB5IxDh+MGZ/C", - "P9nTUmaaF3YiLuqJDD6WM5owGJSlXJut44h2/TOaKTbsAlcvWGkWTbNMLon5tL1QQmfavLNg5L2ckgVV", - "ZMqYIKqa5lxrlo7Jr7LKUsLzIluRlGUMP8sywj5whQNSdanITJY49Hs5HRIqUkNAZF7wzLzD9fhc1Ig+", - "lTJjVMCOrmjWhc/JSi+kIOxDUTKluATgTxkxb1dUs9TASJYpbtCdA4OdNI/Or8ufzbCLGmbYYzGT3YW8", - "ZpqOUqqpHYiRe+ble8HSuhjfOXp7UINB+5Se13+Ze7RcUB2fxFDkVJr1k2MgzzRT0mBIaih2kdGELWQG", - "8GAftAGKQSVEUzNgTkVFM8JFUWky48ycqSILnqZMkO+mLKGVQvCOpBjh+df4oOV8nrGUSOG4gcHN7xtn", - "WkPTzPyKi8s/V1q3IBBF1RfCoLSqN27mwSXcs1OTKYxFpmxBr7gsu8dKnrZeXfIsMyjjr9SfMyZSVt5T", - "OLYFq79eBMhRvdMhrGdi1jMJDwLGbWKcXcM9hTg3Jq8B2tkquHQ1veSwU0GEJJkUc1aSQirFpxnDe8OF", - "0oymQFdFeGK4onsB8O456mcAYfY5PhdPzbWheZHBIdnZiJajKRuVAAGWkllJc0ZKKuZsSJYLnizMwbqb", - "Qystc6p5AnuYSUM/cBiVMOG/m1aaJNQcCpFXrCwRmXK3d0silWFj8dvf4nMtvGmiSYxbXbJV98Yep0xo", - "PuOs9FfWQn5I8kpps9xK8L9XyD8srX1v+VeUPGR0yiJE6pV5DJOkXBUZXXX4ADmeESE1UQVLzJLsEV6y", - "lTkXuL1akjkTrKSaEUpKRpWE60Bg0jFKKbKg5TzCQZ+KFWEfdEkJLedVbuQSx6WmxWpsPlTjU5mzE6RP", - "q+++J+ZQ/dRJyczEsGhLw1YBCGpQ1+e0A+Phec5STjXLVqRkZihCAdIpm3HBzQdDg+YwvZlyCEciK21X", - "REvNkyqjpYdoDxdR1dQJXetktYh4c2q/9ALCziOc2c+vOFzia4zwV/Mlz4zY1r4TBsXtyraU105rULTE", - "tmo6Mr8gxBHlPaI+q8qSCZ2tiDQCFnXjAnoHIpYak8lPT09/evH84uXxqxcXJ0/Pfpqg+pDykiValitS", - "UL0g/0om54O9f4H/nQ8mhBaFoT6WFDBR5WZ/M56xC/O+ue68dP+Ex1bUXVC1YOlF/eZvkSvady5dyctC", - "INh9QBdQrqSKHD93Vwa2HfCPMflFEsGUEUKULqtEVyVT5DuQK9WQpDwxU9GSM/U9oSUjqioKWer21u3i", - "h0blODwwm84k1YMh4PW2mwxQpyFpOGQcxmRuJx00adXEfjM5IjRb0hWylDGZ1OxycoToAV9byvnuGDUA", - "AKiVG0vyXcYvDUGzQCM0TUdSfD8mkyWbxoZZsmnNjAHrcironBmihqzGEFLgKXYWx1ffy+mYTFCUmRwR", - "wa5YCUP/qY3LljSalaJoal4E4IDaa2YXNGvSGndaNUBxpgEQHQuXwXCwZNONZxbHSKc61XiCQhZXRo6g", - "c1ZauUADRaS5kT3UFlLnZyscMUlZ04hG+BNVi5CsACc1zK9FZxSxHBmYG0kWKEjAXs3IKFzh4zE5M48d", - "n5SixjCvETChqtKwLys2e72lOam5hFUBmgLVrEdq9Ux+e/OBm2Br00dMve5opi0OYKkgLi+Y057FJq5g", - "cC4iObziSjsyCHS9H/u6mOYsC9fb+FmD3fbsup4itkFLVU6oXjxbsOTyLVNWk2+ZHoxW0918R+taOXlD", - "LwzCfSek/t4yg+gtAKE8fslQXgeMXFKF5g2DeTMuUpzF8ZHowOoCp41aS1CuWjC/UMuvZGmI4zgqGQHH", - "jK4UBvELnclKpNE1KVmVyUaxJjiSU/ygfaQINLsiP2y456E9sA1H/pKLtD7xrfCvB2EiVqHuPo4+NqUV", - "qpRMONVI981uLpi4uqLlwCJGv5TiTJ+d87A/kJIZPRPkeEoU2tmswQ7o3QeWVJptMsn22zs9+wh+djCO", - "053gk9ixvChLWXb386NRaXhCmPmZlEwVUigWMx6nEVT/6ezshKCFk5g3vI7gByLHhl8nWZWiKQgvxSqT", - "NCVKIlZ7AOJqG7DNMrs0LtAWy6XRnZ+ZyR7uH3qu4+0nKdV0SlGfnlZqZbgTI7BQtyjLvKTQlAtCyb23", - "TJer0dOZZuU9fHXBKJhozPK4SHlCNVPWCIdauOY52hTMUTDlFeyS6ZKzdExegjbuZB87IFcgHRk0oUYC", - "dwLDPWX5nnk3yTgTYBpKJVEyZ0b5nTdUTiOzsQ94eTjNyJQml3I2Q47pjdZOXu1azHOmFJ3HcK+FXHDu", - "9ftRzLpiQr+kZX66lRm+fvMtM3zMD/GznL4rDN+PakSKaW/AHhKDHWDLIKcyuWT6+M3e6387O0M0QBEX", - "hRNlDqIkgi3NQzUkk6JkV1xW6gLxduLtT+wDoikCsS2yZUyzC3vWLL2gEa5yPLM6c8aAYxlq7b+wwpOz", - "8vCcKU3zghiqjghlcM0hk/lUaVmiPPUyozkTifSMvnnMBmYjM2KUUUWI2Lt3x8+dFPgzOCs2+Dlq0ao5", - "0C80D7XU2IctcG/CDiNveR9N6PXxGtPD/RhCl2xWMrW4ABt35Gj8HfYiqL1lagF2c/s9EBy7m3sKLea1", - "fAtYhxqPMhfWAF4NDdKB3JpSUHUYTRZANK54WtEMvXVLmMUbkLSUhgis3CDWal6UNAFrXq/5ZHcg9vu4", - "YOoIepx55JQzklGl7Sq3xrklVRd4Y9IeZxJeUYPl741Gb1+u74i57VqSiS4rNrEKiv2lttCB0giWVp7e", - "q23liumhpczmJrnbnRd6tZV1Ey6AA07gwLNuucBx10S6Xtr4iir91hp0+yicRVBZ1ghqIF8bgnlO5zV/", - "ddCzy4xL/lu5MIcDvajyqaA82wKtwq0cmxWBMyamE+BcVF3af/lJ+sHEZ+zZKomJ1J4AZnzGRol5ibAr", - "MDhY/4LRHoErqkWFFodULsXQCCcl/FkVQ8J0EiPu25gT/eJgqagZtXbda/vDT6i6fCXnfecPzv1Mzkmy", - "qMSlZXBaEkqAr2lZ8GTP8TpSSpmTlCFNS/E9K0MZkA/hyZXkqRknBRmkRXBicMhkxGLwzKzH0XhtVzkm", - "r+nKS1B5lWlegFgimIJ32QcdVVEcQqxlSRAGMdzR916jmtnG2mPYRso4AzBuEDMAHB05A6jBdQUNQ/+v", - "moEO2/Py7QA33IU4bOb7Gif9XMbfjM64zjc3xc9i7MFTOKt8RdiFP8leXESt8Iz2EgV8gZzR+QZU5Nqj", - "YYy+oSVwHST9UrZl32AD3JJ9b2a5ffaxAEzbXFp8c+O1XSJY10AsoeLCSA+01OvsO1zZKUH5o5WWI/tV", - "3MRj4RRVHpyMifZ2pmuN1i7XQNsOMP5i0j8ufxuaYe7NhWJMxNyrSjt9mKtwveZ9ZwMJjJTbrX0z6Vm6", - "1X8u8UEw7Ep+4l9dIF7t8vEz+OIt6n43K5pfsVJZv8MWZK6furlxho27ErvDTcuAM9ABdQSjYgr2xCWF", - "+AtDN1XGWAEmOnMlqX2vEpdCLgWuAUS6qOGuY10wc2KUBQRd2oXgtJ/a917taMHoRkbg4ygcrAz71/oE", - "goXNOTgDD8cHo8ePRvM0PXyQPjz8wZ3B0eD/lVXp7tAAQndK7Q9zcDg+HNGsWND94GzCx+S7ztjfd/cP", - "q9jJsdJYxse1+NbEZAsGr9F4D1rOqNWyF1VOhZEyVZXDZyhjlSxjVDEyrXiWuiBYcCoZ0kAVmYSrmqCK", - "IIFk159AVJY1TOLXkznXE2K/AnNj1P/UOvD6HjRA4a+OgWgMG37GAFqaZW9mg6O/rUe4U+ctM199Gn5c", - "IzOu9Z84rZK4L4gUXp+MyusYdhKzg5sfwLnnKNLWJOif3pZ2DSPOzgxh/BnCrTv0DWLtp98Qj/+cyeQy", - "40r3Oy+RUVvjGy0ZGMEh2pWlJGElqJGgTaGLUxoxzVp6EoecW/mPwvW8ELpcxVxH3Zc6Dsn14eG4n211", - "KPt2DxFtnUA9dBgN3kNCntvrEQ+JNU8JncpKY7yq0z+tFOkkTGtO4g3xssUXFzSn4iJZsORSVnq9z/MU", - "Xibu5SDcyC2gZLm8YimhmRRzDA538SHbBB8219IDmrilqrPwF0JW80XoXQJ2QQMnTMFZwoiWc9xiymcz", - "VoLpGE4QbLfma0LJQoLJLgOhhbx7+8q5dCK2vDE5k8DcIDQJI3TevhqaRwnVTFDNyPng45Qq9mnvoxRe", - "6lXVbMY/MPXpfBDTXcwHTbQssygVssM0XLMbYvFbRwFTBSP1HMVrqpTD1FOWsSQe+XLiHZgYKm5+mzJL", - "0d/LqXK2+hqFDboEQhToKJZmXeT0w+BocLB/cDjafzTav392//Do/oOj+w//df/gaH+/K/x0v+5EcWYZ", - "LgSd8axkIck1C5vJErz8jq/WvKl1+Xagz1GQMk1Tqimw/zSFCE2anUTMmg3G29hMOeW6pOWK5HYwh9Bj", - "8tpsw1DXjH0IY+esjzOXZhcQf1IpLuZkQsfTcTIxZL2+QzaAtnVGRSlhH0eD06LkmpGXJZ8vtGE2ipVj", - "loMheqBW05KJ/3tqQzBkOXdvWHn4FF4gp/p//68rlg164HRijfXPvE7WPPPQw5TTDzw32sn9/f3hIOcC", - "/4q4m1rXwA/Sg/+nQfRR/LB0WbGeb/s1p4SKxBwDpgoVaK8ZDmaU48OCVgr+8feKVfgafDHyctQA98Eq", - "hqpXZWA98jSpGc1d45FfVh9U0VMdD2bB34K0ABs9gKFkX0RciutkQ7esvlPSsuxlE/ZH4BM+itIF5HuR", - "0lyPSkH4IrI48xbyA5aSGc+YQqYrWMKUouUqRsBbDC5qLr/3zHHX4+f3gggIEN1czEGbEYeZP2PylBtN", - "SOBK3Scxpu3sUFZIcMx7Vsrcb71PVYoB+oyqS3Va5TktV7GctbzIwMFHMis9Yt6Sg/qYPEO/A0aHWGu7", - "izs1j9whgSPW/D6OmEStm3groRLszHbBW8TD9TJC9W8Vwz2HTIvnRut+OBzkAVHvI5OfhgPIprqYriDj", - "0LIrCEeujQ/WEsVFg2B4OmBJxG9dFohr+VhTv/vx6JHP5j4veaaNQl5zn6HjJa+O//KiZiXRJAc5mynW", - "XGg0KqAG1ccd8g3VlvS6b0dhSOsuuwpOrX0r3jJdlQKNwyCBgNBMHfXkVtyALeyiK7XDBAKk7kfgviBO", - "QP1t7xSaMq55lyLe2IBDYjx6OQJDYVUMhvWTRaVTuYyzNWsQeCbFjM+rkjoptblJrl7yUum3ldjgGeAK", - "pHuOIr8hoDPzYR04ZucjZSWCGBOfsAbiFSUztiQzakixGhIbqy+kGEFWp9FCknC9wGSMAOqUah9aPWUQ", - "m5IX2pB085ZesJUVqcU9TaasN+gE+Agm/6Vb6X6wCl1SoWasJE9PjiHxxIUWj3tCW4DFvpIJjesHzz1L", - "An5nuJm5aTCX/Xi80cDRnqW9u2F4wDHUs6f2V1pyF/7bRpALvZRLGuFtbwQbLemKXNmPMeAdsj6l0hA/", - "Ks0lt/mFkJLCIUGwZJA5mkMAkmG8k49GDv40sQomLzGj0YkkC0jiUc7j5UoH+CBn5ysbk7OljKwJzKN2", - "0rSTzOGlH2aXX2RUG21m5G02mNML4oIdZLryi+5DNPhos4nEmlZrQLsvtzivp1XKmWgGC1vrlFUw1Dri", - "4IZR61jfOrLXRp8OY3xNi8LAGE7ZHQoxW4ZEPe3T/zim8Ec2vPoLY8XbSohoUYA6FG4ZXFzrtMvpilwy", - "VhiiJJxQGBeh8s483QOtFYEeqb7h+YoRl1bgHm3qC7VJ2GucS4vXxz60DyTyBSOTpXe5sQmxviVMT6mz", - "hPH6mEkA3nNp/ivYB90IQkPH9pBMmkCYkNfvTs+MhjyBjMvJVvFmLUB6qPXBKIblPl7+2CU8tPRcm1yw", - "/mK1wuEjw996/sZXS7MATYilmzmKzZLYLjniLZsbtl2y1HreO5CkaVoypXYsj2Lpb/ymyZle0pKtuYY7", - "e7pdCtKFN1Gr3WTszyqwYhmAA1VYZMUBYjhIMFH2wsYneSj0rD52WqcsqUquVz53okUBtw2iXxc9f8p0", - "VTxViitNhUbhM5Z2Egp5cmpkO6eDg9xlRiF+mC61toa0F5CXQrfIfu5PxPlaglp3C1F4gjj3rNdTcYrB", - "QtYYY10PvCSnPz09ePgIr72q8iFR/B+QTTxdQZC3EchsjQSS2UW5hJau1aRl9ITZwM2L5GdQ59WP5xKF", - "0MHR4PDhdP/Bk/vJwePp/uHhYXp/Nn3wcJbsP/7hCb1/kND9R9P76aMH++nBw0dPHv+wP/1h/3HKHu4/", - "SB/vHzxh+2Yg/g82OLr/4OAB+IlxtkzO51zMw6keHU4fHySPDqdPHhw8mKX3D6dPDh/vz6aP9vcfPdn/", - "YT85pPcfPr7/OJkd0vTBg4NHhw+n9394nDyiPzx5uP/4ST3VweNPXUOCg8hJlNqap4H06BQhy6/DUgdu", - "HFdMxftWrF+lbeICGk6VV4rQ5xuGH5FjQbD+ivXVK+dXsWNhDJMLbTM/nPvtkOPn5wM0NjmV2wcM+Awg", - "iqsAXW1i7TgjlVXzPSjKMTLUaw8LW4yOn096slwtymypTePaX/KMnRYs2ahY4+DD5jFtvk0194/Zdc1v", - "aKVrnUqs0tQ10MO6pduIAYqzBX3tm9MLKqzXsxk5QFVjUHDL2Oxk6sqN1NeYnAXSxecj3xYBJVseiT/q", - "LoGzKhh1UhdFymtplV10QIfjkmLLkS/r8dCUUY/oPbHRCkM0ssImqQ3HjI4BdOZj19zGmjR6sNFRY1Zj", - "xxv2C7tNAP/K9aJ2wmwFaqeEJ85bGQX90IqpQ5KywkbpAx1xPpFv/Gy2lT2D4+jx73ROdbguDq8zXmAJ", - "qIMMqyKTNEV9DIOHomYBHOwtrgbK+rgozusKHiBoNGDXK0vckNBwKwLCLbC3/sNvnhcmBce5Gp4WiNmU", - "lMFnjqUMw6O0tgnZvO6svDJyx0uesSACChDNcBL7mnnmEkNquT5MyL4tHKgvpr8PN4MW4UT+un1hXAnI", - "9+diDVbTbBKOtpcYz39XnvulCOFaoley9HST5tZmJQo+qzkWTY1QbHW6IEKPWqsqOa/29w8eeXuwlc4q", - "ZTC/Y2jW0g4YmQuFKX8PrAB1TzXdHdEMqsDCu4Ml1huGPw0HWQCgHW0tt+AqaZ16VmvIfusNQ0hzTVHs", - "sFkyp9V0TWWiUybAiu+zEDFETkHI9Z4Kvp1gcqatFKelrRDlqGTwpvnxvZz6rETyzI2Jha3mTIe/o+oF", - "pl6qLn3ytPs7k3OFbi3BmK3DUWQ84TpbuWmnDKPIwbFifloN/UaMFoH5N+5dM4YUGPvwHVQA1M2pZy5j", - "972cfg+827xuXrmnIJ8TjNaa52x8LpyPT0iNppHpCtI7QSuxfIRqUpRSy0RmrlKShxb6ZhCYvtwzZDZN", - "SwmZT2bkZkxG83LIYiOVieDCG2cr37b4XmwQV03IWf76w6ix3IWWzWPYI5WoHxjKMN45SVQW62r0rd96", - "ICb6ZUDMVP1XVELsA0WEOFBNLrlIbU7E1jDwkWFZ9rOcQpB2lv3qnVq2MANVl5mc449hcGz4+hmdx91f", - "jQyEaGG02qIVFPfSssbGpgSzTazL54cE2h8Of///yH/9++//8ft//v4/fv+P//r33//n7//5+/8f5vJD", - "VYkw7gNmAa3naLCHgbt7arb3Xk4VmnHuHxyO4SUwo1Ti8gLlmsMAJ09++dGgaKEGR0asglquRtq5P7q/", - "j/USLyBRjS2Vr9EJscFYQ5F90EzYTJ5xYV1DZiUXstK+fFFjfTiFX+FefOe22GNnvFJKvXY8W8ETSwde", - "1JxwkHFRfQiuH3itR/aobOBzN+I2RIINsSI+4HXbKvEb6oWEZ70pRsa9Wtu+t4qsqcMJe6DWCQ9AWiPm", - "RK2UZnkd8G2/bVXagzDDRM4FV6wrXtmX65hpSjK5ZOUooYp5s6Wdwi3Khpic44GeD4bkfLDkIpVLhX+k", - "tFxygf+WBRNTlZo/mE7G5NRPJfOCau4rv/8o7ykyKSsBfPDHN29OJ38iZSXIBPyrMiMpVxri/SbEclnq", - "w/9c0WW/SDU+F0+Vkz9pRsyOho19kHMX83M+cMZBW8AebTMuHBuKKBYl5ENQRc4HTWnTjXc+qGGfS2Xk", - "CRBrLhnRTOm9lE2ruS1RqQijikMxSCuNuLhQ9F7zhKQygSLAkOiSZY2dRcsm9CWimAcX25d6HJJEFjxU", - "MCftgn9jM9rE1xjuFos8s3/VyRyGeLOUcOsfx0IsqWRK3NMkpzrB9A6a6IpmfqSOYf4MaxuD6KjaNSQB", - "j2SWBoF1zZL47TqhviS6K5FyLo4bC+SKyBz51LC2lUHZsFVBlWrVwu6k80SBbtPBNZ2jKGdvnysHV0ff", - "Bmn0x899aI6taWN5N6qPVBNfcHPKiCExaZXh9TdLQaMhhCdgdJcsg40Z7HLZVwYN3Rd+Jc30t62kKOt+", - "7dbDiRC5mJwVb3Ny5uqLYGMTiG9TToN25npX3W1I+JiNXcKFD5MJwqTGu5XW+JLNUW4iaRJDdi+mqwsX", - "rbRL8LINNoisdcsUth0qhkAajZaVwdMN+YoYnSZWvmSA+b+0Tp6xcUe7lQv4+r1jbipX05GeXU582/zO", - "dkGTWNuasDmNv0wb+tTYskcbExQhSU7aHjVBKaPPqmwV904YQgMG9lZRo2HD4t7FlKB20caZqzKLT/zu", - "7aswTbmenXCtWDbznky5FJmk6TYRSHXpI3+KmPMH++87lc/ILPKJBErO9KidcBTTH+sJ71LOUHirr5E0", - "FKaFdHXiSmnCutmlNbpjvrNsFFevyw6C+NvF/h3LNt0lYnjddPQtKZKbqe+k1lVew998iUcIvHeinLRU", - "GlUxxDxr5gZ7I1AsODEo44qiHja6MZK9Pz2w3ckCA4b/RKQ1kbRe4HMBlQq+A/lGuojriaO3toqYkJqw", - "ktrIVl/OoS21m2V9v6nMWDdGPePC9gWx0bcQSXFPkcQ3n8AAcx6mbwO5Jm+uWLksuWYoy3NZKShoJIKq", - "Ey7PNCo+xIrQvZJzW1zO0wCsc+ekYtezwiwaTgUmZLTMeE8Bb90ggTtQiShy1dGcUX2gZBCWkjDQCUF5", - "5wKj8nGciLN/XSDo51GBNZfMTRq7RPUet6taYoNGfd5cJ1GiuAj22JIMToj9rVOpaq1DZjuDSv9Ynx/Y", - "qmms/88ZRUrh+H5dOQw6suQsnyKebiXSN6q1dReA2tU2A6jL7UhucFQN11JQ/SYaU/vpt2Ekhb7LDh21", - "rdHs1Tb1RLqXZlflqI2j6z3EbvT+24Hx3YHHoLZ4W1u0fTLytcsiVlTFkpIBp5QjIfVIsywbUbGSgoWR", - "zEeDw/FBH+yP/uYCZo3kNssLNrftekZ1v5bBcJBzlUQyQa8Zam4X/vHL36y2fIYzNR2dsSksMvcf2Smf", - "izftw2oUALSWeXuAT0+Oof9KcBIXdcUttaTzOStHFb+hg2mVJuwmOPTX6uqs9uaPyRGS+Ml0VrTmlDLG", - "ilNr+4r4ps3P3jbmwhNQjXSZbqcGZuCiZSLFNEwv37g6Uj5tPKWrpp7mxzYEGxSlMXlaFBlntmYj5slL", - "8yEHu9UkpSt1IWcXS8YuJxDuB+80n5uXXW3qyApBJhTk4MFoIauS/PTT0evXdRYxNj6q0TYceXA0yCXR", - "FYE4CnATphcgdR8N7v9wtL+PSStW6bMpzYBX7q39J9E6Kc1JujGRNGEjxQpaYrTuUo4yBq2mXL0cC3Uo", - "0kxXyBcZu+wBM/nufJBL9Djoyjkbvh+TF2DtzBkVipwP2BUrV2Y8VxWn2xHJ7z8QnQCgPZlHDjQf44XY", - "PaA2D9fmsX7sYROajXGDFa+5F5pq1qdT24TyMkyv2z7NJ6oRB4Nttai0rwAjXdLLa1dg3GKhG5bXtHz4", - "kpJDu66gDCW0HzFHypR9Rc5mRhkB40C77mWNQP0FPiPZ/VipDslWrXjaJMc6JBiK6tpy0hHbgLrI6D9W", - "68OOmvmT1j+B2lzYBhLIVe1hQWml1gCtwqvIjAuuFn19Q4df8DyHfn9rTrbPGvNnqniyRvAcf0YJ4OUu", - "JYB3MaJ/lWq7XypD8IvVwt2mgqivwNPSrEqfU3sNO9P2JW5rfSym+IUKC3mKzkoqvCkoW9k4ypWTNuic", - "cB047qEqC9g2xt41aM3EhREY5KwuwW/UT6K4+ZsKBsaXrpTQ0cga9RnN0KkkP568Ixi44a08L1789cWL", - "cV2T9seTdyN4FhESmj0Ody6lqel8TJ7ZnsXWm9kqcURttX003NuUCwpu9pKKVOYEBvQmIqX4XDhK9YVs", - "Jxt0izM635L019TeI4Hq2AnsDgwiNE9U0/kFT0G3eHB4/yB99EMyYvRROnrw8NGj0ZPp7NGIPZntP5my", - "Bz8kbBpRK/wIgai/uXPIOtHfjbgWOk7N7yxmVxU+agz5tGZqNJJsZ8lq1n/6eF2HVLxLSsRIcoZucH/a", - "AZv6hFo2pCUbdSgP7R4XtIolCL1TrIQCErZgrmUZx8+HpKBKLWWZ+hLKoFbbOiFG/3H2y9qsYVAPAAOc", - "zfDVeqcLrYvBp0/QeBEdftAjJNGBAcTT6jNGc+uqwi/V0d7ezIULBmF+e90qGRi8SF7SMrfxsBA7PRgO", - "Mp4wm87hqdSrq8PORMvlcjwXFYxvv1F78yIbHY73x0yMFzrHqoJcZ41l574Gd6313x/vj0FTkgUTtOBg", - "mjGPMCEJjmiPFnzv6nAvadcXmqPFxBekOE6hL59uFiICYRNyQWC0g/19B14m4HtqlFEMBd97b11piMBb", - "RsI354NTbAJdGPTOfE4K4qKTuMyKMYymmao+67Qoxdv9N4j+A0pUj/FCpIXktvz33Lbh7wzYKeFsIB8F", - "7x7E9Ow5e0sfsF9ykf7ZZ5efYArZjYE73iAzAu+XshJ1sjnoyb4lKbxsIxy/0LqwykFkHae+BeHSiP7L", - "Uor5uHX6L7kNfZclyWXJyLNXx64hJnptIABOkSWF0DkQptx2YkhRSBU5KchEjhwVMNE/y3T1xaDRqqgS", - "AYtrBSpL6/SDECSsIiIxmgxr4Nw8HjUqNHRX+kvz4g5xkRjvBkc644LdPZz6K804eF5piE3XQaYWnlr3", - "7VU9vut+Xh/kRqKC+UqjICJ4Dco28q++Ktae3Bp+/lMgJqap1RjZzGLbwO52GKcXGTFHYUsp4iWmcX/W", - "ke9QwfjTsDHWiuZZc6y2gLwJQdoH8Raa7V6xuODRlRPWnsbTJGFK+Sa8kbKKkSFJmNOFG7sHzv03BRNP", - "T45dxlqWyaXtMwIh54Jme1aStAc6IQVNLs1hn4v+41ZMV8WIukI//WTnlF6xaG2hmyE80amiTDMEq6Hd", - "9ArRu4WUDyKtn1rIAKHoSzalReGsJanRlWZVltUNXbUtOWbkyrtHSt7VsUU9Oa5Yesian6DbjYAdrsis", - "EgneRKjIvgG9DULEMLu3hFQ/DjY4395Hl3b6ae+j88Z+WkeSGsyw2bncaOLcwM7WcbAqXJDYWmvQ1mO1", - "i4rTTfY16nxkwsCr3D9hm3r9doPMNJ7AvTvFdFpaK9s6ayR+h+2YGinf5ktrG3AZ3wY5fbo3OgF21O/W", - "LadRZLw3C7wfVX021O5YWpf6/G8MvcYG1GcgZ10ioG0+IO9UnfnshHaapiNkJmvS4ZCM+iqhbIqpXzMK", - "vV0M44hlkZApVXUZp2kpl6qRF3Z9jK/3uDuOu0LbPZwfsnCwF9WNsPpGN7LuIf8spzZxOee6g543qXGs", - "WRD4xyoj4SHvtOliRlSzca5Bt3YF0H5w/+DmZYQzT1F9XhzTdA7pcyBT1vlzzRei2XMcm2BnK5JWvkyZ", - "7WSU0GThkM8PBfdBSpIZ0eRc3Kp4BD8QVxuzSQkQx6yLB4pHyrJzR7DAA2TWhbIPVo1vDPdzM5mQ2UvZ", - "uVSo2m9xtUCv/br3KwmWsO56PYjn6+94IXzap6Gi2JBjYQTKX96cYZql7bBn8xjqPD29kNV88d8X6o9y", - "oQCtNlwnwH6/bzMSmNKglsqSmxPXtZuWR65Zox1av1me6WTxYyantFGwAnLJbpaLxJvHbSXQDONX7sy1", - "2XN50XB7qFhFW8P1yEXQUA7Sill5ZduWRj5XG47vDZQPxjY5dTrSHADds5zW+eVUqRF2MsOtun81DxCa", - "vjHbAe6GqGVvf7mo7bPZYa5Z9B07u0nboW18bdKqsDNcSFxzComt5qa4jqaWIj66FYpYMlyTkEH/upoQ", - "2nMZ3xlq9ZqWl7jSEGTDWhp37U2SkmtWcroB42G83Ny2nQZFHuCkhTrzCisZGKYAqOIooS1PBRXNzImb", - "53nz0LskFwYtSom2xwXz7/rc9ylNLuelrEQ6Phe/SJiP4p2dtHsWTohXVSH+yXzFUlIVICsJzUvw8UuR", - "uvogOUX0RK9dBzxYSHclK8I+FCzRQyzzwHhJJnXzqUmd0a5sEV6jpGW4JwrdXGHWlm0TiMnfXVOsuMwF", - "LYdsXaMbIiC2L1fMhNeu8NokFXOmx7et4TR6MPWzJIBq4FmxAWNYIgJKq/CZQWYQYYAU2C5F8OHdIQUg", - "BPhaMAbw23G3ukvWDBpzQcSYSImSEOnb5WlGfNv7aP77C83ZWtOQLZWylWHIDXhn7DTtgi+9Kgb+1pZD", - "bFKFF3gNTKErjYfEhvMJkv6bPZ6xvkz0XNQWp6EGtwi0qHXLv+R3oyIADFDZdrsGlQpI6tZArKfyDMWP", - "1wXhRww1+7SVrLYVVvtCA/04vSkY7rdtxKnnSIICOuYZky/wo0s+nxtp9XaJ1juBHJGlBFIEur5JjOwM", - "OCmqAEPCRZJVKSpHymrT0PDLqANyjlWHUeW2RZP8IIZdu2j9jnhAfpG+04bqtPv+bsX0902Dpcesfv3r", - "q2LErZgGOep2XabTUpBce/L1Zib8SKQkSObru49702br/PjNfAsNVxuN9m/zQG5E4qq3ElNYqsLg73cY", - "fDq0hTJWBfveyFxB/3jvu/Rw3NKT7O4mTRJWQJ0sJnTJmTVqAVmxk9w1ogJthd1qbWFyc+cDEOx6v78O", - "Xt3cRV+LXGBLWYNgRrWaS43wDIpRwe2/S6iANApMQM2s+LrGvNsDoEkqIZjW6rh+y6q5w/VSB0bIeFTz", - "7jkHnDiV28Ha17a9oanvW0DKP7hJsXnU1zAvRgdtdCTvRyDFdFi3qMc3A5rASV0c6A/OIt1ObHJvj6tD", - "sCVxsLmmydJN5BOQqPKMEa2UBwd9dblc9023BBcJh9/7ONqvTDTXIKuXBOotWDA04102ImidJrkOPU99", - "Eas/NnI2arn1oGYz0xiiM6yZ+VpoetoY7jpI2lyQxVTwXPnDdunNynfy8JL/HwSNm5vcBYlBD93Ins/g", - "rW+DJ8NefGJfXFZEGHOmwppqqiP53DGxkNp1QyU4mmXhqhvYsI28F99xHImWC6pHS1llqfUPjlLZi1Pe", - "5vTrgupfzUfH+vm3IvA5j2SfnIdNE6xZJ2KDMMgXyFDYy9ClhDubDmRE4ygQieDKS7toDSwqOgQ7Uybn", - "NgquVx4Dk5FtvVLPUg+HhiUoZCi8+ysliRQuJyBbuSm4CnpsW++DK1uP7RFR8JSV7jFKfRlYhLiKrXD2", - "XFe8PayEu4ZpN5vJ3lC8T3OSmBcqbB3nYjSI7ax5e86naDPQWIy/a4gJfbRt187AHY78ev/JzRNLvxKa", - "lYymK1tV3AoMD27V946nByFoYg6BrGSiWhCt+8tNgmuCKM+TBZHCmvdvjd1ULXbTIlLPsFcvrVum4vVX", - "qzzj4tJHF0DbZIQAxpdpJCoWKJURXbIssL5hQzikFrZTli32ntAs8xe8juSr6QcCtZ39YBdEiQovEyym", - "0cKZloyupRlhF8BtKUd4sjdKRWKdKLclKF+BlkQbMcbWW03tsUGTDwnifHgQw7ComHnHdi60rpQ7dWWg", - "0WfdJTmEgW0fiwk/hSy1she/Zrx2YxsR/ilmnFEXrejZRntA32vORUBiw0pcRU124F2ljYDgl9C9JTDs", - "3kfXzPTT3kd4wv+xxqEe9jWUJXOhtS0ZcOs2tVBFtSswuld38sMPO/MGdeNdh0dfMj4yq9v9NrPWXYt/", - "u/GL1+lluaUh8k5dorCeWd1zM9p9tSFgBvdlHfH2GPnPjYzDmFHFEhVXP9P6HGwP/JTNWEl8S1fXdCez", - "GZvng4P9H84HHrHquDpQKsC/p6tSOJG+3p7ychyGVfoeup0Dx0g8mimJYyiZMykYYZmCcepC5rFlArYA", - "ABeMYkkBC8L/Z4TTjJ5RMXpu9jl6BwMMIjAMOnbGYChLPueCZjCnGR96+GCl9EyGldV9r2Gug8ZVtlcw", - "D6m2VfJcMSxBKIc3oD/VnGNM+qa9vbELG720CxtsjFXaRp6RiWZ6pHTJaN6kEF5Tn3Jh7vdwc2L4M5xD", - "tRqUX8Ou6MTQrknxYP+HTa9bdGwgoiU5GN/7ODpCaT836gCG4U6ZXjKL7BacQTSQ19ptOMjMN1iXZYfu", - "eNHZ4TIoOw8j7YjwErvU6fW31t3A+uZYxHOxq3JGpsx86Oefrhr3DiWKSe8VOiLmzCa2lCFQl0Z08i1n", - "U2zgQMAZbD5FP98hzXjdxo9wP2eyTPg0W5Ekk7abw09nZyckkUJgILvrkiSh4qQlvLbspmqcFyPsA000", - "UTRnVpLU0nVUI6msjJCHHyjoRotvYaoh3qa66GDkBMhUpqteVhrmtJspau2iC5aG5OgdJ30Bfi9pmZ/W", - "/VhuSDCqZ3kLovf1K2CFzgOu6gi9GS3zDUn6OHVnFNYeJIAfWGf3PtomQJ/WG/Ch7t1WYau+p9DdNLDa", - "3gVRxxPWphUzeUct883uVmvMnpEv1pz8nm2dsv70XTOubwUJ3H7W4QK013L40BMQ1pY44cMFVURARxmy", - "YvpuoVMYwdHpZIaR7jnDrA7c+wYHoq2k0wrbcEOONyCehh7NWyDfmXnx7iCfZh/0XpFRLnasTHTWBs63", - "gldBXBlVmszY0rZeCpAMe9tvRb3CT/x4rp3TWqzaLqgi6M50q1j15S24nR5533xcBbLAbyCwAluf+Xw6", - "cGOw2Ywl2qkF0M4YR6CKLFmWtbMLzbeM2kohiyqnQmEMOQj34IK/4rRbvaSuCW7uCHQIcDcKA0LhYtX3", - "akK4UJrRdi5eUGe9tySOr4h+c1K4lXPdVNcWwr3A3Oh0XpeSWS+Ho2qsfOdubDnnTOjalgbweaC0ni6i", - "4eAxjPK53tN0bk5ivl02Tl3aeltDhqbzOjHmLkewh70LoNY7XIZKYNVr1ehb7cP8ze7QN2LGUFBaoD7G", - "GswbQt7XgPXLIXJQljxOxoPNR1DYC/3ha7173Ybvzb8A2yuqCEyxhF0TqF+eO26Ep81GbgHsmgZBg2m2", - "7ae/Tljh5O5kxtrSgVRgVAPUGdwGWRqINrTbhH4vNp2dNnGzj5BtiBX0B6Zu5Zq96sn3qDvyq/GabMxl", - "+Fr/PYtX+IUgiK9+AXZD/FukdOYyBaFAaE92cUHQ7UR5l8+QKFnbSxOaZdZQeinkEsLY3r07fn53LqEP", - "gBFsuev1Q0mkiXrx2xa0tdx04W7htvVdtb+AF8StddNdU1vByCaTuE+dqNtwuMTaAHSBt/fRNsnYQfTa", - "SqX0w958OnSnXrbFHc+jbCzk3ZT4nLa0tA0ZjzXe/ETmue/eDD7gBEKWwQFla9zWBpSl74fDBZnYXmwT", - "UK7Qg9p8CUNWbCOooWHiBeGazHip9Jg8FSu0yOBrYc+VYBjncwWyXvlmZ9eTO78qTn1pUrCG426bVr30", - "Ddi2kVdIyjSFOnXLepodbv42ViWr83e7kt320d2UEBHttHYXjE13xA7Ui4DbWYMcRu+ElE6g7jV0NuTp", - "bwINO93RenCwK6OT4+eqYUKo/daumTqRs39OHA0qyhtIITTUghfeAvbr7viZMVaMVNB+eROXa/Zr/pZY", - "XnNn2zQ1AW9+o0H1uqRuFgp1Qsa+vJsouIFyfVWMuDFOugkZXI52+xSvbZnyDbK/ql3qmrTJCHCydJa1", - "RmPhCJq33BjYhJCVI/x7nfyGL3p5++bO/23QGHGd9UkSt/pbNc04SLC0X1zvuFPuToydW37DvNJRFDoy", - "Wn0khuXVX6oIUhl9byRnszWiF5+LN7PZVi6YuwdL2yoUSGyjSejfoO9oq0RqoPNSReo+52sB/oxmGUZ7", - "OuuMliSzbjhX5hTMd3rBVvdKRuZQisYOP+49FbHhUMSNXm07Rf+lzpmmKdX0Kxhbw67/f4grvTUaPq30", - "ggkNWQWuT5/BBheK2mct+GycxEBuLWEGm8MsA07F6wOPYqy2icRRwTg4tcHXRg5YqdNufBBHr0AqJOn/", - "4m5j1e4Y4jLkXHd/VmLWiVj1AKEXFUb4ZtpPwjqHlQ5u2ubjJ4ppLbX/Qnk83VlC/QNTHkvV7bk5ezKE", - "JSTeuKAITQzZyFiKtR0x8cxSlFEzJsqhC/hWuagTniyVYeUokwnNgMDRTH1pqnbFGrupYu4lCA5aw2et", - "PG7jxm+uvq41vPeGdUO5uqDdSx+5+kW6eqo+rdUXGQvsHg/2D79g60NEsV7EPGGl6zzznAmOpNPWP4ib", - "zjGEzrI8mmh+hZZYBu5RV2Mry+QSfRUWLHbrJZ8vNBFyaQP4Dm+XwbiLRAXk9KEDz0jhsDrMzIOM/7mE", - "3vY2swUv3I6X1roHqR8/gMam2wQ45RTOMt4UKBpB139dzJBof/sWglHtTvquo5WNuMAlusDAa1k17Fjd", - "6NPYLalzPFTDY+cwyZX1VNLmw/mx69J0t20w+Uzm1DDqqssh0auCJxB7aLs1gcBclHJeMqWG0M7JNbiQ", - "JZlRnlUl28hhHF9RTKQNR50Btxsdqm+zkm2+KXs5XY34qKz6w0pf05U1pVTim0hKeU1Xf2GseIse529M", - "PcPAbyvG1NnfgcQcuN4DBlVWguyRS8YK54qvA8DJm8LVjoJERMqFIpSgqz2USb1TJuZ/70HkjkQPyl6w", - "staauKqj0tejtqx0UelRUcq0StYJ+oZYvoGXT9y7d4I5QM2vvfcFm++ajT203xZi/rUSuQ+2TOQG6c+m", - "KLu2Hw/u37/5i/aKible+OJHfwo7x6U8xX7hhspSYkEwsp9gXr5d6eHNr/SEriBfF9rW0dL2+3pw/+Ft", - "uBFUVRSyNAf1mqWckrNVYT1mgGIEMcoJk1Ofbl53gQ2jvx4cPLmdDoOu/gVySiAdUmKHqZm52LbQnnVL", - "60Uptc6YLcf3h5I8MM/dADqXSpOSJZj970sHwn5RHgiy3TkAB/tOmY9rRwgTCmv/YQ4FSO/2lM2X9xRJ", - "+ZwpKB7cPmPyzFcfgDixk19+BDj/fPLiR2JRyQxaZFSIeJzWOoFHL6p8KijP1F5RsivOlo4s8RILJjpq", - "T5D6OzEIIFpeOWpeldngaLA3CIxQbWJ13AyC6rQFc5ji2QEkqXQLifwsp85MCjLa3ytWcoN+dbvTYasd", - "xbhRRVNFBn16ctzsDxmayGSeVwLFTShQ0l76uO3AjUxgseG1XxN5enI87O/OjM2szDbMXSll5lbUmQyc", - "jpFSOVh+wM8CfKKunWAh6HtWvpdTXxEunMOWO/j026f/EwAA//9t3o1qzhEBAA==", + "iPmI8ydnT8R+2PO0f6Dmj3ZgLQAESTAvsiWr3NMP1VaSxGVhYd0vHweJzAspmNBqcPRxoJIFyyn886lS", + "fC5YekbVpfk7ZSopeaG5FIOjxlPCFaFEm39RRbg2f5csYfyKpWS6InrByK+yvGTleDAcFKUsWKk5g1kS", + "medUpPBvrlkO//i/SjYbHA3+Za9e3J5d2d4z/GDwaTjQq4INjga0LOnK/P1eTs3X9melSy7m9veLouSy", + "5HoVvMCFZnNWujfw18jngubxB+vHVJrqauN2DPxO8U2zI6ou+xdSVTw1D2ayzKkeHOEPw/aLn4aDkv29", + "4iVLB0d/cy8Z4Ni9+LUFW2hBKQBJuKphfV6/+Xnl9D1LtFng0yvKMzrN2M9yesq0NsvpYM4pF/OMEYXP", + "iZwRSn6WU2JGUxEEWUie4D+b4/y6YILM+RUTQ5LxnGvAsyua8dT8t2KKaGl+U4zYQcbkjchWpFJmjWTJ", + "9YIg0GByM7dHwQ7w28iWshmtMt1d19mCEfsQ10HUQi6FXQypFCvJ0qw9ZZqVORcw/4IrB5IxDh+MGZ/C", + "/7Knpcw0L+xEXNQTGXwsZzRhMChLuTZbxxHt+mc0U2zYBa5esNIsmmaZXBLzaXuhhM60eWfByHs5JQuq", + "yJQxQVQ1zbnWLB2TX2WVpYTnRbYiKcsYfpZlhH3gCgek6lKRmSxx6PdyOiRUpIaAyLzgmXmH6/G5qBF9", + "KmXGqIAdXdGsC5+TlV5IQdiHomRKcQnAnzJi3q6oZqmBkSxT3KA7BwY7aR6dX5c/m2EXNcywx2Imuwt5", + "zTQdpVRTOxAj98zL94KldTG+c/T2oAaD9ik9r/8y92i5oDo+iaHIqTTrJ8dAnmmmpMGQ1FDsIqMJW8gM", + "4ME+aAMUg0qIpmbAnIqKZoSLotJkxpk5U0UWPE2ZIN9NWUIrheAdSTHC86/xQcv5PGMpkcJxA4Ob3zfO", + "tIammfkVF5d/rrRuQSCKqi+EQWlVb9zMg0u4Z6cmUxiLTNmCXnFZdo+VPG29uuRZZlDGX6k/Z0ykrLyn", + "cGwLVn+9CJCjeqdDWM/ErGcSHgSM28Q4u4Z7CnFuTF4DtLNVcOlqeslhp4IISTIp5qwkhVSKTzOG94YL", + "pRlNga6K8MRwRfcC4N1z1M8AwuxzfC6emmtD8yKDQ7KzES1HUzYqAQIsJbOS5oyUVMzZkCwXPFmYg3U3", + "h1Za5lTzBPYwk4Z+4DAqYcJ/N600Sag5FCKvWFkiMuVu75ZEKsPG4re/xedaeNNEkxi3umSr7o09TpnQ", + "fMZZ6a+shfyQ5JXSZrmV4H+vkH9YWvve8q8oecjolEWI1CvzM0ySclVkdNXhA+R4RoTURBUsMUuyR3jJ", + "VuZc4PZqSeZMsJJqRigpGVUSrgOBSccopciClvMIB30qVoR90CUltJxXuZFLHJeaFqux+VCNT2XOTpA+", + "rb77nphD9VMnJTMTw6ItDVsFIKhBXZ/TDoyH5zlLOdUsW5GSmaEIBUinbMYFNx8MDZrD9GbKIRyJrLRd", + "ES01T6qMlh6iPVxEVVMndK2T1SLizan90gsIO49wZj+/4nCJrzHCX82XPDNiW/tOGBS3K9tSXjutQdES", + "26rpyDxBiCPKe0R9VpUlEzpbEWkELOrGBfQORCw1JpOfnp7+9OL5xcvjVy8uTp6e/TRB9SHlJUu0LFek", + "oHpB/pVMzgd7/wL/Ox9MCC0KQ30sKWCiys3+ZjxjF+Z9c9156f4JP1tRd0HVgqUX9Zu/Ra5o37l0JS8L", + "gWD3AV1AuZIqcvzcXRnYdsA/xuQXSQRTRghRuqwSXZVMke9ArlRDkvLETEVLztT3hJaMqKooZKnbW7eL", + "HxqV4/DAbDqTVA+GgNfbbjJAnYak4ZBxGJO5nXTQpFUT+83kiNBsSVfIUsZkUrPLyRGiB3xtKee7Y9QA", + "AKBWbizJdxm/NATNAo3QNB1J8f2YTJZsGhtmyaY1Mwasy6mgc2aIGrIaQ0iBp9hZHF99L6djMkFRZnJE", + "BLtiJQz9pzYuW9JoVoqiqXkRgANqr5ld0KxJa9xp1QDFmQZAdCxcBsPBkk03nlkcI53qVOMJCllcGTmC", + "zllp5QINFJHmRvZQW0idn61wxCRlTSMa4U9ULUKyApzUML8WnVHEcmRgbiRZoCABezUjo3CFP4/JmfnZ", + "8UkpagzzGgETqioN+7Jis9dbmpOaS1gVoClQzXqkVs/ktzcfuAm2Nn3E1OuOZtriAJYK4vKCOe1ZbOIK", + "BuciksMrrrQjg0DX+7Gvi2nOsnC9jZ812G3PruspYhu0VOWE6sWzBUsu3zJlNfmW6cFoNd3Nd7SulZM3", + "9MIg3HdC6u8tM4jeAhDK45cM5XXAyCVVaN4wmDfjIsVZHB+JDqwucNqotQTlqgXzC7X8SpaGOI6jkhFw", + "zOhKYRC/0JmsRBpdk5JVmWwUa4IjOcUP2keKQLMr8sOGex7aA9tw5C+5SOsT3wr/ehAmYhXq7uPoY1Na", + "oUrJhFONdN/s5oKJqytaDixi9EspzvTZOQ/7gJTM6Jkgx1Oi0M5mDXZA7z6wpNJsk0m2397p2Ufw2ME4", + "TneCT2LH8qIsZdndz49GpeEJYeYxKZkqpFAsZjxOI6j+09nZCUELJzFveB3BD0SODb9OsipFUxBeilUm", + "aUqURKz2AMTVNmCbZXZpXKAtlkujOz8zkz3cP/Rcx9tPUqrplKI+Pa3UynAnRmChblGWeUmhKReEkntv", + "mS5Xo6czzcp7+OqCUTDRmOVxkfKEaqasEQ61cM1ztCmYo2DKK9gl0yVn6Zi8BG3cyT52QK5AOjJoQo0E", + "7gSGe8ryPfNuknEmwDSUSqJkzozyO2+onEZmYx/w8nCakSlNLuVshhzTG62dvNq1mOdMKTqP4V4LueDc", + "6/ejmHXFhH5Jy/x0KzN8/eZbZviYH+JnOX1XGL4f1YgU096APSQGO8CWQU5lcsn08Zu91/92doZogCIu", + "CifKHERJBFuaH9WQTIqSXXFZqQvE24m3P7EPiKYIxLbIljHNLuxZs/SCRrjK8czqzBkDjmWotf/CCk/O", + "ysNzpjTNC2KoOiKUwTWHTOZTpWWJ8tTLjOZMJNIz+uYxG5iNzIhRRhUhYu/eHT93UuDP4KzY4OeoRavm", + "QL/QPNRSYx+2wL0JO4y85X00odfHa0wP92MIXbJZydTiAmzckaPxd9iLoPaWqQXYze33QHDsbu4ptJjX", + "8i1gHWo8ylxYA3g1NEgHcmtKQdVhNFkA0bjiaUUz9NYtYRZvQNJSGiKwcoNYq3lR0gSseb3mk92B2O/j", + "gqkj6HHmkVPOSEaVtqvcGueWVF3gjUl7nEl4RQ2WvzcavX25viPmtmtJJrqs2MQqKPZJbaEDpREsrTy9", + "V9vKFdNDS5nNTXK3Oy/0aivrJlwAB5zAgWfdcoHjrol0vbTxFVX6rTXo9lE4i6CyrBHUQL42BPOczmv+", + "6qBnlxmX/LdyYQ4HelHlU0F5tgVahVs5NisCZ0xMJ8C5qLq0//KT9IOJz9izVRITqT0BzPiMjRLzEmFX", + "YHCw/gWjPQJXVIsKLQ6pXIqhEU5K+LMqhoTpJEbctzEn+sXBUlEzau261/aHn1B1+UrO+84fnPuZnJNk", + "UYlLy+C0JJQAX9Oy4Mme43WklDInKUOaluJ7VoYyIB/CL1eSp2acFGSQFsGJwSGTEYvBM7MeR+O1XeWY", + "vKYrL0HlVaZ5AWKJYAreZR90VEVxCLGWJUEYxHBH33uNamYba49hGynjDMC4QcwAcHTkDKAG1xU0DP2/", + "agY6bM/LtwPccBfisJnva5z0cxl/MzrjOt/cFD+LsQdP4azyFWEX/iR7cRG1wjPaSxTwBXJG5xtQkWuP", + "hjH6hpbAdZD0S9mWfYMNcEv2vZnl9tnHAjBtc2nxzY3XdolgXQOxhIoLIz3QUq+z73BlpwTlj1ZajuxX", + "cROPhVNUeXAyJtrbma41WrtcA207wPiLSf+4/G1ohrk3F4oxEXOvKu30Ya7C9Zr3nQ0kMFJut/bNpGfp", + "Vv+5xAfBsCv5iX91gXi1y8fP4Iu3qPvdrGh+xUpl/Q5bkLl+6ubGGTbuSuwONy0DzkAH1BGMiinYE5cU", + "4i8M3VQZYwWY6MyVpPa9SlwKuRS4BhDpooa7jnXBzIlRFhB0aReC035q33u1owWjGxmBP0fhYGXYv9Yn", + "ECxszsEZeDg+GD1+NJqn6eGD9OHhD+4Mjgb/r6xKd4cGELpTan+Yg8Px4YhmxYLuB2cT/ky+64z9fXf/", + "sIqdHCuNZXxci29NTLZg8BqN96DljFote1HlVBgpU1U5fIYyVskyRhUj04pnqQuCBaeSIQ1UkUm4qgmq", + "CBJIdv0JRGVZwyR+PZlzPSH2KzA3Rv1PrQOv70EDFP7qGIjGsOFnDKClWfZmNjj623qEO3XeMvPVp+HH", + "NTLjWv+J0yqJ+4JI4fXJqLyOYScxO7h5AM49R5G2JkH/9La0axhxdmYI488Qbt2hbxBrP/2GePznTCaX", + "GVe633mJjNoa32jJwAgO0a4sJQkrQY0EbQpdnNKIadbSkzjk3Mp/FK7nhdDlKuY66r7UcUiuDw/H/Wyr", + "Q9m3e4ho6wTqocNo8B4S8txej3hIrPmV0KmsNMarOv3TSpFOwrTmJN4QL1t8cUFzKi6SBUsuZaXX+zxP", + "4WXiXg7CjdwCSpbLK5YSmkkxx+BwFx+yTfBhcy09oIlbqjoLfyFkNV+E3iVgFzRwwhScJYxoOcctpnw2", + "YyWYjuEEwXZrviaULCSY7DIQWsi7t6+cSydiyxuTMwnMDUKTMELn7auh+SmhmgmqGTkffJxSxT7tfZTC", + "S72qms34B6Y+nQ9iuov5oImWZRalQnaYhmt2Qyx+6yhgqmCknqN4TZVymHrKMpbEI19OvAMTQ8XNsymz", + "FP29nCpnq69R2KBLIESBjmJp1kVOPwyOBgf7B4ej/Uej/ftn9w+P7j84uv/wX/cPjvb3u8JP9+tOFGeW", + "4ULQGc9KFpJcs7CZLMHL7/hqzZtal28H+hwFKdM0pZoC+09TiNCk2UnErNlgvI3NlFOuS1quSG4Hcwg9", + "Jq/NNgx1zdiHMHbO+jhzaXYB8SeV4mJOJnQ8HScTQ9brO2QDaFtnVJQS9nE0OC1Krhl5WfL5Qhtmo1g5", + "ZjkYogdqNS2Z+L+nNgRDlnP3hpWHT+EFcqr/9/+6YtmgB04n1lj/zOtkzTMPPUw5/cBzo53c398fDnIu", + "8K+Iu6l1DfwgPfh/GkQfxQ9LlxXr+bZfc0qoSMwxYKpQgfaa4WBGOf5Y0ErV/xh56WkwHPy9YhV+CGM0", + "nsG/K4bKWGWgP/JUqhnfXWOWX2gfnNF3HQ9vwWdBooCNJ8Dgsi8iQMW1tKFbVt+5aVn2Mg77EDiHj6t0", + "IfpeyDQXplIQ0IhMz7yFHIKlZMYzppANC5YwpWi5ipH0FsuLGtDvPXP89vj5vSAmAoQ5F4XQZs1hLtCY", + "POVGNxK4UvdJjI07y5QVGxw7n5Uy91vvU55igD6j6lKdVnlOy1Usiy0vMnD5kczKk5jJ5KA+Js/QE4Hx", + "Itb+7iJRzU/ukMA1a56PI0ZS6zjeSswEy7Nd8BYRcr2sUf1bxXDPIRvjudHDHw4HeUDm+wjnp+EA8qsu", + "pivIQbQMDAKUa3OEtU1x0SAhng5YovFblyniWj7W9PB+PJ7ks/nRS55po6LX/GjouMur47+8qJlLNO1B", + "zmaKNRcajROoQfVxhwxEtSUF79tRGOS6y66CU2vfirdMV6VAczHIJCBGU0c9uRVAYAu7aE/twIEAqfsR", + "uC+sE1B/2zuFxo1r3qWIfzbgmRihXo7AdFgVg2H9y6LSqVzG2Zo1ETyTYsbnVUmd3NrcJFcvean020ps", + "8BVwBfI+RyXAENCZ+bAOJbPzkbISQdSJT2EDgYuSGVuSGTWkWA2Jjd4XUowgz9PoJUm4XmAyRiR1arYP", + "tp4yiFbJC21IunlLL9jKCtniniZT1huGAnwE0wHTrbRBWIUuqVAzVpKnJ8eQiuKCjcc9wS7AYl/JhMY1", + "hueeJQG/M9zM3DSYy3483mjyaM/S3t0wPOAY6tlT+ystuQsIbiPIhV7KJY3wtjeCjZZ0Ra7sxxgCD3mg", + "UmmIKJXmktuMQ0hS4ZAyWDLIJc0hJMkw3slHIxl/mliVk5eY4+hEkgWk9SjnA3PFBHzYs/OejcnZUkbW", + "BAZTO2naSe/w0g+zyy8yqo1+M/JWHMzyBXHBDjJd+UX3IRp8tNloYo2tNaDdl1uc19Mq5Uw0w4etvcqq", + "HGodcXDDqHWsbx3Za6NPhzG+pkVhYAyn7A6FmC1D6p72CYEck/ojG179hbHibSVEtExAHRy3DC6udePl", + "dEUuGSsMURJOKIyLUHlnnu6B1opAj1Tf8IXFiEsrlI829YXaSOx10KXF62Mf7AcS+YKRydI74diEWG8T", + "JqzUecN4fcwkAO+5NP8V7INuhKWhq3tIJk0gTMjrd6dnRmeeQA7mZKsItBYgPdT6YBTDch9Bf+xSIFqa", + "r003WH+xWgHykeFvPaPjqyVegCbE0s0cxeZNbJcu8ZbNDdsuWWp98R1I0jQtmVI7Fkyx9Dd+0+RML2nJ", + "1lzDnX3fLinpwhut1W4y9meVXLEMwIEqLLviADEcJJg6e2EjljwUelYfO61TllQl1yufTdGigNuG1a+L", + "pz9luiqeKsWVpkKj8BlLRAmFPDk1sp3TwUHuMqMQP0yXWlvT2gvIVKFb5EP3p+Z8LUGtu4UoPEGce9br", + "uzjF8CFrjLHOCF6S05+eHjx8hNdeVfmQKP4PyC+eriDs2whktmoCyeyiXIpL12rSMoPCbOD4RfIzqDPt", + "x3OJQujgaHD4cLr/4Mn95ODxdP/w8DC9P5s+eDhL9h//8ITeP0jo/qPp/fTRg/304OGjJ49/2J/+sP84", + "ZQ/3H6SP9w+esH0zEP8HGxzdf3DwADzHOFsm53Mu5uFUjw6njw+SR4fTJw8OHszS+4fTJ4eP92fTR/v7", + "j57s/7CfHNL7Dx/ff5zMDmn64MHBo8OH0/s/PE4e0R+ePNx//KSe6uDxp64hwUHkJEptza+B9OgUIcuv", + "w+IHbhxXXsV7W6ynpW3iAhpOlVeK0AscBiSRY0GwIov13ivnabFjYVSTC3YzD879dsjx8/MBGpucyu1D", + "CHxOEMVVgK42sXackcqq+R6U6RgZ6rWHpS5Gx88nPXmvFmW21KZx7S95xk4LlmxUrHHwYfOYNt+mmvvH", + "7LrmGVrpWqcSqz11DfSwjuo2YoDibEFfe+v0ggrrB23GElDVGBQcNTZfmboCJPU1JmeBdPH5yLdFiMmW", + "R+KPukvgrApGndRFkfJaWmUXHdDhuKTYcu3Lejw0ZdQjet9stOYQjaywSWrDMaNjAJ352DW3sSaNHmx0", + "3ZjV2PGG/cJuE8C/cr2o3TJbgdop4YnzX0ZBP7Ri6pCkrLBx+0BHnE/kGz+bbWXP4Dh6/DudUx2ui8zr", + "jBdYAuqww6rIJE1RH8NwoqhZAAd7i6uBQj8urvO6ggcIGg3Y9coSNyQ03IqAcAvsrf/wm+eFacJxroan", + "BWI2JWXwmWMpw/AorW1CNq87K6+M3PGSZyyIiQJEM5zEvmZ+c6kitVwfpmjfFg7UF9Pfh5tBi3Aif92+", + "MK4E5PtzsQbrazYJR9tLjOe/K8/9UoRwLdErWXq6SXNrsxIFn9Uci6ZGKLY6XRCzR61VlZxX+/sHj7w9", + "2EpnlTKY3zE0a2kHjMyFwpS/B1aAuqea7o5oTlVg4d3BEusNw5+GgywA0I62lltwlbROPas1ZL/1hiGk", + "uaYodti8mdNquqZW0SkTYMX3eYkYNKcgCHtPBd9OMF3T1o7T0taMclQyeNM8fC+nPk+RPHNjYqmrOdPh", + "c1S9wNRL1aVPp3Z/Z3Ku0K0lGLOVOYqMJ1xnKzftlGFcOThWzKPV0G/EaBGYkePeNWNIgbEP30FNQN2c", + "euZyeN/L6ffAu83r5pV7CjI8wWitec7G58L5+ITUaBqZriDhE7QSy0eoJkUptUxk5moneWihbwaB6QtA", + "Q67TtJSQC2VGbsZkNC+HLDZSmQguvHG28m3L8cUGcfWFnOWvP7AaC2Bo2TyGPVKJ+gdDGcY7p43KYl3V", + "vvVbD8REvwyImar/ikqIfaCIEAeqySUXqc2S2BoGPlYsy36WUwjbzrJfvVPLlmqg6jKTc3wYhsuGr5/R", + "edz91chJiJZKqy1aQbkvLWtsbEow28S6fH6QoH1w+Pv/R/7r33//j9//8/f/8ft//Ne///4/f//P3///", + "MLsf6kyEcR8wC2g9R4M9DOXdU7O993Kq0Ixz/+BwDC+BGaUSlxco1xwGOHnyy48GRQs1ODJiFVR3NdLO", + "/dH9faygeAGpa2ypfNVOiBbGqorsg2bC5vaMC+saMiu5kJX2BY0a68Mp/Ar34ju35R8745VS6rXj2Zqe", + "WEzwouaEg4yL6kNw/cBrPbJHZUOhuzG4IRJsiBXxIbDb1o3fUEEkPOtNMTLu1dr2vVVkTR1O2AO1TngA", + "0hoxJ2qlNMvrEHD7bav2HoQZJnIuuGJd8cq+XEdRU5LJJStHCVXMmy3tFG5RNsTkHA/0fDAk54MlF6lc", + "KvwjpeWSC/y3LJiYqtT8wXQyJqd+KpkXVHNfC/5HeU+RSVkJ4IM/vnlzOvkTKStBJuBflRlJudIQ7zch", + "lstSH/7nyjD7RarxuXiqnPxJM2J2NGzsg5y7mJ/zgTMO2pL2aJtxAdpQVrEoIUOCKnI+aEqbbrzzQQ37", + "XCojT4BYc8mIZkrvpWxazW3RSkUYVRzKQ1ppxMWFoveaJySVCZQFhtSXLGvsLFpIoS81xfxwsX3xxyFJ", + "ZMFDBXPSLgE4NqNNfNXhbvnIM/tXnd5hiDdLCbf+cSzNkkqmxD1NcqoTTPigia5o5kfqGObPsNoxiI6q", + "XVUS8EhmaRBY1yyS364c6ouku6Ip5+K4sUCuiMyRTw1rWxkUElsVVKlWdexOgk8U6DZBXNM5inL29rkC", + "cXX0bZBYf/zch+bYKjeWd6P6SDXxJTinjBgSk1YZXn+zFDQaQngCRnfJMtiYwS6Xj2XQ0H3hV9JMiNtK", + "irLu126FnAiRi8lZ8cYnZ67iCLY6gfg25TRoZ6539d6GhI/Z2KVg+DCZIExqvFuxjS/ZLuUm0igxZPdi", + "urpw0Uq7BC/bYIPIWrdMatuhhggk1mhZGTzdkMGI0Wli5YsImP9L63QaG3e0WwGBr99N5qayNx3p2eXE", + "t834bJc4iTWyCdvV+Mu0oXONLYS0MWUR0uak7VoTFDf6rFpXce+EITRgYG+VORo2LO5dTAmqGW2cuSqz", + "+MTv3r4KE5fr2QnXimUz78mUS5FJmm4TgVQXQ/KniFmAsP++U/mMXCOfSKDkTI/aKUgx/bGe8C7lDIW3", + "+hpJQ2FaSFcnrpQmrJtvWqM7ZkDLRrn1uhAhiL9d7N+xkNNdIobXTVDfkiK5mfpOal0tNnzmiz5C4L0T", + "5aSl0qiKIeZZMzfYG4FiwYlBYVcU9bD1jZHs/emB7U4WGDD8JyKtiaT1Ap8LqF3wHcg30kVcTxy9tXXF", + "hNSEldRGtvoCD22p3Szr+02Fx7ox6hkXtlOIjb6FSIp7iiS+HQUGmPMwoRvINXlzxcplyTVDWZ7LSkGJ", + "IxHUoXCZp1HxIVaW7pWc23JzngZg5TsnFbsuFmbRcCowIaNlxntKeusGCdyBSkSRq47mjOoDJYOwlISB", + "TgjKOxcYlY/jRJz96wJBP48KrLlkbtLYJar3uF0dExs06vPmOokSxUWwx5ZkcELss07tqrUOme0MKv1j", + "fX5gq6axjkBnFCmF4/t1LTHo0ZKzfIp4upVI36jf1l0AalfbDKAutyO5wVE1XEtBPZxoTO2n34aRpPou", + "O3TUtkazV9tUGOleml2VozaOrvcQu9H7bwfGdwceg9ribW3R9peRr2YWsaIqlpQMOKUcCalHmmXZiIqV", + "FCyMZD4aHI4P+mB/9DcXMGskt1lesLlt4DOqO7gMhoOcqySSCXrNUHO78I9f/ma15TOcqenojE1hkbn/", + "yE75XLxpH1ajJKC1zNsDfHpyDB1ZgpO4qGtwqSWdz1k5qvgNHUyrWGE3waG/eldntTd/TI6QxE+ms6I1", + "p5QxVpxa21fEN20ee9uYC09ANdJlup0amIGLlokU0zC9fOMqS/m08ZSumnqaH9sQbFCUxuRpUWSc2SqO", + "mCcvzYcc7FaTlK7UhZxdLBm7nEC4H7zT/N287KpVR1YIMqEgBw9GC1mV5Kefjl6/rrOIsRVSjbbhyIOj", + "QS6JrgjEUYCbML0AqftocP+Ho/19TFqxSp9NaQa8cm/tP4lWTmlO0o2JpAkbKVbQEqN1l3KUMWg+5Sro", + "WKhD2Wa6Qr7I2GUPmMl354NcosdBV87Z8P2YvABrZ86oUOR8wK5YuTLjuTo53R5Jfv+B6AQA7ck8cqD5", + "GC/N7gG1ebg2j/VjD5vQbIwbrHjNvdBUsz6d2iaUl2F63fZpPlGNOBhsq0WlfSUZ6ZJeXrsm4xYL3bC8", + "puXDF5kc2nUFhSmhIYk5UqbsK3I2M8oIGAfalTBrBOov+RnJ7sfadUi2asXTJjnWIcFQZtcWmI7YBtRF", + "Rv+xWh921MyftP4J1ObCxpBArmoPC0ortQZoFV5FZlxwtejrJDr8guc59Ptbc7J91pg/U8WTNYLn+DOK", + "Ai93KQq8ixH9q9Tf/VIZgl+sOu42NUV9BZ6WZlX6nNpr2Jm2L3pb62MxxS9UWMhTdFZS4U1B2crGUa6c", + "tEHnhOvAcQ9VWcC2MfauQWsmLozAIGd1UX6jfhLFzd9UMDC+dKWEjkbWqNhohk4l+fHkHcHADW/lefHi", + "ry9ejOsqtT+evBvBbxEhodn1cOfimprOx+SZ7WJsvZmtEkfU1t9Hw71NuaDgZi+pSGVOYEBvIlKKz4Wj", + "VF/IdrJBtzij8y1Jf03tPRKojp3A7sAgQvNENZ1f8BR0iweH9w/SRz8kI0YfpaMHDx89Gj2Zzh6N2JPZ", + "/pMpe/BDwqYRtcKPEIj6m3uJrBP93YhroePU/M5idlXho8aQT2umRiPJdpasZv2nj9d1SMX7pkSMJGfo", + "BvenHbCpT6hlQ1qyUYfy0O5xQatYgtA7xUooIGFL6FqWcfx8SAqq1FKWqS+qDGq1rRNi9B9nv6zNGgb1", + "ADDA2QxfrXe60LoYfPoErRjR4QddQxIdGEA8rT5jNLeuKvxSHe3tzVy4YBDmt9etkoHBi+QlLXMbDwux", + "04PhIOMJs+kcnkq9ujrsTLRcLsdzUcH49hu1Ny+y0eF4f8zEeKFzrDPIddZYdu6rctda//3x/hg0JVkw", + "QQsOphnzEyYkwRHt0YLvXR3uJe36QnO0mPiCFMcpdOrTzUJEIGxCLgiMdrC/78DLBHxPjTKKoeB7760r", + "DRF4y0j45nxwik2gC4Pemc9JQVx0EpdZMYbRNFPVZ52mpXi7/wbRf0CJ6jFeiLSQ3BYEn9vG/J0BO0Wd", + "DeSj4N2DmJ49Z2/pA/ZLLtI/++zyE0whuzFwx1tmRuD9UlaiTjYHPdk3KYWXbYTjF1oXVjmIrOPUNyVc", + "GtF/WUoxH7dO/yW3oe+yJLksGXn26ti1yESvDQTAKbKkEDoHwpTbTgwpCqkiJwWZyJGjAib6Z5muvhg0", + "WhVVImBxzUFlaZ1+EIKEVUQkRpNhDZybx6NGhYbuSn9pXtwhLhLj3eBIZ1ywu4dTf6UZB88rDbHpOsjU", + "wlPrvr2qx3f90OuD3EhUMF9pFEQEr0HZRv7VV8Xak1vDz38KxMQ0tRojm1lsG9jdDuP0IiPmKGwpRbzE", + "NO7POvIdahp/GjbGWtE8a47VFpA3IUj7IN5C+90rFhc8unLC2tN4miRMKd+WN1JWMTIkCXO6cGP3wLn/", + "pmDi6cmxy1jLMrm0nUcg5FzQbM9KkvZAJ6SgyaU57HPRf9yK6aoYUVfop5/snNIrFq0tdDOEJzpVlGmG", + "YDW0m14hereQ8kGkGVQLGSAUfcmmtCictSQ1utKsyrK6xau2JceMXHn3SMm7OraoJ8cVSw9Z8xP0vxGw", + "wxWZVSLBmwg12jegt0GIGGb3lpDqx8EG59v76NJOP+19dN7YT+tIUoMZNnuZG02cG9jZOg5WhQsSW2sN", + "2nqsdlFxusm+Rp2PTBh4lfsnbFOv326QmcYTuHenmE5La2VbZ43E77BBUyPl23xpbQMu49sgp0/3RifA", + "jvrduuU0ioz3ZoH3o6rPhtodS+tSn/+NodfYgPoM5KxLBLTNB+SdqjOfndBO03SEzGRNOhySUV8llE0x", + "9WtGoduLYRyxLBIypaou4zQt5VI18sKuj/H1HnfHcVdou4fzQxYOdqe6EVbf6E/WPeSf5dQmLudcd9Dz", + "JjWONQsC/1hlJDzknTZdzIhqNs416N+uANoP7h/cvIxw5imqz4tjms4hfQ5kyjp/rvlCNHuOY1vsbEXS", + "ypcps72NEposHPL5oeA+SEkyI5qci1sVj+ABcbUxm5QAccy6eKB4pCw7dwQLPEBmXSj7YNX4xnA/N5MJ", + "mb2UnUuFqv0WVwv02q97v5JgCeuu14N4vv6OF8KnfRoqig05Fkag/OXNGaZZ2p57No+hztPTC1nNF/99", + "of4oFwrQasN1Auz3+zYjgSkNaqksuTlxXbtpeeSaNRqk9ZvlmU4WP2ZyShsFKyCX7Ga5SLyd3FYCzTB+", + "5c5c4z2XFw23h4pVtFlcj1wELeYgrZiVV7aRaeRzteH43kD5YGyTU6cjzQHQPctpnV9OlRphbzPcqvtX", + "8wChDRyzPeFuiFr2dpyL2j6bPeeaRd+x15u0PdvG1yatCnvFhcQ1p5DYam6K63FqKeKjW6GIJcM1CRl0", + "tKsJoT2X8Z2hVq9peYkrDUE2rKVx194kKblmJacbMB7Gy81t22lQ5AFOWqgzr7CSgWEKgCqOEtryVFDR", + "zJy4+T1vHnqX5MKgRSnR9rhg/l2f+z6lyeW8lJVIx+fiFwnzUbyzk3YXwwnxqirEP5mvWEqqAmQloXkJ", + "Pn4pUlcfJKeInui164AHC+muZEXYh4IleohlHhgvyaRuPjWpM9qVLcJrlLQM90ShvyvM2rJtAjH5u2uK", + "FZe5oOWQrWt0QwTE9uWKmfDaFV6bpGLO9Pi2NZxGD6Z+lgRQDTwrNmAMS0RAaRU+M8gMIgyQAtulCD68", + "O6QAhABfC8YAfjvuVnfJmkFjLogYEylREiJ9uzzNiG97H81/f6E5W2sasqVStjIMuQHvjJ2mXfClV8XA", + "Z205xCZVeIHXwBS60nhIbDifIOm/2fUZ68tEz0VtcRpqcItAi1q3/Et+NyoCwACVbf9rUKmApG4NxHoq", + "z1D8eF0QfsRQs09byWpbYbUvNNCP05uC4X7bRpx6jiQooGOeMfkCP7rk87mRVm+XaL0TyBFZSiBFoOub", + "xMjOgJOiCjAkXCRZlaJypKw2DQ2/jDog51h1GFVuWzTJD2LYtYvW74gH5BfpO22oTgPw71ZMf980WHrM", + "6te/vipG3IppkKNu12U6LQXJNSxfb2bCj0RKgmS+vvu4N20204/fzLfQcLXRev82D+RGJK56KzGFpSoM", + "/n6HwadDWyhjVbDvjcwVdJT3vksPxy09ye5u0iRhBdTJYkKXnFmjFpAVO8ldIyrQVtit1hYmN3c+AMGu", + "9/vr4NXNXfS1yAW2lDUIZlSrudQIz6AYFdz+u4QKSKPABNTMiq9rzLs9AJqkEoJprY7rt6yaO1wvdWCE", + "jEc1755zwIlTuR2sfW3bG5r6vgWk/IObFJtHfQ3zYnTQRkfyfgRSTId1i3p8M6AJnNTFgf7gLNLtxCb3", + "9rg6BFsSB5trmizdRD4BiSrPGNFKeXDQV5fLdd90S3CRcPi9j6P9ykRzDbJ6SaDeggVDM95lI4LWaZLr", + "0PPUF7H6YyNno5ZbD2o2M40hOsOama+FpqeN4a6DpM0FWUwFz5U/bJferHwnDy/5/0HQuLnJXZAY9NCN", + "7PkM3vo2eDLsxSf2xWVFhDFnKqyppjqSzx0TC6ldN1SCo1kWrrqBDdvIe/Edx5FouaB6tJRVllr/4CiV", + "vTjlbU6/Lqj+1Xx0rJ9/KwKf80j2yXnYNMGadSI2CIN8gQyFvQxdSriz6UBGNI4CkQiuvLSL1sCiokOw", + "M2VybqPgeuUxMBnZ1iv1LPVwaFiCQobCu79SkkjhcgKylZuCq6DHtvU+uLL12B4RBU9Z6R6j1JeBRYir", + "2Apnz3XF28NKuGuYdrOZ7A3F+zQniXmhwtZxLkaD2M6at+d8ijYDjcX4u4aY0Efbdu0M3OHIr/ef3Dyx", + "9CuhWclourJVxa3A8OBWfe94ehCCJuYQyEomqgXRur/cJLgmiPI8WRAprHn/1thN1WI3LSL1DHv10rpl", + "Kl5/tcozLi59dAG0TUYIYHyZRqJigVIZ0SXLAusbNoRDamE7Zdli7wnNMn/B60i+mn4gUNvZD3ZBlKjw", + "MsFiGi2cacnoWpoRdgHclnKEJ3ujVCTWiXJbgvIVaEm0EWNsvdXUHhs0+ZAgzocHMQyLipl3bOdC60q5", + "U1cGGn3WXZJDGNj2sZjwU8hSK3vxa8ZrN7YR4Z9ixhl10YqebbQH9L3mXAQkNqzEVdRkB95V2ggIfgnd", + "WwLD7n10zUw/7X2EX/g/1jjUw76GsmQutLYlA27dphaqqHYFRvfqTn74YWfeoG686/DoS8ZHZnW732bW", + "umvxbzd+8Tq9LLc0RN6pSxTWM6t7bka7rzYEzOC+rCPeHiP/uZFxGDOqWKLi6mdan4PtgZ+yGSuJb+nq", + "mu5kNmPzfHCw/8P5wCNWHVcHSgX493RVCifS19tTXo7DsErfQ7dz4BiJRzMlcQwlcyYFIyxTME5dyDy2", + "TMAWAOCCUSwpYEH4/4xwmtEzKkbPzT5H72CAQQSGQcfOGAxlyedc0AzmNONDDx+slJ7JsLK67zXMddC4", + "yvYK5iHVtkqeK4YlCOXwBvSnmnOMSd+0tzd2YaOXdmGDjbFK28gzMtFMj5QuGc2bFMJr6lMuzP0ebk4M", + "f4ZzqFaD8mvYFZ0Y2jUpHuz/sOl1i44NRLQkB+N7H0dHKO3nRh3AMNwp00tmkd2CM4gG8lq7DQeZ+Qbr", + "suzQHS86O1wGZedhpB0RXmKXOr3+1robWN8ci3gudlXOyJSZD/3801Xj3qFEMem9QkfEnNnEljIE6tKI", + "Tr7lbIoNHAg4g82n6Oc7pBmv23gI93Mmy4RPsxVJMmm7Ofx0dnZCEikEBrK7LkkSKk5awmvLbqrGeTHC", + "PtBEE0VzZiVJLV1HNZLKygh5+IGCbrT4FqYa4m2qiw5GToBMZbrqZaVhTruZotYuumBpSI7ecdIX4PeS", + "lvlp3Y/lhgSjepa3IHpfvwJW6Dzgqo7Qm9Ey35Ckj1N3RmHtQQL4gXV276NtAvRpvQEf6t5tFbbqewrd", + "TQOr7V0QdTxhbVoxk3fUMt/sbrXG7Bn5Ys3J79nWKetP3zXj+laQwO1nHS5Aey2HDz0BYW2JEz5cUEUE", + "dJQhK6bvFjqFERydTmYY6Z4zzOrAvW9wINpKOq2wDTfkeAPiaejRvAXynZkX7w7yafZB7xUZ5WLHykRn", + "beB8K3gVxJVRpcmMLW3rpQDJsLf9VtQr/MSP59o5rcWq7YIqgu5Mt4pVX96C2+mR983HVSAL/AYCK7D1", + "mc+nAzcGm81Yop1aAO2McQSqyJJlWTu70HzLqK0UsqhyKhTGkINwDy74K0671UvqmuDmjkCHAHejMCAU", + "LlZ9ryaEC6UZbefiBXXWe0vi+IroNyeFWznXTXVtIdwLzI1O53UpmfVyOKrGynfuxpZzzoSubWkAnwdK", + "6+kiGg4ewyif6z1N5+Yk5ttl49Slrbc1ZGg6rxNj7nIEe9i7AGq9w2WoBFa9Vo2+1T7M3+wOfSNmDAWl", + "BepjrMG8IeR9DVi/HCIHZcnjZDzYfASFvdAfvta712343vwLsL2iisAUS9g1gfrlueNGeNps5BbArmkQ", + "NJhm237664QVTu5OZqwtHUgFRjVAncFtkKWBaEO7Tej3YtPZaRM3+wjZhlhBf2DqVq7Zq558j7ojvxqv", + "ycZchq/137N4hV8IgvjqF2A3xL9FSmcuUxAKhPZkFxcE3U6Ud/kMiZK1vTShWWYNpZdCLiGM7d274+d3", + "5xL6ABjBlrteP5REmqgXv21BW8tNF+4WblvfVfsLeEHcWjfdNbUVjGwyifvUiboNh0usDUAXeHsfbZOM", + "HUSvrVRKP+zNp0N36mVb3PE8ysZC3k2Jz2lLS9uQ8VjjzU9knvvuzeADTiBkGRxQtsZtbUBZ+n44XJCJ", + "7cU2AeUKPajNlzBkxTaCGhomXhCuyYyXSo/JU7FCiwy+FvZcCYZxPlcg65VvdnY9ufOr4tSXJgVrOO62", + "adVL34BtG3mFpExTqFO3rKfZ4eZvY1WyOn+3K9ltH91NCRHRTmt3wdh0R+xAvQi4nTXIYfROSOkE6l5D", + "Z0Oe/ibQsNMdrQcHuzI6OX6uGiaE2m/tmqkTOfvnxNGgoryBFEJDLXjhLWC/7o6fGWPFSAXtlzdxuWa/", + "5m+J5TV3tk1TE/DmNxpUr0vqZqFQJ2Tsy7uJghso11fFiBvjpJuQweVot0/x2pYp3yD7q9qlrkmbjAAn", + "S2dZazQWjqB5y42BTQhZOcK/18lv+KKXt2/u/N8GjRHXWZ8kcau/VdOMgwRL+8X1jjvl7sTYueU3zCsd", + "RaEjo9VHYlhe/aWKIJXR90ZyNlsjevG5eDObbeWCuXuwtK1CgcQ2moT+DfqOtkqkBjovVaTuc74W4M9o", + "lmG0p7POaEky64ZzZU7BfKcXbHWvZGQOpWjs8OPeUxEbDkXc6NW2U/Rf6pxpmlJNv4KxNez6/4e40luj", + "4dNKL5jQkFXg+vQZbHChqH3Wgs/GSQzk1hJmsDnMMuBUvD7wKMZqm0gcFYyDUxt8beSAlTrtxgdx9Aqk", + "QpL+L+42Vu2OIS5DznX3ZyVmnYhVDxB6UWGEb6b9JKxzWOngpm0+fqKY1lL7L5TH050l1D8w5bFU3Z6b", + "sydDWELijQuK0MSQjYylWNsRE88sRRk1Y6IcuoBvlYs64clSGVaOMpnQDAgczdSXpmpXrLGbKuZeguCg", + "NXzWyuM2bvzm6utaw3tvWDeUqwvavfSRq1+kq6fq01p9kbHA7vFg//ALtj5EFOtFzBNWus4zz5ngSDpt", + "/YO46RxD6CzLo4nmV2iJZeAedTW2skwu0VdhwWK3XvL5QhMhlzaA7/B2GYy7SFRATh868IwUDqvDzDzI", + "+J9L6G1vM1vwwu14aa17kPrxA2hsuk2AU07hLONNgaIRdP3XxQyJ9rdvIRjV7qTvOlrZiAtcogsMvJZV", + "w47VjT6N3ZI6x0M1PHYOk1xZTyVtPpwfuy5Nd9sGk89kTg2jrrocEr0qeAKxh7ZbEwjMRSnnJVNqCO2c", + "XIMLWZIZ5VlVso0cxvEVxUTacNQZcLvRofo2K9nmm7KX09WIj8qqP6z0NV1ZU0olvomklNd09RfGirfo", + "cf7G1DMM/LZiTJ39HUjMges9YFBlJcgeuWSscK74OgCcvClc7ShIRKRcKEIJutpDmdQ7ZWL+9x5E7kj0", + "oOwFK2utias6Kn09astKF5UeFaVMq2SdoG+I5Rt4+cS9eyeYA9T82ntfsPmu2dhD+20h5l8rkftgy0Ru", + "kP5sirJr+/Hg/v2bv2ivmJjrhS9+9Kewc1zKU+wXbqgsJRYEI/sJ5uXblR7e/EpP6ArydaFtHS1tv68H", + "9x/ehhtBVUUhS3NQr1nKKTlbFdZjBihGEKOcMDn16eZ1F9gw+uvBwZPb6TDo6l8gpwTSISV2mJqZi20L", + "7Vm3tF6UUuuM2XJ8fyjJA/PcDaBzqTQpWYLZ/750IOwX5YEg250DcLDvlPm4doQwobD2H+ZQgPRuT9l8", + "eU+RlM+ZguLB7TMmz3z1AYgTO/nlR4DzzycvfiQWlcygRUaFiMdprRN49KLKp4LyTO0VJbvibOnIEi+x", + "YKKj9gSpvxODAKLllaPmVZkNjgZ7g8AI1SZWx80gqE5bMIcpnh1Akkq3kMjPcurMpCCj/b1iJTfoV7c7", + "HbbaUYwbVTRVZNCnJ8fN/pChiUzmeSVQ3IQCJe2lj9sO3MgEFhte+zWRpyfHw/7uzNjMymzD3JVSZm5F", + "ncnA6RgplYPlB/wswCfq2gkWgr5n5Xs59RXhwjlsuYNPv336PwEAAP//BXEShOARAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/api/openapi_types.gen.go b/pkg/api/openapi_types.gen.go index 80b49695..251670e2 100644 --- a/pkg/api/openapi_types.gen.go +++ b/pkg/api/openapi_types.gen.go @@ -84,6 +84,8 @@ const ( JobStatusFailed JobStatus = "failed" + JobStatusPauseRequested JobStatus = "pause-requested" + JobStatusPaused JobStatus = "paused" JobStatusQueued JobStatus = "queued" diff --git a/web/app/src/components/jobs/JobActionsBar.vue b/web/app/src/components/jobs/JobActionsBar.vue index 615ffae7..eeb24eb9 100644 --- a/web/app/src/components/jobs/JobActionsBar.vue +++ b/web/app/src/components/jobs/JobActionsBar.vue @@ -8,6 +8,9 @@ + diff --git a/web/app/src/manager-api/model/JobStatus.js b/web/app/src/manager-api/model/JobStatus.js index 169d161e..546532d6 100644 --- a/web/app/src/manager-api/model/JobStatus.js +++ b/web/app/src/manager-api/model/JobStatus.js @@ -54,6 +54,13 @@ export default class JobStatus { "paused" = "paused"; + /** + * value: "pause-requested" + * @const + */ + "pause-requested" = "pause-requested"; + + /** * value: "queued" * @const diff --git a/web/app/src/stores/tasks.js b/web/app/src/stores/tasks.js index afcdf393..b5e1bdb4 100644 --- a/web/app/src/stores/tasks.js +++ b/web/app/src/stores/tasks.js @@ -18,6 +18,9 @@ export const useTasks = defineStore('tasks', { activeTaskID: '', }), getters: { + canPause() { + return this._anyTaskWithStatus(['active', 'queued', 'cancelled']); + }, canCancel() { return this._anyTaskWithStatus(['queued', 'active', 'soft-failed']); }, -- 2.30.2 From 562f77bd7475332f16370439829e34bc1f81e7fe Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 1 Jun 2024 17:09:52 +0800 Subject: [PATCH 02/25] Manager: basic test case for paused status with all queued tasks --- .../task_state_machine_test.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index e936bd23..476a4ded 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -336,6 +336,29 @@ func TestJobCancelWithSomeCompletedTasks(t *testing.T) { require.NoError(t, sm.JobStatusChange(ctx, job, api.JobStatusCancelRequested, "someone wrote a unittest")) } +func TestJobPauseWithAllQueuedTasks(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + task1 := taskWithStatus(api.JobStatusActive, api.TaskStatusQueued) + task2 := taskOfSameJob(task1, api.TaskStatusQueued) + task3 := taskOfSameJob(task2, api.TaskStatusQueued) + job := task3.Job + + mocks.expectSaveJobWithStatus(t, job, api.JobStatusPauseRequested) + + // Expect pausing of the job to trigger pausing of all its queued tasks. + mocks.persist.EXPECT().UpdateJobsTaskStatuses(ctx, job, api.TaskStatusPaused, + "Paused because job transitioned status from \"active\" to \"paused\"") + + mocks.expectSaveJobWithStatus(t, job, api.JobStatusPaused) + + mocks.expectBroadcastJobChangeWithTaskRefresh(job, api.JobStatusActive, api.JobStatusPauseRequested) + mocks.expectBroadcastJobChange(job, api.JobStatusPauseRequested, api.JobStatusPaused) + + require.NoError(t, sm.JobStatusChange(ctx, job, api.JobStatusPauseRequested, "someone wrote a unittest")) +} + func TestCheckStuck(t *testing.T) { mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) defer mockCtrl.Finish() -- 2.30.2 From 716f86c0f42322df6644ae7a8d31efd1018d4866 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 1 Jun 2024 17:13:11 +0800 Subject: [PATCH 03/25] Manager: leave out updateJobOnTaskStatusPaused --- .../task_state_machine/task_state_machine.go | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 085837aa..2b4c2663 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -119,7 +119,7 @@ func (sm *StateMachine) updateJobAfterTaskStatusChange( return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusCompleted, api.JobStatusRequeueing, "task was queued") case api.TaskStatusPaused: - return sm.updateJobOnTaskStatusPaused(ctx, logger, job) + return nil case api.TaskStatusCanceled: return sm.updateJobOnTaskStatusCanceled(ctx, logger, job) @@ -182,38 +182,6 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge return nil } -// updateJobOnTaskStatusPaused conditionally escalates the pausing of a task to pause the job. -func (sm *StateMachine) updateJobOnTaskStatusPaused(ctx context.Context, logger zerolog.Logger, job *persistence.Job) error { - // If no more tasks can run, pause the job. - numRunnable, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed) - if err != nil { - return err - } - if numRunnable == 0 { - logger.Info().Msg("paused task was last runnable task of job, pausing job") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "paused task was last runnable task of job, pausing job") - } - - if job.Status == api.JobStatusPauseRequested { - // if the job is in pause-requested state, and all other tasks are paused, - // then the job can be paused. - numPaused, numTotal, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusPaused) - if err != nil { - return err - } - if numPaused == numTotal { - logger.Info().Msg("all tasks of job are paused, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks paused") - } - } else { - // if the job is not in pause-requested state, then some error occurred and the job should be failed. - logger.Info().Msg("task cannot be changed to paused when job is not in pause-requested state") - } - - return nil -} - // updateJobOnTaskStatusFailed conditionally escalates the failure of a task to fail the entire job. func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger zerolog.Logger, job *persistence.Job) error { // Count the number of failed tasks. If it is over the threshold, fail the job. -- 2.30.2 From e30832aab4dd4bb4992531cc97bc051773146159 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 1 Jun 2024 17:18:22 +0800 Subject: [PATCH 04/25] Manager: taskStatusesToPause should include soft-failed --- internal/manager/task_state_machine/task_state_machine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 2b4c2663..5520331c 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -455,6 +455,7 @@ func (sm *StateMachine) pauseTasks( api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusCanceled, + api.TaskStatusSoftFailed, } err := sm.persist.UpdateJobsTaskStatusesConditional( ctx, job, taskStatusesToPause, api.TaskStatusPaused, -- 2.30.2 From a84097e59f6578622e211ecb475a468cc03152ee Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 1 Jun 2024 17:27:06 +0800 Subject: [PATCH 05/25] Manager: improve error handling of pauseTasks() --- internal/manager/task_state_machine/task_state_machine.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 5520331c..3992ded7 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -5,6 +5,7 @@ package task_state_machine import ( "context" "fmt" + "projects.blender.org/studio/flamenco/pkg/website" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -471,9 +472,9 @@ func (sm *StateMachine) pauseTasks( return api.JobStatusPaused, nil } - // This could mean pause was triggered by failure of the job, in which case the - // job is already in the correct status. - return "", nil + // This could mean state transition entered a non-recoverable error state. + log.Warn().Str("jobStatus", string(job.Status)).Msgf("unexpected job status in StateMachine::pauseTasks(), please report this at %s", website.BugReportURL) + return "", fmt.Errorf("unexpected job status %q in StateMachine::pauseTasks()", job.Status) } // requeueTasks re-queues all tasks of the job. -- 2.30.2 From ba454cd4a0ba7ed244a7fb92622031f3ff21baca Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 4 Jun 2024 21:55:46 +0800 Subject: [PATCH 06/25] Webapp: interactive `Pause Job` button --- web/app/src/components/jobs/JobActionsBar.vue | 3 +++ web/app/src/stores/jobs.js | 6 ++++++ web/app/src/stores/tasks.js | 3 --- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/web/app/src/components/jobs/JobActionsBar.vue b/web/app/src/components/jobs/JobActionsBar.vue index eeb24eb9..d69deff6 100644 --- a/web/app/src/components/jobs/JobActionsBar.vue +++ b/web/app/src/components/jobs/JobActionsBar.vue @@ -72,6 +72,9 @@ export default { onButtonRequeue() { return this._handleJobActionPromise(this.jobs.requeueJobs(), 'requeueing'); }, + onButtonPause() { + return this._handleJobActionPromise(this.jobs.pauseJobs(), 'marked for pausing'); + }, _handleJobActionPromise(promise, description) { return promise.then(() => { diff --git a/web/app/src/stores/jobs.js b/web/app/src/stores/jobs.js index 84c31552..3e524717 100644 --- a/web/app/src/stores/jobs.js +++ b/web/app/src/stores/jobs.js @@ -33,6 +33,9 @@ export const useJobs = defineStore('jobs', { canRequeue() { return this._anyJobWithStatus(['canceled', 'completed', 'failed', 'paused']); }, + canPause() { + return this._anyJobWithStatus(['active', 'queued', 'canceled']); + }, }, actions: { setIsJobless(isJobless) { @@ -74,6 +77,9 @@ export const useJobs = defineStore('jobs', { cancelJobs() { return this._setJobStatus('cancel-requested'); }, + pauseJobs() { + return this._setJobStatus('pause-requested'); + }, requeueJobs() { return this._setJobStatus('requeueing'); }, diff --git a/web/app/src/stores/tasks.js b/web/app/src/stores/tasks.js index b5e1bdb4..afcdf393 100644 --- a/web/app/src/stores/tasks.js +++ b/web/app/src/stores/tasks.js @@ -18,9 +18,6 @@ export const useTasks = defineStore('tasks', { activeTaskID: '', }), getters: { - canPause() { - return this._anyTaskWithStatus(['active', 'queued', 'cancelled']); - }, canCancel() { return this._anyTaskWithStatus(['queued', 'active', 'soft-failed']); }, -- 2.30.2 From 050c779995b3c20ccd92b5056511ec5046c1ca92 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 7 Jun 2024 01:05:03 +0800 Subject: [PATCH 07/25] Manager: improve state transition logic --- .../task_state_machine/task_state_machine.go | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 3992ded7..44f18647 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -5,8 +5,6 @@ package task_state_machine import ( "context" "fmt" - "projects.blender.org/studio/flamenco/pkg/website" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -120,6 +118,7 @@ func (sm *StateMachine) updateJobAfterTaskStatusChange( return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusCompleted, api.JobStatusRequeueing, "task was queued") case api.TaskStatusPaused: + // Pausing a task has no impact on the job. return nil case api.TaskStatusCanceled: @@ -214,10 +213,19 @@ func (sm *StateMachine) updateJobOnTaskStatusCompleted(ctx context.Context, logg if err != nil { return err } + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return err + } if numComplete == numTotal { logger.Info().Msg("all tasks of job are completed, job is completed") return sm.JobStatusChange(ctx, job, api.JobStatusCompleted, "all tasks completed") } + if numActive == 0 && job.Status == api.JobStatusPauseRequested { + // there is no active task, and the job is in pause-requested status, so we can pause the job + logger.Info().Msg("all tasks of job are completed, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + } logger.Info(). Int("taskNumTotal", numTotal). Int("taskNumComplete", numComplete). @@ -453,7 +461,6 @@ func (sm *StateMachine) pauseTasks( // Any task that might run in the future should get paused. // Active jobs should remain active until finished taskStatusesToPause := []api.TaskStatus{ - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusCanceled, api.TaskStatusSoftFailed, @@ -466,15 +473,7 @@ func (sm *StateMachine) pauseTasks( return "", fmt.Errorf("pausing tasks of job %s: %w", job.UUID, err) } - // If pause was requested, it has now happened, so the job can transition. - if job.Status == api.JobStatusPauseRequested { - logger.Info().Msg("all tasks of job paused, job can go to 'paused' status") - return api.JobStatusPaused, nil - } - - // This could mean state transition entered a non-recoverable error state. - log.Warn().Str("jobStatus", string(job.Status)).Msgf("unexpected job status in StateMachine::pauseTasks(), please report this at %s", website.BugReportURL) - return "", fmt.Errorf("unexpected job status %q in StateMachine::pauseTasks()", job.Status) + return api.JobStatusPauseRequested, nil } // requeueTasks re-queues all tasks of the job. -- 2.30.2 From 71c7715eab7dbe04c458b9413c480e582621afd7 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 7 Jun 2024 14:35:09 +0800 Subject: [PATCH 08/25] Manager: cancelling tasks should not cause a pause-requested job to be cancelled --- .../task_state_machine/task_state_machine.go | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 44f18647..6810a2fc 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -169,7 +169,7 @@ func (sm *StateMachine) jobStatusIfAThenB( func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logger zerolog.Logger, job *persistence.Job) error { // If no more tasks can run, cancel the job. numRunnable, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed) + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused) if err != nil { return err } @@ -179,6 +179,16 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge return sm.JobStatusChange(ctx, job, api.JobStatusCanceled, "canceled task was last runnable task of job, canceling job") } + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return err + } + if numActive == 0 && job.Status == api.JobStatusPauseRequested { + // there is no active task, and the job is in pause-requested status, so we can pause the job + logger.Info().Msg("all tasks of job are completed, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + } + return nil } @@ -203,6 +213,16 @@ func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger } // If the job didn't fail, this failure indicates that at least the job is active. failLogger.Info().Msg("task failed, but not enough to fail the job") + + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return err + } + if numActive == 0 && job.Status == api.JobStatusPauseRequested { + // there is no active task, and the job is in pause-requested status, so we can pause the job + logger.Info().Msg("all tasks of job are completed, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + } return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, "task failed, but not enough to fail the job") } @@ -473,6 +493,16 @@ func (sm *StateMachine) pauseTasks( return "", fmt.Errorf("pausing tasks of job %s: %w", job.UUID, err) } + // If pausing was requested, it has now happened, so the job can transition. + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return "", fmt.Errorf("error when accessing number of active tasks") + } + if job.Status == api.JobStatusPauseRequested && numActive == 0 { + logger.Info().Msg("all tasks of job paused, job can go to 'paused' status") + return api.JobStatusPaused, nil + } + return api.JobStatusPauseRequested, nil } -- 2.30.2 From 0e523a9b3132873e46e874005231ea906c64c88b Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 7 Jun 2024 16:28:16 +0800 Subject: [PATCH 09/25] Manager: passing TestTaskStatusChangeActiveToCompleted --- internal/manager/task_state_machine/task_state_machine_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index 476a4ded..7ee71850 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -76,6 +76,7 @@ func TestTaskStatusChangeActiveToCompleted(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task, "task changed status active -> completed") mocks.expectBroadcastTaskChange(task, api.TaskStatusActive, api.TaskStatusCompleted) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(1, 3, nil) // 1 of 3 complete. + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(2, 3, nil) // 2 of 3 active. require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCompleted)) // Second task hickup: T: active > soft-failed --> J: active > active @@ -89,6 +90,7 @@ func TestTaskStatusChangeActiveToCompleted(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task2, "task changed status soft-failed -> completed") mocks.expectBroadcastTaskChange(task2, api.TaskStatusSoftFailed, api.TaskStatusCompleted) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(2, 3, nil) // 2 of 3 complete. + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(1, 3, nil) // 1 of 3 active. require.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusCompleted)) // Third task completing: T: active > completed --> J: active > completed @@ -96,6 +98,7 @@ func TestTaskStatusChangeActiveToCompleted(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task3, "task changed status active -> completed") mocks.expectBroadcastTaskChange(task3, api.TaskStatusActive, api.TaskStatusCompleted) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(3, 3, nil) // 3 of 3 complete. + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(0, 3, nil) // 0 of 3 active. mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusCompleted) mocks.expectBroadcastJobChange(task.Job, api.JobStatusActive, api.JobStatusCompleted) -- 2.30.2 From 1c35617cb151e739082d19d5907e86218c8bba73 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Thu, 13 Jun 2024 00:18:33 +0800 Subject: [PATCH 10/25] Manager: fix manager warning --- internal/manager/task_state_machine/task_state_machine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 6810a2fc..7ec9c033 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -397,7 +397,7 @@ func (sm *StateMachine) updateTasksAfterJobStatusChange( // Every case in this switch MUST return, for sanity sake. switch job.Status { - case api.JobStatusCompleted, api.JobStatusCanceled: + case api.JobStatusCompleted, api.JobStatusCanceled, api.JobStatusPaused: // Nothing to do; this will happen as a response to all tasks receiving this status. return tasksUpdateResult{}, nil -- 2.30.2 From c3af84f6a66756fc327dec14fc393f3da9edc484 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Fri, 14 Jun 2024 20:14:46 +0900 Subject: [PATCH 11/25] Manager: pass TestTaskStatusChangeQueuedToFailed and TestTaskStatusChangeCancelSingleTask --- .../manager/task_state_machine/task_state_machine_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index f0bcd3bb..8380c2af 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -116,6 +116,7 @@ func TestTaskStatusChangeQueuedToFailed(t *testing.T) { mocks.expectBroadcastTaskChange(task, api.TaskStatusQueued, api.TaskStatusFailed) mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusActive) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusFailed).Return(1, 100, nil) // 1 out of 100 failed. + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(0, 100, nil) // 0 out of 100 active. mocks.expectBroadcastJobChange(task.Job, api.JobStatusQueued, api.JobStatusActive) require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusFailed)) @@ -188,8 +189,10 @@ func TestTaskStatusChangeCancelSingleTask(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task, "task changed status active -> canceled") mocks.expectBroadcastTaskChange(task, api.TaskStatusActive, api.TaskStatusCanceled) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed). + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused). Return(1, 2, nil) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, + api.TaskStatusActive).Return(0, 2, nil) require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCanceled)) // T2: queued > cancelled --> J: cancel-requested > canceled @@ -197,7 +200,7 @@ func TestTaskStatusChangeCancelSingleTask(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task2, "task changed status queued -> canceled") mocks.expectBroadcastTaskChange(task2, api.TaskStatusQueued, api.TaskStatusCanceled) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed). + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused). Return(0, 2, nil) mocks.expectSaveJobWithStatus(t, job, api.JobStatusCanceled) mocks.expectBroadcastJobChange(task.Job, api.JobStatusCancelRequested, api.JobStatusCanceled) -- 2.30.2 From c32a591e5566531f2bf64aa61504afaa14f89790 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sat, 15 Jun 2024 11:52:21 -0400 Subject: [PATCH 12/25] Manager: disallow task cancellation when the job has `pause-requested` status --- web/app/src/stores/tasks.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/app/src/stores/tasks.js b/web/app/src/stores/tasks.js index afcdf393..99f16f5c 100644 --- a/web/app/src/stores/tasks.js +++ b/web/app/src/stores/tasks.js @@ -2,6 +2,7 @@ import { defineStore } from 'pinia'; import * as API from '@/manager-api'; import { getAPIClient } from '@/api-client'; +import { useJobs } from '@/stores/jobs'; const jobsAPI = new API.JobsApi(getAPIClient()); @@ -19,6 +20,14 @@ export const useTasks = defineStore('tasks', { }), getters: { canCancel() { + const jobs = useJobs(); + const activeJob = jobs.activeJob; + + // Check if the job status is 'pause-requested' + if (activeJob && activeJob.status === 'pause-requested') { + return false; + } + // Allow cancellation for specified task statuses return this._anyTaskWithStatus(['queued', 'active', 'soft-failed']); }, canRequeue() { -- 2.30.2 From 5a4e225a6b9cc424a96f89446e61294bec9d69f5 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 18 Jun 2024 13:36:07 -0400 Subject: [PATCH 13/25] Manager: keep the newline between built-in packages and 3rd party packages --- internal/manager/task_state_machine/task_state_machine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 7ec9c033..1abefe24 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -5,6 +5,7 @@ package task_state_machine import ( "context" "fmt" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" -- 2.30.2 From 4b4511f9b6d87788a78e6bb56ec1cc377d1e6e3b Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 18 Jun 2024 16:12:59 -0400 Subject: [PATCH 14/25] Manager: reorder updateJobOnTaskStatusCanceled logic --- .../task_state_machine/task_state_machine.go | 22 ++++++++++++------- .../task_state_machine_test.go | 2 -- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 1abefe24..7c4700cf 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -180,16 +180,22 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge return sm.JobStatusChange(ctx, job, api.JobStatusCanceled, "canceled task was last runnable task of job, canceling job") } - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return err - } - if numActive == 0 && job.Status == api.JobStatusPauseRequested { - // there is no active task, and the job is in pause-requested status, so we can pause the job - logger.Info().Msg("all tasks of job are completed, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + // Deal with the special case when the job is in pause-requested status. + if job.Status != api.JobStatusPauseRequested { + return nil + } else { + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return err + } + if numActive == 0 { + // there is no active task, and the job is in pause-requested status, so we can pause the job + logger.Info().Msg("all tasks of job are completed, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + } } + // Execution should not reach here. return nil } diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index 8380c2af..da150724 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -191,8 +191,6 @@ func TestTaskStatusChangeCancelSingleTask(t *testing.T) { mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused). Return(1, 2, nil) - mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive).Return(0, 2, nil) require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCanceled)) // T2: queued > cancelled --> J: cancel-requested > canceled -- 2.30.2 From c13cc884cec764161d314b4632a2ed2a6135b3b3 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 18 Jun 2024 16:16:32 -0400 Subject: [PATCH 15/25] Webapp: end comment with a period --- web/app/src/stores/tasks.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/src/stores/tasks.js b/web/app/src/stores/tasks.js index 99f16f5c..67803473 100644 --- a/web/app/src/stores/tasks.js +++ b/web/app/src/stores/tasks.js @@ -23,11 +23,11 @@ export const useTasks = defineStore('tasks', { const jobs = useJobs(); const activeJob = jobs.activeJob; - // Check if the job status is 'pause-requested' + // Check if the job status is 'pause-requested'. if (activeJob && activeJob.status === 'pause-requested') { return false; } - // Allow cancellation for specified task statuses + // Allow cancellation for specified task statuses. return this._anyTaskWithStatus(['queued', 'active', 'soft-failed']); }, canRequeue() { -- 2.30.2 From a569a1ce00d50d18196cc17d613cc9362713e942 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 18 Jun 2024 16:27:24 -0400 Subject: [PATCH 16/25] Manager: pauseTasks keeps cancelled tasks as cancelled --- internal/manager/task_state_machine/task_state_machine.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 7c4700cf..f0042688 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -486,10 +486,9 @@ func (sm *StateMachine) pauseTasks( logger.Info().Msg("pausing tasks of job") // Any task that might run in the future should get paused. - // Active jobs should remain active until finished + // Active tasks should remain active until finished. taskStatusesToPause := []api.TaskStatus{ api.TaskStatusQueued, - api.TaskStatusCanceled, api.TaskStatusSoftFailed, } err := sm.persist.UpdateJobsTaskStatusesConditional( -- 2.30.2 From 05a1388106f82e9083031b27e79da4b2d5d80653 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 18 Jun 2024 16:31:08 -0400 Subject: [PATCH 17/25] Webapp: update canCancel to properly expose bugs --- web/app/src/stores/tasks.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/web/app/src/stores/tasks.js b/web/app/src/stores/tasks.js index 67803473..351dff6e 100644 --- a/web/app/src/stores/tasks.js +++ b/web/app/src/stores/tasks.js @@ -23,10 +23,17 @@ export const useTasks = defineStore('tasks', { const jobs = useJobs(); const activeJob = jobs.activeJob; - // Check if the job status is 'pause-requested'. - if (activeJob && activeJob.status === 'pause-requested') { + if (!activeJob) { + console.warn('no active job, unable to determine whether the active task is cancellable'); return false; } + + if (activeJob.status == 'pause-requested') { + // Cancelling a task should not be possible while the job is being paused. + // In the future this might be supported, see issue #104315. + return false; + } + // Allow cancellation for specified task statuses. return this._anyTaskWithStatus(['queued', 'active', 'soft-failed']); }, -- 2.30.2 From 01754e9e3c2ef7ad0a8d6440311ddbbf1745004a Mon Sep 17 00:00:00 2001 From: David Zhang Date: Tue, 18 Jun 2024 16:47:55 -0400 Subject: [PATCH 18/25] Manager: correct logger message and reorder logic --- .../task_state_machine/task_state_machine.go | 53 ++++++++++++------- .../task_state_machine_test.go | 4 -- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index f0042688..c46621f3 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -189,13 +189,12 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge return err } if numActive == 0 { - // there is no active task, and the job is in pause-requested status, so we can pause the job - logger.Info().Msg("all tasks of job are completed, job is paused") + // There is no active task, and the job is in pause-requested status, so we can pause the job. + logger.Info().Msg("No more active tasks, job is paused") return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") } } - // Execution should not reach here. return nil } @@ -221,15 +220,21 @@ func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger // If the job didn't fail, this failure indicates that at least the job is active. failLogger.Info().Msg("task failed, but not enough to fail the job") - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return err - } - if numActive == 0 && job.Status == api.JobStatusPauseRequested { - // there is no active task, and the job is in pause-requested status, so we can pause the job - logger.Info().Msg("all tasks of job are completed, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + if job.Status != api.JobStatusPauseRequested { + return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, + "task failed, but not enough to fail the job") + } else { + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return err + } + if numActive == 0 { + // There is no active task, and the job is in pause-requested status, so we can pause the job. + failLogger.Info().Msg("No more active tasks, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + } } + return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, "task failed, but not enough to fail the job") } @@ -240,19 +245,29 @@ func (sm *StateMachine) updateJobOnTaskStatusCompleted(ctx context.Context, logg if err != nil { return err } - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return err - } if numComplete == numTotal { logger.Info().Msg("all tasks of job are completed, job is completed") return sm.JobStatusChange(ctx, job, api.JobStatusCompleted, "all tasks completed") } - if numActive == 0 && job.Status == api.JobStatusPauseRequested { - // there is no active task, and the job is in pause-requested status, so we can pause the job - logger.Info().Msg("all tasks of job are completed, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + + if job.Status != api.JobStatusPauseRequested { + logger.Info(). + Int("taskNumTotal", numTotal). + Int("taskNumComplete", numComplete). + Msg("task completed; there are more tasks to do") + return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, "no more tasks to do") + } else { + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return err + } + if numActive == 0 { + // There is no active task, and the job is in pause-requested status, so we can pause the job. + logger.Info().Msg("No more active tasks, job is paused") + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") + } } + logger.Info(). Int("taskNumTotal", numTotal). Int("taskNumComplete", numComplete). diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index da150724..bec7c9fc 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -76,7 +76,6 @@ func TestTaskStatusChangeActiveToCompleted(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task, "task changed status active -> completed") mocks.expectBroadcastTaskChange(task, api.TaskStatusActive, api.TaskStatusCompleted) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(1, 3, nil) // 1 of 3 complete. - mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(2, 3, nil) // 2 of 3 active. require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCompleted)) // Second task hickup: T: active > soft-failed --> J: active > active @@ -90,7 +89,6 @@ func TestTaskStatusChangeActiveToCompleted(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task2, "task changed status soft-failed -> completed") mocks.expectBroadcastTaskChange(task2, api.TaskStatusSoftFailed, api.TaskStatusCompleted) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(2, 3, nil) // 2 of 3 complete. - mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(1, 3, nil) // 1 of 3 active. require.NoError(t, sm.TaskStatusChange(ctx, task2, api.TaskStatusCompleted)) // Third task completing: T: active > completed --> J: active > completed @@ -98,7 +96,6 @@ func TestTaskStatusChangeActiveToCompleted(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task3, "task changed status active -> completed") mocks.expectBroadcastTaskChange(task3, api.TaskStatusActive, api.TaskStatusCompleted) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusCompleted).Return(3, 3, nil) // 3 of 3 complete. - mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(0, 3, nil) // 0 of 3 active. mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusCompleted) mocks.expectBroadcastJobChange(task.Job, api.JobStatusActive, api.JobStatusCompleted) @@ -116,7 +113,6 @@ func TestTaskStatusChangeQueuedToFailed(t *testing.T) { mocks.expectBroadcastTaskChange(task, api.TaskStatusQueued, api.TaskStatusFailed) mocks.expectSaveJobWithStatus(t, task.Job, api.JobStatusActive) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusFailed).Return(1, 100, nil) // 1 out of 100 failed. - mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, task.Job, api.TaskStatusActive).Return(0, 100, nil) // 0 out of 100 active. mocks.expectBroadcastJobChange(task.Job, api.JobStatusQueued, api.JobStatusActive) require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusFailed)) -- 2.30.2 From d510eb390ca155929d83bba0a1b630e1aee5b6f9 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 19 Jun 2024 11:21:15 -0400 Subject: [PATCH 19/25] Manager: reorder logic --- .../task_state_machine/task_state_machine.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index c46621f3..cc446f9c 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -181,9 +181,7 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge } // Deal with the special case when the job is in pause-requested status. - if job.Status != api.JobStatusPauseRequested { - return nil - } else { + if job.Status == api.JobStatusPauseRequested { numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) if err != nil { return err @@ -220,10 +218,7 @@ func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger // If the job didn't fail, this failure indicates that at least the job is active. failLogger.Info().Msg("task failed, but not enough to fail the job") - if job.Status != api.JobStatusPauseRequested { - return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, - "task failed, but not enough to fail the job") - } else { + if job.Status == api.JobStatusPauseRequested { numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) if err != nil { return err @@ -250,13 +245,7 @@ func (sm *StateMachine) updateJobOnTaskStatusCompleted(ctx context.Context, logg return sm.JobStatusChange(ctx, job, api.JobStatusCompleted, "all tasks completed") } - if job.Status != api.JobStatusPauseRequested { - logger.Info(). - Int("taskNumTotal", numTotal). - Int("taskNumComplete", numComplete). - Msg("task completed; there are more tasks to do") - return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, "no more tasks to do") - } else { + if job.Status == api.JobStatusPauseRequested { numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) if err != nil { return err -- 2.30.2 From 8f067da901a148e4cb2a8bbb45cce17a45c4a118 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Thu, 20 Jun 2024 15:08:21 -0400 Subject: [PATCH 20/25] Manager: wrap job pause condition inside a function --- .../task_state_machine/task_state_machine.go | 67 ++++++++++--------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index cc446f9c..6e0113cb 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -166,11 +166,26 @@ func (sm *StateMachine) jobStatusIfAThenB( return sm.JobStatusChange(ctx, job, thenStatus, reason) } +func (sm *StateMachine) shouldJobBePaused(ctx context.Context, logger zerolog.Logger, job *persistence.Job) (bool, error) { + if job.Status == api.JobStatusPauseRequested { + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return false, err + } + if numActive == 0 { + // There is no active task, and the job is in pause-requested status, so we can pause the job. + logger.Info().Msg("No more active tasks, job is paused") + return true, nil + } + } + return false, nil +} + // updateJobOnTaskStatusCanceled conditionally escalates the cancellation of a task to cancel the job. func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logger zerolog.Logger, job *persistence.Job) error { // If no more tasks can run, cancel the job. numRunnable, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused) + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed) if err != nil { return err } @@ -181,16 +196,12 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge } // Deal with the special case when the job is in pause-requested status. - if job.Status == api.JobStatusPauseRequested { - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return err - } - if numActive == 0 { - // There is no active task, and the job is in pause-requested status, so we can pause the job. - logger.Info().Msg("No more active tasks, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") - } + toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + if err != nil { + return err + } + if toBePaused { + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "no more active tasks after task cancellation") } return nil @@ -218,16 +229,13 @@ func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger // If the job didn't fail, this failure indicates that at least the job is active. failLogger.Info().Msg("task failed, but not enough to fail the job") - if job.Status == api.JobStatusPauseRequested { - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return err - } - if numActive == 0 { - // There is no active task, and the job is in pause-requested status, so we can pause the job. - failLogger.Info().Msg("No more active tasks, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") - } + // Deal with the special case when the job is in pause-requested status. + toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + if err != nil { + return err + } + if toBePaused { + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "no more active tasks after task failure") } return sm.jobStatusIfAThenB(ctx, logger, job, api.JobStatusQueued, api.JobStatusActive, @@ -245,16 +253,13 @@ func (sm *StateMachine) updateJobOnTaskStatusCompleted(ctx context.Context, logg return sm.JobStatusChange(ctx, job, api.JobStatusCompleted, "all tasks completed") } - if job.Status == api.JobStatusPauseRequested { - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return err - } - if numActive == 0 { - // There is no active task, and the job is in pause-requested status, so we can pause the job. - logger.Info().Msg("No more active tasks, job is paused") - return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "all tasks completed") - } + // Deal with the special case when the job is in pause-requested status. + toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + if err != nil { + return err + } + if toBePaused { + return sm.JobStatusChange(ctx, job, api.JobStatusPaused, "no more active tasks after task completion") } logger.Info(). -- 2.30.2 From 009ad23da133f9391218229ce8cc7f4d51860239 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Thu, 20 Jun 2024 15:13:43 -0400 Subject: [PATCH 21/25] Manager: all original test cases passed --- .../manager/task_state_machine/task_state_machine_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index bec7c9fc..f48f50a6 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -185,7 +185,7 @@ func TestTaskStatusChangeCancelSingleTask(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task, "task changed status active -> canceled") mocks.expectBroadcastTaskChange(task, api.TaskStatusActive, api.TaskStatusCanceled) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused). + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed). Return(1, 2, nil) require.NoError(t, sm.TaskStatusChange(ctx, task, api.TaskStatusCanceled)) @@ -194,7 +194,7 @@ func TestTaskStatusChangeCancelSingleTask(t *testing.T) { mocks.expectWriteTaskLogTimestamped(t, task2, "task changed status queued -> canceled") mocks.expectBroadcastTaskChange(task2, api.TaskStatusQueued, api.TaskStatusCanceled) mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, - api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed, api.TaskStatusPaused). + api.TaskStatusActive, api.TaskStatusQueued, api.TaskStatusSoftFailed). Return(0, 2, nil) mocks.expectSaveJobWithStatus(t, job, api.JobStatusCanceled) mocks.expectBroadcastJobChange(task.Job, api.JobStatusCancelRequested, api.JobStatusCanceled) -- 2.30.2 From 484f746f92da3912e02b67f672fa3fc2e1979068 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Thu, 20 Jun 2024 15:56:18 -0400 Subject: [PATCH 22/25] Manager: add more test cases for paused status --- .../task_state_machine/task_state_machine.go | 4 +- .../task_state_machine_test.go | 77 +++++++++++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 6e0113cb..d0bcd596 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -509,11 +509,11 @@ func (sm *StateMachine) pauseTasks( } // If pausing was requested, it has now happened, so the job can transition. - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) if err != nil { return "", fmt.Errorf("error when accessing number of active tasks") } - if job.Status == api.JobStatusPauseRequested && numActive == 0 { + if toBePaused { logger.Info().Msg("all tasks of job paused, job can go to 'paused' status") return api.JobStatusPaused, nil } diff --git a/internal/manager/task_state_machine/task_state_machine_test.go b/internal/manager/task_state_machine/task_state_machine_test.go index f48f50a6..531bd7ca 100644 --- a/internal/manager/task_state_machine/task_state_machine_test.go +++ b/internal/manager/task_state_machine/task_state_machine_test.go @@ -340,7 +340,7 @@ func TestJobPauseWithAllQueuedTasks(t *testing.T) { mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) defer mockCtrl.Finish() - task1 := taskWithStatus(api.JobStatusActive, api.TaskStatusQueued) + task1 := taskWithStatus(api.JobStatusQueued, api.TaskStatusQueued) task2 := taskOfSameJob(task1, api.TaskStatusQueued) task3 := taskOfSameJob(task2, api.TaskStatusQueued) job := task3.Job @@ -348,17 +348,82 @@ func TestJobPauseWithAllQueuedTasks(t *testing.T) { mocks.expectSaveJobWithStatus(t, job, api.JobStatusPauseRequested) // Expect pausing of the job to trigger pausing of all its queued tasks. - mocks.persist.EXPECT().UpdateJobsTaskStatuses(ctx, job, api.TaskStatusPaused, - "Paused because job transitioned status from \"active\" to \"paused\"") - + mocks.persist.EXPECT().UpdateJobsTaskStatusesConditional(ctx, job, + []api.TaskStatus{ + api.TaskStatusQueued, + api.TaskStatusSoftFailed, + }, + api.TaskStatusPaused, + "Manager paused this task because the job got status \"pause-requested\".", + ) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, + api.TaskStatusActive). + Return(0, 3, nil) mocks.expectSaveJobWithStatus(t, job, api.JobStatusPaused) - - mocks.expectBroadcastJobChangeWithTaskRefresh(job, api.JobStatusActive, api.JobStatusPauseRequested) + mocks.expectBroadcastJobChangeWithTaskRefresh(job, api.JobStatusQueued, api.JobStatusPauseRequested) mocks.expectBroadcastJobChange(job, api.JobStatusPauseRequested, api.JobStatusPaused) require.NoError(t, sm.JobStatusChange(ctx, job, api.JobStatusPauseRequested, "someone wrote a unittest")) } +func TestJobPauseWithSomeCompletedTasks(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + task1 := taskWithStatus(api.JobStatusQueued, api.TaskStatusCompleted) + task2 := taskOfSameJob(task1, api.TaskStatusQueued) + task3 := taskOfSameJob(task2, api.TaskStatusQueued) + job := task3.Job + + mocks.expectSaveJobWithStatus(t, job, api.JobStatusPauseRequested) + + // Expect pausing of the job to trigger pausing of all its queued tasks. + mocks.persist.EXPECT().UpdateJobsTaskStatusesConditional(ctx, job, + []api.TaskStatus{ + api.TaskStatusQueued, + api.TaskStatusSoftFailed, + }, + api.TaskStatusPaused, + "Manager paused this task because the job got status \"pause-requested\".", + ) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, + api.TaskStatusActive). + Return(0, 3, nil) + mocks.expectSaveJobWithStatus(t, job, api.JobStatusPaused) + mocks.expectBroadcastJobChangeWithTaskRefresh(job, api.JobStatusQueued, api.JobStatusPauseRequested) + mocks.expectBroadcastJobChange(job, api.JobStatusPauseRequested, api.JobStatusPaused) + + require.NoError(t, sm.JobStatusChange(ctx, job, api.JobStatusPauseRequested, "someone wrote a unittest")) +} + +func TestJobPauseWithSomeActiveTasks(t *testing.T) { + mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) + defer mockCtrl.Finish() + + task1 := taskWithStatus(api.JobStatusActive, api.TaskStatusActive) + task2 := taskOfSameJob(task1, api.TaskStatusCompleted) + task3 := taskOfSameJob(task2, api.TaskStatusQueued) + job := task3.Job + + mocks.expectSaveJobWithStatus(t, job, api.JobStatusPauseRequested) + + // Expect pausing of the job to trigger pausing of all its queued tasks. + mocks.persist.EXPECT().UpdateJobsTaskStatusesConditional(ctx, job, + []api.TaskStatus{ + api.TaskStatusQueued, + api.TaskStatusSoftFailed, + }, + api.TaskStatusPaused, + "Manager paused this task because the job got status \"pause-requested\".", + ) + mocks.persist.EXPECT().CountTasksOfJobInStatus(ctx, job, + api.TaskStatusActive). + Return(1, 3, nil) + mocks.expectBroadcastJobChangeWithTaskRefresh(job, api.JobStatusActive, api.JobStatusPauseRequested) + + require.NoError(t, sm.JobStatusChange(ctx, job, api.JobStatusPauseRequested, "someone wrote a unittest")) +} + func TestCheckStuck(t *testing.T) { mockCtrl, ctx, sm, mocks := taskStateMachineTestFixtures(t) defer mockCtrl.Finish() -- 2.30.2 From adac7bbb372b1bf34913444f49fdd826fe078bce Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sun, 30 Jun 2024 22:17:45 -0400 Subject: [PATCH 23/25] Manager: function rename --- .../manager/task_state_machine/task_state_machine.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index d0bcd596..778d5e25 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -166,7 +166,8 @@ func (sm *StateMachine) jobStatusIfAThenB( return sm.JobStatusChange(ctx, job, thenStatus, reason) } -func (sm *StateMachine) shouldJobBePaused(ctx context.Context, logger zerolog.Logger, job *persistence.Job) (bool, error) { +// isJobPausingComplete returns true when the job status is pause-requested and there are no more active tasks. +func (sm *StateMachine) isJobPausingComplete(ctx context.Context, logger zerolog.Logger, job *persistence.Job) (bool, error) { if job.Status == api.JobStatusPauseRequested { numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) if err != nil { @@ -196,7 +197,7 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge } // Deal with the special case when the job is in pause-requested status. - toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) if err != nil { return err } @@ -230,7 +231,7 @@ func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger failLogger.Info().Msg("task failed, but not enough to fail the job") // Deal with the special case when the job is in pause-requested status. - toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) if err != nil { return err } @@ -254,7 +255,7 @@ func (sm *StateMachine) updateJobOnTaskStatusCompleted(ctx context.Context, logg } // Deal with the special case when the job is in pause-requested status. - toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) if err != nil { return err } @@ -509,7 +510,7 @@ func (sm *StateMachine) pauseTasks( } // If pausing was requested, it has now happened, so the job can transition. - toBePaused, err := sm.shouldJobBePaused(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) if err != nil { return "", fmt.Errorf("error when accessing number of active tasks") } -- 2.30.2 From 95f3472e6d82fa3dfcdd920ab6c9e94dfa29045e Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sun, 30 Jun 2024 22:21:50 -0400 Subject: [PATCH 24/25] Manager: flip condition --- .../task_state_machine/task_state_machine.go | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index 778d5e25..fe80bcc4 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -168,16 +168,17 @@ func (sm *StateMachine) jobStatusIfAThenB( // isJobPausingComplete returns true when the job status is pause-requested and there are no more active tasks. func (sm *StateMachine) isJobPausingComplete(ctx context.Context, logger zerolog.Logger, job *persistence.Job) (bool, error) { - if job.Status == api.JobStatusPauseRequested { - numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) - if err != nil { - return false, err - } - if numActive == 0 { - // There is no active task, and the job is in pause-requested status, so we can pause the job. - logger.Info().Msg("No more active tasks, job is paused") - return true, nil - } + if job.Status != api.JobStatusPauseRequested { + return false, nil + } + numActive, _, err := sm.persist.CountTasksOfJobInStatus(ctx, job, api.TaskStatusActive) + if err != nil { + return false, err + } + if numActive == 0 { + // There is no active task, and the job is in pause-requested status, so we can pause the job. + logger.Info().Msg("No more active tasks, job is paused") + return true, nil } return false, nil } -- 2.30.2 From 4b20f8fad8a58ea78a11edca4e385119ecafb9dd Mon Sep 17 00:00:00 2001 From: David Zhang Date: Sun, 30 Jun 2024 22:31:16 -0400 Subject: [PATCH 25/25] Manager: remove logging in isJobPausingComplete --- .../task_state_machine/task_state_machine.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/internal/manager/task_state_machine/task_state_machine.go b/internal/manager/task_state_machine/task_state_machine.go index fe80bcc4..1ab687ea 100644 --- a/internal/manager/task_state_machine/task_state_machine.go +++ b/internal/manager/task_state_machine/task_state_machine.go @@ -167,7 +167,7 @@ func (sm *StateMachine) jobStatusIfAThenB( } // isJobPausingComplete returns true when the job status is pause-requested and there are no more active tasks. -func (sm *StateMachine) isJobPausingComplete(ctx context.Context, logger zerolog.Logger, job *persistence.Job) (bool, error) { +func (sm *StateMachine) isJobPausingComplete(ctx context.Context, job *persistence.Job) (bool, error) { if job.Status != api.JobStatusPauseRequested { return false, nil } @@ -175,12 +175,7 @@ func (sm *StateMachine) isJobPausingComplete(ctx context.Context, logger zerolog if err != nil { return false, err } - if numActive == 0 { - // There is no active task, and the job is in pause-requested status, so we can pause the job. - logger.Info().Msg("No more active tasks, job is paused") - return true, nil - } - return false, nil + return numActive == 0, nil } // updateJobOnTaskStatusCanceled conditionally escalates the cancellation of a task to cancel the job. @@ -198,7 +193,7 @@ func (sm *StateMachine) updateJobOnTaskStatusCanceled(ctx context.Context, logge } // Deal with the special case when the job is in pause-requested status. - toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, job) if err != nil { return err } @@ -232,7 +227,7 @@ func (sm *StateMachine) updateJobOnTaskStatusFailed(ctx context.Context, logger failLogger.Info().Msg("task failed, but not enough to fail the job") // Deal with the special case when the job is in pause-requested status. - toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, job) if err != nil { return err } @@ -256,7 +251,7 @@ func (sm *StateMachine) updateJobOnTaskStatusCompleted(ctx context.Context, logg } // Deal with the special case when the job is in pause-requested status. - toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, job) if err != nil { return err } @@ -511,9 +506,9 @@ func (sm *StateMachine) pauseTasks( } // If pausing was requested, it has now happened, so the job can transition. - toBePaused, err := sm.isJobPausingComplete(ctx, logger, job) + toBePaused, err := sm.isJobPausingComplete(ctx, job) if err != nil { - return "", fmt.Errorf("error when accessing number of active tasks") + return "", err } if toBePaused { logger.Info().Msg("all tasks of job paused, job can go to 'paused' status") -- 2.30.2