WIP: 103268-job-task-progress #104185
@ -9,6 +9,7 @@ bugs in actually-released versions.
|
||||
- Improve speed of queueing up >100 simultaneous job deletions.
|
||||
- Improve logging of job deletion.
|
||||
- Add Worker Cluster support. Workers can be members of any number of clusters. Workers will only work on jobs that are assigned to that cluster. Jobs that do not have a cluster will be available to all workers, regardless of their cluster assignment. As a result, clusterless workers will only work on clusterless jobs.
|
||||
- Fix limitation where a job could have no more than 1000 tasks ([#104201](https://projects.blender.org/studio/flamenco/issues/104201))
|
||||
|
||||
|
||||
## 3.2 - released 2023-02-21
|
||||
|
@ -36,6 +36,10 @@ def refresh(context: bpy.types.Context, api_client: _ApiClient) -> None:
|
||||
rna_cluster.name = cluster.name
|
||||
rna_cluster.description = getattr(cluster, "description", "")
|
||||
|
||||
# Preferences have changed, so make sure that Blender saves them (assuming
|
||||
# auto-save here).
|
||||
context.preferences.is_dirty = True
|
||||
|
||||
|
||||
def _get_enum_items(self, context):
|
||||
global _enum_items
|
||||
|
@ -208,11 +208,19 @@ func (db *DB) StoreAuthoredJob(ctx context.Context, authoredJob job_compilers.Au
|
||||
}
|
||||
deps[i] = depTask
|
||||
}
|
||||
|
||||
dbTask.Dependencies = deps
|
||||
subQuery := tx.Model(dbTask).Updates(Task{Dependencies: deps})
|
||||
dependenciesbatchsize := 1000
|
||||
for j := 0; j < len(deps); j += dependenciesbatchsize {
|
||||
end := j + dependenciesbatchsize
|
||||
if end > len(deps) {
|
||||
end = len(deps)
|
||||
}
|
||||
currentDeps := deps[j:end]
|
||||
dbTask.Dependencies = currentDeps
|
||||
tx.Model(&dbTask).Where("UUID = ?", dbTask.UUID)
|
||||
subQuery := tx.Model(dbTask).Updates(Task{Dependencies: currentDeps})
|
||||
if subQuery.Error != nil {
|
||||
return taskError(subQuery.Error, "unable to store dependencies of task %q", authoredTask.UUID)
|
||||
return taskError(subQuery.Error, "error with storing dependencies of task %q issue exists in dependencies %d to %d", authoredTask.UUID, j, end)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -258,6 +258,21 @@ func TestCountTasksOfJobInStatus(t *testing.T) {
|
||||
assert.Equal(t, 3, numTotal)
|
||||
}
|
||||
|
||||
func TestCheckIfJobsHoldLargeNumOfTasks(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping test in short mode")
|
||||
}
|
||||
numtasks := 3500
|
||||
ctx, close, db, job, _ := jobTasksTestFixturesWithTaskNum(t, numtasks)
|
||||
defer close()
|
||||
|
||||
numQueued, numTotal, err := db.CountTasksOfJobInStatus(ctx, job, api.TaskStatusQueued)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, numtasks, numQueued)
|
||||
assert.Equal(t, numtasks, numTotal)
|
||||
|
||||
}
|
||||
|
||||
func TestFetchJobsInStatus(t *testing.T) {
|
||||
ctx, close, db, job1, _ := jobTasksTestFixtures(t)
|
||||
defer close()
|
||||
@ -594,6 +609,36 @@ func createTestAuthoredJobWithTasks() job_compilers.AuthoredJob {
|
||||
return createTestAuthoredJob("263fd47e-b9f8-4637-b726-fd7e47ecfdae", task1, task2, task3)
|
||||
}
|
||||
|
||||
func createTestAuthoredJobWithNumTasks(numTasks int) job_compilers.AuthoredJob {
|
||||
//Generates all of the render jobs
|
||||
prevtasks := make([]*job_compilers.AuthoredTask, 0)
|
||||
for i := 0; i < numTasks-1; i++ {
|
||||
currtask := job_compilers.AuthoredTask{
|
||||
Name: "render-" + fmt.Sprintf("%d", i),
|
||||
Type: "blender-render",
|
||||
UUID: uuid.New(),
|
||||
Commands: []job_compilers.AuthoredCommand{},
|
||||
}
|
||||
prevtasks = append(prevtasks, &currtask)
|
||||
}
|
||||
// Generates the preview video command with Dependencies
|
||||
videoJob := job_compilers.AuthoredTask{
|
||||
Name: "preview-video",
|
||||
Type: "ffmpeg",
|
||||
UUID: uuid.New(),
|
||||
Commands: []job_compilers.AuthoredCommand{},
|
||||
Dependencies: prevtasks,
|
||||
}
|
||||
// convert pointers to values and generate job
|
||||
taskvalues := make([]job_compilers.AuthoredTask, len(prevtasks))
|
||||
for i, ptr := range prevtasks {
|
||||
taskvalues[i] = *ptr
|
||||
}
|
||||
taskvalues = append(taskvalues, videoJob)
|
||||
return createTestAuthoredJob(uuid.New(), taskvalues...)
|
||||
|
||||
}
|
||||
|
||||
func createTestAuthoredJob(jobID string, tasks ...job_compilers.AuthoredTask) job_compilers.AuthoredJob {
|
||||
job := job_compilers.AuthoredJob{
|
||||
JobID: jobID,
|
||||
@ -676,6 +721,16 @@ func jobTasksTestFixtures(t *testing.T) (context.Context, context.CancelFunc, *D
|
||||
return ctx, cancel, db, dbJob, authoredJob
|
||||
}
|
||||
|
||||
// This created Test Jobs using the new function createTestAuthoredJobWithNumTasks so that you can set the number of tasks
|
||||
func jobTasksTestFixturesWithTaskNum(t *testing.T, numtasks int) (context.Context, context.CancelFunc, *DB, *Job, job_compilers.AuthoredJob) {
|
||||
ctx, cancel, db := persistenceTestFixtures(t, schedulerTestTimeoutlong)
|
||||
|
||||
authoredJob := createTestAuthoredJobWithNumTasks(numtasks)
|
||||
dbJob := persistAuthoredJob(t, ctx, db, authoredJob)
|
||||
|
||||
return ctx, cancel, db, dbJob, authoredJob
|
||||
}
|
||||
|
||||
func createWorker(ctx context.Context, t *testing.T, db *DB, updaters ...func(*Worker)) *Worker {
|
||||
w := Worker{
|
||||
UUID: "f0a123a9-ab05-4ce2-8577-94802cfe74a4",
|
||||
|
@ -16,6 +16,7 @@ import (
|
||||
)
|
||||
|
||||
const schedulerTestTimeout = 100 * time.Millisecond
|
||||
const schedulerTestTimeoutlong = 5000 * time.Millisecond
|
||||
|
||||
func TestNoTasks(t *testing.T) {
|
||||
ctx, cancel, db := persistenceTestFixtures(t, schedulerTestTimeout)
|
||||
|
111
web/project-website/content/design-principles/_index.md
Normal file
111
web/project-website/content/design-principles/_index.md
Normal file
@ -0,0 +1,111 @@
|
||||
---
|
||||
title: Design Principles
|
||||
weight: 25
|
||||
---
|
||||
|
||||
This page describes some of the design ideas & principles behind Flamenco.
|
||||
|
||||
## Target Audience
|
||||
|
||||
Flamenco is meant for **smaller animation studios and individuals** at home.
|
||||
Think of roughly **1-10 artists** using it, and **1-100 computers** attached to
|
||||
the farm to execute tasks. [Blender Studio][studio] uses a handful of servers,
|
||||
and combines those with various desktop machines when they're not used by the
|
||||
artists.
|
||||
|
||||
## Design Principles
|
||||
|
||||
The following principles guide the design of Flamenco:
|
||||
|
||||
Blender.org Project
|
||||
: Flamenco is a true blender.org project. This means that it's Free and Open
|
||||
Source, made by the community, lead by Blender HQ. Its development will fall
|
||||
under the umbrella of the [Pipline, Assets & IO][PAIO] module.
|
||||
|
||||
[PAIO]: https://projects.blender.org/blender/blender/wiki/Module:%20Pipeline,%20Assets%20&%20I/O
|
||||
|
||||
Minimal Authentication & Organisation
|
||||
: Because Flamenco is aimed at small studios and individuals, it won't offer
|
||||
much in terms of user authentication, nor the organisation of users into groups.
|
||||
[Custom job types][jobtypes] can be used to attach arbitrary metadata to jobs,
|
||||
such as the submitter's name, a project identifier, etc.
|
||||
|
||||
[jobtypes]: {{< ref "/usage/job-types" >}}
|
||||
|
||||
Minimize External Components
|
||||
: Running Flamenco should be extremely simple. This means that it should depend
|
||||
on as few external packages as possible. Apart from the Flamenco components
|
||||
themselves, all you need to install is [Blender][blender].
|
||||
: The downside of this is that development might take longer, as some things
|
||||
that an external service could solve need to be implemented. This trade-off of
|
||||
developer time for simplicity of use is considered a good thing, though.
|
||||
|
||||
[blender]: https://www.blender.org/
|
||||
|
||||
No Errors, Guide Users To Success
|
||||
: Instead of stopping with a description of what's wrong, like "no database
|
||||
configured", Flamenco should show something helpful in which you're guided
|
||||
towards a working system.
|
||||
|
||||
Customisable
|
||||
: Studio pipeline developers / TDs should be able to customise the behaviour of
|
||||
Flamenco. They should be able to create new [job types][jobtypes], and adjust
|
||||
existing job types to their needs. For this, Flamenco uses JavaScript to convert
|
||||
a job definition like "*render this blend file, frames 1-100*" into individual
|
||||
tasks for computers to execute.
|
||||
|
||||
Work offline
|
||||
: Like Blender itself, Flamenco should be able to fully work offline. That is,
|
||||
work without internet connection. If any future feature should need such a
|
||||
connection, that feature should always be optional, and be disabled by default.
|
||||
|
||||
Data Storage
|
||||
: Data should be stored as plain files whenever possible. Where a higher level
|
||||
of coordination is required, an embedded database can be used; currently
|
||||
Flamenco uses [SQLite][sqlite] for this.
|
||||
|
||||
[sqlite]: https://pkg.go.dev/modernc.org/sqlite
|
||||
|
||||
## Infrastructure & Supported Platforms
|
||||
|
||||
Setting up a render farm is not as simple as pushing a button, but Flamenco aims
|
||||
to keep things as simple as possible. What you need to run Flamenco is:
|
||||
|
||||
- One or more computers to do the work, i.e. running Flamenco Worker.
|
||||
- A computer to run the central software, Flamenco Manager. This could be one of
|
||||
the above computers, or a dedicated one.
|
||||
- A local network with file sharing already set up, so that the above computers
|
||||
can all reach the same set of files.
|
||||
|
||||
Since Blender Studio fully runs on Open Source software, Linux is the main
|
||||
platform Flamenco is developed for. Windows and macOS will also be supported,
|
||||
but will need help from the community to get tested & developed well.
|
||||
|
||||
## Software Design
|
||||
|
||||
The Flamenco software follows an **API-first** approach. All the functionality
|
||||
of Flamenco Manager is exposed via [the OpenAPI interface][openapi] ([more
|
||||
info](openapi-info)). The web interface is no exception; anything you can do
|
||||
with the web interface, you can do with any other OpenAPI client.
|
||||
|
||||
- The API can be browsed by following the 'API' link in the top-right corner of
|
||||
the Flamenco Manager web interface. That's a link to
|
||||
`http://your.manager.address/api/v3/swagger-ui/`
|
||||
- The web interface, Flamenco Worker, and the Blender add-on are all using that
|
||||
same API.
|
||||
|
||||
[openapi]: https://projects.blender.org/studio/flamenco/src/branch/main/pkg/api/flamenco-openapi.yaml
|
||||
[openapi-info]: https://www.openapis.org/
|
||||
|
||||
## New Features
|
||||
|
||||
To add a new feature to Flamenco, these steps are recommended:
|
||||
|
||||
1. Define which changes to the API are necessary, and update the [flamenco-openapi.yaml][openapi] file for this.
|
||||
1. Run `go generate ./pkg/...` to generate the OpenAPI Go code.
|
||||
1. Implement any new operations in a minimal way, so that the code compiles (but doesn't do anything else).
|
||||
1. Run `make generate` to regenerate all the code (so also the JavaScript and Python client, and Go mocks).
|
||||
1. Write unit tests that test the new functionality.
|
||||
1. Write the code necessary to make the unit tests pass.
|
||||
1. Now that you know how it can work, refactor to clean it up.
|
||||
1. Send in a pull request!
|
Loading…
Reference in New Issue
Block a user