CHANGELOG.md

@@ -9,6 +9,7 @@ bugs in actually-released versions.
 - Improve speed of queueing up >100 simultaneous job deletions.
 - Improve logging of job deletion.
 - Add Worker Cluster support. Workers can be members of any number of clusters. Workers will only work on jobs that are assigned to that cluster. Jobs that do not have a cluster will be available to all workers, regardless of their cluster assignment. As a result, clusterless workers will only work on clusterless jobs.
+- Fix limitation where a job could have no more than 1000 tasks ([#104201](https://projects.blender.org/studio/flamenco/issues/104201))
 
 
 ## 3.2 - released 2023-02-21

addon/flamenco/worker_clusters.py

@@ -36,6 +36,10 @@ def refresh(context: bpy.types.Context, api_client: _ApiClient) -> None:
         rna_cluster.name = cluster.name
         rna_cluster.description = getattr(cluster, "description", "")
 
+    # Preferences have changed, so make sure that Blender saves them (assuming
+    # auto-save here).
+    context.preferences.is_dirty = True
+
 
 def _get_enum_items(self, context):
     global _enum_items

internal/manager/persistence/jobs.go

@@ -208,11 +208,19 @@ func (db *DB) StoreAuthoredJob(ctx context.Context, authoredJob job_compilers.Au
 				}
 				deps[i] = depTask
 			}
-
-			dbTask.Dependencies = deps
-			subQuery := tx.Model(dbTask).Updates(Task{Dependencies: deps})
-			if subQuery.Error != nil {
-				return taskError(subQuery.Error, "unable to store dependencies of task %q", authoredTask.UUID)
+			dependenciesbatchsize := 1000
+			for j := 0; j < len(deps); j += dependenciesbatchsize {
+				end := j + dependenciesbatchsize
+				if end > len(deps) {
+					end = len(deps)
+				}
+				currentDeps := deps[j:end]
+				dbTask.Dependencies = currentDeps
+				tx.Model(&dbTask).Where("UUID = ?", dbTask.UUID)
+				subQuery := tx.Model(dbTask).Updates(Task{Dependencies: currentDeps})
+				if subQuery.Error != nil {
+					return taskError(subQuery.Error, "error with storing dependencies of task %q issue exists in dependencies %d to %d", authoredTask.UUID, j, end)
+				}
 			}
 		}
 

internal/manager/persistence/jobs_test.go

@@ -258,6 +258,21 @@ func TestCountTasksOfJobInStatus(t *testing.T) {
 	assert.Equal(t, 3, numTotal)
 }
 
+func TestCheckIfJobsHoldLargeNumOfTasks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test in short mode")
+	}
+	numtasks := 3500
+	ctx, close, db, job, _ := jobTasksTestFixturesWithTaskNum(t, numtasks)
+	defer close()
+
+	numQueued, numTotal, err := db.CountTasksOfJobInStatus(ctx, job, api.TaskStatusQueued)
+	assert.NoError(t, err)
+	assert.Equal(t, numtasks, numQueued)
+	assert.Equal(t, numtasks, numTotal)
+
+}
+
 func TestFetchJobsInStatus(t *testing.T) {
 	ctx, close, db, job1, _ := jobTasksTestFixtures(t)
 	defer close()
@@ -594,6 +609,36 @@ func createTestAuthoredJobWithTasks() job_compilers.AuthoredJob {
 	return createTestAuthoredJob("263fd47e-b9f8-4637-b726-fd7e47ecfdae", task1, task2, task3)
 }
 
+func createTestAuthoredJobWithNumTasks(numTasks int) job_compilers.AuthoredJob {
+	//Generates all of the render jobs
+	prevtasks := make([]*job_compilers.AuthoredTask, 0)
+	for i := 0; i < numTasks-1; i++ {
+		currtask := job_compilers.AuthoredTask{
+			Name:     "render-" + fmt.Sprintf("%d", i),
+			Type:     "blender-render",
+			UUID:     uuid.New(),
+			Commands: []job_compilers.AuthoredCommand{},
+		}
+		prevtasks = append(prevtasks, &currtask)
+	}
+	//	Generates the preview video command with Dependencies
+	videoJob := job_compilers.AuthoredTask{
+		Name:         "preview-video",
+		Type:         "ffmpeg",
+		UUID:         uuid.New(),
+		Commands:     []job_compilers.AuthoredCommand{},
+		Dependencies: prevtasks,
+	}
+	//	convert pointers to values and generate job
+	taskvalues := make([]job_compilers.AuthoredTask, len(prevtasks))
+	for i, ptr := range prevtasks {
+		taskvalues[i] = *ptr
+	}
+	taskvalues = append(taskvalues, videoJob)
+	return createTestAuthoredJob(uuid.New(), taskvalues...)
+
+}
+
 func createTestAuthoredJob(jobID string, tasks ...job_compilers.AuthoredTask) job_compilers.AuthoredJob {
 	job := job_compilers.AuthoredJob{
 		JobID:    jobID,
@@ -676,6 +721,16 @@ func jobTasksTestFixtures(t *testing.T) (context.Context, context.CancelFunc, *D
 	return ctx, cancel, db, dbJob, authoredJob
 }
 
+// This created Test Jobs using the new function createTestAuthoredJobWithNumTasks so that you can set the number of tasks
+func jobTasksTestFixturesWithTaskNum(t *testing.T, numtasks int) (context.Context, context.CancelFunc, *DB, *Job, job_compilers.AuthoredJob) {
+	ctx, cancel, db := persistenceTestFixtures(t, schedulerTestTimeoutlong)
+
+	authoredJob := createTestAuthoredJobWithNumTasks(numtasks)
+	dbJob := persistAuthoredJob(t, ctx, db, authoredJob)
+
+	return ctx, cancel, db, dbJob, authoredJob
+}
+
 func createWorker(ctx context.Context, t *testing.T, db *DB, updaters ...func(*Worker)) *Worker {
 	w := Worker{
 		UUID:               "f0a123a9-ab05-4ce2-8577-94802cfe74a4",

internal/manager/persistence/task_scheduler_test.go

@@ -16,6 +16,7 @@ import (
 )
 
 const schedulerTestTimeout = 100 * time.Millisecond
+const schedulerTestTimeoutlong = 5000 * time.Millisecond
 
 func TestNoTasks(t *testing.T) {
 	ctx, cancel, db := persistenceTestFixtures(t, schedulerTestTimeout)

web/project-website/content/design-principles/_index.md

@@ -0,0 +1,111 @@
+---
+title: Design Principles
+weight: 25
+---
+
+This page describes some of the design ideas & principles behind Flamenco.
+
+## Target Audience
+
+Flamenco is meant for **smaller animation studios and individuals** at home.
+Think of roughly **1-10 artists** using it, and **1-100 computers** attached to
+the farm to execute tasks. [Blender Studio][studio] uses a handful of servers,
+and combines those with various desktop machines when they're not used by the
+artists.
+
+## Design Principles
+
+The following principles guide the design of Flamenco:
+
+Blender.org Project
+: Flamenco is a true blender.org project. This means that it's Free and Open
+Source, made by the community, lead by Blender HQ. Its development will fall
+under the umbrella of the [Pipline, Assets & IO][PAIO] module.
+
+[PAIO]: https://projects.blender.org/blender/blender/wiki/Module:%20Pipeline,%20Assets%20&%20I/O
+
+Minimal Authentication & Organisation
+: Because Flamenco is aimed at small studios and individuals, it won't offer
+much in terms of user authentication, nor the organisation of users into groups.
+[Custom job types][jobtypes] can be used to attach arbitrary metadata to jobs,
+such as the submitter's name, a project identifier, etc.
+
+[jobtypes]: {{< ref "/usage/job-types" >}}
+
+Minimize External Components
+: Running Flamenco should be extremely simple. This means that it should depend
+on as few external packages as possible. Apart from the Flamenco components
+themselves, all you need to install is [Blender][blender].
+: The downside of this is that development might take longer, as some things
+that an external service could solve need to be implemented. This trade-off of
+developer time for simplicity of use is considered a good thing, though.
+
+[blender]: https://www.blender.org/
+
+No Errors, Guide Users To Success
+: Instead of stopping with a description of what's wrong, like "no database
+configured", Flamenco should show something helpful in which you're guided
+towards a working system.
+
+Customisable
+: Studio pipeline developers / TDs should be able to customise the behaviour of
+Flamenco. They should be able to create new [job types][jobtypes], and adjust
+existing job types to their needs. For this, Flamenco uses JavaScript to convert
+a job definition like "*render this blend file, frames 1-100*" into individual
+tasks for computers to execute.
+
+Work offline
+: Like Blender itself, Flamenco should be able to fully work offline. That is,
+work without internet connection. If any future feature should need such a
+connection, that feature should always be optional, and be disabled by default.
+
+Data Storage
+: Data should be stored as plain files whenever possible. Where a higher level
+of coordination is required, an embedded database can be used; currently
+Flamenco uses [SQLite][sqlite] for this.
+
+[sqlite]: https://pkg.go.dev/modernc.org/sqlite
+
+## Infrastructure & Supported Platforms
+
+Setting up a render farm is not as simple as pushing a button, but Flamenco aims
+to keep things as simple as possible. What you need to run Flamenco is:
+
+- One or more computers to do the work, i.e. running Flamenco Worker.
+- A computer to run the central software, Flamenco Manager. This could be one of
+  the above computers, or a dedicated one.
+- A local network with file sharing already set up, so that the above computers
+  can all reach the same set of files.
+
+Since Blender Studio fully runs on Open Source software, Linux is the main
+platform Flamenco is developed for. Windows and macOS will also be supported,
+but will need help from the community to get tested & developed well.
+
+## Software Design
+
+The Flamenco software follows an **API-first** approach. All the functionality
+of Flamenco Manager is exposed via [the OpenAPI interface][openapi] ([more
+info](openapi-info)). The web interface is no exception; anything you can do
+with the web interface, you can do with any other OpenAPI client.
+
+- The API can be browsed by following the 'API' link in the top-right corner of
+  the Flamenco Manager web interface. That's a link to
+  `http://your.manager.address/api/v3/swagger-ui/`
+- The web interface, Flamenco Worker, and the Blender add-on are all using that
+  same API.
+
+[openapi]: https://projects.blender.org/studio/flamenco/src/branch/main/pkg/api/flamenco-openapi.yaml
+[openapi-info]: https://www.openapis.org/
+
+## New Features
+
+To add a new feature to Flamenco, these steps are recommended:
+
+1. Define which changes to the API are necessary, and update the [flamenco-openapi.yaml][openapi] file for this.
+1. Run `go generate ./pkg/...` to generate the OpenAPI Go code.
+1. Implement any new operations in a minimal way, so that the code compiles (but doesn't do anything else).
+1. Run `make generate` to regenerate all the code (so also the JavaScript and Python client, and Go mocks).
+1. Write unit tests that test the new functionality.
+1. Write the code necessary to make the unit tests pass.
+1. Now that you know how it can work, refactor to clean it up.
+1. Send in a pull request!