.arcconfig

@@ -1,8 +0,0 @@
-{
-  "project_id": "Flamenco",
-  "conduit_uri": "https://developer.blender.org/",
-  "phabricator.uri": "https://developer.blender.org/",
-  "git.default-relative-commit": "origin/main",
-  "arc.land.update.default": "rebase",
-  "arc.land.onto.default": "main"
-}

CHANGELOG.md

@@ -4,6 +4,12 @@ This file contains the history of changes to Flamenco. Only changes that might
 be interesting for users are listed here, such as new features and fixes for
 bugs in actually-released versions.
 
+## 3.3 - in development
+
+- Improve speed of queueing up >100 simultaneous job deletions.
+- Improve logging of job deletion.
+
+
 ## 3.2 - released 2023-02-21
 
 - When rendering EXR files, use Blender's preview JPEG files to generate the preview video ([43bc22f10fae](https://developer.blender.org/rF43bc22f10fae0fcaed6a4a3b3ace1be617193e21)).

Makefile

@@ -2,13 +2,13 @@ PKG := git.blender.org/flamenco
 
 # To update the version number in all the relevant places, update the VERSION
 # variable below and run `make update-version`.
-VERSION := 3.2
+VERSION := 3.3-alpha0
-RELEASE_CYCLE := release
+RELEASE_CYCLE := alpha
 
 # _GIT_DESCRIPTION_OR_TAG is either something like '16-123abc' (when we're 16
 # commits since the last tag) or it's something like `v3.0-beta2` (when exactly
 # on a tagged version).
-_GIT_DESCRIPTION_OR_TAG := $(subst v${VERSION}-,,$(shell git describe --dirty --always))
+_GIT_DESCRIPTION_OR_TAG := $(subst v${VERSION}-,,$(shell git describe --tag --dirty --always))
 # In the above cases, GITHASH is either `16-123abc` (in the same case above) or
 # `123abc` (when the tag matches the current commit exactly) or `dirty` (when
 # the tag matches the current commit exactly, and there are subsequent

addon/flamenco/__init__.py

@@ -5,14 +5,14 @@
 bl_info = {
     "name": "Flamenco 3",
     "author": "Sybren A. Stüvel",
-    "version": (3, 2),
+    "version": (3, 3),
     "blender": (3, 1, 0),
     "description": "Flamenco client for Blender.",
     "location": "Output Properties > Flamenco",
     "doc_url": "https://flamenco.blender.org/",
     "category": "System",
     "support": "COMMUNITY",
-    "warning": "",
+    "warning": "This is version 3.3-alpha0 of the add-on, which is not a stable release",
 }
 
 from pathlib import Path

addon/flamenco/manager/__init__.py

@@ -10,7 +10,7 @@
 """
 
 
-__version__ = "3.2"
+__version__ = "3.3-alpha0"
 
 # import ApiClient
 from flamenco.manager.api_client import ApiClient

addon/flamenco/manager/api_client.py

@@ -76,7 +76,7 @@ class ApiClient(object):
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'Flamenco/3.2 (Blender add-on)'
+        self.user_agent = 'Flamenco/3.3-alpha0 (Blender add-on)'
 
     def __enter__(self):
         return self

addon/flamenco/manager/configuration.py

@@ -404,7 +404,7 @@ conf = flamenco.manager.Configuration(
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
                "Version of the API: 1.0.0\n"\
-               "SDK Package Version: 3.2".\
+               "SDK Package Version: 3.3-alpha0".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self):

addon/flamenco/manager_README.md

@@ -4,7 +4,7 @@ Render Farm manager API
 The `flamenco.manager` package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
 - API version: 1.0.0
-- Package version: 3.2
+- Package version: 3.3-alpha0
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 For more information, please visit [https://flamenco.io/](https://flamenco.io/)
 

go.mod

@@ -30,7 +30,7 @@ require (
 	golang.org/x/image v0.5.0
 	golang.org/x/net v0.7.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/vuln v0.0.0-20230224180816-edec1fb0a9c7
+	golang.org/x/vuln v0.0.0-20230320232729-bfc1eaef17a4
 	gopkg.in/yaml.v2 v2.4.0
 	gorm.io/gorm v1.23.8
 	honnef.co/go/tools v0.4.2
@@ -62,7 +62,6 @@ require (
 	github.com/urfave/cli/v2 v2.3.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
-	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
 	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sys v0.5.0 // indirect

go.sum

@@ -200,8 +200,6 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU=
 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
 golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a h1:Jw5wfR+h9mnIYH+OtGT2im5wV1YGGDora5vTv/aa5bE=
 golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -286,8 +284,8 @@ golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.1-0.20230217175706-3102dad5faf9 h1:IuFp2CklNBim6OdHXn/1P4VoeKt5pA2jcDKWlboqtlQ=
 golang.org/x/tools v0.6.1-0.20230217175706-3102dad5faf9/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/vuln v0.0.0-20230224180816-edec1fb0a9c7 h1:76OHcDuEUuIsFcMcJeefTj3z6YC2Q2llE4GF3RALw+I=
+golang.org/x/vuln v0.0.0-20230320232729-bfc1eaef17a4 h1:E/sS+2T8wsKgQNbdkQFdIFrytP7CK17WA5z0wbVoFgU=
-golang.org/x/vuln v0.0.0-20230224180816-edec1fb0a9c7/go.mod h1:LTLnfk/dpXDNKsX6aCg/cI4LyCVnTyrQhgV/yLJuly0=
+golang.org/x/vuln v0.0.0-20230320232729-bfc1eaef17a4/go.mod h1:ydpjOTRSBwOBFJRP/w5NF2HSPnFg1JxobEZQGOirxgo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/manager/api_impl/meta.go

@@ -221,7 +221,7 @@ func (f *Flamenco) FindBlenderExePath(e echo.Context) error {
 func (f *Flamenco) CheckBlenderExePath(e echo.Context) error {
 	logger := requestLogger(e)
 
-	var toCheck api.CheckSharedStoragePathJSONBody
+	var toCheck api.CheckBlenderExePathJSONBody
 	if err := e.Bind(&toCheck); err != nil {
 		logger.Warn().Err(err).Msg("bad request received")
 		return sendAPIError(e, http.StatusBadRequest, "invalid format")

internal/manager/job_deleter/job_deleter.go

@@ -28,7 +28,13 @@ import (
 // memory at a time. This is variable to allow unit testing with lower limits.
 var jobDeletionQueueSize = defaultJobDeletionQueueSize
 
-const defaultJobDeletionQueueSize = 100
+const (
+	defaultJobDeletionQueueSize = 100
+
+	// jobDeletionCheckInterval determines how often the database is checked for
+	// jobs that have been requested to be deleted.
+	jobDeletionCheckInterval = 1 * time.Minute
+)
 
 // Service can mark jobs as "deletion requested", as well as delete those jobs
 // in a background goroutine.
@@ -106,7 +112,7 @@ func (s *Service) Run(ctx context.Context) {
 			return
 		case jobUUID := <-s.queue:
 			s.deleteJob(ctx, jobUUID)
-		case <-time.After(1 * time.Minute):
+		case <-time.After(jobDeletionCheckInterval):
 			// Inspect the database to see if there was anything marked for deletion
 			// without getting into our queue. This can happen when lots of jobs are
 			// queued in quick succession, as then the queue channel gets full.
@@ -126,13 +132,20 @@ func (s *Service) queuePendingDeletions(ctx context.Context) {
 		return
 	}
 
-	for _, jobUUID := range jobUUIDs {
+	numDeletionsQueued := len(jobUUIDs)
+queueLoop:
+	for index, jobUUID := range jobUUIDs {
 		select {
 		case s.queue <- jobUUID:
 			log.Debug().Str("job", jobUUID).Msg("job deleter: job queued for deletion")
 		case <-time.After(100 * time.Millisecond):
-			log.Info().Msg("job deleter: job deletion queue is full")
+			numRemaining := numDeletionsQueued - index
-			break
+			log.Info().
+				Int("deletionsQueued", len(s.queue)).
+				Int("deletionsRemaining", numRemaining).
+				Stringer("checkInterval", jobDeletionCheckInterval).
+				Msg("job deleter: job deletion queue is full, remaining deletions will be picked up later")
+			break queueLoop
 		}
 	}
 }
@@ -145,13 +158,13 @@ func (s *Service) deleteJob(ctx context.Context, jobUUID string) error {
 		return err
 	}
 
-	logger.Info().Msg("job deleter: removing logs, last-rendered images, etc.")
+	logger.Debug().Msg("job deleter: removing logs, last-rendered images, etc.")
 	if err := s.storage.RemoveJobStorage(ctx, jobUUID); err != nil {
 		logger.Error().Err(err).Msg("job deleter: error removing job logs, job deletion aborted")
 		return err
 	}
 
-	logger.Info().Msg("job deleter: removing job from database")
+	logger.Debug().Msg("job deleter: removing job from database")
 	if err := s.persist.DeleteJob(ctx, jobUUID); err != nil {
 		logger.Error().Err(err).Msg("job deleter: unable to remove job from database")
 		return err

web/app/src/manager-api/ApiClient.js

@@ -55,7 +55,7 @@ class ApiClient {
          * @default {}
          */
         this.defaultHeaders = {
-            'User-Agent': 'Flamenco/3.2 / webbrowser'
+            'User-Agent': 'Flamenco/3.3-alpha0 / webbrowser'
         };
 
         /**