CHANGELOG.md

@@ -4,6 +4,12 @@ This file contains the history of changes to Flamenco. Only changes that might
 be interesting for users are listed here, such as new features and fixes for
 bugs in actually-released versions.
 
+## 3.3 - in development
+
+- Improve speed of queueing up >100 simultaneous job deletions.
+- Improve logging of job deletion.
+
+
 ## 3.2 - released 2023-02-21
 
 - When rendering EXR files, use Blender's preview JPEG files to generate the preview video ([43bc22f10fae](https://developer.blender.org/rF43bc22f10fae0fcaed6a4a3b3ace1be617193e21)).

Makefile

@@ -2,13 +2,13 @@ PKG := git.blender.org/flamenco
 
 # To update the version number in all the relevant places, update the VERSION
 # variable below and run `make update-version`.
-VERSION := 3.2
+VERSION := 3.3-alpha0
-RELEASE_CYCLE := release
+RELEASE_CYCLE := alpha
 
 # _GIT_DESCRIPTION_OR_TAG is either something like '16-123abc' (when we're 16
 # commits since the last tag) or it's something like `v3.0-beta2` (when exactly
 # on a tagged version).
-_GIT_DESCRIPTION_OR_TAG := $(subst v${VERSION}-,,$(shell git describe --dirty --always))
+_GIT_DESCRIPTION_OR_TAG := $(subst v${VERSION}-,,$(shell git describe --tag --dirty --always))
 # In the above cases, GITHASH is either `16-123abc` (in the same case above) or
 # `123abc` (when the tag matches the current commit exactly) or `dirty` (when
 # the tag matches the current commit exactly, and there are subsequent

addon/flamenco/__init__.py

@@ -5,14 +5,14 @@
 bl_info = {
     "name": "Flamenco 3",
     "author": "Sybren A. Stüvel",
-    "version": (3, 2),
+    "version": (3, 3),
     "blender": (3, 1, 0),
     "description": "Flamenco client for Blender.",
     "location": "Output Properties > Flamenco",
     "doc_url": "https://flamenco.blender.org/",
     "category": "System",
     "support": "COMMUNITY",
-    "warning": "",
+    "warning": "This is version 3.3-alpha0 of the add-on, which is not a stable release",
 }
 
 from pathlib import Path

addon/flamenco/manager/__init__.py

@@ -10,7 +10,7 @@
 """
 
 
-__version__ = "3.2"
+__version__ = "3.3-alpha0"
 
 # import ApiClient
 from flamenco.manager.api_client import ApiClient

addon/flamenco/manager/api_client.py

@@ -76,7 +76,7 @@ class ApiClient(object):
             self.default_headers[header_name] = header_value
         self.cookie = cookie
         # Set default User-Agent.
-        self.user_agent = 'Flamenco/3.2 (Blender add-on)'
+        self.user_agent = 'Flamenco/3.3-alpha0 (Blender add-on)'
 
     def __enter__(self):
         return self

addon/flamenco/manager/configuration.py

@@ -404,7 +404,7 @@ conf = flamenco.manager.Configuration(
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
                "Version of the API: 1.0.0\n"\
-               "SDK Package Version: 3.2".\
+               "SDK Package Version: 3.3-alpha0".\
                format(env=sys.platform, pyversion=sys.version)
 
     def get_host_settings(self):

addon/flamenco/manager_README.md

@@ -4,7 +4,7 @@ Render Farm manager API
 The `flamenco.manager` package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
 - API version: 1.0.0
-- Package version: 3.2
+- Package version: 3.3-alpha0
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 For more information, please visit [https://flamenco.io/](https://flamenco.io/)
 

go.mod

@@ -51,11 +51,12 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
-	golang.org/x/mod v0.7.0 // indirect
+	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2 // indirect
+	golang.org/x/tools v0.6.1-0.20230217175706-3102dad5faf9 // indirect
+	golang.org/x/vuln v0.0.0-20230320232729-bfc1eaef17a4 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	modernc.org/libc v1.16.17 // indirect
 	modernc.org/mathutil v1.4.1 // indirect

go.sum

@@ -185,6 +185,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -254,6 +256,10 @@ golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2 h1:v0FhRDmSCNH/0EurAT6T8KRY4aNuUhz6/WwBMxG+gvQ=
 golang.org/x/tools v0.5.1-0.20230117180257-8aba49bb5ea2/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
+golang.org/x/tools v0.6.1-0.20230217175706-3102dad5faf9 h1:IuFp2CklNBim6OdHXn/1P4VoeKt5pA2jcDKWlboqtlQ=
+golang.org/x/tools v0.6.1-0.20230217175706-3102dad5faf9/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/vuln v0.0.0-20230320232729-bfc1eaef17a4 h1:E/sS+2T8wsKgQNbdkQFdIFrytP7CK17WA5z0wbVoFgU=
+golang.org/x/vuln v0.0.0-20230320232729-bfc1eaef17a4/go.mod h1:ydpjOTRSBwOBFJRP/w5NF2HSPnFg1JxobEZQGOirxgo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/manager/job_deleter/job_deleter.go

@@ -28,7 +28,13 @@ import (
 // memory at a time. This is variable to allow unit testing with lower limits.
 var jobDeletionQueueSize = defaultJobDeletionQueueSize
 
-const defaultJobDeletionQueueSize = 100
+const (
+	defaultJobDeletionQueueSize = 100
+
+	// jobDeletionCheckInterval determines how often the database is checked for
+	// jobs that have been requested to be deleted.
+	jobDeletionCheckInterval = 1 * time.Minute
+)
 
 // Service can mark jobs as "deletion requested", as well as delete those jobs
 // in a background goroutine.
@@ -106,7 +112,7 @@ func (s *Service) Run(ctx context.Context) {
 			return
 		case jobUUID := <-s.queue:
 			s.deleteJob(ctx, jobUUID)
-		case <-time.After(1 * time.Minute):
+		case <-time.After(jobDeletionCheckInterval):
 			// Inspect the database to see if there was anything marked for deletion
 			// without getting into our queue. This can happen when lots of jobs are
 			// queued in quick succession, as then the queue channel gets full.
@@ -126,13 +132,20 @@ func (s *Service) queuePendingDeletions(ctx context.Context) {
 		return
 	}
 
-	for _, jobUUID := range jobUUIDs {
+	numDeletionsQueued := len(jobUUIDs)
+queueLoop:
+	for index, jobUUID := range jobUUIDs {
 		select {
 		case s.queue <- jobUUID:
 			log.Debug().Str("job", jobUUID).Msg("job deleter: job queued for deletion")
 		case <-time.After(100 * time.Millisecond):
-			log.Info().Msg("job deleter: job deletion queue is full")
+			numRemaining := numDeletionsQueued - index
-			break
+			log.Info().
+				Int("deletionsQueued", len(s.queue)).
+				Int("deletionsRemaining", numRemaining).
+				Stringer("checkInterval", jobDeletionCheckInterval).
+				Msg("job deleter: job deletion queue is full, remaining deletions will be picked up later")
+			break queueLoop
 		}
 	}
 }
@@ -145,13 +158,13 @@ func (s *Service) deleteJob(ctx context.Context, jobUUID string) error {
 		return err
 	}
 
-	logger.Info().Msg("job deleter: removing logs, last-rendered images, etc.")
+	logger.Debug().Msg("job deleter: removing logs, last-rendered images, etc.")
 	if err := s.storage.RemoveJobStorage(ctx, jobUUID); err != nil {
 		logger.Error().Err(err).Msg("job deleter: error removing job logs, job deletion aborted")
 		return err
 	}
 
-	logger.Info().Msg("job deleter: removing job from database")
+	logger.Debug().Msg("job deleter: removing job from database")
 	if err := s.persist.DeleteJob(ctx, jobUUID); err != nil {
 		logger.Error().Err(err).Msg("job deleter: unable to remove job from database")
 		return err

web/app/src/manager-api/ApiClient.js

@@ -55,7 +55,7 @@ class ApiClient {
          * @default {}
          */
         this.defaultHeaders = {
-            'User-Agent': 'Flamenco/3.2 / webbrowser'
+            'User-Agent': 'Flamenco/3.3-alpha0 / webbrowser'
         };
 
         /**