studio/flamenco
studio/flamenco
12
30
Fork 38
Code Issues 63 Pull Requests 5 Releases Wiki Activity

Web Interface for Tags #104244

Merged
Sybren A. Stüvel merged 30 commits from Evelinealy/flamenco:tag-interface into main 2023-09-04 13:06:10 +02:00
Conversation 6 Commits 30 Files Changed 3 +1342 -716
180 changed files with 1342 additions and 716 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 88f3ea7eff - Show all commits

3
CHANGELOG.md
View File

@ -6,6 +6,8 @@ bugs in actually-released versions.
## 3.3 - in development
- Upgrade bundled FFmpeg from 5.0 to 5.1.
- Rename the add-on download to `flamenco-addon.zip` (it used to be `flamenco3-addon.zip`). It still contains the same files as before, and in Blender the name of the add-on has not changed.
- Improve speed of queueing up >100 simultaneous job deletions.
- Improve logging of job deletion.
- Add Worker Tag support. Workers can be members of any number of tags. Workers will only work on jobs that are assigned to that tag. Jobs that do not have a tag will be available to all workers, regardless of their tag assignment. As a result, tagless workers will only work on tagless jobs.
@ -18,6 +20,7 @@ bugs in actually-released versions.
- Database integrity tests. These are always run at startup of Flamenco Manager, and by default run periodically every hour. This can be configured by adding/changing the `database_check_period: 1h` setting in `flamenco-manager.yaml`. Setting it to `0` will disable the periodic check. When a database consistency error is found, Flamenco Manager will immediately shut down.
- The webapp automatically reloads after a disconnect, when it reconnects to Flamenco Manager and sees the Manager version changed [#104235](https://projects.blender.org/studio/flamenco/pulls/104235).
- Show the configured Flamenco Manager name in the webapp's browser window title.
- Workers can be marked as 'restartable' by using the `-restart-exit-code N` commandline option. More info in the [Worker Actions documentation](https://flamenco.blender.org/usage/worker-actions/).
## 3.2 - released 2023-02-21

39
Makefile
View File

@ -1,6 +1,6 @@
-include .env
PKG := git.blender.org/flamenco
PKG := projects.blender.org/studio/flamenco
# To update the version number in all the relevant places, update the VERSION
# variable below and run `make update-version`.
@ -18,6 +18,9 @@ _GIT_DESCRIPTION_OR_TAG := $(subst v${VERSION}-,,$(shell git describe --tag --di
# in the "extended version" of Flamenco, which combines ${VERSION} and
# ${GITHASH}.
GITHASH := $(subst v${VERSION},$(shell git rev-parse --short=9 HEAD),${_GIT_DESCRIPTION_OR_TAG})
ifeq (${GITHASH},dirty)
GITHASH := $(shell git rev-parse --short=9 HEAD)
endif
LDFLAGS := ${LDFLAGS} -X ${PKG}/internal/appinfo.ApplicationVersion=${VERSION} \
-X ${PKG}/internal/appinfo.ApplicationGitHash=${GITHASH} \
@ -38,11 +41,6 @@ WEB_STATIC=web/static
# Prevent any dependency that requires a C compiler, i.e. only work with pure-Go libraries.
export CGO_ENABLED=0
# FFmpeg version to bundle.
FFMPEG_VERSION=5.0.1
TOOLS=./tools
TOOLS_DOWNLOAD=./tools/download
all: application
# Install generators and build the software.
@ -96,7 +94,7 @@ webapp-static: addon-packer
# in `cmd/flamenco-manager/main.go`
yarn --cwd web/app build --outDir ../static --base=/app/ --logLevel warn
# yarn --cwd web/app build --outDir ../static --base=/app/ --minify false
./addon-packer -filename ${WEB_STATIC}/flamenco3-addon.zip
./addon-packer -filename ${WEB_STATIC}/flamenco-addon.zip
@echo "Web app has been installed into ${WEB_STATIC}"
generate:
@ -264,18 +262,29 @@ tools:
$(MAKE) -s tools-darwin
$(MAKE) -s tools-windows
FFMPEG_PACKAGE_LINUX=$(TOOLS_DOWNLOAD)/ffmpeg-$(FFMPEG_VERSION)-linux-amd64-static.tar.xz
FFMPEG_PACKAGE_DARWIN=$(TOOLS_DOWNLOAD)/ffmpeg-$(FFMPEG_VERSION)-darwin-amd64.zip
FFMPEG_PACKAGE_WINDOWS=$(TOOLS_DOWNLOAD)/ffmpeg-$(FFMPEG_VERSION)-windows-amd64.zip
# FFmpeg version to bundle.
# Version 5.1.3 is the last release in the 5.x series, but binaries have not
# been made available. And then there are different 'latest' binaries for the
# different platforms.
FFMPEG_VERSION_LINUX=5.1.1
FFMPEG_VERSION_WINDOWS=5.1.2
FFMPEG_VERSION_DARWIN=5.1.2
TOOLS=./tools
TOOLS_DOWNLOAD=./tools/download
FFMPEG_PACKAGE_LINUX=$(TOOLS_DOWNLOAD)/ffmpeg-$(FFMPEG_VERSION_LINUX)-linux-amd64-static.tar.xz
FFMPEG_PACKAGE_DARWIN=$(TOOLS_DOWNLOAD)/ffmpeg-$(FFMPEG_VERSION_DARWIN)-darwin-amd64.zip
FFMPEG_PACKAGE_WINDOWS=$(TOOLS_DOWNLOAD)/ffmpeg-$(FFMPEG_VERSION_WINDOWS)-windows-amd64.zip
.PHONY: tools-linux
tools-linux:
[ -e $(FFMPEG_PACKAGE_LINUX) ] || curl \
--create-dirs -o $(FFMPEG_PACKAGE_LINUX) \
https://johnvansickle.com/ffmpeg/releases/ffmpeg-$(FFMPEG_VERSION)-amd64-static.tar.xz
https://www.johnvansickle.com/ffmpeg/old-releases/ffmpeg-$(FFMPEG_VERSION_LINUX)-amd64-static.tar.xz
tar xvf \
$(FFMPEG_PACKAGE_LINUX) \
ffmpeg-$(FFMPEG_VERSION)-amd64-static/ffmpeg \
ffmpeg-$(FFMPEG_VERSION_LINUX)-amd64-static/ffmpeg \
--strip-components=1
mv ffmpeg $(TOOLS)/ffmpeg-linux-amd64
@ -283,7 +292,7 @@ tools-linux:
tools-darwin:
[ -e $(FFMPEG_PACKAGE_DARWIN) ] || curl \
--create-dirs -o $(FFMPEG_PACKAGE_DARWIN) \
https://evermeet.cx/ffmpeg/ffmpeg-$(FFMPEG_VERSION).zip
https://evermeet.cx/ffmpeg/ffmpeg-$(FFMPEG_VERSION_DARWIN).zip
unzip $(FFMPEG_PACKAGE_DARWIN)
mv ffmpeg $(TOOLS)/ffmpeg-darwin-amd64
@ -291,8 +300,8 @@ tools-darwin:
tools-windows:
[ -e $(FFMPEG_PACKAGE_WINDOWS) ] || curl \
--create-dirs -o $(FFMPEG_PACKAGE_WINDOWS) \
https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-$(FFMPEG_VERSION)-essentials_build.zip
unzip -j $(FFMPEG_PACKAGE_WINDOWS) ffmpeg-5.0.1-essentials_build/bin/ffmpeg.exe -d .
https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-$(FFMPEG_VERSION_WINDOWS)-essentials_build.zip
unzip -j $(FFMPEG_PACKAGE_WINDOWS) ffmpeg-$(FFMPEG_VERSION_WINDOWS)-essentials_build/bin/ffmpeg.exe -d .
mv ffmpeg.exe $(TOOLS)/ffmpeg-windows-amd64.exe

1
addon/flamenco/manager/docs/SocketIOWorkerUpdate.md generated
View File

@ -10,6 +10,7 @@ Name | Type | Description | Notes
**updated** | **datetime** | Timestamp of last update |
**status** | [**WorkerStatus**](WorkerStatus.md) | |
**version** | **str** | |
**can_restart** | **bool** | Whether this Worker can auto-restart. |
**last_seen** | **datetime** | Last time this worker was seen by the Manager. | [optional]
**previous_status** | [**WorkerStatus**](WorkerStatus.md) | | [optional]
**status_change** | [**WorkerStatusChangeRequest**](WorkerStatusChangeRequest.md) | | [optional]

1
addon/flamenco/manager/docs/Worker.md generated
View File

@ -9,6 +9,7 @@ Name | Type | Description | Notes
**name** | **str** | |
**status** | [**WorkerStatus**](WorkerStatus.md) | |
**version** | **str** | Version of Flamenco this Worker is running |
**can_restart** | **bool** | Whether this worker can auto-restart. |
**ip_address** | **str** | IP address of the Worker |
**platform** | **str** | Operating system of the Worker |
**supported_task_types** | **[str]** | |

1
addon/flamenco/manager/docs/WorkerApi.md generated
View File

@ -362,6 +362,7 @@ with flamenco.manager.ApiClient(configuration) as api_client:
"supported_task_types_example",
],
software_version="software_version_example",
can_restart=True,
) # WorkerSignOn | Worker metadata
# example passing only required values which don't have defaults set

1
addon/flamenco/manager/docs/WorkerSignOn.md generated
View File

@ -7,6 +7,7 @@ Name | Type | Description | Notes
**name** | **str** | |
**supported_task_types** | **[str]** | |
**software_version** | **str** | |
**can_restart** | **bool** | | [optional]
**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

2
addon/flamenco/manager/docs/WorkerStatus.md generated
View File

@ -4,7 +4,7 @@
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**value** | **str** | | must be one of ["starting", "awake", "asleep", "error", "testing", "offline", ]
**value** | **str** | | must be one of ["starting", "awake", "asleep", "error", "testing", "offline", "restart", ]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

1
addon/flamenco/manager/docs/WorkerSummary.md generated
View File

@ -9,6 +9,7 @@ Name | Type | Description | Notes
**name** | **str** | |
**status** | [**WorkerStatus**](WorkerStatus.md) | |
**version** | **str** | Version of Flamenco this Worker is running |
**can_restart** | **bool** | Whether this worker can auto-restart. |
**status_change** | [**WorkerStatusChangeRequest**](WorkerStatusChangeRequest.md) | | [optional]
**last_seen** | **datetime** | Last time this worker was seen by the Manager. | [optional]
**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional]

10
addon/flamenco/manager/model/socket_io_worker_update.py generated
View File

@ -94,6 +94,7 @@ class SocketIOWorkerUpdate(ModelNormal):
'updated': (datetime,), # noqa: E501
'status': (WorkerStatus,), # noqa: E501
'version': (str,), # noqa: E501
'can_restart': (bool,), # noqa: E501
'last_seen': (datetime,), # noqa: E501
'previous_status': (WorkerStatus,), # noqa: E501
'status_change': (WorkerStatusChangeRequest,), # noqa: E501
@ -111,6 +112,7 @@ class SocketIOWorkerUpdate(ModelNormal):
'updated': 'updated', # noqa: E501
'status': 'status', # noqa: E501
'version': 'version', # noqa: E501
'can_restart': 'can_restart', # noqa: E501
'last_seen': 'last_seen', # noqa: E501
'previous_status': 'previous_status', # noqa: E501
'status_change': 'status_change', # noqa: E501
@ -124,7 +126,7 @@ class SocketIOWorkerUpdate(ModelNormal):
@classmethod
@convert_js_args_to_python_args
def _from_openapi_data(cls, id, name, updated, status, version, *args, **kwargs): # noqa: E501
def _from_openapi_data(cls, id, name, updated, status, version, can_restart, *args, **kwargs): # noqa: E501
"""SocketIOWorkerUpdate - a model defined in OpenAPI
Args:
@ -133,6 +135,7 @@ class SocketIOWorkerUpdate(ModelNormal):
updated (datetime): Timestamp of last update
status (WorkerStatus):
version (str):
can_restart (bool): Whether this Worker can auto-restart.
Keyword Args:
_check_type (bool): if True, values for parameters in openapi_types
@ -201,6 +204,7 @@ class SocketIOWorkerUpdate(ModelNormal):
self.updated = updated
self.status = status
self.version = version
self.can_restart = can_restart
for var_name, var_value in kwargs.items():
if var_name not in self.attribute_map and \
self._configuration is not None and \
@ -221,7 +225,7 @@ class SocketIOWorkerUpdate(ModelNormal):
])
@convert_js_args_to_python_args
def __init__(self, id, name, updated, status, version, *args, **kwargs): # noqa: E501
def __init__(self, id, name, updated, status, version, can_restart, *args, **kwargs): # noqa: E501
"""SocketIOWorkerUpdate - a model defined in OpenAPI
Args:
@ -230,6 +234,7 @@ class SocketIOWorkerUpdate(ModelNormal):
updated (datetime): Timestamp of last update
status (WorkerStatus):
version (str):
can_restart (bool): Whether this Worker can auto-restart.
Keyword Args:
_check_type (bool): if True, values for parameters in openapi_types
@ -296,6 +301,7 @@ class SocketIOWorkerUpdate(ModelNormal):
self.updated = updated
self.status = status
self.version = version
self.can_restart = can_restart
for var_name, var_value in kwargs.items():
if var_name not in self.attribute_map and \
self._configuration is not None and \

4
addon/flamenco/manager/model/worker.py generated
View File

@ -101,6 +101,7 @@ class Worker(ModelComposed):
'name': (str,), # noqa: E501
'status': (WorkerStatus,), # noqa: E501
'version': (str,), # noqa: E501
'can_restart': (bool,), # noqa: E501
'ip_address': (str,), # noqa: E501
'platform': (str,), # noqa: E501
'supported_task_types': ([str],), # noqa: E501
@ -120,6 +121,7 @@ class Worker(ModelComposed):
'name': 'name', # noqa: E501
'status': 'status', # noqa: E501
'version': 'version', # noqa: E501
'can_restart': 'can_restart', # noqa: E501
'ip_address': 'ip_address', # noqa: E501
'platform': 'platform', # noqa: E501
'supported_task_types': 'supported_task_types', # noqa: E501
@ -142,6 +144,7 @@ class Worker(ModelComposed):
name (str):
status (WorkerStatus):
version (str): Version of Flamenco this Worker is running
can_restart (bool): Whether this worker can auto-restart.
ip_address (str): IP address of the Worker
platform (str): Operating system of the Worker
supported_task_types ([str]):
@ -252,6 +255,7 @@ class Worker(ModelComposed):
name (str):
status (WorkerStatus):
version (str): Version of Flamenco this Worker is running
can_restart (bool): Whether this worker can auto-restart.
ip_address (str): IP address of the Worker
platform (str): Operating system of the Worker
supported_task_types ([str]):

4
addon/flamenco/manager/model/worker_sign_on.py generated
View File

@ -84,6 +84,7 @@ class WorkerSignOn(ModelNormal):
'name': (str,), # noqa: E501
'supported_task_types': ([str],), # noqa: E501
'software_version': (str,), # noqa: E501
'can_restart': (bool,), # noqa: E501
}
@cached_property
@ -95,6 +96,7 @@ class WorkerSignOn(ModelNormal):
'name': 'name', # noqa: E501
'supported_task_types': 'supported_task_types', # noqa: E501
'software_version': 'software_version', # noqa: E501
'can_restart': 'can_restart', # noqa: E501
}
read_only_vars = {
@ -143,6 +145,7 @@ class WorkerSignOn(ModelNormal):
Animal class but this time we won't travel
through its discriminator because we passed in
_visited_composed_classes = (Animal,)
can_restart (bool): [optional] # noqa: E501
"""
_check_type = kwargs.pop('_check_type', True)
@ -232,6 +235,7 @@ class WorkerSignOn(ModelNormal):
Animal class but this time we won't travel
through its discriminator because we passed in
_visited_composed_classes = (Animal,)
can_restart (bool): [optional] # noqa: E501
"""
_check_type = kwargs.pop('_check_type', True)

9
addon/flamenco/manager/model/worker_status.py generated
View File

@ -58,6 +58,7 @@ class WorkerStatus(ModelSimple):
'ERROR': "error",
'TESTING': "testing",
'OFFLINE': "offline",
'RESTART': "restart",
},
}
@ -109,10 +110,10 @@ class WorkerStatus(ModelSimple):
Note that value can be passed either in args or in kwargs, but not in both.
Args:
args[0] (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", ] # noqa: E501
args[0] (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", "restart", ] # noqa: E501
Keyword Args:
value (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", ] # noqa: E501
value (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", "restart", ] # noqa: E501
_check_type (bool): if True, values for parameters in openapi_types
will be type checked and a TypeError will be
raised if the wrong type is input.
@ -199,10 +200,10 @@ class WorkerStatus(ModelSimple):
Note that value can be passed either in args or in kwargs, but not in both.
Args:
args[0] (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", ] # noqa: E501
args[0] (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", "restart", ] # noqa: E501
Keyword Args:
value (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", ] # noqa: E501
value (str):, must be one of ["starting", "awake", "asleep", "error", "testing", "offline", "restart", ] # noqa: E501
_check_type (bool): if True, values for parameters in openapi_types
will be type checked and a TypeError will be
raised if the wrong type is input.

10
addon/flamenco/manager/model/worker_summary.py generated
View File

@ -93,6 +93,7 @@ class WorkerSummary(ModelNormal):
'name': (str,), # noqa: E501
'status': (WorkerStatus,), # noqa: E501
'version': (str,), # noqa: E501
'can_restart': (bool,), # noqa: E501
'status_change': (WorkerStatusChangeRequest,), # noqa: E501
'last_seen': (datetime,), # noqa: E501
}
@ -107,6 +108,7 @@ class WorkerSummary(ModelNormal):
'name': 'name', # noqa: E501
'status': 'status', # noqa: E501
'version': 'version', # noqa: E501
'can_restart': 'can_restart', # noqa: E501
'status_change': 'status_change', # noqa: E501
'last_seen': 'last_seen', # noqa: E501
}
@ -118,7 +120,7 @@ class WorkerSummary(ModelNormal):
@classmethod
@convert_js_args_to_python_args
def _from_openapi_data(cls, id, name, status, version, *args, **kwargs): # noqa: E501
def _from_openapi_data(cls, id, name, status, version, can_restart, *args, **kwargs): # noqa: E501
"""WorkerSummary - a model defined in OpenAPI
Args:
@ -126,6 +128,7 @@ class WorkerSummary(ModelNormal):
name (str):
status (WorkerStatus):
version (str): Version of Flamenco this Worker is running
can_restart (bool): Whether this worker can auto-restart.
Keyword Args:
_check_type (bool): if True, values for parameters in openapi_types
@ -191,6 +194,7 @@ class WorkerSummary(ModelNormal):
self.name = name
self.status = status
self.version = version
self.can_restart = can_restart
for var_name, var_value in kwargs.items():
if var_name not in self.attribute_map and \
self._configuration is not None and \
@ -211,7 +215,7 @@ class WorkerSummary(ModelNormal):
])
@convert_js_args_to_python_args
def __init__(self, id, name, status, version, *args, **kwargs): # noqa: E501
def __init__(self, id, name, status, version, can_restart, *args, **kwargs): # noqa: E501
"""WorkerSummary - a model defined in OpenAPI
Args:
@ -219,6 +223,7 @@ class WorkerSummary(ModelNormal):
name (str):
status (WorkerStatus):
version (str): Version of Flamenco this Worker is running
can_restart (bool): Whether this worker can auto-restart.
Keyword Args:
_check_type (bool): if True, values for parameters in openapi_types
@ -282,6 +287,7 @@ class WorkerSummary(ModelNormal):
self.name = name
self.status = status
self.version = version
self.can_restart = can_restart
for var_name, var_value in kwargs.items():
if var_name not in self.attribute_map and \
self._configuration is not None and \

4
cmd/addon-packer/addon-packer.go
View File

@ -18,7 +18,7 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/appinfo"
)
var cliArgs struct {
@ -147,7 +147,7 @@ func parseCliArgs() {
flag.BoolVar(&cliArgs.quiet, "quiet", false, "Only log warning-level and worse.")
flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug-level logging.")
flag.BoolVar(&cliArgs.trace, "trace", false, "Enable trace-level logging.")
flag.StringVar(&cliArgs.filename, "filename", "web/static/flamenco3-addon.zip", "Filename to save the add-on to.")
flag.StringVar(&cliArgs.filename, "filename", "web/static/flamenco-addon.zip", "Filename to save the add-on to.")
flag.Parse()
}

34
cmd/flamenco-manager/main.go
View File

@ -22,23 +22,23 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/internal/manager/api_impl"
"git.blender.org/flamenco/internal/manager/api_impl/dummy"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/manager/job_deleter"
"git.blender.org/flamenco/internal/manager/last_rendered"
"git.blender.org/flamenco/internal/manager/local_storage"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/sleep_scheduler"
"git.blender.org/flamenco/internal/manager/task_logs"
"git.blender.org/flamenco/internal/manager/task_state_machine"
"git.blender.org/flamenco/internal/manager/timeout_checker"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/internal/own_url"
"git.blender.org/flamenco/internal/upnp_ssdp"
"git.blender.org/flamenco/pkg/shaman"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/manager/api_impl"
"projects.blender.org/studio/flamenco/internal/manager/api_impl/dummy"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/manager/job_deleter"
"projects.blender.org/studio/flamenco/internal/manager/last_rendered"
"projects.blender.org/studio/flamenco/internal/manager/local_storage"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/sleep_scheduler"
"projects.blender.org/studio/flamenco/internal/manager/task_logs"
"projects.blender.org/studio/flamenco/internal/manager/task_state_machine"
"projects.blender.org/studio/flamenco/internal/manager/timeout_checker"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/internal/own_url"
"projects.blender.org/studio/flamenco/internal/upnp_ssdp"
"projects.blender.org/studio/flamenco/pkg/shaman"
)
var cliArgs struct {

20
cmd/flamenco-manager/webservice.go
View File

@ -19,13 +19,13 @@ import (
"github.com/rs/zerolog/log"
"github.com/ziflex/lecho/v3"
"git.blender.org/flamenco/internal/manager/api_impl"
"git.blender.org/flamenco/internal/manager/local_storage"
"git.blender.org/flamenco/internal/manager/swagger_ui"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/internal/upnp_ssdp"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/web"
"projects.blender.org/studio/flamenco/internal/manager/api_impl"
"projects.blender.org/studio/flamenco/internal/manager/local_storage"
"projects.blender.org/studio/flamenco/internal/manager/swagger_ui"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/internal/upnp_ssdp"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/web"
)
func buildWebService(
@ -136,7 +136,11 @@ func buildWebService(
})
// Serve the Blender add-on. It's contained in the static files of the webapp.
e.GET("/flamenco3-addon.zip", echo.WrapHandler(webAppHandler))
e.GET("/flamenco-addon.zip", echo.WrapHandler(webAppHandler))
e.GET("/flamenco3-addon.zip", func(c echo.Context) error {
return c.Redirect(http.StatusPermanentRedirect, "/flamenco-addon.zip")
})
// The favicons are also in the static files of the webapp.
e.GET("/favicon.png", echo.WrapHandler(webAppHandler))
e.GET("/favicon.ico", echo.WrapHandler(webAppHandler))

4
cmd/flamenco-worker/find_exes.go
View File

@ -9,8 +9,8 @@ import (
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/find_blender"
"git.blender.org/flamenco/internal/find_ffmpeg"
"projects.blender.org/studio/flamenco/internal/find_blender"
"projects.blender.org/studio/flamenco/internal/find_ffmpeg"
)
// findFFmpeg tries to find FFmpeg, in order to show its version (if found) or a warning (if not).

38
cmd/flamenco-worker/main.go
View File

@ -20,9 +20,9 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/internal/worker"
"git.blender.org/flamenco/internal/worker/cli_runner"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/worker"
"projects.blender.org/studio/flamenco/internal/worker/cli_runner"
)
var (
@ -47,6 +47,8 @@ var cliArgs struct {
manager string
register bool
restartExitCode int
}
func main() {
@ -84,17 +86,22 @@ func main() {
// Load configuration, and override things from the CLI arguments if necessary.
configWrangler := worker.NewConfigWrangler()
if cliArgs.managerURL != nil {
url := cliArgs.managerURL.String()
log.Info().Str("manager", url).Msg("using Manager URL from commandline")
// Before the config can be overridden, it has to be loaded.
if _, err := configWrangler.WorkerConfig(); err != nil {
log.Fatal().Err(err).Msg("error loading worker configuration")
}
if cliArgs.managerURL != nil {
url := cliArgs.managerURL.String()
log.Info().Str("manager", url).Msg("using Manager URL from commandline")
configWrangler.SetManagerURL(url)
}
if cliArgs.restartExitCode != 0 {
log.Info().Int("exitCode", cliArgs.restartExitCode).
Msg("will tell Manager this Worker can restart")
configWrangler.SetRestartExitCode(cliArgs.restartExitCode)
}
findBlender()
findFFmpeg()
@ -163,7 +170,8 @@ func main() {
go w.Start(workerCtx, startupState)
if w.WaitForShutdown(workerCtx) {
shutdownReason := w.WaitForShutdown(workerCtx)
if shutdownReason != worker.ReasonContextClosed {
go shutdown()
}
<-shutdownComplete
@ -172,6 +180,8 @@ func main() {
wg.Wait()
log.Debug().Msg("process shutting down")
config, _ := configWrangler.WorkerConfig()
stopProcess(config, shutdownReason)
}
func shutdown() {
@ -203,6 +213,17 @@ func shutdown() {
close(shutdownComplete)
}
func stopProcess(config worker.WorkerConfig, shutdownReason worker.ShutdownReason) {
switch shutdownReason {
case worker.ReasonContextClosed:
os.Exit(1)
case worker.ReasonShutdownReq:
os.Exit(0)
case worker.ReasonRestartReq:
os.Exit(config.RestartExitCode)
}
}
func parseCliArgs() {
flag.BoolVar(&cliArgs.version, "version", false, "Shows the application version, then exits.")
flag.BoolVar(&cliArgs.flush, "flush", false, "Flush any buffered task updates to the Manager, then exits.")
@ -216,6 +237,9 @@ func parseCliArgs() {
flag.BoolVar(&cliArgs.register, "register", false, "(Re-)register at the Manager.")
flag.BoolVar(&cliArgs.findManager, "find-manager", false, "Autodiscover a Manager, then quit.")
flag.IntVar(&cliArgs.restartExitCode, "restart-exit-code", 0,
"Mark this Worker as restartable. It will exit with this code to signify it needs to be restarted.")
flag.Parse()
if cliArgs.manager != "" {

10
cmd/job-creator/main.go
View File

@ -18,11 +18,11 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
var cliArgs struct {

8
cmd/shaman-checkout-id-setter/main.go
View File

@ -18,10 +18,10 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func main() {

4
cmd/stresser/main.go
View File

@ -15,8 +15,8 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/internal/stresser"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/stresser"
)
var cliArgs struct {

2
deploy.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
MY_DIR="$(dirname "$(readlink -e "$0")")"
ADDON_ZIP="$MY_DIR/web/static/flamenco3-addon.zip"
ADDON_ZIP="$MY_DIR/web/static/flamenco-addon.zip"
WORKER_TARGET=/shared/software/flamenco3-worker/flamenco-worker
TIMESTAMP=$(date +'%Y-%m-%d-%H%M%S')

2
go.mod
View File

@ -1,4 +1,4 @@
module git.blender.org/flamenco
module projects.blender.org/studio/flamenco
go 1.20

4
internal/find_blender/find_blender.go
View File

@ -12,9 +12,9 @@ import (
"strings"
"time"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/crosspath"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
var (

2
internal/manager/api_impl/api_impl.go
View File

@ -11,8 +11,8 @@ import (
"sync"
"time"
"git.blender.org/flamenco/pkg/api"
"github.com/labstack/echo/v4"
"projects.blender.org/studio/flamenco/pkg/api"
)
type Flamenco struct {

4
internal/manager/api_impl/dummy/shaman.go
View File

@ -7,8 +7,8 @@ import (
"errors"
"io"
"git.blender.org/flamenco/internal/manager/api_impl"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/api_impl"
"projects.blender.org/studio/flamenco/pkg/api"
)
// DummyShaman implements the Shaman interface from `internal/manager/api_impl/interfaces.go`

22
internal/manager/api_impl/interfaces.go
View File

@ -13,20 +13,20 @@ import (
"github.com/benbjohnson/clock"
"github.com/rs/zerolog"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/manager/job_deleter"
"git.blender.org/flamenco/internal/manager/last_rendered"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/sleep_scheduler"
"git.blender.org/flamenco/internal/manager/task_state_machine"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/shaman"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/manager/job_deleter"
"projects.blender.org/studio/flamenco/internal/manager/last_rendered"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/sleep_scheduler"
"projects.blender.org/studio/flamenco/internal/manager/task_state_machine"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/shaman"
)
// Generate mock implementations of these interfaces.
//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks git.blender.org/flamenco/internal/manager/api_impl PersistenceService,ChangeBroadcaster,JobCompiler,LogStorage,ConfigService,TaskStateMachine,Shaman,LastRendered,LocalStorage,WorkerSleepScheduler,JobDeleter
//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/api_impl PersistenceService,ChangeBroadcaster,JobCompiler,LogStorage,ConfigService,TaskStateMachine,Shaman,LastRendered,LocalStorage,WorkerSleepScheduler,JobDeleter
type PersistenceService interface {
StoreAuthoredJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error

12
internal/manager/api_impl/jobs.go
View File

@ -15,12 +15,12 @@ import (
"github.com/labstack/echo/v4"
"github.com/rs/zerolog"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/crosspath"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
// JobFilesURLPrefix is the URL prefix that the Flamenco API expects to serve

6
internal/manager/api_impl/jobs_query.go
View File

@ -9,9 +9,9 @@ import (
"github.com/labstack/echo/v4"
"github.com/rs/zerolog"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
// fetchJob fetches the job from the database, and sends the appropriate error

4
internal/manager/api_impl/jobs_query_test.go
View File

@ -7,10 +7,10 @@ import (
"testing"
"time"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestQueryJobs(t *testing.T) {

12
internal/manager/api_impl/jobs_test.go
View File

@ -9,15 +9,15 @@ import (
"os"
"testing"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/manager/last_rendered"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/moremock"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/manager/last_rendered"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/moremock"
)
func ptr[T any](value T) *T {

18
internal/manager/api_impl/meta.go
View File

@ -14,11 +14,11 @@ import (
"strconv"
"strings"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/internal/find_blender"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/pkg/api"
"github.com/labstack/echo/v4"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/internal/find_blender"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/pkg/api"
)
func (f *Flamenco) GetVersion(e echo.Context) error {
@ -71,18 +71,12 @@ func (f *Flamenco) GetVariables(e echo.Context, audience api.ManagerVariableAudi
func (f *Flamenco) GetSharedStorage(e echo.Context, audience api.ManagerVariableAudience, platform string) error {
location := f.config.EffectiveStoragePath()
feeder := make(chan string, 1)
receiver := make(chan string, 1)
feeder <- location
close(feeder)
f.config.ExpandVariables(feeder, receiver, config.VariableAudience(audience), config.VariablePlatform(platform))
varExpand := f.config.NewVariableExpander(config.VariableAudience(audience), config.VariablePlatform(platform))
return e.JSON(http.StatusOK, api.SharedStorageLocation{
Audience: audience,
Platform: platform,
Location: <-receiver,
Location: varExpand.Expand(location),
ShamanEnabled: f.isShamanEnabled(),
})
}

85
internal/manager/api_impl/meta_test.go
View File

@ -11,12 +11,12 @@ import (
"runtime"
"testing"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/pkg/api"
"github.com/golang/mock/gomock"
"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestGetVariables(t *testing.T) {
@ -90,12 +90,10 @@ func TestGetSharedStorage(t *testing.T) {
mf.config.EXPECT().EffectiveStoragePath().Return(`S:\storage\flamenco`).AnyTimes()
{ // Test user client on Linux.
mf.config.EXPECT().
ExpandVariables(gomock.Any(), gomock.Any(), config.VariableAudienceUsers, config.VariablePlatformLinux).
Do(func(inputChannel <-chan string, outputChannel chan<- string, audience config.VariableAudience, platform config.VariablePlatform) {
// Defer to the actual ExpandVariables() implementation of the above config.
conf.ExpandVariables(inputChannel, outputChannel, audience, platform)
})
mf.config.EXPECT().
NewVariableExpander(config.VariableAudienceUsers, config.VariablePlatformLinux).
DoAndReturn(conf.NewVariableExpander)
mf.shaman.EXPECT().IsEnabled().Return(false)
echoCtx := mf.prepareMockedRequest(nil)
@ -109,12 +107,10 @@ func TestGetSharedStorage(t *testing.T) {
}
{ // Test worker client on Linux with Shaman enabled.
mf.config.EXPECT().
ExpandVariables(gomock.Any(), gomock.Any(), config.VariableAudienceWorkers, config.VariablePlatformLinux).
Do(func(inputChannel <-chan string, outputChannel chan<- string, audience config.VariableAudience, platform config.VariablePlatform) {
// Defer to the actual ExpandVariables() implementation of the above config.
conf.ExpandVariables(inputChannel, outputChannel, audience, platform)
})
mf.config.EXPECT().
NewVariableExpander(config.VariableAudienceWorkers, config.VariablePlatformLinux).
DoAndReturn(conf.NewVariableExpander)
mf.shaman.EXPECT().IsEnabled().Return(true)
echoCtx := mf.prepareMockedRequest(nil)
@ -129,12 +125,10 @@ func TestGetSharedStorage(t *testing.T) {
}
{ // Test user client on Windows.
mf.config.EXPECT().
ExpandVariables(gomock.Any(), gomock.Any(), config.VariableAudienceUsers, config.VariablePlatformWindows).
Do(func(inputChannel <-chan string, outputChannel chan<- string, audience config.VariableAudience, platform config.VariablePlatform) {
// Defer to the actual ExpandVariables() implementation of the above config.
conf.ExpandVariables(inputChannel, outputChannel, audience, platform)
})
mf.config.EXPECT().
NewVariableExpander(config.VariableAudienceUsers, config.VariablePlatformWindows).
DoAndReturn(conf.NewVariableExpander)
mf.shaman.EXPECT().IsEnabled().Return(false)
echoCtx := mf.prepareMockedRequest(nil)
@ -149,6 +143,63 @@ func TestGetSharedStorage(t *testing.T) {
}
// Test shared storage sitting on /mnt/flamenco, where that's mapped to F:\ for Windows.
func TestGetSharedStorageDriveLetterRoot(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
mf := newMockedFlamenco(mockCtrl)
conf := config.GetTestConfig(func(c *config.Conf) {
// Test with a Manager on Linux.
c.MockCurrentGOOSForTests("linux")
// Set up a two-way variable to do the mapping.
c.Variables["shared_storage_mapping"] = config.Variable{
IsTwoWay: true,
Values: []config.VariableValue{
{Value: "/mnt/flamenco", Platform: config.VariablePlatformLinux, Audience: config.VariableAudienceAll},
{Value: `F:\`, Platform: config.VariablePlatformWindows, Audience: config.VariableAudienceAll},
},
}
})
mf.config.EXPECT().Get().Return(&conf).AnyTimes()
mf.config.EXPECT().EffectiveStoragePath().Return(`/mnt/flamenco`).AnyTimes()
{ // Test user client on Linux.
mf.config.EXPECT().
NewVariableExpander(config.VariableAudienceUsers, config.VariablePlatformLinux).
DoAndReturn(conf.NewVariableExpander)
mf.shaman.EXPECT().IsEnabled().Return(false)
echoCtx := mf.prepareMockedRequest(nil)
err := mf.flamenco.GetSharedStorage(echoCtx, api.ManagerVariableAudienceUsers, "linux")
require.NoError(t, err)
assertResponseJSON(t, echoCtx, http.StatusOK, api.SharedStorageLocation{
Location: "/mnt/flamenco",
Audience: api.ManagerVariableAudienceUsers,
Platform: "linux",
})
}
{ // Test user client on Windows.
mf.config.EXPECT().
NewVariableExpander(config.VariableAudienceUsers, config.VariablePlatformWindows).
DoAndReturn(conf.NewVariableExpander)
mf.shaman.EXPECT().IsEnabled().Return(false)
echoCtx := mf.prepareMockedRequest(nil)
err := mf.flamenco.GetSharedStorage(echoCtx, api.ManagerVariableAudienceUsers, "windows")
require.NoError(t, err)
assertResponseJSON(t, echoCtx, http.StatusOK, api.SharedStorageLocation{
Location: `F:\`,
Audience: api.ManagerVariableAudienceUsers,
Platform: "windows",
})
}
}
func TestCheckSharedStoragePath(t *testing.T) {
mf, finish := metaTestFixtures(t)
defer finish()

64
internal/manager/api_impl/mocks/api_impl_mock.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/api_impl (interfaces: PersistenceService,ChangeBroadcaster,JobCompiler,LogStorage,ConfigService,TaskStateMachine,Shaman,LastRendered,LocalStorage,WorkerSleepScheduler,JobDeleter)
// Source: projects.blender.org/studio/flamenco/internal/manager/api_impl (interfaces: PersistenceService,ChangeBroadcaster,JobCompiler,LogStorage,ConfigService,TaskStateMachine,Shaman,LastRendered,LocalStorage,WorkerSleepScheduler,JobDeleter)
// Package mocks is a generated GoMock package.
package mocks
@ -9,13 +9,13 @@ import (
io "io"
reflect "reflect"
config "git.blender.org/flamenco/internal/manager/config"
job_compilers "git.blender.org/flamenco/internal/manager/job_compilers"
last_rendered "git.blender.org/flamenco/internal/manager/last_rendered"
persistence "git.blender.org/flamenco/internal/manager/persistence"
api "git.blender.org/flamenco/pkg/api"
gomock "github.com/golang/mock/gomock"
zerolog "github.com/rs/zerolog"
config "projects.blender.org/studio/flamenco/internal/manager/config"
job_compilers "projects.blender.org/studio/flamenco/internal/manager/job_compilers"
last_rendered "projects.blender.org/studio/flamenco/internal/manager/last_rendered"
persistence "projects.blender.org/studio/flamenco/internal/manager/persistence"
api "projects.blender.org/studio/flamenco/pkg/api"
)
// MockPersistenceService is a mock of PersistenceService interface.
@ -841,18 +841,6 @@ func (m *MockConfigService) EXPECT() *MockConfigServiceMockRecorder {
return m.recorder
}
// ConvertTwoWayVariables mocks base method.
func (m *MockConfigService) ConvertTwoWayVariables(arg0 <-chan string, arg1 chan<- string, arg2 config.VariableAudience, arg3 config.VariablePlatform) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "ConvertTwoWayVariables", arg0, arg1, arg2, arg3)
}
// ConvertTwoWayVariables indicates an expected call of ConvertTwoWayVariables.
func (mr *MockConfigServiceMockRecorder) ConvertTwoWayVariables(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConvertTwoWayVariables", reflect.TypeOf((*MockConfigService)(nil).ConvertTwoWayVariables), arg0, arg1, arg2, arg3)
}
// EffectiveStoragePath mocks base method.
func (m *MockConfigService) EffectiveStoragePath() string {
m.ctrl.T.Helper()
@ -867,18 +855,6 @@ func (mr *MockConfigServiceMockRecorder) EffectiveStoragePath() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EffectiveStoragePath", reflect.TypeOf((*MockConfigService)(nil).EffectiveStoragePath))
}
// ExpandVariables mocks base method.
func (m *MockConfigService) ExpandVariables(arg0 <-chan string, arg1 chan<- string, arg2 config.VariableAudience, arg3 config.VariablePlatform) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "ExpandVariables", arg0, arg1, arg2, arg3)
}
// ExpandVariables indicates an expected call of ExpandVariables.
func (mr *MockConfigServiceMockRecorder) ExpandVariables(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpandVariables", reflect.TypeOf((*MockConfigService)(nil).ExpandVariables), arg0, arg1, arg2, arg3)
}
// ForceFirstRun mocks base method.
func (m *MockConfigService) ForceFirstRun() {
m.ctrl.T.Helper()
@ -920,6 +896,34 @@ func (mr *MockConfigServiceMockRecorder) IsFirstRun() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsFirstRun", reflect.TypeOf((*MockConfigService)(nil).IsFirstRun))
}
// NewVariableExpander mocks base method.
func (m *MockConfigService) NewVariableExpander(arg0 config.VariableAudience, arg1 config.VariablePlatform) *config.VariableExpander {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NewVariableExpander", arg0, arg1)
ret0, _ := ret[0].(*config.VariableExpander)
return ret0
}
// NewVariableExpander indicates an expected call of NewVariableExpander.
func (mr *MockConfigServiceMockRecorder) NewVariableExpander(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewVariableExpander", reflect.TypeOf((*MockConfigService)(nil).NewVariableExpander), arg0, arg1)
}
// NewVariableToValueConverter mocks base method.
func (m *MockConfigService) NewVariableToValueConverter(arg0 config.VariableAudience, arg1 config.VariablePlatform) *config.ValueToVariableReplacer {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NewVariableToValueConverter", arg0, arg1)
ret0, _ := ret[0].(*config.ValueToVariableReplacer)
return ret0
}
// NewVariableToValueConverter indicates an expected call of NewVariableToValueConverter.
func (mr *MockConfigServiceMockRecorder) NewVariableToValueConverter(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewVariableToValueConverter", reflect.TypeOf((*MockConfigService)(nil).NewVariableToValueConverter), arg0, arg1)
}
// ResolveVariables mocks base method.
func (m *MockConfigService) ResolveVariables(arg0 config.VariableAudience, arg1 config.VariablePlatform) map[string]config.ResolvedVariable {
m.ctrl.T.Helper()

32
internal/manager/api_impl/mocks/varrepl.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/api_impl (interfaces: VariableReplacer)
// Source: projects.blender.org/studio/flamenco/internal/manager/api_impl (interfaces: VariableReplacer)
// Package mocks is a generated GoMock package.
package mocks
@ -7,8 +7,8 @@ package mocks
import (
reflect "reflect"
config "git.blender.org/flamenco/internal/manager/config"
gomock "github.com/golang/mock/gomock"
config "projects.blender.org/studio/flamenco/internal/manager/config"
)
// MockVariableReplacer is a mock of VariableReplacer interface.
@ -34,28 +34,32 @@ func (m *MockVariableReplacer) EXPECT() *MockVariableReplacerMockRecorder {
return m.recorder
}
// ConvertTwoWayVariables mocks base method.
func (m *MockVariableReplacer) ConvertTwoWayVariables(arg0 <-chan string, arg1 chan<- string, arg2 config.VariableAudience, arg3 config.VariablePlatform) {
// NewVariableExpander mocks base method.
func (m *MockVariableReplacer) NewVariableExpander(arg0 config.VariableAudience, arg1 config.VariablePlatform) *config.VariableExpander {
m.ctrl.T.Helper()
m.ctrl.Call(m, "ConvertTwoWayVariables", arg0, arg1, arg2, arg3)
ret := m.ctrl.Call(m, "NewVariableExpander", arg0, arg1)
ret0, _ := ret[0].(*config.VariableExpander)
return ret0
}
// ConvertTwoWayVariables indicates an expected call of ConvertTwoWayVariables.
func (mr *MockVariableReplacerMockRecorder) ConvertTwoWayVariables(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
// NewVariableExpander indicates an expected call of NewVariableExpander.
func (mr *MockVariableReplacerMockRecorder) NewVariableExpander(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConvertTwoWayVariables", reflect.TypeOf((*MockVariableReplacer)(nil).ConvertTwoWayVariables), arg0, arg1, arg2, arg3)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewVariableExpander", reflect.TypeOf((*MockVariableReplacer)(nil).NewVariableExpander), arg0, arg1)
}
// ExpandVariables mocks base method.
func (m *MockVariableReplacer) ExpandVariables(arg0 <-chan string, arg1 chan<- string, arg2 config.VariableAudience, arg3 config.VariablePlatform) {
// NewVariableToValueConverter mocks base method.
func (m *MockVariableReplacer) NewVariableToValueConverter(arg0 config.VariableAudience, arg1 config.VariablePlatform) *config.ValueToVariableReplacer {
m.ctrl.T.Helper()
m.ctrl.Call(m, "ExpandVariables", arg0, arg1, arg2, arg3)
ret := m.ctrl.Call(m, "NewVariableToValueConverter", arg0, arg1)
ret0, _ := ret[0].(*config.ValueToVariableReplacer)
return ret0
}
// ExpandVariables indicates an expected call of ExpandVariables.
func (mr *MockVariableReplacerMockRecorder) ExpandVariables(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
// NewVariableToValueConverter indicates an expected call of NewVariableToValueConverter.
func (mr *MockVariableReplacerMockRecorder) NewVariableToValueConverter(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpandVariables", reflect.TypeOf((*MockVariableReplacer)(nil).ExpandVariables), arg0, arg1, arg2, arg3)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewVariableToValueConverter", reflect.TypeOf((*MockVariableReplacer)(nil).NewVariableToValueConverter), arg0, arg1)
}
// ResolveVariables mocks base method.

4
internal/manager/api_impl/shaman.go
View File

@ -7,8 +7,8 @@ import (
"github.com/labstack/echo/v4"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/shaman/fileserver"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/shaman/fileserver"
)
func (f *Flamenco) isShamanEnabled() bool {

16
internal/manager/api_impl/support_test.go
View File

@ -17,10 +17,10 @@ import (
"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/api_impl/mocks"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/api_impl/mocks"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
type mockedFlamenco struct {
@ -101,8 +101,8 @@ func (mf *mockedFlamenco) expectExpandVariables(
// Defer the mocked call to the fake configuration.
return mf.config.EXPECT().
ExpandVariables(gomock.Any(), gomock.Any(), expectAudience, expectPlatform).
DoAndReturn(c.ExpandVariables)
NewVariableExpander(expectAudience, expectPlatform).
DoAndReturn(c.NewVariableExpander)
}
func (mf *mockedFlamenco) expectConvertTwoWayVariables(
@ -126,8 +126,8 @@ func (mf *mockedFlamenco) expectConvertTwoWayVariables(
// Defer the mocked call to the fake configuration.
return mf.config.EXPECT().
ConvertTwoWayVariables(gomock.Any(), gomock.Any(), expectAudience, expectPlatform).
DoAndReturn(c.ConvertTwoWayVariables)
NewVariableToValueConverter(expectAudience, expectPlatform).
DoAndReturn(c.NewVariableToValueConverter)
}
// prepareMockedJSONRequest returns an `echo.Context` that has a JSON request body attached to it.

67
internal/manager/api_impl/varrepl.go
View File

@ -3,45 +3,35 @@ package api_impl
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"sync"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
//go:generate go run github.com/golang/mock/mockgen -destination mocks/varrepl.gen.go -package mocks git.blender.org/flamenco/internal/manager/api_impl VariableReplacer
//go:generate go run github.com/golang/mock/mockgen -destination mocks/varrepl.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/api_impl VariableReplacer
type VariableReplacer interface {
ExpandVariables(inputChannel <-chan string, outputChannel chan<- string, audience config.VariableAudience, platform config.VariablePlatform)
NewVariableExpander(audience config.VariableAudience, platform config.VariablePlatform) *config.VariableExpander
ResolveVariables(audience config.VariableAudience, platform config.VariablePlatform) map[string]config.ResolvedVariable
ConvertTwoWayVariables(inputChannel <-chan string, outputChannel chan<- string, audience config.VariableAudience, platform config.VariablePlatform)
NewVariableToValueConverter(audience config.VariableAudience, platform config.VariablePlatform) *config.ValueToVariableReplacer
}
// replaceTaskVariables performs variable replacement for worker tasks.
func replaceTaskVariables(replacer VariableReplacer, task api.AssignedTask, worker persistence.Worker) api.AssignedTask {
feeder := make(chan string, 1)
receiver := make(chan string, 1)
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
replacer.ExpandVariables(feeder, receiver,
config.VariableAudienceWorkers, config.VariablePlatform(worker.Platform))
}()
varExpander := replacer.NewVariableExpander(
config.VariableAudienceWorkers,
config.VariablePlatform(worker.Platform),
)
for cmdIndex, cmd := range task.Commands {
for key, value := range cmd.Parameters {
switch v := value.(type) {
case string:
feeder <- v
task.Commands[cmdIndex].Parameters[key] = <-receiver
task.Commands[cmdIndex].Parameters[key] = varExpander.Expand(v)
case []string:
replaced := make([]string, len(v))
for idx := range v {
feeder <- v[idx]
replaced[idx] = <-receiver
replaced[idx] = varExpander.Expand(v[idx])
}
task.Commands[cmdIndex].Parameters[key] = replaced
@ -50,8 +40,7 @@ func replaceTaskVariables(replacer VariableReplacer, task api.AssignedTask, work
for idx := range v {
switch itemValue := v[idx].(type) {
case string:
feeder <- itemValue
replaced[idx] = <-receiver
replaced[idx] = varExpander.Expand(itemValue)
default:
replaced[idx] = itemValue
}
@ -64,10 +53,6 @@ func replaceTaskVariables(replacer VariableReplacer, task api.AssignedTask, work
}
}
close(feeder)
wg.Wait()
close(receiver)
return task
}
@ -78,16 +63,10 @@ func replaceTaskVariables(replacer VariableReplacer, task api.AssignedTask, work
//
// NOTE: this updates the job in place.
func replaceTwoWayVariables(replacer VariableReplacer, job *api.SubmittedJob) {
feeder := make(chan string, 1)
receiver := make(chan string, 1)
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
replacer.ConvertTwoWayVariables(feeder, receiver,
config.VariableAudienceWorkers, config.VariablePlatform(job.SubmitterPlatform))
}()
valueToVariable := replacer.NewVariableToValueConverter(
config.VariableAudienceWorkers,
config.VariablePlatform(job.SubmitterPlatform),
)
// Only replace variables in settings and metadata, not in other job fields.
if job.Settings != nil {
@ -96,18 +75,14 @@ func replaceTwoWayVariables(replacer VariableReplacer, job *api.SubmittedJob) {
if !ok {
continue
}
feeder <- stringValue
job.Settings.AdditionalProperties[settingKey] = <-receiver
newValue := valueToVariable.Replace(stringValue)
job.Settings.AdditionalProperties[settingKey] = newValue
}
}
if job.Metadata != nil {
for metaKey, metaValue := range job.Metadata.AdditionalProperties {
feeder <- metaValue
job.Metadata.AdditionalProperties[metaKey] = <-receiver
newValue := valueToVariable.Replace(metaValue)
job.Metadata.AdditionalProperties[metaKey] = newValue
}
}
close(feeder)
wg.Wait()
close(receiver)
}

8
internal/manager/api_impl/varrepl_test.go
View File

@ -9,10 +9,10 @@ import (
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/crosspath"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
func varreplTestTask() api.AssignedTask {

2
internal/manager/api_impl/worker_auth.go
View File

@ -13,7 +13,7 @@ import (
"github.com/labstack/echo/v4"
"golang.org/x/crypto/bcrypt"
"git.blender.org/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
)
type workerContextKey string

16
internal/manager/api_impl/worker_mgt.go
View File

@ -6,11 +6,11 @@ import (
"errors"
"net/http"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"github.com/labstack/echo/v4"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func (f *Flamenco) FetchWorkers(e echo.Context) error {
@ -159,6 +159,13 @@ func (f *Flamenco) RequestWorkerStatusChange(e echo.Context, workerUUID string)
Str("requested", string(change.Status)).
Bool("lazy", change.IsLazy).
Logger()
if change.Status == api.WorkerStatusRestart && !dbWorker.CanRestart {
logger.Error().Msg("worker cannot be restarted, rejecting status change request")
return sendAPIError(e, http.StatusPreconditionFailed,
"worker %q does not know how to restart", workerUUID)
}
logger.Info().Msg("worker status change requested")
if dbWorker.Status == change.Status {
@ -384,6 +391,7 @@ func workerSummary(w persistence.Worker) api.WorkerSummary {
Name: w.Name,
Status: w.Status,
Version: w.Software,
CanRestart: w.CanRestart,
}
if w.StatusRequested != "" {
summary.StatusChange = &api.WorkerStatusChangeRequest{

4
internal/manager/api_impl/worker_mgt_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestFetchWorkers(t *testing.T) {

6
internal/manager/api_impl/worker_sleep_schedule.go
View File

@ -6,10 +6,10 @@ import (
"errors"
"net/http"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"github.com/labstack/echo/v4"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func (f *Flamenco) FetchWorkerSleepSchedule(e echo.Context, workerUUID string) error {

6
internal/manager/api_impl/worker_task_updates.go
View File

@ -12,9 +12,9 @@ import (
"github.com/labstack/echo/v4"
"github.com/rs/zerolog"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func (f *Flamenco) TaskUpdate(e echo.Context, taskID string) error {

6
internal/manager/api_impl/worker_task_updates_test.go
View File

@ -9,9 +9,9 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestTaskUpdate(t *testing.T) {

22
internal/manager/api_impl/workers.go
View File

@ -13,12 +13,12 @@ import (
"github.com/labstack/echo/v4"
"github.com/rs/zerolog"
"git.blender.org/flamenco/internal/manager/last_rendered"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/task_state_machine"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/last_rendered"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/task_state_machine"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
// rememberableWorkerStates contains those worker statuses that should be
@ -30,6 +30,13 @@ var rememberableWorkerStates = map[api.WorkerStatus]bool{
api.WorkerStatusAwake: true,
}
// offlineWorkerStates contains worker statuses that are automatically
// acknowledged on sign-off.
var offlineWorkerStates = map[api.WorkerStatus]bool{
api.WorkerStatusOffline: true,
api.WorkerStatusRestart: true,
}
// RegisterWorker registers a new worker and stores it in the database.
func (f *Flamenco) RegisterWorker(e echo.Context) error {
logger := requestLogger(e)
@ -137,6 +144,7 @@ func (f *Flamenco) workerUpdateAfterSignOn(e echo.Context, update api.SignOnJSON
w.Address = e.RealIP()
w.Name = update.Name
w.Software = update.SoftwareVersion
w.CanRestart = update.CanRestart != nil && *update.CanRestart
// Remove trailing spaces from task types, and convert to lower case.
for idx := range update.SupportedTaskTypes {
@ -168,7 +176,7 @@ func (f *Flamenco) SignOff(e echo.Context) error {
w := requestWorkerOrPanic(e)
prevStatus := w.Status
w.Status = api.WorkerStatusOffline
if w.StatusRequested == api.WorkerStatusOffline {
if offlineWorkerStates[w.StatusRequested] {
w.StatusChangeClear()
}

8
internal/manager/api_impl/workers_test.go
View File

@ -13,10 +13,10 @@ import (
"github.com/labstack/echo/v4"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/config"
"git.blender.org/flamenco/internal/manager/last_rendered"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/config"
"projects.blender.org/studio/flamenco/internal/manager/last_rendered"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestTaskScheduleHappy(t *testing.T) {

57
internal/manager/config/config.go
View File

@ -20,9 +20,9 @@ import (
"github.com/rs/zerolog/log"
yaml "gopkg.in/yaml.v2"
"git.blender.org/flamenco/internal/appinfo"
"git.blender.org/flamenco/pkg/crosspath"
shaman_config "git.blender.org/flamenco/pkg/shaman/config"
"projects.blender.org/studio/flamenco/internal/appinfo"
"projects.blender.org/studio/flamenco/pkg/crosspath"
shaman_config "projects.blender.org/studio/flamenco/pkg/shaman/config"
)
// configFilename is used to specify where flamenco will write its config file.
@ -472,57 +472,6 @@ func (c *Conf) ExpandVariables(inputChannel <-chan string, outputChannel chan<-
}
}
// ConvertTwoWayVariables converts the value of a variable with "{variable
// name}", but only for two-way variables. The function iterates over all
// strings provided by the input channel, and sends the expanded result into the
// output channel. It will return when the input channel is closed.
func (c *Conf) ConvertTwoWayVariables(inputChannel <-chan string, outputChannel chan<- string,
audience VariableAudience, platform VariablePlatform) {
// Get the variables for the given audience & platform.
twoWayVars := c.GetTwoWayVariables(audience, platform)
if len(twoWayVars) == 0 {
log.Debug().
Str("audience", string(audience)).
Str("platform", string(platform)).
Msg("no two-way variables defined for this platform given this audience")
}
doValueReplacement := func(valueToConvert string) string {
for varName, varValue := range twoWayVars {
if !isValueMatch(valueToConvert, varValue) {
continue
}
valueToConvert = fmt.Sprintf("{%s}%s", varName, valueToConvert[len(varValue):])
}
return valueToConvert
}
for valueToExpand := range inputChannel {
outputChannel <- doValueReplacement(valueToExpand)
}
}
// isValueMatch returns whether `valueToMatch` starts with `variableValue`.
// When `variableValue` is a Windows path (with backslash separators), it is
// also tested with forward slashes against `valueToMatch`.
func isValueMatch(valueToMatch, variableValue string) bool {
if strings.HasPrefix(valueToMatch, variableValue) {
return true
}
// If the variable value has a backslash, assume it is a Windows path.
// Convert it to slash notation just to see if that would provide a
// match.
if strings.ContainsRune(variableValue, '\\') {
slashedValue := crosspath.ToSlash(variableValue)
return strings.HasPrefix(valueToMatch, slashedValue)
}
return false
}
// getVariables returns the variable values for this (audience, platform) combination.
// If no variables are found, just returns an empty map. If a value is defined
// for both the "all" platform and specifically the given platform, the specific

31
internal/manager/config/config_test.go
View File

@ -17,31 +17,8 @@ func TestVariablesWithBackslashes(t *testing.T) {
assert.Equal(t, expectSingle, vars["single-backslash"]["blender"])
assert.Equal(t, expectDouble, vars["double-backslash"]["blender"])
assert.Equal(t, expectSingle, vars["quoted-double-backslash"]["blender"])
}
func TestReplaceTwowayVariables(t *testing.T) {
c := DefaultConfig(func(c *Conf) {
c.Variables["shared"] = Variable{
IsTwoWay: true,
Values: []VariableValue{
{Value: "/shared/flamenco", Platform: VariablePlatformLinux},
{Value: `Y:\shared\flamenco`, Platform: VariablePlatformWindows},
},
}
})
feeder := make(chan string, 2)
receiver := make(chan string, 2)
feeder <- `Y:\shared\flamenco\shot\file.blend`
// This is the real reason for this test: forward slashes in the path should
// still be matched to the backslashes in the variable value.
feeder <- `Y:/shared/flamenco/shot/file.blend`
close(feeder)
c.ConvertTwoWayVariables(feeder, receiver, VariableAudienceUsers, VariablePlatformWindows)
assert.Equal(t, `{shared}\shot\file.blend`, <-receiver)
assert.Equal(t, `{shared}/shot/file.blend`, <-receiver)
assert.Equal(t, `C:\Downloads\tab\newline.exe`, vars["single-backslash-common-escapechar"]["blender"])
assert.Equal(t, `C:\Downloads\blender-1.0\`, vars["single-backslash-trailing"]["blender"])
assert.Equal(t, `F:\`, vars["single-backslash-drive-only"]["blender"])
}

2
internal/manager/config/defaults.go
View File

@ -4,7 +4,7 @@ import (
"runtime"
"time"
shaman_config "git.blender.org/flamenco/pkg/shaman/config"
shaman_config "projects.blender.org/studio/flamenco/pkg/shaman/config"
)
// SPDX-License-Identifier: GPL-3.0-or-later

10
internal/manager/config/service.go
View File

@ -70,12 +70,12 @@ func (s *Service) Save() error {
return nil
}
// Expose some functions on Conf here, for easier mocking of functionality via interfaces.
func (s *Service) ExpandVariables(inputChannel <-chan string, outputChannel chan<- string, audience VariableAudience, platform VariablePlatform) {
s.config.ExpandVariables(inputChannel, outputChannel, audience, platform)
// Expose some functions of Conf here, for easier mocking of functionality via interfaces.
func (s *Service) NewVariableExpander(audience VariableAudience, platform VariablePlatform) *VariableExpander {
return s.config.NewVariableExpander(audience, platform)
}
func (s *Service) ConvertTwoWayVariables(inputChannel <-chan string, outputChannel chan<- string, audience VariableAudience, platform VariablePlatform) {
s.config.ConvertTwoWayVariables(inputChannel, outputChannel, audience, platform)
func (s *Service) NewVariableToValueConverter(audience VariableAudience, platform VariablePlatform) *ValueToVariableReplacer {
return s.config.NewVariableToValueConverter(audience, platform)
}
func (s *Service) ResolveVariables(audience VariableAudience, platform VariablePlatform) map[string]ResolvedVariable {
return s.config.ResolveVariables(audience, platform)

2
internal/manager/config/settings_test.go
View File

@ -6,8 +6,8 @@ import (
"sync"
"testing"
"git.blender.org/flamenco/pkg/crosspath"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
func TestDefaultSettings(t *testing.T) {

6
internal/manager/config/test_files/config_with_backslashes.yaml
View File

@ -22,3 +22,9 @@ variables:
value: C:\\Downloads\\blender-1.0\\blender.exe
- platform: quoted-double-backslash
value: "C:\\Downloads\\blender-1.0\\blender.exe"
- platform: single-backslash-common-escapechar
value: C:\Downloads\tab\newline.exe
- platform: single-backslash-trailing
value: C:\Downloads\blender-1.0\
- platform: single-backslash-drive-only
value: F:\

155
internal/manager/config/variables.go Normal file
View File

@ -0,0 +1,155 @@
package config
import (
"fmt"
"strings"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
type ValueToVariableReplacer struct {
twoWayVars map[string]string // Mapping from variable name to value.
}
// VariableExpander expands variables and applies two-way variable replacement to the values.
type VariableExpander struct {
oneWayVars map[string]string // Mapping from variable name to value.
managerTwoWayVars map[string]string // Mapping from variable name to value for the Manager platform.
targetTwoWayVars map[string]string // Mapping from variable name to value for the target platform.
targetPlatform VariablePlatform
}
// NewVariableToValueConverter returns a ValueToVariableReplacer for the given audience & platform.
func (c *Conf) NewVariableToValueConverter(audience VariableAudience, platform VariablePlatform) *ValueToVariableReplacer {
// Get the variables for the given audience & platform.
twoWayVars := c.GetTwoWayVariables(audience, platform)
if len(twoWayVars) == 0 {
log.Debug().
Str("audience", string(audience)).
Str("platform", string(platform)).
Msg("no two-way variables defined for this platform given this audience")
}
return &ValueToVariableReplacer{
twoWayVars: twoWayVars,
}
}
// NewVariableExpander returns a new VariableExpander for the given audience & platform.
func (c *Conf) NewVariableExpander(audience VariableAudience, platform VariablePlatform) *VariableExpander {
// Get the variables for the given audience & platform.
varsForPlatform := c.getVariables(audience, platform)
if len(varsForPlatform) == 0 {
log.Warn().
Str("audience", string(audience)).
Str("platform", string(platform)).
Msg("no variables defined for this platform given this audience")
}
return &VariableExpander{
oneWayVars: varsForPlatform,
managerTwoWayVars: c.GetTwoWayVariables(audience, c.currentGOOS),
targetTwoWayVars: c.GetTwoWayVariables(audience, platform),
targetPlatform: platform,
}
}
// ValueToVariableReplacer replaces any variable values it recognises in
// valueToConvert to the actual variable. For example, `/path/to/file.blend` can
// be changed to `{my_storage}/file.blend`.
func (vvc *ValueToVariableReplacer) Replace(valueToConvert string) string {
result := valueToConvert
for varName, varValue := range vvc.twoWayVars {
if !isValueMatch(result, varValue) {
continue
}
result = vvc.join(varName, result[len(varValue):])
}
log.Debug().
Str("from", valueToConvert).
Str("to", result).
Msg("first step of two-way variable replacement")
return result
}
func (vvc *ValueToVariableReplacer) join(varName, value string) string {
return fmt.Sprintf("{%s}%s", varName, value)
}
// isValueMatch returns whether `valueToMatch` starts with `variableValue`.
// When `variableValue` is a Windows path (with backslash separators), it is
// also tested with forward slashes against `valueToMatch`.
func isValueMatch(valueToMatch, variableValue string) bool {
if strings.HasPrefix(valueToMatch, variableValue) {
return true
}
// If the variable value has a backslash, assume it is a Windows path.
// Convert it to slash notation just to see if that would provide a
// match.
if strings.ContainsRune(variableValue, '\\') {
slashedValue := crosspath.ToSlash(variableValue)
return strings.HasPrefix(valueToMatch, slashedValue)
}
return false
}
// Replace converts "{variable name}" to the value that belongs to the audience and platform.
func (ve *VariableExpander) Expand(valueToExpand string) string {
expanded := valueToExpand
// Expand variables from {varname} to their value for the target platform.
for varname, varvalue := range ve.oneWayVars {
placeholder := fmt.Sprintf("{%s}", varname)
expanded = strings.Replace(expanded, placeholder, varvalue, -1)
}
// Go through the two-way variables, to make sure that the result of
// expanding variables gets the two-way variables applied as well. This is
// necessary to make implicitly-defined variable, which are only defined for
// the Manager's platform, usable for the target platform.
//
// Practically, this replaces "value for the Manager platform" with "value
// for the target platform".
isPathValue := false
for varname, managerValue := range ve.managerTwoWayVars {
targetValue, ok := ve.targetTwoWayVars[varname]
if !ok {
continue
}
if !isValueMatch(expanded, managerValue) {
continue
}
expanded = ve.join(targetValue, expanded[len(managerValue):], ve.targetPlatform)
// Since two-way variables are meant for path replacement, we know this
// should be a path.
isPathValue = true
}
if isPathValue {
expanded = crosspath.ToPlatform(expanded, string(ve.targetPlatform))
}
return expanded
}
func (ve *VariableExpander) join(valueFromVariable, suffix string, platform VariablePlatform) string {
result := valueFromVariable + suffix
if platform == VariablePlatformWindows {
// 'result' may now be of the form `F:some\path\to\file`, where `F:` comes
// from `valueFromVariable` and the rest is the suffix. This is not an
// absolute path, and needs a separator between the drive letter and the
// rest of the path.
return crosspath.EnsureDriveAbsolute(result)
}
return result
}

26
internal/manager/config/variables_test.go Normal file
View File

@ -0,0 +1,26 @@
package config
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestReplaceTwowayVariables(t *testing.T) {
c := DefaultConfig(func(c *Conf) {
c.Variables["shared"] = Variable{
IsTwoWay: true,
Values: []VariableValue{
{Value: "/shared/flamenco", Platform: VariablePlatformLinux},
{Value: `Y:\shared\flamenco`, Platform: VariablePlatformWindows},
},
}
})
replacer := c.NewVariableToValueConverter(VariableAudienceUsers, VariablePlatformWindows)
// This is the real reason for this test: forward slashes in the path should
// still be matched to the backslashes in the variable value.
assert.Equal(t, `{shared}\shot\file.blend`, replacer.Replace(`Y:\shared\flamenco\shot\file.blend`))
assert.Equal(t, `{shared}/shot/file.blend`, replacer.Replace(`Y:/shared/flamenco/shot/file.blend`))
}

4
internal/manager/job_compilers/author.go
View File

@ -10,8 +10,8 @@ import (
"github.com/dop251/goja"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
// Author allows scripts to author tasks and commands.

4
internal/manager/job_compilers/job_compilers.go
View File

@ -19,8 +19,8 @@ import (
"github.com/dop251/goja_nodejs/require"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
var ErrJobTypeUnknown = errors.New("job type unknown")

2
internal/manager/job_compilers/job_compilers_test.go
View File

@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/api"
)
// The example job is expected to result in these arguments for FFmpeg.

2
internal/manager/job_compilers/js_path.go
View File

@ -3,9 +3,9 @@ package job_compilers
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"git.blender.org/flamenco/pkg/crosspath"
"github.com/dop251/goja"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
// PathModule provides file path manipulation functions by wrapping Go's `path`.

12
internal/manager/job_deleter/interfaces.go
View File

@ -5,15 +5,15 @@ package job_deleter
import (
"context"
"git.blender.org/flamenco/internal/manager/local_storage"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/shaman"
"projects.blender.org/studio/flamenco/internal/manager/local_storage"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/shaman"
)
// Generate mock implementations of these interfaces.
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks git.blender.org/flamenco/internal/manager/job_deleter PersistenceService,Storage,ChangeBroadcaster,Shaman
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/job_deleter PersistenceService,Storage,ChangeBroadcaster,Shaman
type PersistenceService interface {
FetchJob(ctx context.Context, jobUUID string) (*persistence.Job, error)

8
internal/manager/job_deleter/job_deleter.go
View File

@ -16,12 +16,12 @@ import (
"fmt"
"time"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"git.blender.org/flamenco/pkg/shaman"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/shaman"
)
// jobDeletionQueueSize determines how many job deletion requests can be kept in

6
internal/manager/job_deleter/job_deleter_test.go
View File

@ -7,11 +7,11 @@ import (
"errors"
"testing"
"git.blender.org/flamenco/internal/manager/job_deleter/mocks"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/shaman"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/internal/manager/job_deleter/mocks"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/shaman"
)
type JobDeleterMocks struct {

6
internal/manager/job_deleter/mocks/interfaces_mock.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/job_deleter (interfaces: PersistenceService,Storage,ChangeBroadcaster,Shaman)
// Source: projects.blender.org/studio/flamenco/internal/manager/job_deleter (interfaces: PersistenceService,Storage,ChangeBroadcaster,Shaman)
// Package mocks is a generated GoMock package.
package mocks
@ -8,9 +8,9 @@ import (
context "context"
reflect "reflect"
persistence "git.blender.org/flamenco/internal/manager/persistence"
api "git.blender.org/flamenco/pkg/api"
gomock "github.com/golang/mock/gomock"
persistence "projects.blender.org/studio/flamenco/internal/manager/persistence"
api "projects.blender.org/studio/flamenco/pkg/api"
)
// MockPersistenceService is a mock of PersistenceService interface.

2
internal/manager/last_rendered/last_rendered_test.go
View File

@ -9,8 +9,8 @@ import (
"path/filepath"
"testing"
"git.blender.org/flamenco/internal/manager/local_storage"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/internal/manager/local_storage"
)
func TestNew(t *testing.T) {

2
internal/manager/local_storage/local_storage.go
View File

@ -9,8 +9,8 @@ import (
"os"
"path/filepath"
"git.blender.org/flamenco/pkg/crosspath"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/pkg/crosspath"
)
type StorageInfo struct {

2
internal/manager/persistence/db.go
View File

@ -11,7 +11,7 @@ import (
"github.com/rs/zerolog/log"
"gorm.io/gorm"
// sqlite "git.blender.org/flamenco/pkg/gorm-modernc-sqlite"
// sqlite "projects.blender.org/studio/flamenco/pkg/gorm-modernc-sqlite"
"github.com/glebarez/sqlite"
)

4
internal/manager/persistence/jobs.go
View File

@ -15,8 +15,8 @@ import (
"gorm.io/gorm"
"gorm.io/gorm/clause"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/pkg/api"
)
type Job struct {

2
internal/manager/persistence/jobs_query.go
View File

@ -5,8 +5,8 @@ import (
"context"
"strings"
"git.blender.org/flamenco/pkg/api"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/pkg/api"
)
func (db *DB) QueryJobs(ctx context.Context, apiQ api.JobsQuery) ([]*Job, error) {

6
internal/manager/persistence/jobs_query_test.go
View File

@ -8,9 +8,9 @@ import (
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestSimpleQuery(t *testing.T) {

6
internal/manager/persistence/jobs_test.go
View File

@ -13,9 +13,9 @@ import (
"github.com/stretchr/testify/require"
"golang.org/x/net/context"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestStoreAuthoredJob(t *testing.T) {

2
internal/manager/persistence/task_scheduler.go
View File

@ -9,7 +9,7 @@ import (
"github.com/rs/zerolog/log"
"gorm.io/gorm"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/api"
)
var (

6
internal/manager/persistence/task_scheduler_test.go
View File

@ -10,9 +10,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.blender.org/flamenco/internal/manager/job_compilers"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/job_compilers"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
const schedulerTestTimeout = 100 * time.Millisecond

4
internal/manager/persistence/test_support.go
View File

@ -10,13 +10,13 @@ import (
"testing"
"time"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"github.com/glebarez/sqlite"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
// Change this to a filename if you want to run a single test and inspect the

2
internal/manager/persistence/timeout.go
View File

@ -6,7 +6,7 @@ import (
"context"
"time"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/pkg/api"
)
// This file contains functions for dealing with task/worker timeouts. Not database timeouts.

2
internal/manager/persistence/timeout_test.go
View File

@ -4,8 +4,8 @@ import (
"testing"
"time"
"git.blender.org/flamenco/pkg/api"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/pkg/api"
)
// SPDX-License-Identifier: GPL-3.0-or-later

4
internal/manager/persistence/worker_sleep_schedule_test.go
View File

@ -6,10 +6,10 @@ import (
"testing"
"time"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestFetchWorkerSleepSchedule(t *testing.T) {

2
internal/manager/persistence/worker_tag_test.go
View File

@ -6,9 +6,9 @@ import (
"testing"
"time"
"git.blender.org/flamenco/internal/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"projects.blender.org/studio/flamenco/internal/uuid"
)
func TestCreateFetchTag(t *testing.T) {

3
internal/manager/persistence/workers.go
View File

@ -8,8 +8,8 @@ import (
"strings"
"time"
"git.blender.org/flamenco/pkg/api"
"gorm.io/gorm"
"projects.blender.org/studio/flamenco/pkg/api"
)
type Worker struct {
@ -25,6 +25,7 @@ type Worker struct {
Software string `gorm:"type:varchar(32);default:''"`
Status api.WorkerStatus `gorm:"type:varchar(16);default:''"`
LastSeenAt time.Time `gorm:"index"` // Should contain UTC timestamps.
CanRestart bool `gorm:"type:smallint;default:false"`
StatusRequested api.WorkerStatus `gorm:"type:varchar(16);default:''"`
LazyStatusRequest bool `gorm:"type:smallint;default:false"`

4
internal/manager/persistence/workers_test.go
View File

@ -10,8 +10,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.blender.org/flamenco/internal/uuid"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/uuid"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestCreateFetchWorker(t *testing.T) {

4
internal/manager/sleep_scheduler/calculations.go
View File

@ -6,8 +6,8 @@ import (
"strings"
"time"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
// scheduledWorkerStatus returns the expected worker status at the given date/time.

4
internal/manager/sleep_scheduler/calculations_test.go
View File

@ -7,8 +7,8 @@ import (
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestCalculateNextCheck(t *testing.T) {

8
internal/manager/sleep_scheduler/interfaces.go
View File

@ -5,13 +5,13 @@ package sleep_scheduler
import (
"context"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
)
// Generate mock implementations of these interfaces.
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks git.blender.org/flamenco/internal/manager/sleep_scheduler PersistenceService,ChangeBroadcaster
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/sleep_scheduler PersistenceService,ChangeBroadcaster
type PersistenceService interface {
FetchWorkerSleepSchedule(ctx context.Context, workerUUID string) (*persistence.SleepSchedule, error)

6
internal/manager/sleep_scheduler/mocks/interfaces_mock.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/sleep_scheduler (interfaces: PersistenceService,ChangeBroadcaster)
// Source: projects.blender.org/studio/flamenco/internal/manager/sleep_scheduler (interfaces: PersistenceService,ChangeBroadcaster)
// Package mocks is a generated GoMock package.
package mocks
@ -8,9 +8,9 @@ import (
context "context"
reflect "reflect"
persistence "git.blender.org/flamenco/internal/manager/persistence"
api "git.blender.org/flamenco/pkg/api"
gomock "github.com/golang/mock/gomock"
persistence "projects.blender.org/studio/flamenco/internal/manager/persistence"
api "projects.blender.org/studio/flamenco/pkg/api"
)
// MockPersistenceService is a mock of PersistenceService interface.

4
internal/manager/sleep_scheduler/sleep_scheduler.go
View File

@ -12,8 +12,8 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
// Time period for checking the schedule of every worker.

6
internal/manager/sleep_scheduler/sleep_scheduler_test.go
View File

@ -11,9 +11,9 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/sleep_scheduler/mocks"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/sleep_scheduler/mocks"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestFetchSchedule(t *testing.T) {

4
internal/manager/task_logs/mocks/interfaces_mock.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/task_logs (interfaces: LocalStorage,ChangeBroadcaster)
// Source: projects.blender.org/studio/flamenco/internal/manager/task_logs (interfaces: LocalStorage,ChangeBroadcaster)
// Package mocks is a generated GoMock package.
package mocks
@ -7,8 +7,8 @@ package mocks
import (
reflect "reflect"
api "git.blender.org/flamenco/pkg/api"
gomock "github.com/golang/mock/gomock"
api "projects.blender.org/studio/flamenco/pkg/api"
)
// MockLocalStorage is a mock of LocalStorage interface.

6
internal/manager/task_logs/task_logs.go
View File

@ -11,10 +11,10 @@ import (
"sync"
"time"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"github.com/benbjohnson/clock"
"github.com/rs/zerolog"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
)
const (
@ -35,7 +35,7 @@ type Storage struct {
}
// Generate mock implementations of these interfaces.
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks git.blender.org/flamenco/internal/manager/task_logs LocalStorage,ChangeBroadcaster
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/task_logs LocalStorage,ChangeBroadcaster
type LocalStorage interface {
// ForJob returns the absolute directory path for storing job-related files.

2
internal/manager/task_logs/task_logs_test.go
View File

@ -14,12 +14,12 @@ import (
"testing"
"time"
"git.blender.org/flamenco/internal/manager/task_logs/mocks"
"github.com/benbjohnson/clock"
"github.com/golang/mock/gomock"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/internal/manager/task_logs/mocks"
)
func TestLogWriting(t *testing.T) {

10
internal/manager/task_state_machine/interfaces.go
View File

@ -5,15 +5,15 @@ package task_state_machine
import (
"context"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/task_logs"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"github.com/rs/zerolog"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/task_logs"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
)
// Generate mock implementations of these interfaces.
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks git.blender.org/flamenco/internal/manager/task_state_machine PersistenceService,ChangeBroadcaster,LogStorage
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/task_state_machine PersistenceService,ChangeBroadcaster,LogStorage
type PersistenceService interface {
SaveTask(ctx context.Context, task *persistence.Task) error

6
internal/manager/task_state_machine/mocks/interfaces_mock.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/task_state_machine (interfaces: PersistenceService,ChangeBroadcaster,LogStorage)
// Source: projects.blender.org/studio/flamenco/internal/manager/task_state_machine (interfaces: PersistenceService,ChangeBroadcaster,LogStorage)
// Package mocks is a generated GoMock package.
package mocks
@ -8,10 +8,10 @@ import (
context "context"
reflect "reflect"
persistence "git.blender.org/flamenco/internal/manager/persistence"
api "git.blender.org/flamenco/pkg/api"
gomock "github.com/golang/mock/gomock"
zerolog "github.com/rs/zerolog"
persistence "projects.blender.org/studio/flamenco/internal/manager/persistence"
api "projects.blender.org/studio/flamenco/pkg/api"
)
// MockPersistenceService is a mock of PersistenceService interface.

6
internal/manager/task_state_machine/task_state_machine.go
View File

@ -9,9 +9,9 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
)
// taskFailJobPercentage is the percentage of a job's tasks that need to fail to

6
internal/manager/task_state_machine/task_state_machine_test.go
View File

@ -11,9 +11,9 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/task_state_machine/mocks"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/task_state_machine/mocks"
"projects.blender.org/studio/flamenco/pkg/api"
)
type StateMachineMocks struct {

2
internal/manager/task_state_machine/task_statuses.go
View File

@ -2,7 +2,7 @@ package task_state_machine
// SPDX-License-Identifier: GPL-3.0-or-later
import "git.blender.org/flamenco/pkg/api"
import "projects.blender.org/studio/flamenco/pkg/api"
var (
// Task statuses that always get requeued when the job is requeueing.

4
internal/manager/task_state_machine/worker_requeue.go
View File

@ -5,9 +5,9 @@ package task_state_machine
import (
"context"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
// RequeueActiveTasksOfWorker re-queues all active tasks (should be max one) of this worker.

4
internal/manager/task_state_machine/worker_requeue_test.go
View File

@ -5,10 +5,10 @@ package task_state_machine
import (
"testing"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func TestRequeueActiveTasksOfWorker(t *testing.T) {

10
internal/manager/timeout_checker/interfaces.go
View File

@ -6,15 +6,15 @@ import (
"context"
"time"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/internal/manager/task_state_machine"
"git.blender.org/flamenco/internal/manager/webupdates"
"git.blender.org/flamenco/pkg/api"
"github.com/rs/zerolog"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/internal/manager/task_state_machine"
"projects.blender.org/studio/flamenco/internal/manager/webupdates"
"projects.blender.org/studio/flamenco/pkg/api"
)
// Generate mock implementations of these interfaces.
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks git.blender.org/flamenco/internal/manager/timeout_checker PersistenceService,TaskStateMachine,LogStorage,ChangeBroadcaster
//go:generate go run github.com/golang/mock/mockgen -destination mocks/interfaces_mock.gen.go -package mocks projects.blender.org/studio/flamenco/internal/manager/timeout_checker PersistenceService,TaskStateMachine,LogStorage,ChangeBroadcaster
type PersistenceService interface {
FetchTimedOutTasks(ctx context.Context, untouchedSince time.Time) ([]*persistence.Task, error)

6
internal/manager/timeout_checker/mocks/interfaces_mock.gen.go generated
View File

@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: git.blender.org/flamenco/internal/manager/timeout_checker (interfaces: PersistenceService,TaskStateMachine,LogStorage,ChangeBroadcaster)
// Source: projects.blender.org/studio/flamenco/internal/manager/timeout_checker (interfaces: PersistenceService,TaskStateMachine,LogStorage,ChangeBroadcaster)
// Package mocks is a generated GoMock package.
package mocks
@ -9,10 +9,10 @@ import (
reflect "reflect"
time "time"
persistence "git.blender.org/flamenco/internal/manager/persistence"
api "git.blender.org/flamenco/pkg/api"
gomock "github.com/golang/mock/gomock"
zerolog "github.com/rs/zerolog"
persistence "projects.blender.org/studio/flamenco/internal/manager/persistence"
api "projects.blender.org/studio/flamenco/pkg/api"
)
// MockPersistenceService is a mock of PersistenceService interface.

4
internal/manager/timeout_checker/tasks.go
View File

@ -10,8 +10,8 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
func (ttc *TimeoutChecker) checkTasks(ctx context.Context) {

4
internal/manager/timeout_checker/tasks_test.go
View File

@ -11,8 +11,8 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/persistence"
"git.blender.org/flamenco/pkg/api"
"projects.blender.org/studio/flamenco/internal/manager/persistence"
"projects.blender.org/studio/flamenco/pkg/api"
)
const taskTimeout = 20 * time.Minute

2
internal/manager/timeout_checker/timeout_checker_test.go
View File

@ -12,7 +12,7 @@ import (
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"git.blender.org/flamenco/internal/manager/timeout_checker/mocks"
"projects.blender.org/studio/flamenco/internal/manager/timeout_checker/mocks"
)
type TimeoutCheckerMocks struct {

Some files were not shown because too many files have changed in this diff Show More