diff --git a/.gitignore b/.gitignore index fd6febb2..29b56aec 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /shaman-checkout-id-setter /stresser /job-creator +/mage /addon-packer flamenco-manager.yaml flamenco-worker.yaml diff --git a/Makefile b/Makefile index 224fa4c2..21f8eba8 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,11 @@ ifeq (${GITHASH},dirty) GITHASH := $(shell git rev-parse --short=9 HEAD) endif -LDFLAGS := ${LDFLAGS} -X ${PKG}/internal/appinfo.ApplicationVersion=${VERSION} \ - -X ${PKG}/internal/appinfo.ApplicationGitHash=${GITHASH} \ - -X ${PKG}/internal/appinfo.ReleaseCycle=${RELEASE_CYCLE} -BUILD_FLAGS = -ldflags="${LDFLAGS}" +BUILDTOOL := mage +ifeq ($(OS),Windows_NT) + BUILDTOOL := $(BUILDTOOL).exe +endif +BUILDTOOL_PATH := ${PWD}/${BUILDTOOL} # Package name of the generated Python/JavaScript code for the Flamenco API. PY_API_PKG_NAME=flamenco.manager @@ -48,28 +49,29 @@ export CGO_ENABLED=0 all: application # Install generators and build the software. -with-deps: - go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.9.0 - go install github.com/golang/mock/mockgen@v1.6.0 +with-deps: buildtool + "${BUILDTOOL_PATH}" installGenerators $(MAKE) application -vet: - go vet ./... - go run golang.org/x/vuln/cmd/govulncheck@latest ./... +vet: buildtool + "${BUILDTOOL_PATH}" vet -application: webapp flamenco-manager flamenco-worker +application: flamenco-manager flamenco-worker -flamenco-manager: - $(MAKE) webapp-static - go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager +flamenco-manager: buildtool + "${BUILDTOOL_PATH}" flamencoManager -.PHONY: flamenco-manager-without-webapp -flamenco-manager-without-webapp: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager +flamenco-manager-without-webapp: buildtool + "${BUILDTOOL_PATH}" flamencoManagerWithoutWebapp -flamenco-worker: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-worker +flamenco-worker: buildtool + "${BUILDTOOL_PATH}" flamencoWorker +# Builds the buildtool itself, for faster rebuilds of Skyfill. +buildtool: ${BUILDTOOL} +${BUILDTOOL}: mage.go $(wildcard magefiles/*.go) go.mod + @echo "Building build tool $@" + @go run mage.go -compile "${BUILDTOOL_PATH}" # NOTE: these database migration commands are just for reference / debugging / # development purposes. Flamenco Manager and Worker each perform their own @@ -85,118 +87,20 @@ db-migrate-down: goose -dir ./internal/manager/persistence/migrations/ sqlite3 flamenco-manager.sqlite down .PHONY: db-migrate-status db-migrate-up db-migrate-down -.PHONY: stresser -stresser: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/stresser +webapp-static: buildtool + "${BUILDTOOL_PATH}" webappStatic -.PHONY: job-creator -job-creator: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/job-creator +generate: buildtool + "${BUILDTOOL_PATH}" generate -flamenco-addon.zip: addon-packer - ./addon-packer -filename ./flamenco-addon.zip +generate-go: buildtool + "${BUILDTOOL_PATH}" generateGo -addon-packer: cmd/addon-packer/addon-packer.go - go build -v ${BUILD_FLAGS} ${PKG}/cmd/addon-packer +generate-py: buildtool + "${BUILDTOOL_PATH}" generatePy -flamenco-manager_race: - CGO_ENABLED=1 go build -race -o $@ -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager - -flamenco-worker_race: - CGO_ENABLED=1 go build -race -o $@ -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-worker - -.PHONY: shaman-checkout-id-setter -shaman-checkout-id-setter: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/shaman-checkout-id-setter - -webapp: - yarn --cwd web/app install - -webapp-static: addon-packer - $(MAKE) clean-webapp-static -# When changing the base URL, also update the line -# e.GET("/app/*", echo.WrapHandler(webAppHandler)) -# in `cmd/flamenco-manager/main.go` - MSYS2_ARG_CONV_EXCL="*" yarn --cwd web/app build --outDir ../static --base=/app/ --logLevel warn -# yarn --cwd web/app build --outDir ../static --base=/app/ --minify false - ./addon-packer -filename ${WEB_STATIC}/flamenco-addon.zip - @echo "Web app has been installed into ${WEB_STATIC}" - -generate: - $(MAKE) generate-go - $(MAKE) generate-py - $(MAKE) generate-js - -generate-go: - go generate ./pkg/api/... - go generate ./internal/... -# The generators always produce UNIX line-ends. This creates false file -# modifications with Git. Convert them to DOS line-ends to avoid this. -ifeq ($(OS),Windows_NT) - git status --porcelain | grep '^ M .*.gen.go' | cut -d' ' -f3 | xargs unix2dos --keepdate -endif - -generate-py: -# The generator doesn't consistently overwrite existing files, nor does it -# remove no-longer-generated files. - rm -rf addon/flamenco/manager - -# See https://openapi-generator.tech/docs/generators/python for the options. - java -jar addon/openapi-generator-cli.jar \ - generate \ - -i pkg/api/flamenco-openapi.yaml \ - -g python \ - -o addon/ \ - --package-name "${PY_API_PKG_NAME}" \ - --http-user-agent "Flamenco/${VERSION} (Blender add-on)" \ - -p generateSourceCodeOnly=true \ - -p projectName=Flamenco \ - -p packageVersion="${VERSION}" > .openapi-generator-py.log - -# The generator outputs files so that we can write our own tests. We don't, -# though, so it's better to just remove those placeholders. - rm -rf addon/flamenco/manager/test -# The generators always produce UNIX line-ends. This creates false file -# modifications with Git. Convert them to DOS line-ends to avoid this. -ifeq ($(OS),Windows_NT) - git status --porcelain | grep '^ M addon/flamenco/manager' | cut -d' ' -f3 | xargs unix2dos --keepdate -endif - -generate-js: -# The generator doesn't consistently overwrite existing files, nor does it -# remove no-longer-generated files. - rm -rf web/app/src/manager-api - rm -rf web/_tmp-manager-api-javascript - -# See https://openapi-generator.tech/docs/generators/javascript for the options. -# Version '0.0.0' is used as NPM doesn't like Git hashes as versions. -# -# -p modelPropertyNaming=original is needed because otherwise the generator will -# use original naming internally, but generate docs with camelCase, and then -# things don't work properly. - java -jar addon/openapi-generator-cli.jar \ - generate \ - -i pkg/api/flamenco-openapi.yaml \ - -g javascript \ - -o web/_tmp-manager-api-javascript \ - --http-user-agent "Flamenco/${VERSION} / webbrowser" \ - -p projectName=flamenco-manager \ - -p projectVersion="0.0.0" \ - -p apiPackage="${JS_API_PKG_NAME}" \ - -p disallowAdditionalPropertiesIfNotPresent=false \ - -p usePromises=true \ - -p moduleName=flamencoManager > .openapi-generator-js.log - -# Cherry-pick the generated sources, and remove everything else. -# The only relevant bit is that the generated code depends on `superagent`, -# which is listed in our `. - mv web/_tmp-manager-api-javascript/src web/app/src/manager-api - rm -rf web/_tmp-manager-api-javascript -# The generators always produce UNIX line-ends. This creates false file -# modifications with Git. Convert them to DOS line-ends to avoid this. -ifeq ($(OS),Windows_NT) - git status --porcelain | grep '^ M web/app/src/manager-api' | cut -d' ' -f3 | xargs unix2dos --keepdate -endif +generate-js: buildtool + "${BUILDTOOL_PATH}" generateJS .PHONY: update-version: @@ -213,6 +117,7 @@ update-version: addon/flamenco/__init__.py \ addon/flamenco/manager \ addon/flamenco/manager_README.md \ + magefiles/version.go \ web/app/src/manager-api \ web/project-website/data/flamenco.yaml @echo 'git tag -a -m "Tagged version ${VERSION}" v${VERSION}' @@ -237,28 +142,17 @@ swagger-ui: @echo @echo 'Now update pkg/api/static/swagger-ui/index.html to have url: "/api/openapi3.json",' -test: -# Ensure the web-static directory exists, so that `web/web_app.go` can embed something. - mkdir -p ${WEB_STATIC} - go test -short -failfast ./... +test: buildtool + "${BUILDTOOL_PATH}" test -clean: - @go clean -i -x - rm -f flamenco*-v* flamenco-manager flamenco-worker *.exe flamenco-*_race addon-packer stresser - $(MAKE) clean-webapp-static - -clean-webapp-static: -# Start with `./` to avoid horrors when WEB_STATIC is absolute (like / or /home/yourname). - rm -rf ./${WEB_STATIC} -# Make sure there is at least something to embed by Go, or it may cause some errors. - mkdir -p ./${WEB_STATIC} - touch ${WEB_STATIC}/emptyfile +clean: buildtool + "${BUILDTOOL_PATH}" clean devserver-website: go run ${HUGO_PKG} -s web/project-website serve -devserver-webapp: - yarn --cwd web/app run dev --host +devserver-webapp: buildtool + "${BUILDTOOL_PATH}" devServerWebapp deploy-website: $(MAKE) -s check-environment @@ -417,4 +311,4 @@ publish-release-packages: ${RELEASE_PACKAGE_LINUX} ${RELEASE_PACKAGE_DARWIN} ${RELEASE_PACKAGE_DARWIN_ARM64} ${RELEASE_PACKAGE_WINDOWS} ${RELEASE_PACKAGE_SHAFILE} \ ${WEBSERVER_SSH}:${WEBSERVER_ROOT}/downloads/ -.PHONY: application version flamenco-manager flamenco-worker flamenco-manager_race flamenco-worker_race webapp webapp-static generate generate-go generate-py with-deps swagger-ui list-embedded test clean clean-webapp-static +.PHONY: application version flamenco-manager flamenco-worker flamenco-manager_race flamenco-worker_race webapp webapp-static generate generate-go generate-py with-deps swagger-ui list-embedded test clean flamenco-manager-without-webapp diff --git a/cmd/addon-packer/addon-packer.go b/cmd/addon-packer/addon-packer.go deleted file mode 100644 index 5b842ae3..00000000 --- a/cmd/addon-packer/addon-packer.go +++ /dev/null @@ -1,167 +0,0 @@ -package main - -// SPDX-License-Identifier: GPL-3.0-or-later - -import ( - "archive/zip" - "compress/flate" - "flag" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "github.com/mattn/go-colorable" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - - "projects.blender.org/studio/flamenco/internal/appinfo" -) - -var cliArgs struct { - // Do-and-quit flags. - version bool - - // Logging level flags. - quiet, debug, trace bool - - filename string -} - -func main() { - parseCliArgs() - if cliArgs.version { - fmt.Println(appinfo.ApplicationVersion) - return - } - - output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339} - log.Logger = log.Output(output) - configLogLevel() - - outfile, err := filepath.Abs(cliArgs.filename) - if err != nil { - log.Fatal().Err(err).Str("filepath", cliArgs.filename).Msg("unable make output file path absolute") - } - - // Open the output file. - logger := log.With().Str("zipname", outfile).Logger() - logger.Info().Msg("creating ZIP file") - zipFile, err := os.Create(outfile) - if err != nil { - logger.Fatal().Err(err).Msg("error creating file") - } - defer zipFile.Close() - zipWriter := zip.NewWriter(zipFile) - - zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(out, flate.BestCompression) - }) - - // CD to the addon dir to get the relative paths nice. - if err := os.Chdir("addon"); err != nil { - log.Fatal().Err(err).Msg("unable to cd to addon") - } - - basePath, err := os.Getwd() - if err != nil { - logger.Fatal().Err(err).Msg("error getting current working directory") - } - - // Copy all the files into the ZIP. - addToZip := func(path string, d fs.DirEntry, err error) error { - sublog := log.With().Str("path", path).Logger() - - if err != nil { - sublog.Error().Err(err).Msg("error received from filepath.WalkDir, aborting") - return err - } - - // Construct the path inside the ZIP file. - relpath, err := filepath.Rel(basePath, path) - if err != nil { - return fmt.Errorf("making %s relative to %s: %w", path, basePath, err) - } - - if d.IsDir() { - switch { - case filepath.Base(path) == "__pycache__": - return fs.SkipDir - case relpath == filepath.Join("flamenco", "manager", "docs"): - return fs.SkipDir - case strings.HasPrefix(filepath.Base(path), "."): - // Skip directories like .mypy_cache, etc. - return fs.SkipDir - default: - // Just recurse into this directory. - return nil - } - } - - sublog.Debug().Str("path", relpath).Msg("adding file to ZIP") - - // Read the file's contents. These are just Python files and maybe a Wheel, - // nothing huge. - contents, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("reading %s: %w", path, err) - } - - // Write into the ZIP file. - fileInZip, err := zipWriter.Create(relpath) - if err != nil { - return fmt.Errorf("creating %s in ZIP: %w", relpath, err) - } - _, err = fileInZip.Write(contents) - if err != nil { - return fmt.Errorf("writing to %s in ZIP: %w", relpath, err) - } - - return nil - } - - logger.Debug().Str("cwd", basePath).Msg("walking directory") - - if err := filepath.WalkDir(filepath.Join(basePath, "flamenco"), addToZip); err != nil { - logger.Fatal().Err(err).Msg("error filling ZIP file") - } - - comment := fmt.Sprintf("%s add-on for Blender, version %s", - appinfo.ApplicationName, - appinfo.ApplicationVersion, - ) - if err := zipWriter.SetComment(comment); err != nil { - logger.Fatal().Err(err).Msg("error setting ZIP comment") - } - - if err := zipWriter.Close(); err != nil { - logger.Fatal().Err(err).Msg("error closing ZIP file") - } -} - -func parseCliArgs() { - flag.BoolVar(&cliArgs.version, "version", false, "Shows the application version, then exits.") - flag.BoolVar(&cliArgs.quiet, "quiet", false, "Only log warning-level and worse.") - flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug-level logging.") - flag.BoolVar(&cliArgs.trace, "trace", false, "Enable trace-level logging.") - flag.StringVar(&cliArgs.filename, "filename", "web/static/flamenco-addon.zip", "Filename to save the add-on to.") - flag.Parse() -} - -func configLogLevel() { - var logLevel zerolog.Level - switch { - case cliArgs.trace: - logLevel = zerolog.TraceLevel - case cliArgs.debug: - logLevel = zerolog.DebugLevel - case cliArgs.quiet: - logLevel = zerolog.WarnLevel - default: - logLevel = zerolog.InfoLevel - } - zerolog.SetGlobalLevel(logLevel) -} diff --git a/cmd/update-version/magefiles.go b/cmd/update-version/magefiles.go new file mode 100644 index 00000000..8171e498 --- /dev/null +++ b/cmd/update-version/magefiles.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +const mageFile = "magefiles/version.go" + +// updateMagefiles changes the version number in the Mage files. +// Returns whether the file actually changed. +func updateMagefiles() bool { + logger := log.With().Str("filename", mageFile).Logger() + + // Parse the mage file as AST. + fset := token.NewFileSet() + astFile, err := parser.ParseFile(fset, mageFile, nil, parser.SkipObjectResolution|parser.ParseComments) + if err != nil { + logger.Fatal().Err(err).Msgf("could not update mage version file") + return false + } + + // Perform replacements on the AST. + replacements := map[string]string{ + "version": cliArgs.newVersion, + "releaseCycle": releaseCycle, + } + + var ( + lastIdent *ast.Ident // Last-seen identifier. + anyFieldChanged bool + ) + + ast.Inspect(astFile, func(node ast.Node) bool { + switch x := node.(type) { + case *ast.Ident: + lastIdent = x + + case *ast.BasicLit: + replacement, ok := replacements[lastIdent.Name] + if ok { + newValue := fmt.Sprintf("%q", replacement) + if x.Value != newValue { + logger.Info(). + Str("old", x.Value). + Str("new", newValue). + Msg("updating mage version file") + x.Value = newValue + anyFieldChanged = true + } + } + } + + return true + }) + + // Open a temporary file for writing. + mageDir := filepath.Dir(mageFile) + writer, err := os.CreateTemp(mageDir, filepath.Base(mageFile)+"*.go") + if err != nil { + log.Fatal().Err(err).Msgf("cannot create file in %s", mageDir) + } + defer writer.Close() + + // Write the altered AST to the temp file. + if err := format.Node(writer, fset, astFile); err != nil { + log.Fatal().Err(err).Msgf("cannot write updated version of %s to %s", mageFile, writer.Name()) + } + + // Close the file. + if err := writer.Close(); err != nil { + log.Fatal().Err(err).Msgf("cannot close %s", writer.Name()) + } + + // Overwrite the original mage file with the temp file. + if err := os.Rename(writer.Name(), mageFile); err != nil { + log.Fatal().Err(err).Msgf("cannot rename %s to %s", writer.Name(), mageFile) + } + + return anyFieldChanged +} diff --git a/cmd/update-version/main.go b/cmd/update-version/main.go index 01de5567..70f9f0fe 100644 --- a/cmd/update-version/main.go +++ b/cmd/update-version/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "os" + "strings" "time" "github.com/mattn/go-colorable" @@ -13,13 +14,18 @@ import ( "github.com/rs/zerolog/log" ) -var cliArgs struct { - // Logging level flags. - quiet, debug, trace bool +// Global variables used by the updateXXX() functions. +var ( + cliArgs struct { + // Logging level flags. + quiet, debug, trace bool - newVersion string - updateMakefile bool -} + newVersion string + updateMakefile bool + } + + releaseCycle string +) func main() { parseCliArgs() @@ -29,11 +35,23 @@ func main() { log.Info().Str("version", cliArgs.newVersion).Msg("updating Flamenco version") + switch { + case strings.Contains(cliArgs.newVersion, "alpha"), strings.Contains(cliArgs.newVersion, "dev"): + releaseCycle = "alpha" + case strings.Contains(cliArgs.newVersion, "beta"): + releaseCycle = "beta" + case strings.Contains(cliArgs.newVersion, "rc"): + releaseCycle = "rc" + default: + releaseCycle = "release" + } + var anyFileWasChanged bool if cliArgs.updateMakefile { anyFileWasChanged = updateMakefile() || anyFileWasChanged } anyFileWasChanged = updateAddon() || anyFileWasChanged + anyFileWasChanged = updateMagefiles() || anyFileWasChanged if !anyFileWasChanged { log.Warn().Msg("nothing changed") diff --git a/go.mod b/go.mod index cd1c7e2f..7ac3d647 100644 --- a/go.mod +++ b/go.mod @@ -19,22 +19,27 @@ require ( github.com/google/uuid v1.5.0 github.com/graarh/golang-socketio v0.0.0-20170510162725-2c44953b9b5f github.com/labstack/echo/v4 v4.9.1 + github.com/magefile/mage v1.15.0 github.com/mattn/go-colorable v0.1.12 - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pressly/goose/v3 v3.15.1 github.com/rs/zerolog v1.26.1 github.com/stretchr/testify v1.8.4 github.com/zcalusic/sysinfo v1.0.1 github.com/ziflex/lecho/v3 v3.1.0 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.25.0 golang.org/x/image v0.18.0 - golang.org/x/net v0.25.0 - golang.org/x/sys v0.20.0 + golang.org/x/net v0.27.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.22.0 + golang.org/x/vuln v1.1.3 gopkg.in/yaml.v2 v2.4.0 + honnef.co/go/tools v0.5.1 modernc.org/sqlite v1.28.0 ) require ( + github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -54,11 +59,12 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.23.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.41.0 // indirect @@ -70,3 +76,7 @@ require ( modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect ) + +// Replace staticcheck release with a specific revision of their `main` branch, +// so that it includes my PR https://github.com/dominikh/go-tools/pull/1597 +replace honnef.co/go/tools v0.5.1 => honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a diff --git a/go.sum b/go.sum index efbc1c53..2a7c1825 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= +github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= @@ -115,6 +117,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.7/go.mod h1:bw24IXWbavc0R2RsOtpXL7RtMyP589yZ1+L7kd09ZGA= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= @@ -138,8 +142,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -191,8 +195,10 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ= +golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= @@ -200,8 +206,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -211,8 +217,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -235,7 +241,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -247,9 +252,12 @@ golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 h1:FemxDzfMUcK2f3YY4H+05K9CDzbSVr2+q/JKN45pey0= +golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -271,8 +279,10 @@ golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/vuln v1.1.3 h1:NPGnvPOTgnjBc9HTaUx+nj+EaUYxl5SJOWqaDYGaFYw= +golang.org/x/vuln v1.1.3/go.mod h1:7Le6Fadm5FOqE9C926BCD0g12NWyhg7cxV4BwcPFuNY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -294,6 +304,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a h1:P0jdAtRz/UFVIXrgf9EdWApQJnbQrmsUwyARz4FF+D8= +honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= diff --git a/mage.go b/mage.go new file mode 100644 index 00000000..6a3b958e --- /dev/null +++ b/mage.go @@ -0,0 +1,11 @@ +//go:build ignore + +package main + +import ( + "os" + + "github.com/magefile/mage/mage" +) + +func main() { os.Exit(mage.Main()) } diff --git a/magefiles/addonpacker.go b/magefiles/addonpacker.go new file mode 100644 index 00000000..ee78417e --- /dev/null +++ b/magefiles/addonpacker.go @@ -0,0 +1,108 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "archive/zip" + "compress/flate" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "projects.blender.org/studio/flamenco/internal/appinfo" +) + +func packAddon(filename string) error { + outfile, err := filepath.Abs(filename) + if err != nil { + return fmt.Errorf("unable make output file path absolute: %w", err) + } + + // Open the output file. + zipFile, err := os.Create(outfile) + if err != nil { + return fmt.Errorf("error creating file %s: %w", outfile, err) + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(out, flate.BestCompression) + }) + + basePath, err := filepath.Abs("./addon") // os.Getwd() + if err != nil { + return fmt.Errorf("error getting current working directory: %w", err) + } + + // Copy all the files into the ZIP. + addToZip := func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("error received from filepath.WalkDir: %w", err) + } + + // Construct the path inside the ZIP file. + relpath, err := filepath.Rel(basePath, path) + if err != nil { + return fmt.Errorf("making %s relative to %s: %w", path, basePath, err) + } + + if d.IsDir() { + switch { + case filepath.Base(path) == "__pycache__": + return fs.SkipDir + case relpath == filepath.Join("flamenco", "manager", "docs"): + return fs.SkipDir + case strings.HasPrefix(filepath.Base(path), "."): + // Skip directories like .mypy_cache, etc. + return fs.SkipDir + default: + // Just recurse into this directory. + return nil + } + } + + // Read the file's contents. These are just Python files and maybe a Wheel, + // nothing huge. + contents, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + // Write into the ZIP file. + fileInZip, err := zipWriter.Create(relpath) + if err != nil { + return fmt.Errorf("creating %s in ZIP: %w", relpath, err) + } + _, err = fileInZip.Write(contents) + if err != nil { + return fmt.Errorf("writing to %s in ZIP: %w", relpath, err) + } + + return nil + } + + if err := filepath.WalkDir(filepath.Join(basePath, "flamenco"), addToZip); err != nil { + return fmt.Errorf("error filling ZIP file: %w", err) + } + + comment := fmt.Sprintf("%s add-on for Blender, version %s", + appinfo.ApplicationName, + appinfo.ApplicationVersion, + ) + if err := zipWriter.SetComment(comment); err != nil { + return fmt.Errorf("error setting ZIP comment: %w", err) + } + + if err := zipWriter.Close(); err != nil { + return fmt.Errorf("error closing ZIP file: %w", err) + } + return nil +} diff --git a/magefiles/build.go b/magefiles/build.go new file mode 100644 index 00000000..63438d47 --- /dev/null +++ b/magefiles/build.go @@ -0,0 +1,134 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "fmt" + "path/filepath" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/magefile/mage/target" +) + +const ( + goPkg = "projects.blender.org/studio/flamenco" +) + +var ( + // The directory that will contain the built webapp files, and some other + // files that will be served as static files by the Flamenco Manager web + // server. + webStatic = filepath.Join("web", "static") +) + +// Build Flamenco Manager and Flamenco Worker, including the webapp and the add-on +func Build() { + mg.Deps(FlamencoManager, FlamencoWorker) +} + +// Build Flamenco Manager with the webapp and add-on ZIP embedded +func FlamencoManager() error { + mg.Deps(WebappStatic) + mg.Deps(flamencoManager) + return nil +} + +// Only build the Flamenco Manager executable, do not rebuild the webapp +func FlamencoManagerWithoutWebapp() error { + mg.Deps(flamencoManager) + return nil +} + +func flamencoManager() error { + return build("./cmd/flamenco-manager") +} + +// Build the Flamenco Worker executable +func FlamencoWorker() error { + return build("./cmd/flamenco-worker") +} + +// Build the webapp as static files that can be served +func WebappStatic() error { + runInstall, err := target.Dir("web/app/node_modules") + if err != nil { + return err + } + if runInstall { + mg.SerialDeps(WebappInstallDeps) + } + if err := cleanWebappStatic(); err != nil { + return err + } + + env := map[string]string{ + "MSYS2_ARG_CONV_EXCL": "*", + } + + // When changing the base URL, also update the line + // e.GET("/app/*", echo.WrapHandler(webAppHandler)) + // in `cmd/flamenco-manager/main.go` + err = sh.RunWithV(env, + "yarn", + "--cwd", "web/app", + "build", + "--outDir", "../static", + "--base=/app/", + "--logLevel", "warn", + // For debugging you can add: + // "--minify", "false", + ) + if err != nil { + return err + } + + fmt.Printf("Web app has been installed into %s\n", webStatic) + + // Build the add-on ZIP as it's part of the static web files. + zipPath := filepath.Join(webStatic, "flamenco3-addon.zip") + return packAddon(zipPath) +} + +// Use Yarn to install the webapp's NodeJS dependencies +func WebappInstallDeps() error { + env := map[string]string{ + "MSYS2_ARG_CONV_EXCL": "*", + } + return sh.RunWithV(env, + "yarn", + "--cwd", "web/app", + "install", + ) +} + +func build(exePackage string) error { + flags, err := buildFlags() + if err != nil { + return err + } + + args := []string{"build", "-v"} + args = append(args, flags...) + args = append(args, exePackage) + return sh.RunV(mg.GoCmd(), args...) +} + +func buildFlags() ([]string, error) { + hash, err := gitHash() + if err != nil { + return nil, err + } + + ldflags := "" + + fmt.Sprintf(" -X %s/internal/appinfo.ApplicationVersion=%s", goPkg, version) + + fmt.Sprintf(" -X %s/internal/appinfo.ApplicationGitHash=%s", goPkg, hash) + + fmt.Sprintf(" -X %s/internal/appinfo.ReleaseCycle=%s", goPkg, releaseCycle) + + flags := []string{ + "-ldflags=" + ldflags, + } + return flags, nil +} diff --git a/magefiles/check.go b/magefiles/check.go new file mode 100644 index 00000000..4e179653 --- /dev/null +++ b/magefiles/check.go @@ -0,0 +1,61 @@ +//go:build mage + +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "golang.org/x/vuln/scan" + "honnef.co/go/tools/lintcmd" + lintcmdversion "honnef.co/go/tools/lintcmd/version" + "honnef.co/go/tools/simple" + "honnef.co/go/tools/staticcheck" + "honnef.co/go/tools/stylecheck" + "honnef.co/go/tools/unused" +) + +// Run unit tests, check for vulnerabilities, and run the linter +func Check(ctx context.Context) { + mg.CtxDeps(ctx, Test, Govulncheck, Staticcheck, Vet) +} + +// Run unit tests +func Test(ctx context.Context) error { + return sh.RunV(mg.GoCmd(), "test", "-short", "-failfast", "./...") +} + +// Check for known vulnerabilities. +func Govulncheck(ctx context.Context) error { + cmd := scan.Command(ctx, "./...") + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() +} + +// Analyse the source code. +func Staticcheck() error { + cmd := lintcmd.NewCommand("staticcheck") + cmd.SetVersion(lintcmdversion.Version, lintcmdversion.MachineVersion) + cmd.ParseFlags([]string{"./..."}) + cmd.AddAnalyzers(simple.Analyzers...) + cmd.AddAnalyzers(staticcheck.Analyzers...) + cmd.AddAnalyzers(stylecheck.Analyzers...) + cmd.AddAnalyzers(unused.Analyzer) + + exitCode := cmd.Execute() + if exitCode != 0 { + return errors.New("staticcheck failed") + } + fmt.Println("staticcheck ok") + return nil +} + +// Run `go vet` +func Vet() error { + return sh.RunV(mg.GoCmd(), "vet", "./...") +} diff --git a/magefiles/clean.go b/magefiles/clean.go new file mode 100644 index 00000000..659aaa6a --- /dev/null +++ b/magefiles/clean.go @@ -0,0 +1,64 @@ +//go:build mage + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/magefile/mage/sh" +) + +// Remove executables and other build output +func Clean() error { + if err := cleanWebappStatic(); err != nil { + return err + } + + if err := sh.Run("go", "clean"); err != nil { + return err + } + + if err := rm( + "flamenco-manager", "flamenco-manager.exe", + "flamenco-manager_race", "flamenco-manager_race.exe", + "flamenco-worker", "flamenco-worker.exe", + "flamenco-worker_race", "flamenco-worker_race.exe", + ); err != nil { + return err + } + return nil +} + +func cleanWebappStatic() error { + // Just a simple heuristic to avoid deleting things like "/" or "C:\" + if len(webStatic) < 4 { + panic(fmt.Sprintf("webStatic path is too short, I don't trust it: %q", webStatic)) + } + + if err := sh.Rm(webStatic); err != nil { + return fmt.Errorf("unable to remove old web static dir %q: %w", webStatic, err) + } + if err := os.MkdirAll(webStatic, os.ModePerm); err != nil { + return fmt.Errorf("unable to create web static dir %q: %w", webStatic, err) + } + + // Make sure there is at least something to embed by Go, or it may cause some + // errors. This is done in the 'clean' function so that the Go code can be + // built before building the webapp. + emptyfile := filepath.Join(webStatic, "emptyfile") + if err := os.WriteFile(emptyfile, []byte{}, os.ModePerm); err != nil { + return err + } + return nil +} + +func rm(path ...string) error { + for _, p := range path { + if err := sh.Rm(p); err != nil { + return err + } + } + return nil +} diff --git a/magefiles/devserver.go b/magefiles/devserver.go new file mode 100644 index 00000000..d63c1478 --- /dev/null +++ b/magefiles/devserver.go @@ -0,0 +1,13 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "github.com/magefile/mage/sh" +) + +func DevServerWebapp() error { + return sh.RunV("yarn", "--cwd", "web/app", "run", "dev", "--host") +} diff --git a/magefiles/generate.go b/magefiles/generate.go new file mode 100644 index 00000000..11d8cacd --- /dev/null +++ b/magefiles/generate.go @@ -0,0 +1,172 @@ +//go:build mage + +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +// Generate code (OpenAPI and test mocks) +func Generate() { + mg.Deps(GenerateGo, GeneratePy, GenerateJS) +} + +// Generate Go code for Flamenco Manager and Worker +func GenerateGo(ctx context.Context) error { + r := NewRunner(ctx) + r.Run(mg.GoCmd(), "generate", "./pkg/api/...") + r.Run(mg.GoCmd(), "generate", "./internal/...") + if err := r.Wait(); err != nil { + return err + } + + // The generators always produce UNIX line-ends. This creates false file + // modifications with Git. Convert them to DOS line-ends to avoid this. + if runtime.GOOS == "windows" { + unix2dosModifiedFiles(".gen.go$") + } + return nil +} + +// Generate Python code for the add-on +func GeneratePy() error { + // The generator doesn't consistently overwrite existing files, nor does it + // remove no-longer-generated files. + sh.Rm("addon/flamenco/manager") + + // See https://openapi-generator.tech/docs/generators/python for the options. + err := sh.Run("java", + "-jar", "addon/openapi-generator-cli.jar", + "generate", + "-i", "pkg/api/flamenco-openapi.yaml", + "-g", "python", + "-o", "addon/", + "--package-name", "flamenco.manager", + "--http-user-agent", fmt.Sprintf("Flamenco/%s (Blender add-on)", version), + "-p", "generateSourceCodeOnly=true", + "-p", "projectName=Flamenco", + "-p", fmt.Sprintf("packageVersion=%s", version)) + if err != nil { + return err + } + + // The generator outputs files so that we can write our own tests. We don't, + // though, so it's better to just remove those placeholders. + sh.Rm("addon/flamenco/manager/test") + + // The generators always produce UNIX line-ends. This creates false file + // modifications with Git. Convert them to DOS line-ends to avoid this. + if runtime.GOOS == "windows" { + unix2dosModifiedFiles("addon/flamenco/manager") + } + + return nil +} + +// Generate JavaScript code for the webapp +func GenerateJS() error { + const ( + jsOutDir = "web/app/src/manager-api" + jsTempDir = "web/_tmp-manager-api-javascript" + ) + sh.Rm(jsOutDir) + sh.Rm(jsTempDir) + + // See https://openapi-generator.tech/docs/generators/javascript for the options. + // Version '0.0.0' is used as NPM doesn't like Git hashes as versions. + // + // -p modelPropertyNaming=original is needed because otherwise the generator will + // use original naming internally, but generate docs with camelCase, and then + // things don't work properly. + err := sh.Run("java", + "-jar", "addon/openapi-generator-cli.jar", + "generate", + "-i", "pkg/api/flamenco-openapi.yaml", + "-g", "javascript", + "-o", jsTempDir, + "--http-user-agent", fmt.Sprintf("Flamenco/%s / webbrowser", version), + "-p", "projectName=flamenco-manager", + "-p", "projectVersion=0.0.0", + "-p", "apiPackage=manager", + "-p", "disallowAdditionalPropertiesIfNotPresent=false", + "-p", "usePromises=true", + "-p", "moduleName=flamencoManager") + if err != nil { + return err + } + + // Cherry-pick the generated sources, and remove everything else. + if err := os.Rename(filepath.Join(jsTempDir, "src"), jsOutDir); err != nil { + return err + } + sh.Rm(jsTempDir) + + if runtime.GOOS == "windows" { + unix2dosModifiedFiles(jsOutDir) + } + + return nil +} + +// unix2dosModifiedFiles changes line ends in files Git considers modified that match the given pattern. +func unix2dosModifiedFiles(pattern string) { + // Get modified files from Git. Expected lines like: + // + // M pkg/api/openapi_client.gen.go + // M pkg/api/openapi_server.gen.go + // M pkg/api/openapi_spec.gen.go + // M pkg/api/openapi_types.gen.go + // ?? magefiles/generate.go + + gitStatus, err := sh.Output("git", "status", "--porcelain") + if err != nil { + panic(fmt.Sprintf("error running 'git status': %s", err)) + } + + // Construct a list of modified files that match the pattern. + patternRe := regexp.MustCompile(pattern) + modified := []string{} + for _, line := range strings.Split(gitStatus, "\n") { + if line[0:3] != " M " { + continue + } + if !patternRe.MatchString(line[3:]) { + continue + } + modified = append(modified, line[3:]) + } + + // Run unix2dos on all found files. + for _, path := range modified { + unix2dos(path) + } +} + +func unix2dos(filename string) { + if mg.Verbose() { + fmt.Printf("unix2dos %s\n", filename) + } + + // TODO: rewrite to stream the data instead of reading, copying, and writing + // everything. + contents, err := os.ReadFile(filename) + if err != nil { + panic(fmt.Sprintf("error converting UNIX to DOS line ends: %v", err)) + } + + lines := bytes.Split(contents, []byte("\n")) + err = os.WriteFile(filename, bytes.Join(lines, []byte("\r\n")), os.ModePerm) + if err != nil { + panic(fmt.Sprintf("error writing DOS line ends: %v", err)) + } +} diff --git a/magefiles/install.go b/magefiles/install.go new file mode 100644 index 00000000..a90cb668 --- /dev/null +++ b/magefiles/install.go @@ -0,0 +1,45 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "context" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +var ( + generators = []string{ + "github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.9.0", + "github.com/golang/mock/mockgen@v1.6.0", + } +) + +// Install build-time dependencies: code generators and NodeJS dependencies. +func InstallDeps() { + mg.SerialDeps(InstallGenerators, InstallDepsWebapp) +} + +// Install code generators. +func InstallGenerators(ctx context.Context) error { + r := NewRunner(ctx) + for _, pkg := range generators { + r.Run(mg.GoCmd(), "install", pkg) + } + return r.Wait() +} + +// Use Yarn to install the webapp's NodeJS dependencies +func InstallDepsWebapp() error { + env := map[string]string{ + "MSYS2_ARG_CONV_EXCL": "*", + } + return sh.RunWithV(env, + "yarn", + "--cwd", "web/app", + "install", + ) +} diff --git a/magefiles/runner.go b/magefiles/runner.go new file mode 100644 index 00000000..5f02017d --- /dev/null +++ b/magefiles/runner.go @@ -0,0 +1,48 @@ +//go:build mage + +package main + +import ( + "context" + + "github.com/magefile/mage/sh" + "golang.org/x/sync/errgroup" +) + +// Runner allows running a group of commands sequentially, stopping at the first +// failure. +// See https://github.com/magefile/mage/issues/455 for the feature request +// to include this in Mage. +type Runner struct { + group *errgroup.Group + ctx context.Context +} + +// NewRunner constructs a new runner that's bound to the given context. If the +// context is done, no new command will be executed. It does NOT abort an +// already-running command. +func NewRunner(ctx context.Context) *Runner { + group, groupctx := errgroup.WithContext(ctx) + group.SetLimit(1) + + return &Runner{ + group: group, + ctx: groupctx, + } +} + +// Run the given command. +// This only runs a command if no previous command has failed yet. +func (r *Runner) Run(cmd string, args ...string) { + r.group.Go(func() error { + if err := r.ctx.Err(); err != nil { + return err + } + return sh.RunV(cmd, args...) + }) +} + +// Wait for the commands to finish running, and return any error. +func (r *Runner) Wait() error { + return r.group.Wait() +} diff --git a/magefiles/version.go b/magefiles/version.go new file mode 100644 index 00000000..ff5ef66e --- /dev/null +++ b/magefiles/version.go @@ -0,0 +1,33 @@ +//go:build mage + +package main + +import ( + "fmt" + + "github.com/magefile/mage/sh" +) + +// To update the version number in all the relevant places, update the VERSION +// variable below and run `mage update-version`. +const ( + version = "3.6-alpha5" + releaseCycle = "alpha" +) + +func gitHash() (string, error) { + return sh.Output("git", "rev-parse", "--short", "HEAD") +} + +// Show which version information would be embedded in executables +func Version() error { + fmt.Printf("Package : %s\n", goPkg) + fmt.Printf("Version : %s\n", version) + + hash, err := gitHash() + if err != nil { + return err + } + fmt.Printf("Git Hash : %s\n", hash) + return nil +}