Compare commits
7 Commits
temp-geome
...
tmp-buildb
Author | SHA1 | Date | |
---|---|---|---|
aa75e383ba | |||
![]() |
ee54a8ace7 | ||
![]() |
ffe7a41540 | ||
![]() |
74c7e21f6c | ||
![]() |
fc2b56e68c | ||
![]() |
51bbdfbef3 | ||
![]() |
5baf1dddd5 |
@@ -1,70 +1,4 @@
|
||||
Blender Buildbot
|
||||
================
|
||||
Buildbot Configuration
|
||||
=====================
|
||||
|
||||
Code signing
|
||||
------------
|
||||
|
||||
Code signing is done as part of INSTALL target, which makes it possible to sign
|
||||
files which are aimed into a bundle and coming from a non-signed source (such as
|
||||
libraries SVN).
|
||||
|
||||
This is achieved by specifying `worker_codesign.cmake` as a post-install script
|
||||
run by CMake. This CMake script simply involves an utility script written in
|
||||
Python which takes care of an actual signing.
|
||||
|
||||
### Configuration
|
||||
|
||||
Client configuration doesn't need anything special, other than variable
|
||||
`SHARED_STORAGE_DIR` pointing to a location which is watched by a server.
|
||||
This is done in `config_builder.py` file and is stored in Git (which makes it
|
||||
possible to have almost zero-configuration buildbot machines).
|
||||
|
||||
Server configuration requires copying `config_server_template.py` under the
|
||||
name of `config_server.py` and tweaking values, which are platform-specific.
|
||||
|
||||
#### Windows configuration
|
||||
|
||||
There are two things which are needed on Windows in order to have code signing
|
||||
to work:
|
||||
|
||||
- `TIMESTAMP_AUTHORITY_URL` which is most likely set http://timestamp.digicert.com
|
||||
- `CERTIFICATE_FILEPATH` which is a full file path to a PKCS #12 key (.pfx).
|
||||
|
||||
## Tips
|
||||
|
||||
### Self-signed certificate on Windows
|
||||
|
||||
It is easiest to test configuration using self-signed certificate.
|
||||
|
||||
The certificate manipulation utilities are coming with Windows SDK.
|
||||
Unfortunately, they are not added to PATH. Here is an example of how to make
|
||||
sure they are easily available:
|
||||
|
||||
```
|
||||
set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH%
|
||||
set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64;%PATH%
|
||||
```
|
||||
|
||||
Generate CA:
|
||||
|
||||
```
|
||||
makecert -r -pe -n "CN=Blender Test CA" -ss CA -sr CurrentUser -a sha256 ^
|
||||
-cy authority -sky signature -sv BlenderTestCA.pvk BlenderTestCA.cer
|
||||
```
|
||||
|
||||
Import the generated CA:
|
||||
|
||||
```
|
||||
certutil -user -addstore Root BlenderTestCA.cer
|
||||
```
|
||||
|
||||
Create self-signed certificate and pack it into PKCS #12:
|
||||
|
||||
```
|
||||
makecert -pe -n "CN=Blender Test SPC" -a sha256 -cy end ^
|
||||
-sky signature ^
|
||||
-ic BlenderTestCA.cer -iv BlenderTestCA.pvk ^
|
||||
-sv BlenderTestSPC.pvk BlenderTestSPC.cer
|
||||
|
||||
pvk2pfx -pvk BlenderTestSPC.pvk -spc BlenderTestSPC.cer -pfx BlenderTestSPC.pfx
|
||||
```
|
||||
Files used by Buildbot's `compile-code` step.
|
||||
|
@@ -1,127 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def is_tool(name):
|
||||
"""Check whether `name` is on PATH and marked as executable."""
|
||||
|
||||
# from whichcraft import which
|
||||
from shutil import which
|
||||
|
||||
return which(name) is not None
|
||||
|
||||
|
||||
class Builder:
|
||||
def __init__(self, name, branch, codesign):
|
||||
self.name = name
|
||||
self.branch = branch
|
||||
self.is_release_branch = re.match("^blender-v(.*)-release$", branch) is not None
|
||||
self.codesign = codesign
|
||||
|
||||
# Buildbot runs from build/ directory
|
||||
self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git'))
|
||||
self.build_dir = os.path.abspath(os.path.join('..', 'build'))
|
||||
self.install_dir = os.path.abspath(os.path.join('..', 'install'))
|
||||
self.upload_dir = os.path.abspath(os.path.join('..', 'install'))
|
||||
|
||||
# Detect platform
|
||||
if name.startswith('mac'):
|
||||
self.platform = 'mac'
|
||||
self.command_prefix = []
|
||||
elif name.startswith('linux'):
|
||||
self.platform = 'linux'
|
||||
if is_tool('scl'):
|
||||
self.command_prefix = ['scl', 'enable', 'devtoolset-9', '--']
|
||||
else:
|
||||
self.command_prefix = []
|
||||
elif name.startswith('win'):
|
||||
self.platform = 'win'
|
||||
self.command_prefix = []
|
||||
else:
|
||||
raise ValueError('Unkonw platform for builder ' + self.platform)
|
||||
|
||||
# Always 64 bit now
|
||||
self.bits = 64
|
||||
|
||||
|
||||
def create_builder_from_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('builder_name')
|
||||
parser.add_argument('branch', default='master', nargs='?')
|
||||
parser.add_argument("--codesign", action="store_true")
|
||||
args = parser.parse_args()
|
||||
return Builder(args.builder_name, args.branch, args.codesign)
|
||||
|
||||
|
||||
class VersionInfo:
|
||||
def __init__(self, builder):
|
||||
# Get version information
|
||||
buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h")
|
||||
blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
|
||||
|
||||
version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION'))
|
||||
version_number_patch = int(self._parse_header_file(blender_h, 'BLENDER_VERSION_PATCH'))
|
||||
version_numbers = (version_number // 100, version_number % 100, version_number_patch)
|
||||
self.short_version = "%d.%d" % (version_numbers[0], version_numbers[1])
|
||||
self.version = "%d.%d.%d" % version_numbers
|
||||
self.version_cycle = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE')
|
||||
self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
|
||||
|
||||
if self.version_cycle == "release":
|
||||
# Final release
|
||||
self.full_version = self.version
|
||||
self.is_development_build = False
|
||||
elif self.version_cycle == "rc":
|
||||
# Release candidate
|
||||
self.full_version = self.version + self.version_cycle
|
||||
self.is_development_build = False
|
||||
else:
|
||||
# Development build
|
||||
self.full_version = self.version + '-' + self.hash
|
||||
self.is_development_build = True
|
||||
|
||||
def _parse_header_file(self, filename, define):
|
||||
import re
|
||||
regex = re.compile(r"^#\s*define\s+%s\s+(.*)" % define)
|
||||
with open(filename, "r") as file:
|
||||
for l in file:
|
||||
match = regex.match(l)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def call(cmd, env=None, exit_on_error=True):
|
||||
print(' '.join(cmd))
|
||||
|
||||
# Flush to ensure correct order output on Windows.
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
|
||||
retcode = subprocess.call(cmd, env=env)
|
||||
if exit_on_error and retcode != 0:
|
||||
sys.exit(retcode)
|
||||
return retcode
|
@@ -1,81 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
|
||||
@dataclass
|
||||
class AbsoluteAndRelativeFileName:
|
||||
"""
|
||||
Helper class which keeps track of absolute file path for a direct access and
|
||||
corresponding relative path against given base.
|
||||
|
||||
The relative part is used to construct a file name within an archive which
|
||||
contains files which are to be signed or which has been signed already
|
||||
(depending on whether the archive is addressed to signing server or back
|
||||
to the buildbot worker).
|
||||
"""
|
||||
|
||||
# Base directory which is where relative_filepath is relative to.
|
||||
base_dir: Path
|
||||
|
||||
# Full absolute path of the corresponding file.
|
||||
absolute_filepath: Path
|
||||
|
||||
# Derived from full file path, contains part of the path which is relative
|
||||
# to a desired base path.
|
||||
relative_filepath: Path
|
||||
|
||||
def __init__(self, base_dir: Path, filepath: Path):
|
||||
self.base_dir = base_dir
|
||||
self.absolute_filepath = filepath.resolve()
|
||||
self.relative_filepath = self.absolute_filepath.relative_to(
|
||||
self.base_dir)
|
||||
|
||||
@classmethod
|
||||
def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName':
|
||||
assert path.is_absolute()
|
||||
assert path.is_file()
|
||||
|
||||
base_dir = path.parent
|
||||
return AbsoluteAndRelativeFileName(base_dir, path)
|
||||
|
||||
@classmethod
|
||||
def recursively_from_directory(cls, base_dir: Path) \
|
||||
-> List['AbsoluteAndRelativeFileName']:
|
||||
"""
|
||||
Create list of AbsoluteAndRelativeFileName for all the files in the
|
||||
given directory.
|
||||
|
||||
NOTE: Result will be pointing to a resolved paths.
|
||||
"""
|
||||
assert base_dir.is_absolute()
|
||||
assert base_dir.is_dir()
|
||||
|
||||
base_dir = base_dir.resolve()
|
||||
|
||||
result = []
|
||||
for filename in base_dir.glob('**/*'):
|
||||
if not filename.is_file():
|
||||
continue
|
||||
result.append(AbsoluteAndRelativeFileName(base_dir, filename))
|
||||
return result
|
@@ -1,245 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import codesign.util as util
|
||||
|
||||
|
||||
class ArchiveStateError(Exception):
|
||||
message: str
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ArchiveState:
|
||||
"""
|
||||
Additional information (state) of the archive
|
||||
|
||||
Includes information like expected file size of the archive file in the case
|
||||
the archive file is expected to be successfully created.
|
||||
|
||||
If the archive can not be created, this state will contain error message
|
||||
indicating details of error.
|
||||
"""
|
||||
|
||||
# Size in bytes of the corresponding archive.
|
||||
file_size: Optional[int] = None
|
||||
|
||||
# Non-empty value indicates that error has happenned.
|
||||
error_message: str = ''
|
||||
|
||||
def has_error(self) -> bool:
|
||||
"""
|
||||
Check whether the archive is at error state
|
||||
"""
|
||||
|
||||
return self.error_message
|
||||
|
||||
def serialize_to_string(self) -> str:
|
||||
payload = dataclasses.asdict(self)
|
||||
return json.dumps(payload, sort_keys=True, indent=4)
|
||||
|
||||
def serialize_to_file(self, filepath: Path) -> None:
|
||||
string = self.serialize_to_string()
|
||||
filepath.write_text(string)
|
||||
|
||||
@classmethod
|
||||
def deserialize_from_string(cls, string: str) -> 'ArchiveState':
|
||||
try:
|
||||
object_as_dict = json.loads(string)
|
||||
except json.decoder.JSONDecodeError:
|
||||
raise ArchiveStateError('Error parsing JSON')
|
||||
|
||||
return cls(**object_as_dict)
|
||||
|
||||
@classmethod
|
||||
def deserialize_from_file(cls, filepath: Path):
|
||||
string = filepath.read_text()
|
||||
return cls.deserialize_from_string(string)
|
||||
|
||||
|
||||
class ArchiveWithIndicator:
|
||||
"""
|
||||
The idea of this class is to wrap around logic which takes care of keeping
|
||||
track of a name of an archive and synchronization routines between buildbot
|
||||
worker and signing server.
|
||||
|
||||
The synchronization is done based on creating a special file after the
|
||||
archive file is knowingly ready for access.
|
||||
"""
|
||||
|
||||
# Base directory where the archive is stored (basically, a basename() of
|
||||
# the absolute archive file name).
|
||||
#
|
||||
# For example, 'X:\\TEMP\\'.
|
||||
base_dir: Path
|
||||
|
||||
# Absolute file name of the archive.
|
||||
#
|
||||
# For example, 'X:\\TEMP\\FOO.ZIP'.
|
||||
archive_filepath: Path
|
||||
|
||||
# Absolute name of a file which acts as an indication of the fact that the
|
||||
# archive is ready and is available for access.
|
||||
#
|
||||
# This is how synchronization between buildbot worker and signing server is
|
||||
# done:
|
||||
# - First, the archive is created under archive_filepath name.
|
||||
# - Second, the indication file is created under ready_indicator_filepath
|
||||
# name.
|
||||
# - Third, the colleague of whoever created the indicator name watches for
|
||||
# the indication file to appear, and once it's there it access the
|
||||
# archive.
|
||||
ready_indicator_filepath: Path
|
||||
|
||||
def __init__(
|
||||
self, base_dir: Path, archive_name: str, ready_indicator_name: str):
|
||||
"""
|
||||
Construct the object from given base directory and name of the archive
|
||||
file:
|
||||
ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY')
|
||||
"""
|
||||
|
||||
self.base_dir = base_dir
|
||||
self.archive_filepath = self.base_dir / archive_name
|
||||
self.ready_indicator_filepath = self.base_dir / ready_indicator_name
|
||||
|
||||
def is_ready_unsafe(self) -> bool:
|
||||
"""
|
||||
Check whether the archive is ready for access.
|
||||
|
||||
No guarding about possible network failres is done here.
|
||||
"""
|
||||
if not self.ready_indicator_filepath.exists():
|
||||
return False
|
||||
|
||||
try:
|
||||
archive_state = ArchiveState.deserialize_from_file(
|
||||
self.ready_indicator_filepath)
|
||||
except ArchiveStateError as error:
|
||||
print(f'Error deserializing archive state: {error.message}')
|
||||
return False
|
||||
|
||||
if archive_state.has_error():
|
||||
# If the error did happen during codesign procedure there will be no
|
||||
# corresponding archive file.
|
||||
# The caller code will deal with the error check further.
|
||||
return True
|
||||
|
||||
# Sometimes on macOS indicator file appears prior to the actual archive
|
||||
# despite the order of creation and os.sync() used in tag_ready().
|
||||
# So consider archive not ready if there is an indicator without an
|
||||
# actual archive.
|
||||
if not self.archive_filepath.exists():
|
||||
print('Found indicator without actual archive, waiting for archive '
|
||||
f'({self.archive_filepath}) to appear.')
|
||||
return False
|
||||
|
||||
# Wait for until archive is fully stored.
|
||||
actual_archive_size = self.archive_filepath.stat().st_size
|
||||
if actual_archive_size != archive_state.file_size:
|
||||
print('Partial/invalid archive size (expected '
|
||||
f'{archive_state.file_size} got {actual_archive_size})')
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
"""
|
||||
Check whether the archive is ready for access.
|
||||
|
||||
Will tolerate possible network failures: if there is a network failure
|
||||
or if there is still no proper permission on a file False is returned.
|
||||
"""
|
||||
|
||||
# There are some intermitten problem happening at a random which is
|
||||
# translates to "OSError : [WinError 59] An unexpected network error occurred".
|
||||
# Some reports suggests it might be due to lack of permissions to the file,
|
||||
# which might be applicable in our case since it's possible that file is
|
||||
# initially created with non-accessible permissions and gets chmod-ed
|
||||
# after initial creation.
|
||||
try:
|
||||
return self.is_ready_unsafe()
|
||||
except OSError as e:
|
||||
print(f'Exception checking archive: {e}')
|
||||
return False
|
||||
|
||||
def tag_ready(self, error_message='') -> None:
|
||||
"""
|
||||
Tag the archive as ready by creating the corresponding indication file.
|
||||
|
||||
NOTE: It is expected that the archive was never tagged as ready before
|
||||
and that there are no subsequent tags of the same archive.
|
||||
If it is violated, an assert will fail.
|
||||
"""
|
||||
assert not self.is_ready()
|
||||
|
||||
# Try the best to make sure everything is synced to the file system,
|
||||
# to avoid any possibility of stamp appearing on a network share prior to
|
||||
# an actual file.
|
||||
if util.get_current_platform() != util.Platform.WINDOWS:
|
||||
os.sync()
|
||||
|
||||
archive_size = -1
|
||||
if self.archive_filepath.exists():
|
||||
archive_size = self.archive_filepath.stat().st_size
|
||||
|
||||
archive_info = ArchiveState(
|
||||
file_size=archive_size, error_message=error_message)
|
||||
|
||||
self.ready_indicator_filepath.write_text(
|
||||
archive_info.serialize_to_string())
|
||||
|
||||
def get_state(self) -> ArchiveState:
|
||||
"""
|
||||
Get state object for this archive
|
||||
|
||||
The state is read from the corresponding state file.
|
||||
"""
|
||||
|
||||
try:
|
||||
return ArchiveState.deserialize_from_file(self.ready_indicator_filepath)
|
||||
except ArchiveStateError as error:
|
||||
return ArchiveState(error_message=f'Error in information format: {error}')
|
||||
|
||||
def clean(self) -> None:
|
||||
"""
|
||||
Remove both archive and the ready indication file.
|
||||
"""
|
||||
util.ensure_file_does_not_exist_or_die(self.ready_indicator_filepath)
|
||||
util.ensure_file_does_not_exist_or_die(self.archive_filepath)
|
||||
|
||||
def is_fully_absent(self) -> bool:
|
||||
"""
|
||||
Check whether both archive and its ready indicator are absent.
|
||||
Is used for a sanity check during code signing process by both
|
||||
buildbot worker and signing server.
|
||||
"""
|
||||
return (not self.archive_filepath.exists() and
|
||||
not self.ready_indicator_filepath.exists())
|
@@ -1,501 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# Signing process overview.
|
||||
#
|
||||
# From buildbot worker side:
|
||||
# - Files which needs to be signed are collected from either a directory to
|
||||
# sign all signable files in there, or by filename of a single file to sign.
|
||||
# - Those files gets packed into an archive and stored in a location location
|
||||
# which is watched by the signing server.
|
||||
# - A marker READY file is created which indicates the archive is ready for
|
||||
# access.
|
||||
# - Wait for the server to provide an archive with signed files.
|
||||
# This is done by watching for the READY file which corresponds to an archive
|
||||
# coming from the signing server.
|
||||
# - Unpack the signed signed files from the archives and replace original ones.
|
||||
#
|
||||
# From code sign server:
|
||||
# - Watch special location for a READY file which indicates the there is an
|
||||
# archive with files which are to be signed.
|
||||
# - Unpack the archive to a temporary location.
|
||||
# - Run codesign tool and make sure all the files are signed.
|
||||
# - Pack the signed files and store them in a location which is watched by
|
||||
# the buildbot worker.
|
||||
# - Create a READY file which indicates that the archive with signed files is
|
||||
# ready.
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import tarfile
|
||||
import uuid
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Iterable, List
|
||||
|
||||
import codesign.util as util
|
||||
|
||||
from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
|
||||
from codesign.archive_with_indicator import ArchiveWithIndicator
|
||||
from codesign.exception import CodeSignException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger_builder = logger.getChild('builder')
|
||||
logger_server = logger.getChild('server')
|
||||
|
||||
|
||||
def pack_files(files: Iterable[AbsoluteAndRelativeFileName],
|
||||
archive_filepath: Path) -> None:
|
||||
"""
|
||||
Create tar archive from given files for the signing pipeline.
|
||||
Is used by buildbot worker to create an archive of files which are to be
|
||||
signed, and by signing server to send signed files back to the worker.
|
||||
"""
|
||||
with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle:
|
||||
for file_info in files:
|
||||
tar_file_handle.add(file_info.absolute_filepath,
|
||||
arcname=file_info.relative_filepath)
|
||||
|
||||
|
||||
def extract_files(archive_filepath: Path,
|
||||
extraction_dir: Path) -> None:
|
||||
"""
|
||||
Extract all files form the given archive into the given direcotry.
|
||||
"""
|
||||
|
||||
# TODO(sergey): Verify files in the archive have relative path.
|
||||
|
||||
with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle:
|
||||
tar_file_handle.extractall(path=extraction_dir)
|
||||
|
||||
|
||||
class BaseCodeSigner(metaclass=abc.ABCMeta):
|
||||
"""
|
||||
Base class for a platform-specific signer of binaries.
|
||||
|
||||
Contains all the logic shared across platform-specific implementations, such
|
||||
as synchronization and notification logic.
|
||||
|
||||
Platform specific bits (such as actual command for signing the binary) are
|
||||
to be implemented as a subclass.
|
||||
|
||||
Provides utilities code signing as a whole, including functionality needed
|
||||
by a signing server and a buildbot worker.
|
||||
|
||||
The signer and builder may run on separate machines, the only requirement is
|
||||
that they have access to a directory which is shared between them. For the
|
||||
security concerns this is to be done as a separate machine (or as a Shared
|
||||
Folder configuration in VirtualBox configuration). This directory might be
|
||||
mounted under different base paths, but its underlying storage is to be
|
||||
the same.
|
||||
|
||||
The code signer is short-lived on a buildbot worker side, and is living
|
||||
forever on a code signing server side.
|
||||
"""
|
||||
|
||||
# TODO(sergey): Find a neat way to have config annotated.
|
||||
# config: Config
|
||||
|
||||
# Storage directory where builder puts files which are requested to be
|
||||
# signed.
|
||||
# Consider this an input of the code signing server.
|
||||
unsigned_storage_dir: Path
|
||||
|
||||
# Storage where signed files are stored.
|
||||
# Consider this an output of the code signer server.
|
||||
signed_storage_dir: Path
|
||||
|
||||
# Platform the code is currently executing on.
|
||||
platform: util.Platform
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve()
|
||||
|
||||
# Unsigned (signing server input) configuration.
|
||||
self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned'
|
||||
|
||||
# Signed (signing server output) configuration.
|
||||
self.signed_storage_dir = absolute_shared_storage_dir / 'signed'
|
||||
|
||||
self.platform = util.get_current_platform()
|
||||
|
||||
def cleanup_environment_for_builder(self) -> None:
|
||||
# TODO(sergey): Revisit need of cleaning up the existing files.
|
||||
# In practice it wasn't so helpful, and with multiple clients
|
||||
# talking to the same server it becomes even more tricky.
|
||||
pass
|
||||
|
||||
def cleanup_environment_for_signing_server(self) -> None:
|
||||
# TODO(sergey): Revisit need of cleaning up the existing files.
|
||||
# In practice it wasn't so helpful, and with multiple clients
|
||||
# talking to the same server it becomes even more tricky.
|
||||
pass
|
||||
|
||||
def generate_request_id(self) -> str:
|
||||
"""
|
||||
Generate an unique identifier for code signing request.
|
||||
"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def archive_info_for_request_id(
|
||||
self, path: Path, request_id: str) -> ArchiveWithIndicator:
|
||||
return ArchiveWithIndicator(
|
||||
path, f'{request_id}.tar', f'{request_id}.ready')
|
||||
|
||||
def signed_archive_info_for_request_id(
|
||||
self, request_id: str) -> ArchiveWithIndicator:
|
||||
return self.archive_info_for_request_id(
|
||||
self.signed_storage_dir, request_id)
|
||||
|
||||
def unsigned_archive_info_for_request_id(
|
||||
self, request_id: str) -> ArchiveWithIndicator:
|
||||
return self.archive_info_for_request_id(
|
||||
self.unsigned_storage_dir, request_id)
|
||||
|
||||
############################################################################
|
||||
# Buildbot worker side helpers.
|
||||
|
||||
@abc.abstractmethod
|
||||
def check_file_is_to_be_signed(
|
||||
self, file: AbsoluteAndRelativeFileName) -> bool:
|
||||
"""
|
||||
Check whether file is to be signed.
|
||||
|
||||
Is used by both single file signing pipeline and recursive directory
|
||||
signing pipeline.
|
||||
|
||||
This is where code signer is to check whether file is to be signed or
|
||||
not. This check might be based on a simple extension test or on actual
|
||||
test whether file have a digital signature already or not.
|
||||
"""
|
||||
|
||||
def collect_files_to_sign(self, path: Path) \
|
||||
-> List[AbsoluteAndRelativeFileName]:
|
||||
"""
|
||||
Get all files which need to be signed from the given path.
|
||||
|
||||
NOTE: The path might either be a file or directory.
|
||||
|
||||
This function is run from the buildbot worker side.
|
||||
"""
|
||||
|
||||
# If there is a single file provided trust the buildbot worker that it
|
||||
# is eligible for signing.
|
||||
if path.is_file():
|
||||
file = AbsoluteAndRelativeFileName.from_path(path)
|
||||
if not self.check_file_is_to_be_signed(file):
|
||||
return []
|
||||
return [file]
|
||||
|
||||
all_files = AbsoluteAndRelativeFileName.recursively_from_directory(
|
||||
path)
|
||||
files_to_be_signed = [file for file in all_files
|
||||
if self.check_file_is_to_be_signed(file)]
|
||||
return files_to_be_signed
|
||||
|
||||
def wait_for_signed_archive_or_die(self, request_id) -> None:
|
||||
"""
|
||||
Wait until archive with signed files is available.
|
||||
|
||||
Will only return if the archive with signed files is available. If there
|
||||
was an error during code sign procedure the SystemExit exception is
|
||||
raised, with the message set to the error reported by the codesign
|
||||
server.
|
||||
|
||||
Will only wait for the configured time. If that time exceeds and there
|
||||
is still no responce from the signing server the application will exit
|
||||
with a non-zero exit code.
|
||||
|
||||
"""
|
||||
|
||||
signed_archive_info = self.signed_archive_info_for_request_id(
|
||||
request_id)
|
||||
unsigned_archive_info = self.unsigned_archive_info_for_request_id(
|
||||
request_id)
|
||||
|
||||
timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS
|
||||
time_start = time.monotonic()
|
||||
while not signed_archive_info.is_ready():
|
||||
time.sleep(1)
|
||||
time_slept_in_seconds = time.monotonic() - time_start
|
||||
if time_slept_in_seconds > timeout_in_seconds:
|
||||
signed_archive_info.clean()
|
||||
unsigned_archive_info.clean()
|
||||
raise SystemExit("Signing server didn't finish signing in "
|
||||
f'{timeout_in_seconds} seconds, dying :(')
|
||||
|
||||
archive_state = signed_archive_info.get_state()
|
||||
if archive_state.has_error():
|
||||
signed_archive_info.clean()
|
||||
unsigned_archive_info.clean()
|
||||
raise SystemExit(
|
||||
f'Error happenned during codesign procedure: {archive_state.error_message}')
|
||||
|
||||
def copy_signed_files_to_directory(
|
||||
self, signed_dir: Path, destination_dir: Path) -> None:
|
||||
"""
|
||||
Copy all files from signed_dir to destination_dir.
|
||||
|
||||
This function will overwrite any existing file. Permissions are copied
|
||||
from the source files, but other metadata, such as timestamps, are not.
|
||||
"""
|
||||
for signed_filepath in signed_dir.glob('**/*'):
|
||||
if not signed_filepath.is_file():
|
||||
continue
|
||||
|
||||
relative_filepath = signed_filepath.relative_to(signed_dir)
|
||||
destination_filepath = destination_dir / relative_filepath
|
||||
destination_filepath.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
shutil.copy(signed_filepath, destination_filepath)
|
||||
|
||||
def run_buildbot_path_sign_pipeline(self, path: Path) -> None:
|
||||
"""
|
||||
Run all steps needed to make given path signed.
|
||||
|
||||
Path points to an unsigned file or a directory which contains unsigned
|
||||
files.
|
||||
|
||||
If the path points to a single file then this file will be signed.
|
||||
This is used to sign a final bundle such as .msi on Windows or .dmg on
|
||||
macOS.
|
||||
|
||||
NOTE: The code signed implementation might actually reject signing the
|
||||
file, in which case the file will be left unsigned. This isn't anything
|
||||
to be considered a failure situation, just might happen when buildbot
|
||||
worker can not detect whether signing is really required in a specific
|
||||
case or not.
|
||||
|
||||
If the path points to a directory then code signer will sign all
|
||||
signable files from it (finding them recursively).
|
||||
"""
|
||||
|
||||
self.cleanup_environment_for_builder()
|
||||
|
||||
# Make sure storage directory exists.
|
||||
self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Collect all files which needs to be signed and pack them into a single
|
||||
# archive which will be sent to the signing server.
|
||||
logger_builder.info('Collecting files which are to be signed...')
|
||||
files = self.collect_files_to_sign(path)
|
||||
if not files:
|
||||
logger_builder.info('No files to be signed, ignoring.')
|
||||
return
|
||||
logger_builder.info('Found %d files to sign.', len(files))
|
||||
|
||||
request_id = self.generate_request_id()
|
||||
signed_archive_info = self.signed_archive_info_for_request_id(
|
||||
request_id)
|
||||
unsigned_archive_info = self.unsigned_archive_info_for_request_id(
|
||||
request_id)
|
||||
|
||||
pack_files(files=files,
|
||||
archive_filepath=unsigned_archive_info.archive_filepath)
|
||||
unsigned_archive_info.tag_ready()
|
||||
|
||||
# Wait for the signing server to finish signing.
|
||||
logger_builder.info('Waiting signing server to sign the files...')
|
||||
self.wait_for_signed_archive_or_die(request_id)
|
||||
|
||||
# Extract signed files from archive and move files to final location.
|
||||
with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str:
|
||||
unpacked_signed_files_dir = Path(temp_dir_str)
|
||||
|
||||
logger_builder.info('Extracting signed files from archive...')
|
||||
extract_files(
|
||||
archive_filepath=signed_archive_info.archive_filepath,
|
||||
extraction_dir=unpacked_signed_files_dir)
|
||||
|
||||
destination_dir = path
|
||||
if destination_dir.is_file():
|
||||
destination_dir = destination_dir.parent
|
||||
self.copy_signed_files_to_directory(
|
||||
unpacked_signed_files_dir, destination_dir)
|
||||
|
||||
logger_builder.info('Removing archive with signed files...')
|
||||
signed_archive_info.clean()
|
||||
|
||||
############################################################################
|
||||
# Signing server side helpers.
|
||||
|
||||
def wait_for_sign_request(self) -> str:
|
||||
"""
|
||||
Wait for the buildbot to request signing of an archive.
|
||||
|
||||
Returns an identifier of signing request.
|
||||
"""
|
||||
|
||||
# TOOD(sergey): Support graceful shutdown on Ctrl-C.
|
||||
|
||||
logger_server.info(
|
||||
f'Waiting for a request directory {self.unsigned_storage_dir} to appear.')
|
||||
while not self.unsigned_storage_dir.exists():
|
||||
time.sleep(1)
|
||||
|
||||
logger_server.info(
|
||||
'Waiting for a READY indicator of any signing request.')
|
||||
request_id = None
|
||||
while request_id is None:
|
||||
for file in self.unsigned_storage_dir.iterdir():
|
||||
if file.suffix != '.ready':
|
||||
continue
|
||||
request_id = file.stem
|
||||
logger_server.info(f'Found READY for request ID {request_id}.')
|
||||
if request_id is None:
|
||||
time.sleep(1)
|
||||
|
||||
unsigned_archive_info = self.unsigned_archive_info_for_request_id(
|
||||
request_id)
|
||||
while not unsigned_archive_info.is_ready():
|
||||
time.sleep(1)
|
||||
|
||||
return request_id
|
||||
|
||||
@abc.abstractmethod
|
||||
def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
|
||||
"""
|
||||
Sign all files in the given directory.
|
||||
|
||||
NOTE: Signing should happen in-place.
|
||||
"""
|
||||
|
||||
def run_signing_pipeline(self, request_id: str):
|
||||
"""
|
||||
Run the full signing pipeline starting from the point when buildbot
|
||||
worker have requested signing.
|
||||
"""
|
||||
|
||||
# Make sure storage directory exists.
|
||||
self.signed_storage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str:
|
||||
temp_dir = Path(temp_dir_str)
|
||||
|
||||
signed_archive_info = self.signed_archive_info_for_request_id(
|
||||
request_id)
|
||||
unsigned_archive_info = self.unsigned_archive_info_for_request_id(
|
||||
request_id)
|
||||
|
||||
logger_server.info('Extracting unsigned files from archive...')
|
||||
extract_files(
|
||||
archive_filepath=unsigned_archive_info.archive_filepath,
|
||||
extraction_dir=temp_dir)
|
||||
|
||||
logger_server.info('Collecting all files which needs signing...')
|
||||
files = AbsoluteAndRelativeFileName.recursively_from_directory(
|
||||
temp_dir)
|
||||
|
||||
logger_server.info('Signing all requested files...')
|
||||
try:
|
||||
self.sign_all_files(files)
|
||||
except CodeSignException as error:
|
||||
signed_archive_info.tag_ready(error_message=error.message)
|
||||
unsigned_archive_info.clean()
|
||||
logger_server.info('Signing is complete with errors.')
|
||||
return
|
||||
|
||||
logger_server.info('Packing signed files...')
|
||||
pack_files(files=files,
|
||||
archive_filepath=signed_archive_info.archive_filepath)
|
||||
signed_archive_info.tag_ready()
|
||||
|
||||
logger_server.info('Removing signing request...')
|
||||
unsigned_archive_info.clean()
|
||||
|
||||
logger_server.info('Signing is complete.')
|
||||
|
||||
def run_signing_server(self):
|
||||
logger_server.info('Starting new code signing server...')
|
||||
self.cleanup_environment_for_signing_server()
|
||||
logger_server.info('Code signing server is ready')
|
||||
while True:
|
||||
logger_server.info('Waiting for the signing request in %s...',
|
||||
self.unsigned_storage_dir)
|
||||
request_id = self.wait_for_sign_request()
|
||||
|
||||
logger_server.info(
|
||||
f'Beging signign procedure for request ID {request_id}.')
|
||||
self.run_signing_pipeline(request_id)
|
||||
|
||||
############################################################################
|
||||
# Command executing.
|
||||
#
|
||||
# Abstracted to a degree that allows to run commands from a foreign
|
||||
# platform.
|
||||
# The goal with this is to allow performing dry-run tests of code signer
|
||||
# server from other platforms (for example, to test that macOS code signer
|
||||
# does what it is supposed to after doing a refactor on Linux).
|
||||
|
||||
# TODO(sergey): What is the type annotation for the command?
|
||||
def run_command_or_mock(self, command, platform: util.Platform) -> None:
|
||||
"""
|
||||
Run given command if current platform matches given one
|
||||
|
||||
If the platform is different then it will only be printed allowing
|
||||
to verify logic of the code signing process.
|
||||
"""
|
||||
|
||||
if platform != self.platform:
|
||||
logger_server.info(
|
||||
f'Will run command for {platform}: {command}')
|
||||
return
|
||||
|
||||
logger_server.info(f'Running command: {command}')
|
||||
subprocess.run(command)
|
||||
|
||||
# TODO(sergey): What is the type annotation for the command?
|
||||
def check_output_or_mock(self, command,
|
||||
platform: util.Platform,
|
||||
allow_nonzero_exit_code=False) -> str:
|
||||
"""
|
||||
Run given command if current platform matches given one
|
||||
|
||||
If the platform is different then it will only be printed allowing
|
||||
to verify logic of the code signing process.
|
||||
|
||||
If allow_nonzero_exit_code is truth then the output will be returned
|
||||
even if application quit with non-zero exit code.
|
||||
Otherwise an subprocess.CalledProcessError exception will be raised
|
||||
in such case.
|
||||
"""
|
||||
|
||||
if platform != self.platform:
|
||||
logger_server.info(
|
||||
f'Will run command for {platform}: {command}')
|
||||
return
|
||||
|
||||
if allow_nonzero_exit_code:
|
||||
process = subprocess.Popen(command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
output = process.communicate()[0]
|
||||
return output.decode()
|
||||
|
||||
logger_server.info(f'Running command: {command}')
|
||||
return subprocess.check_output(
|
||||
command, stderr=subprocess.STDOUT).decode()
|
@@ -1,62 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# Configuration of a code signer which is specific to the code running from
|
||||
# buildbot's worker.
|
||||
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import codesign.util as util
|
||||
|
||||
from codesign.config_common import *
|
||||
|
||||
platform = util.get_current_platform()
|
||||
if platform == util.Platform.LINUX:
|
||||
SHARED_STORAGE_DIR = Path('/data/codesign')
|
||||
elif platform == util.Platform.WINDOWS:
|
||||
SHARED_STORAGE_DIR = Path('Z:\\codesign')
|
||||
elif platform == util.Platform.MACOS:
|
||||
SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign')
|
||||
|
||||
# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
'formatters': {
|
||||
'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
|
||||
},
|
||||
'handlers': {
|
||||
'console': {
|
||||
'class': 'logging.StreamHandler',
|
||||
'formatter': 'default',
|
||||
'stream': 'ext://sys.stderr',
|
||||
}
|
||||
},
|
||||
'loggers': {
|
||||
'codesign': {'level': 'INFO'},
|
||||
},
|
||||
'root': {
|
||||
'level': 'WARNING',
|
||||
'handlers': [
|
||||
'console',
|
||||
],
|
||||
}
|
||||
}
|
@@ -1,36 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
# Timeout in seconds for the signing process.
|
||||
#
|
||||
# This is how long buildbot packing step will wait signing server to
|
||||
# perform signing.
|
||||
#
|
||||
# NOTE: Notarization could take a long time, hence the rather high value
|
||||
# here. Might consider using different timeout for different platforms.
|
||||
TIMEOUT_IN_SECONDS = 45 * 60 * 60
|
||||
|
||||
# Directory which is shared across buildbot worker and signing server.
|
||||
#
|
||||
# This is where worker puts files requested for signing as well as where
|
||||
# server puts signed files.
|
||||
SHARED_STORAGE_DIR: Path
|
@@ -1,101 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# Configuration of a code signer which is specific to the code signing server.
|
||||
#
|
||||
# NOTE: DO NOT put any sensitive information here, put it in an actual
|
||||
# configuration on the signing machine.
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from codesign.config_common import *
|
||||
|
||||
CODESIGN_DIRECTORY = Path(__file__).absolute().parent
|
||||
BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent
|
||||
|
||||
################################################################################
|
||||
# Common configuration.
|
||||
|
||||
# Directory where folders for codesign requests and signed result are stored.
|
||||
# For example, /data/codesign
|
||||
SHARED_STORAGE_DIR: Path
|
||||
|
||||
################################################################################
|
||||
# macOS-specific configuration.
|
||||
|
||||
MACOS_ENTITLEMENTS_FILE = \
|
||||
BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist'
|
||||
|
||||
# Identity of the Developer ID Application certificate which is to be used for
|
||||
# codesign tool.
|
||||
# Use `security find-identity -v -p codesigning` to find the identity.
|
||||
#
|
||||
# NOTE: This identity is just an example from release/darwin/README.txt.
|
||||
MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97'
|
||||
|
||||
# User name (Apple ID) which will be used to request notarization.
|
||||
MACOS_XCRUN_USERNAME = 'me@example.com'
|
||||
|
||||
# One-time application password which will be used to request notarization.
|
||||
MACOS_XCRUN_PASSWORD = '@keychain:altool-password'
|
||||
|
||||
# Timeout in seconds within which the notarial office is supposed to reply.
|
||||
MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60
|
||||
|
||||
################################################################################
|
||||
# Windows-specific configuration.
|
||||
|
||||
# URL to the timestamping authority.
|
||||
WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com'
|
||||
|
||||
# Full path to the certificate used for signing.
|
||||
#
|
||||
# The path and expected file format might vary depending on a platform.
|
||||
#
|
||||
# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look
|
||||
# like Path('C:\\Secret\\Blender.pfx').
|
||||
WIN_CERTIFICATE_FILEPATH: Path
|
||||
|
||||
################################################################################
|
||||
# Logging configuration, common for all platforms.
|
||||
|
||||
# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
'formatters': {
|
||||
'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'}
|
||||
},
|
||||
'handlers': {
|
||||
'console': {
|
||||
'class': 'logging.StreamHandler',
|
||||
'formatter': 'default',
|
||||
'stream': 'ext://sys.stderr',
|
||||
}
|
||||
},
|
||||
'loggers': {
|
||||
'codesign': {'level': 'INFO'},
|
||||
},
|
||||
'root': {
|
||||
'level': 'WARNING',
|
||||
'handlers': [
|
||||
'console',
|
||||
],
|
||||
}
|
||||
}
|
@@ -1,26 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
class CodeSignException(Exception):
|
||||
message: str
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(self.message)
|
@@ -1,72 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# NOTE: This is a no-op signer (since there isn't really a procedure to sign
|
||||
# Linux binaries yet). Used to debug and verify the code signing routines on
|
||||
# a Linux environment.
|
||||
|
||||
import logging
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
|
||||
from codesign.base_code_signer import BaseCodeSigner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger_server = logger.getChild('server')
|
||||
|
||||
|
||||
class LinuxCodeSigner(BaseCodeSigner):
|
||||
def is_active(self) -> bool:
|
||||
"""
|
||||
Check whether this signer is active.
|
||||
|
||||
if it is inactive, no files will be signed.
|
||||
|
||||
Is used to be able to debug code signing pipeline on Linux, where there
|
||||
is no code signing happening in the actual buildbot and release
|
||||
environment.
|
||||
"""
|
||||
return False
|
||||
|
||||
def check_file_is_to_be_signed(
|
||||
self, file: AbsoluteAndRelativeFileName) -> bool:
|
||||
if file.relative_filepath == Path('blender'):
|
||||
return True
|
||||
if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and
|
||||
file.relative_filepath.name.startwith('python')):
|
||||
return True
|
||||
if file.relative_filepath.suffix == '.so':
|
||||
return True
|
||||
return False
|
||||
|
||||
def collect_files_to_sign(self, path: Path) \
|
||||
-> List[AbsoluteAndRelativeFileName]:
|
||||
if not self.is_active():
|
||||
return []
|
||||
|
||||
return super().collect_files_to_sign(path)
|
||||
|
||||
def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
|
||||
num_files = len(files)
|
||||
for file_index, file in enumerate(files):
|
||||
logger.info('Server: Signed file [%d/%d] %s',
|
||||
file_index + 1, num_files, file.relative_filepath)
|
@@ -1,456 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import logging
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import codesign.util as util
|
||||
|
||||
from buildbot_utils import Builder
|
||||
|
||||
from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
|
||||
from codesign.base_code_signer import BaseCodeSigner
|
||||
from codesign.exception import CodeSignException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger_server = logger.getChild('server')
|
||||
|
||||
# NOTE: Check is done as filename.endswith(), so keep the dot
|
||||
EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'}
|
||||
|
||||
# Prefixes of a file (not directory) name which are to be signed.
|
||||
# Used to sign extra executable files in Contents/Resources.
|
||||
NAME_PREFIXES_TO_BE_SIGNED = {'python'}
|
||||
|
||||
|
||||
class NotarizationException(CodeSignException):
|
||||
pass
|
||||
|
||||
|
||||
def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool:
|
||||
"""
|
||||
Check whether file is coming from an .app bundle
|
||||
"""
|
||||
parts = file.relative_filepath.parts
|
||||
if not parts:
|
||||
return False
|
||||
if not parts[0].endswith('.app'):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_bundle_from_file(
|
||||
file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName:
|
||||
"""
|
||||
Get AbsoluteAndRelativeFileName descriptor of bundle
|
||||
"""
|
||||
assert(is_file_from_bundle(file))
|
||||
|
||||
parts = file.relative_filepath.parts
|
||||
bundle_name = parts[0]
|
||||
|
||||
base_dir = file.base_dir
|
||||
bundle_filepath = file.base_dir / bundle_name
|
||||
return AbsoluteAndRelativeFileName(base_dir, bundle_filepath)
|
||||
|
||||
|
||||
def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool:
|
||||
"""
|
||||
Check whether given file is an executable within an app bundle
|
||||
"""
|
||||
if not is_file_from_bundle(file):
|
||||
return False
|
||||
|
||||
parts = file.relative_filepath.parts
|
||||
num_parts = len(parts)
|
||||
if num_parts < 3:
|
||||
return False
|
||||
|
||||
if parts[1:3] != ('Contents', 'MacOS'):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def xcrun_field_value_from_output(field: str, output: str) -> str:
|
||||
"""
|
||||
Get value of a given field from xcrun output.
|
||||
|
||||
If field is not found empty string is returned.
|
||||
"""
|
||||
|
||||
field_prefix = field + ': '
|
||||
for line in output.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith(field_prefix):
|
||||
return line[len(field_prefix):]
|
||||
return ''
|
||||
|
||||
|
||||
class MacOSCodeSigner(BaseCodeSigner):
|
||||
def check_file_is_to_be_signed(
|
||||
self, file: AbsoluteAndRelativeFileName) -> bool:
|
||||
if file.relative_filepath.name.startswith('.'):
|
||||
return False
|
||||
|
||||
if is_bundle_executable_file(file):
|
||||
return True
|
||||
|
||||
base_name = file.relative_filepath.name
|
||||
if any(base_name.startswith(prefix)
|
||||
for prefix in NAME_PREFIXES_TO_BE_SIGNED):
|
||||
return True
|
||||
|
||||
mode = file.absolute_filepath.lstat().st_mode
|
||||
if mode & stat.S_IXUSR != 0:
|
||||
file_output = subprocess.check_output(
|
||||
("file", file.absolute_filepath)).decode()
|
||||
if "64-bit executable" in file_output:
|
||||
return True
|
||||
|
||||
return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED
|
||||
|
||||
def collect_files_to_sign(self, path: Path) \
|
||||
-> List[AbsoluteAndRelativeFileName]:
|
||||
# Include all files when signing app or dmg bundle: all the files are
|
||||
# needed to do valid signature of bundle.
|
||||
if path.name.endswith('.app'):
|
||||
return AbsoluteAndRelativeFileName.recursively_from_directory(path)
|
||||
if path.is_dir():
|
||||
files = []
|
||||
for child in path.iterdir():
|
||||
if child.name.endswith('.app'):
|
||||
current_files = AbsoluteAndRelativeFileName.recursively_from_directory(
|
||||
child)
|
||||
else:
|
||||
current_files = super().collect_files_to_sign(child)
|
||||
for current_file in current_files:
|
||||
files.append(AbsoluteAndRelativeFileName(
|
||||
path, current_file.absolute_filepath))
|
||||
return files
|
||||
return super().collect_files_to_sign(path)
|
||||
|
||||
############################################################################
|
||||
# Codesign.
|
||||
|
||||
def codesign_remove_signature(
|
||||
self, file: AbsoluteAndRelativeFileName) -> None:
|
||||
"""
|
||||
Make sure given file does not have codesign signature
|
||||
|
||||
This is needed because codesigning is not possible for file which has
|
||||
signature already.
|
||||
"""
|
||||
|
||||
logger_server.info(
|
||||
'Removing codesign signature from %s...', file.relative_filepath)
|
||||
|
||||
command = ['codesign', '--remove-signature', file.absolute_filepath]
|
||||
self.run_command_or_mock(command, util.Platform.MACOS)
|
||||
|
||||
def codesign_file(
|
||||
self, file: AbsoluteAndRelativeFileName) -> None:
|
||||
"""
|
||||
Sign given file
|
||||
|
||||
NOTE: File must not have any signatures.
|
||||
"""
|
||||
|
||||
logger_server.info(
|
||||
'Codesigning %s...', file.relative_filepath)
|
||||
|
||||
entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE
|
||||
command = ['codesign',
|
||||
'--timestamp',
|
||||
'--options', 'runtime',
|
||||
f'--entitlements={entitlements_file}',
|
||||
'--sign', self.config.MACOS_CODESIGN_IDENTITY,
|
||||
file.absolute_filepath]
|
||||
self.run_command_or_mock(command, util.Platform.MACOS)
|
||||
|
||||
def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
|
||||
"""
|
||||
Run codesign tool on all eligible files in the given list.
|
||||
|
||||
Will ignore all files which are not to be signed. For the rest will
|
||||
remove possible existing signature and add a new signature.
|
||||
"""
|
||||
|
||||
num_files = len(files)
|
||||
have_ignored_files = False
|
||||
signed_files = []
|
||||
for file_index, file in enumerate(files):
|
||||
# Ignore file if it is not to be signed.
|
||||
# Allows to manually construct ZIP of a bundle and get it signed.
|
||||
if not self.check_file_is_to_be_signed(file):
|
||||
logger_server.info(
|
||||
'Ignoring file [%d/%d] %s',
|
||||
file_index + 1, num_files, file.relative_filepath)
|
||||
have_ignored_files = True
|
||||
continue
|
||||
|
||||
logger_server.info(
|
||||
'Running codesigning routines for file [%d/%d] %s...',
|
||||
file_index + 1, num_files, file.relative_filepath)
|
||||
|
||||
self.codesign_remove_signature(file)
|
||||
self.codesign_file(file)
|
||||
|
||||
signed_files.append(file)
|
||||
|
||||
if have_ignored_files:
|
||||
logger_server.info('Signed %d files:', len(signed_files))
|
||||
num_signed_files = len(signed_files)
|
||||
for file_index, signed_file in enumerate(signed_files):
|
||||
logger_server.info(
|
||||
'- [%d/%d] %s',
|
||||
file_index + 1, num_signed_files,
|
||||
signed_file.relative_filepath)
|
||||
|
||||
def codesign_bundles(
|
||||
self, files: List[AbsoluteAndRelativeFileName]) -> None:
|
||||
"""
|
||||
Codesign all .app bundles in the given list of files.
|
||||
|
||||
Bundle is deducted from paths of the files, and every bundle is only
|
||||
signed once.
|
||||
"""
|
||||
|
||||
signed_bundles = set()
|
||||
extra_files = []
|
||||
|
||||
for file in files:
|
||||
if not is_file_from_bundle(file):
|
||||
continue
|
||||
bundle = get_bundle_from_file(file)
|
||||
bundle_name = bundle.relative_filepath
|
||||
if bundle_name in signed_bundles:
|
||||
continue
|
||||
|
||||
logger_server.info('Running codesign routines on bundle %s',
|
||||
bundle_name)
|
||||
|
||||
# It is not possible to remove signature from DMG.
|
||||
if bundle.relative_filepath.name.endswith('.app'):
|
||||
self.codesign_remove_signature(bundle)
|
||||
self.codesign_file(bundle)
|
||||
|
||||
signed_bundles.add(bundle_name)
|
||||
|
||||
# Codesign on a bundle adds an extra folder with information.
|
||||
# It needs to be compied to the source.
|
||||
code_signature_directory = \
|
||||
bundle.absolute_filepath / 'Contents' / '_CodeSignature'
|
||||
code_signature_files = \
|
||||
AbsoluteAndRelativeFileName.recursively_from_directory(
|
||||
code_signature_directory)
|
||||
for code_signature_file in code_signature_files:
|
||||
bundle_relative_file = AbsoluteAndRelativeFileName(
|
||||
bundle.base_dir,
|
||||
code_signature_directory /
|
||||
code_signature_file.relative_filepath)
|
||||
extra_files.append(bundle_relative_file)
|
||||
|
||||
files.extend(extra_files)
|
||||
|
||||
############################################################################
|
||||
# Notarization.
|
||||
|
||||
def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str:
|
||||
"""
|
||||
Get bundle ID which will be used to notarize DMG
|
||||
"""
|
||||
name = file.relative_filepath.name
|
||||
app_name = name.split('-', 2)[0].lower()
|
||||
|
||||
app_name_words = app_name.split()
|
||||
if len(app_name_words) > 1:
|
||||
app_name_id = ''.join(word.capitalize() for word in app_name_words)
|
||||
else:
|
||||
app_name_id = app_name_words[0]
|
||||
|
||||
# TODO(sergey): Consider using "alpha" for buildbot builds.
|
||||
return f'org.blenderfoundation.{app_name_id}.release'
|
||||
|
||||
def notarize_request(self, file) -> str:
|
||||
"""
|
||||
Request notarization of the given file.
|
||||
|
||||
Returns UUID of the notarization request. If error occurred None is
|
||||
returned instead of UUID.
|
||||
"""
|
||||
|
||||
bundle_id = self.notarize_get_bundle_id(file)
|
||||
logger_server.info('Bundle ID: %s', bundle_id)
|
||||
|
||||
logger_server.info('Submitting file to the notarial office.')
|
||||
command = [
|
||||
'xcrun', 'altool', '--notarize-app', '--verbose',
|
||||
'-f', file.absolute_filepath,
|
||||
'--primary-bundle-id', bundle_id,
|
||||
'--username', self.config.MACOS_XCRUN_USERNAME,
|
||||
'--password', self.config.MACOS_XCRUN_PASSWORD]
|
||||
|
||||
output = self.check_output_or_mock(
|
||||
command, util.Platform.MACOS, allow_nonzero_exit_code=True)
|
||||
|
||||
for line in output.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith('RequestUUID = '):
|
||||
request_uuid = line[14:]
|
||||
return request_uuid
|
||||
|
||||
# Check whether the package has been already submitted.
|
||||
if 'The software asset has already been uploaded.' in line:
|
||||
request_uuid = re.sub(
|
||||
'.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line)
|
||||
logger_server.warning(
|
||||
f'The package has been already submitted under UUID {request_uuid}')
|
||||
return request_uuid
|
||||
|
||||
logger_server.error(output)
|
||||
logger_server.error('xcrun command did not report RequestUUID')
|
||||
return None
|
||||
|
||||
def notarize_review_status(self, xcrun_output: str) -> bool:
|
||||
"""
|
||||
Review status returned by xcrun's notarization info
|
||||
|
||||
Returns truth if the notarization process has finished.
|
||||
If there are errors during notarization, a NotarizationException()
|
||||
exception is thrown with status message from the notarial office.
|
||||
"""
|
||||
|
||||
# Parse status and message
|
||||
status = xcrun_field_value_from_output('Status', xcrun_output)
|
||||
status_message = xcrun_field_value_from_output(
|
||||
'Status Message', xcrun_output)
|
||||
|
||||
if status == 'success':
|
||||
logger_server.info(
|
||||
'Package successfully notarized: %s', status_message)
|
||||
return True
|
||||
|
||||
if status == 'invalid':
|
||||
logger_server.error(xcrun_output)
|
||||
logger_server.error(
|
||||
'Package notarization has failed: %s', status_message)
|
||||
raise NotarizationException(status_message)
|
||||
|
||||
if status == 'in progress':
|
||||
return False
|
||||
|
||||
logger_server.info(
|
||||
'Unknown notarization status %s (%s)', status, status_message)
|
||||
|
||||
return False
|
||||
|
||||
def notarize_wait_result(self, request_uuid: str) -> None:
|
||||
"""
|
||||
Wait for until notarial office have a reply
|
||||
"""
|
||||
|
||||
logger_server.info(
|
||||
'Waiting for a result from the notarization office.')
|
||||
|
||||
command = ['xcrun', 'altool',
|
||||
'--notarization-info', request_uuid,
|
||||
'--username', self.config.MACOS_XCRUN_USERNAME,
|
||||
'--password', self.config.MACOS_XCRUN_PASSWORD]
|
||||
|
||||
time_start = time.monotonic()
|
||||
timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS
|
||||
|
||||
while True:
|
||||
xcrun_output = self.check_output_or_mock(
|
||||
command, util.Platform.MACOS, allow_nonzero_exit_code=True)
|
||||
|
||||
if self.notarize_review_status(xcrun_output):
|
||||
break
|
||||
|
||||
logger_server.info('Keep waiting for notarization office.')
|
||||
time.sleep(30)
|
||||
|
||||
time_slept_in_seconds = time.monotonic() - time_start
|
||||
if time_slept_in_seconds > timeout_in_seconds:
|
||||
logger_server.error(
|
||||
"Notarial office didn't reply in %f seconds.",
|
||||
timeout_in_seconds)
|
||||
|
||||
def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool:
|
||||
"""
|
||||
Staple notarial label on the file
|
||||
"""
|
||||
|
||||
logger_server.info('Stapling notarial stamp.')
|
||||
|
||||
command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath]
|
||||
self.check_output_or_mock(command, util.Platform.MACOS)
|
||||
|
||||
def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool:
|
||||
"""
|
||||
Run entire pipeline to get DMG notarized.
|
||||
"""
|
||||
logger_server.info('Begin notarization routines on %s',
|
||||
file.relative_filepath)
|
||||
|
||||
# Submit file for notarization.
|
||||
request_uuid = self.notarize_request(file)
|
||||
if not request_uuid:
|
||||
return False
|
||||
logger_server.info('Received Request UUID: %s', request_uuid)
|
||||
|
||||
# Wait for the status from the notarization office.
|
||||
if not self.notarize_wait_result(request_uuid):
|
||||
return False
|
||||
|
||||
# Staple.
|
||||
self.notarize_staple(file)
|
||||
|
||||
def notarize_all_dmg(
|
||||
self, files: List[AbsoluteAndRelativeFileName]) -> bool:
|
||||
"""
|
||||
Notarize all DMG images from the input.
|
||||
|
||||
Images are supposed to be codesigned already.
|
||||
"""
|
||||
for file in files:
|
||||
if not file.relative_filepath.name.endswith('.dmg'):
|
||||
continue
|
||||
if not self.check_file_is_to_be_signed(file):
|
||||
continue
|
||||
|
||||
self.notarize_dmg(file)
|
||||
|
||||
############################################################################
|
||||
# Entry point.
|
||||
|
||||
def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
|
||||
# TODO(sergey): Handle errors somehow.
|
||||
|
||||
self.codesign_all_files(files)
|
||||
self.codesign_bundles(files)
|
||||
self.notarize_all_dmg(files)
|
@@ -1,52 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
|
||||
import logging.config
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import codesign.config_builder
|
||||
import codesign.util as util
|
||||
from codesign.base_code_signer import BaseCodeSigner
|
||||
|
||||
|
||||
class SimpleCodeSigner:
|
||||
code_signer: Optional[BaseCodeSigner]
|
||||
|
||||
def __init__(self):
|
||||
platform = util.get_current_platform()
|
||||
if platform == util.Platform.LINUX:
|
||||
from codesign.linux_code_signer import LinuxCodeSigner
|
||||
self.code_signer = LinuxCodeSigner(codesign.config_builder)
|
||||
elif platform == util.Platform.MACOS:
|
||||
from codesign.macos_code_signer import MacOSCodeSigner
|
||||
self.code_signer = MacOSCodeSigner(codesign.config_builder)
|
||||
elif platform == util.Platform.WINDOWS:
|
||||
from codesign.windows_code_signer import WindowsCodeSigner
|
||||
self.code_signer = WindowsCodeSigner(codesign.config_builder)
|
||||
else:
|
||||
self.code_signer = None
|
||||
|
||||
def sign_file_or_directory(self, path: Path) -> None:
|
||||
logging.config.dictConfig(codesign.config_builder.LOGGING)
|
||||
self.code_signer.run_buildbot_path_sign_pipeline(path)
|
@@ -1,54 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import sys
|
||||
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Platform(Enum):
|
||||
LINUX = 1
|
||||
MACOS = 2
|
||||
WINDOWS = 3
|
||||
|
||||
|
||||
def get_current_platform() -> Platform:
|
||||
if sys.platform == 'linux':
|
||||
return Platform.LINUX
|
||||
elif sys.platform == 'darwin':
|
||||
return Platform.MACOS
|
||||
elif sys.platform == 'win32':
|
||||
return Platform.WINDOWS
|
||||
raise Exception(f'Unknown platform {sys.platform}')
|
||||
|
||||
|
||||
def ensure_file_does_not_exist_or_die(filepath: Path) -> None:
|
||||
"""
|
||||
If the file exists, unlink it.
|
||||
If the file path exists and is not a file an assert will trigger.
|
||||
If the file path does not exists nothing happens.
|
||||
"""
|
||||
if not filepath.exists():
|
||||
return
|
||||
if not filepath.is_file():
|
||||
# TODO(sergey): Provide information about what the filepath actually is.
|
||||
raise SystemExit(f'{filepath} is expected to be a file, but is not')
|
||||
filepath.unlink()
|
@@ -1,117 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import logging
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import codesign.util as util
|
||||
|
||||
from buildbot_utils import Builder
|
||||
|
||||
from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
|
||||
from codesign.base_code_signer import BaseCodeSigner
|
||||
from codesign.exception import CodeSignException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger_server = logger.getChild('server')
|
||||
|
||||
# NOTE: Check is done as filename.endswith(), so keep the dot
|
||||
EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'}
|
||||
|
||||
BLACKLIST_FILE_PREFIXES = (
|
||||
'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime')
|
||||
|
||||
|
||||
class SigntoolException(CodeSignException):
|
||||
pass
|
||||
|
||||
class WindowsCodeSigner(BaseCodeSigner):
|
||||
def check_file_is_to_be_signed(
|
||||
self, file: AbsoluteAndRelativeFileName) -> bool:
|
||||
base_name = file.relative_filepath.name
|
||||
if any(base_name.startswith(prefix)
|
||||
for prefix in BLACKLIST_FILE_PREFIXES):
|
||||
return False
|
||||
|
||||
return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED
|
||||
|
||||
|
||||
def get_sign_command_prefix(self) -> List[str]:
|
||||
return [
|
||||
'signtool', 'sign', '/v',
|
||||
'/f', self.config.WIN_CERTIFICATE_FILEPATH,
|
||||
'/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL]
|
||||
|
||||
|
||||
def run_codesign_tool(self, filepath: Path) -> None:
|
||||
command = self.get_sign_command_prefix() + [filepath]
|
||||
|
||||
try:
|
||||
codesign_output = self.check_output_or_mock(command, util.Platform.WINDOWS)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise SigntoolException(f'Error running signtool {e}')
|
||||
|
||||
logger_server.info(f'signtool output:\n{codesign_output}')
|
||||
|
||||
got_number_of_success = False
|
||||
|
||||
for line in codesign_output.split('\n'):
|
||||
line_clean = line.strip()
|
||||
line_clean_lower = line_clean.lower()
|
||||
|
||||
if line_clean_lower.startswith('number of warnings') or \
|
||||
line_clean_lower.startswith('number of errors'):
|
||||
number = int(line_clean_lower.split(':')[1])
|
||||
if number != 0:
|
||||
raise SigntoolException('Non-clean success of signtool')
|
||||
|
||||
if line_clean_lower.startswith('number of files successfully signed'):
|
||||
got_number_of_success = True
|
||||
number = int(line_clean_lower.split(':')[1])
|
||||
if number != 1:
|
||||
raise SigntoolException('Signtool did not consider codesign a success')
|
||||
|
||||
if not got_number_of_success:
|
||||
raise SigntoolException('Signtool did not report number of files signed')
|
||||
|
||||
|
||||
def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None:
|
||||
# NOTE: Sign files one by one to avoid possible command line length
|
||||
# overflow (which could happen if we ever decide to sign every binary
|
||||
# in the install folder, for example).
|
||||
#
|
||||
# TODO(sergey): Consider doing batched signing of handful of files in
|
||||
# one go (but only if this actually known to be much faster).
|
||||
num_files = len(files)
|
||||
for file_index, file in enumerate(files):
|
||||
# Ignore file if it is not to be signed.
|
||||
# Allows to manually construct ZIP of package and get it signed.
|
||||
if not self.check_file_is_to_be_signed(file):
|
||||
logger_server.info(
|
||||
'Ignoring file [%d/%d] %s',
|
||||
file_index + 1, num_files, file.relative_filepath)
|
||||
continue
|
||||
|
||||
logger_server.info(
|
||||
'Running signtool command for file [%d/%d] %s...',
|
||||
file_index + 1, num_files, file.relative_filepath)
|
||||
self.run_codesign_tool(file.absolute_filepath)
|
@@ -1,37 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# NOTE: This is a no-op signer (since there isn't really a procedure to sign
|
||||
# Linux binaries yet). Used to debug and verify the code signing routines on
|
||||
# a Linux environment.
|
||||
|
||||
import logging.config
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from codesign.linux_code_signer import LinuxCodeSigner
|
||||
import codesign.config_server
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.config.dictConfig(codesign.config_server.LOGGING)
|
||||
code_signer = LinuxCodeSigner(codesign.config_server)
|
||||
code_signer.run_signing_server()
|
@@ -1,41 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import logging.config
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from codesign.macos_code_signer import MacOSCodeSigner
|
||||
import codesign.config_server
|
||||
|
||||
if __name__ == "__main__":
|
||||
entitlements_file = codesign.config_server.MACOS_ENTITLEMENTS_FILE
|
||||
if not entitlements_file.exists():
|
||||
raise SystemExit(
|
||||
'Entitlements file {entitlements_file} does not exist.')
|
||||
if not entitlements_file.is_file():
|
||||
raise SystemExit(
|
||||
'Entitlements file {entitlements_file} is not a file.')
|
||||
|
||||
logging.config.dictConfig(codesign.config_server.LOGGING)
|
||||
code_signer = MacOSCodeSigner(codesign.config_server)
|
||||
code_signer.run_signing_server()
|
@@ -1,11 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem This is an entry point of the codesign server for Windows.
|
||||
rem It makes sure that signtool.exe is within the current PATH and can be
|
||||
rem used by the Python script.
|
||||
|
||||
SETLOCAL
|
||||
|
||||
set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH%
|
||||
|
||||
codesign_server_windows.py
|
@@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# Implementation of codesign server for Windows.
|
||||
#
|
||||
# NOTE: If signtool.exe is not in the PATH use codesign_server_windows.bat
|
||||
|
||||
import logging.config
|
||||
import shutil
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import codesign.util as util
|
||||
|
||||
from codesign.windows_code_signer import WindowsCodeSigner
|
||||
import codesign.config_server
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.config.dictConfig(codesign.config_server.LOGGING)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger_server = logger.getChild('server')
|
||||
|
||||
# TODO(sergey): Consider moving such sanity checks into
|
||||
# CodeSigner.check_environment_or_die().
|
||||
if not shutil.which('signtool.exe'):
|
||||
if util.get_current_platform() == util.Platform.WINDOWS:
|
||||
raise SystemExit("signtool.exe is not found in %PATH%")
|
||||
logger_server.info(
|
||||
'signtool.exe not found, '
|
||||
'but will not be used on this foreign platform')
|
||||
|
||||
code_signer = WindowsCodeSigner(codesign.config_server)
|
||||
code_signer.run_signing_server()
|
@@ -1,551 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory, NamedTemporaryFile
|
||||
from typing import List
|
||||
|
||||
BUILDBOT_DIRECTORY = Path(__file__).absolute().parent
|
||||
CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py'
|
||||
BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent
|
||||
DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin'
|
||||
|
||||
|
||||
# Extra size which is added on top of actual files size when estimating size
|
||||
# of destination DNG.
|
||||
EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024
|
||||
|
||||
################################################################################
|
||||
# Common utilities
|
||||
|
||||
|
||||
def get_directory_size(root_directory: Path) -> int:
|
||||
"""
|
||||
Get size of directory on disk
|
||||
"""
|
||||
|
||||
total_size = 0
|
||||
for file in root_directory.glob('**/*'):
|
||||
total_size += file.lstat().st_size
|
||||
return total_size
|
||||
|
||||
|
||||
################################################################################
|
||||
# DMG bundling specific logic
|
||||
|
||||
def create_argument_parser():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'source_dir',
|
||||
type=Path,
|
||||
help='Source directory which points to either existing .app bundle'
|
||||
'or to a directory with .app bundles.')
|
||||
parser.add_argument(
|
||||
'--background-image',
|
||||
type=Path,
|
||||
help="Optional background picture which will be set on the DMG."
|
||||
"If not provided default Blender's one is used.")
|
||||
parser.add_argument(
|
||||
'--volume-name',
|
||||
type=str,
|
||||
help='Optional name of a volume which will be used for DMG.')
|
||||
parser.add_argument(
|
||||
'--dmg',
|
||||
type=Path,
|
||||
help='Optional argument which points to a final DMG file name.')
|
||||
parser.add_argument(
|
||||
'--applescript',
|
||||
type=Path,
|
||||
help="Optional path to applescript to set up folder looks of DMG."
|
||||
"If not provided default Blender's one is used.")
|
||||
parser.add_argument(
|
||||
'--codesign',
|
||||
action="store_true",
|
||||
help="Code sign and notarize DMG contents.")
|
||||
return parser
|
||||
|
||||
|
||||
def collect_app_bundles(source_dir: Path) -> List[Path]:
|
||||
"""
|
||||
Collect all app bundles which are to be put into DMG
|
||||
|
||||
If the source directory points to FOO.app it will be the only app bundle
|
||||
packed.
|
||||
|
||||
Otherwise all .app bundles from given directory are placed to a single
|
||||
DMG.
|
||||
"""
|
||||
|
||||
if source_dir.name.endswith('.app'):
|
||||
return [source_dir]
|
||||
|
||||
app_bundles = []
|
||||
for filename in source_dir.glob('*'):
|
||||
if not filename.is_dir():
|
||||
continue
|
||||
if not filename.name.endswith('.app'):
|
||||
continue
|
||||
|
||||
app_bundles.append(filename)
|
||||
|
||||
return app_bundles
|
||||
|
||||
|
||||
def collect_and_log_app_bundles(source_dir: Path) -> List[Path]:
|
||||
app_bundles = collect_app_bundles(source_dir)
|
||||
|
||||
if not app_bundles:
|
||||
print('No app bundles found for packing')
|
||||
return
|
||||
|
||||
print(f'Found {len(app_bundles)} to pack:')
|
||||
for app_bundle in app_bundles:
|
||||
print(f'- {app_bundle}')
|
||||
|
||||
return app_bundles
|
||||
|
||||
|
||||
def estimate_dmg_size(app_bundles: List[Path]) -> int:
|
||||
"""
|
||||
Estimate size of DMG to hold requested app bundles
|
||||
|
||||
The size is based on actual size of all files in all bundles plus some
|
||||
space to compensate for different size-on-disk plus some space to hold
|
||||
codesign signatures.
|
||||
|
||||
Is better to be on a high side since the empty space is compressed, but
|
||||
lack of space might cause silent failures later on.
|
||||
"""
|
||||
|
||||
app_bundles_size = 0
|
||||
for app_bundle in app_bundles:
|
||||
app_bundles_size += get_directory_size(app_bundle)
|
||||
|
||||
return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES
|
||||
|
||||
|
||||
def copy_app_bundles_to_directory(app_bundles: List[Path],
|
||||
directory: Path) -> None:
|
||||
"""
|
||||
Copy all bundles to a given directory
|
||||
|
||||
This directory is what the DMG will be created from.
|
||||
"""
|
||||
for app_bundle in app_bundles:
|
||||
print(f'Copying {app_bundle.name}...')
|
||||
shutil.copytree(app_bundle, directory / app_bundle.name)
|
||||
|
||||
|
||||
def get_main_app_bundle(app_bundles: List[Path]) -> Path:
|
||||
"""
|
||||
Get application bundle main for the installation
|
||||
"""
|
||||
return app_bundles[0]
|
||||
|
||||
|
||||
def create_dmg_image(app_bundles: List[Path],
|
||||
dmg_filepath: Path,
|
||||
volume_name: str) -> None:
|
||||
"""
|
||||
Create DMG disk image and put app bundles in it
|
||||
|
||||
No DMG configuration or codesigning is happening here.
|
||||
"""
|
||||
|
||||
if dmg_filepath.exists():
|
||||
print(f'Removing existing writable DMG {dmg_filepath}...')
|
||||
dmg_filepath.unlink()
|
||||
|
||||
print('Preparing directory with app bundles for the DMG...')
|
||||
with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str:
|
||||
# Copy all bundles to a clean directory.
|
||||
content_dir = Path(content_dir_str)
|
||||
copy_app_bundles_to_directory(app_bundles, content_dir)
|
||||
|
||||
# Estimate size of the DMG.
|
||||
dmg_size = estimate_dmg_size(app_bundles)
|
||||
print(f'Estimated DMG size: {dmg_size:,} bytes.')
|
||||
|
||||
# Create the DMG.
|
||||
print(f'Creating writable DMG {dmg_filepath}')
|
||||
command = ('hdiutil',
|
||||
'create',
|
||||
'-size', str(dmg_size),
|
||||
'-fs', 'HFS+',
|
||||
'-srcfolder', content_dir,
|
||||
'-volname', volume_name,
|
||||
'-format', 'UDRW',
|
||||
dmg_filepath)
|
||||
subprocess.run(command)
|
||||
|
||||
|
||||
def get_writable_dmg_filepath(dmg_filepath: Path):
|
||||
"""
|
||||
Get file path for writable DMG image
|
||||
"""
|
||||
parent = dmg_filepath.parent
|
||||
return parent / (dmg_filepath.stem + '-temp.dmg')
|
||||
|
||||
|
||||
def mount_readwrite_dmg(dmg_filepath: Path) -> None:
|
||||
"""
|
||||
Mount writable DMG
|
||||
|
||||
Mounting point would be /Volumes/<volume name>
|
||||
"""
|
||||
|
||||
print(f'Mounting read-write DMG ${dmg_filepath}')
|
||||
command = ('hdiutil',
|
||||
'attach', '-readwrite',
|
||||
'-noverify',
|
||||
'-noautoopen',
|
||||
dmg_filepath)
|
||||
subprocess.run(command)
|
||||
|
||||
|
||||
def get_mount_directory_for_volume_name(volume_name: str) -> Path:
|
||||
"""
|
||||
Get directory under which the volume will be mounted
|
||||
"""
|
||||
|
||||
return Path('/Volumes') / volume_name
|
||||
|
||||
|
||||
def eject_volume(volume_name: str) -> None:
|
||||
"""
|
||||
Eject given volume, if mounted
|
||||
"""
|
||||
mount_directory = get_mount_directory_for_volume_name(volume_name)
|
||||
if not mount_directory.exists():
|
||||
return
|
||||
mount_directory_str = str(mount_directory)
|
||||
|
||||
print(f'Ejecting volume {volume_name}')
|
||||
|
||||
# Figure out which device to eject.
|
||||
mount_output = subprocess.check_output(['mount']).decode()
|
||||
device = ''
|
||||
for line in mount_output.splitlines():
|
||||
if f'on {mount_directory_str} (' not in line:
|
||||
continue
|
||||
tokens = line.split(' ', 3)
|
||||
if len(tokens) < 3:
|
||||
continue
|
||||
if tokens[1] != 'on':
|
||||
continue
|
||||
if device:
|
||||
raise Exception(
|
||||
f'Multiple devices found for mounting point {mount_directory}')
|
||||
device = tokens[0]
|
||||
|
||||
if not device:
|
||||
raise Exception(
|
||||
f'No device found for mounting point {mount_directory}')
|
||||
|
||||
print(f'{mount_directory} is mounted as device {device}, ejecting...')
|
||||
subprocess.run(['diskutil', 'eject', device])
|
||||
|
||||
|
||||
def copy_background_if_needed(background_image_filepath: Path,
|
||||
mount_directory: Path) -> None:
|
||||
"""
|
||||
Copy background to the DMG
|
||||
|
||||
If the background image is not specified it will not be copied.
|
||||
"""
|
||||
|
||||
if not background_image_filepath:
|
||||
print('No background image provided.')
|
||||
return
|
||||
|
||||
print(f'Copying background image {background_image_filepath}')
|
||||
|
||||
destination_dir = mount_directory / '.background'
|
||||
destination_dir.mkdir(exist_ok=True)
|
||||
|
||||
destination_filepath = destination_dir / background_image_filepath.name
|
||||
shutil.copy(background_image_filepath, destination_filepath)
|
||||
|
||||
|
||||
def create_applications_link(mount_directory: Path) -> None:
|
||||
"""
|
||||
Create link to /Applications in the given location
|
||||
"""
|
||||
|
||||
print('Creating link to /Applications')
|
||||
|
||||
command = ('ln', '-s', '/Applications', mount_directory / ' ')
|
||||
subprocess.run(command)
|
||||
|
||||
|
||||
def run_applescript(applescript: Path,
|
||||
volume_name: str,
|
||||
app_bundles: List[Path],
|
||||
background_image_filepath: Path) -> None:
|
||||
"""
|
||||
Run given applescript to adjust look and feel of the DMG
|
||||
"""
|
||||
|
||||
main_app_bundle = get_main_app_bundle(app_bundles)
|
||||
|
||||
with NamedTemporaryFile(
|
||||
mode='w', suffix='.applescript') as temp_applescript:
|
||||
print('Adjusting applescript for volume name...')
|
||||
# Adjust script to the specific volume name.
|
||||
with open(applescript, mode='r') as input:
|
||||
for line in input.readlines():
|
||||
stripped_line = line.strip()
|
||||
if stripped_line.startswith('tell disk'):
|
||||
line = re.sub('tell disk ".*"',
|
||||
f'tell disk "{volume_name}"',
|
||||
line)
|
||||
elif stripped_line.startswith('set background picture'):
|
||||
if not background_image_filepath:
|
||||
continue
|
||||
else:
|
||||
background_image_short = \
|
||||
'.background:' + background_image_filepath.name
|
||||
line = re.sub('to file ".*"',
|
||||
f'to file "{background_image_short}"',
|
||||
line)
|
||||
line = line.replace('blender.app', main_app_bundle.name)
|
||||
temp_applescript.write(line)
|
||||
|
||||
temp_applescript.flush()
|
||||
|
||||
print('Running applescript...')
|
||||
command = ('osascript', temp_applescript.name)
|
||||
subprocess.run(command)
|
||||
|
||||
print('Waiting for applescript...')
|
||||
|
||||
# NOTE: This is copied from bundle.sh. The exact reason for sleep is
|
||||
# still remained a mystery.
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def codesign(subject: Path):
|
||||
"""
|
||||
Codesign file or directory
|
||||
|
||||
NOTE: For DMG it will also notarize.
|
||||
"""
|
||||
|
||||
command = (CODESIGN_SCRIPT, subject)
|
||||
subprocess.run(command)
|
||||
|
||||
|
||||
def codesign_app_bundles_in_dmg(mount_directory: str) -> None:
|
||||
"""
|
||||
Code sign all binaries and bundles in the mounted directory
|
||||
"""
|
||||
|
||||
print(f'Codesigning all app bundles in {mount_directory}')
|
||||
codesign(mount_directory)
|
||||
|
||||
|
||||
def codesign_and_notarize_dmg(dmg_filepath: Path) -> None:
|
||||
"""
|
||||
Run codesign and notarization pipeline on the DMG
|
||||
"""
|
||||
|
||||
print(f'Codesigning and notarizing DMG {dmg_filepath}')
|
||||
codesign(dmg_filepath)
|
||||
|
||||
|
||||
def compress_dmg(writable_dmg_filepath: Path,
|
||||
final_dmg_filepath: Path) -> None:
|
||||
"""
|
||||
Compress temporary read-write DMG
|
||||
"""
|
||||
command = ('hdiutil', 'convert',
|
||||
writable_dmg_filepath,
|
||||
'-format', 'UDZO',
|
||||
'-o', final_dmg_filepath)
|
||||
|
||||
if final_dmg_filepath.exists():
|
||||
print(f'Removing old compressed DMG {final_dmg_filepath}')
|
||||
final_dmg_filepath.unlink()
|
||||
|
||||
print('Compressing disk image...')
|
||||
subprocess.run(command)
|
||||
|
||||
|
||||
def create_final_dmg(app_bundles: List[Path],
|
||||
dmg_filepath: Path,
|
||||
background_image_filepath: Path,
|
||||
volume_name: str,
|
||||
applescript: Path,
|
||||
codesign: bool) -> None:
|
||||
"""
|
||||
Create DMG with all app bundles
|
||||
|
||||
Will take care configuring background, signing all binaries and app bundles
|
||||
and notarizing the DMG.
|
||||
"""
|
||||
|
||||
print('Running all routines to create final DMG')
|
||||
|
||||
writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath)
|
||||
mount_directory = get_mount_directory_for_volume_name(volume_name)
|
||||
|
||||
# Make sure volume is not mounted.
|
||||
# If it is mounted it will prevent removing old DMG files and could make
|
||||
# it so app bundles are copied to the wrong place.
|
||||
eject_volume(volume_name)
|
||||
|
||||
create_dmg_image(app_bundles, writable_dmg_filepath, volume_name)
|
||||
|
||||
mount_readwrite_dmg(writable_dmg_filepath)
|
||||
|
||||
# Run codesign first, prior to copying amything else.
|
||||
#
|
||||
# This allows to recurs into the content of bundles without worrying about
|
||||
# possible interfereice of Application symlink.
|
||||
if codesign:
|
||||
codesign_app_bundles_in_dmg(mount_directory)
|
||||
|
||||
copy_background_if_needed(background_image_filepath, mount_directory)
|
||||
create_applications_link(mount_directory)
|
||||
run_applescript(applescript, volume_name, app_bundles,
|
||||
background_image_filepath)
|
||||
|
||||
print('Ejecting read-write DMG image...')
|
||||
eject_volume(volume_name)
|
||||
|
||||
compress_dmg(writable_dmg_filepath, dmg_filepath)
|
||||
writable_dmg_filepath.unlink()
|
||||
|
||||
if codesign:
|
||||
codesign_and_notarize_dmg(dmg_filepath)
|
||||
|
||||
|
||||
def ensure_dmg_extension(filepath: Path) -> Path:
|
||||
"""
|
||||
Make sure given file have .dmg extension
|
||||
"""
|
||||
|
||||
if filepath.suffix != '.dmg':
|
||||
return filepath.with_suffix(f'{filepath.suffix}.dmg')
|
||||
return filepath
|
||||
|
||||
|
||||
def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path:
|
||||
"""
|
||||
Get full file path for the final DMG image
|
||||
|
||||
Will use the provided one when possible, otherwise will deduct it from
|
||||
app bundles.
|
||||
|
||||
If the name is deducted, the DMG is stored in the current directory.
|
||||
"""
|
||||
|
||||
if requested_name:
|
||||
return ensure_dmg_extension(requested_name.absolute())
|
||||
|
||||
# TODO(sergey): This is not necessarily the main one.
|
||||
main_bundle = app_bundles[0]
|
||||
# Strip .app from the name
|
||||
return Path(main_bundle.name[:-4] + '.dmg').absolute()
|
||||
|
||||
|
||||
def get_background_image(requested_background_image: Path) -> Path:
|
||||
"""
|
||||
Get effective filepath for the background image
|
||||
"""
|
||||
|
||||
if requested_background_image:
|
||||
return requested_background_image.absolute()
|
||||
|
||||
return DARWIN_DIRECTORY / 'background.tif'
|
||||
|
||||
|
||||
def get_applescript(requested_applescript: Path) -> Path:
|
||||
"""
|
||||
Get effective filepath for the applescript
|
||||
"""
|
||||
|
||||
if requested_applescript:
|
||||
return requested_applescript.absolute()
|
||||
|
||||
return DARWIN_DIRECTORY / 'blender.applescript'
|
||||
|
||||
|
||||
def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str:
|
||||
"""
|
||||
Deduct volume name from the DMG path
|
||||
|
||||
Will use first part of the DMG file name prior to dash.
|
||||
"""
|
||||
|
||||
tokens = dmg_filepath.stem.split('-')
|
||||
words = tokens[0].split()
|
||||
|
||||
return ' '.join(word.capitalize() for word in words)
|
||||
|
||||
|
||||
def get_volume_name(requested_volume_name: str,
|
||||
dmg_filepath: Path) -> str:
|
||||
"""
|
||||
Get effective name for DMG volume
|
||||
"""
|
||||
|
||||
if requested_volume_name:
|
||||
return requested_volume_name
|
||||
|
||||
return get_volume_name_from_dmg_filepath(dmg_filepath)
|
||||
|
||||
|
||||
def main():
|
||||
parser = create_argument_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get normalized input parameters.
|
||||
source_dir = args.source_dir.absolute()
|
||||
background_image_filepath = get_background_image(args.background_image)
|
||||
applescript = get_applescript(args.applescript)
|
||||
codesign = args.codesign
|
||||
|
||||
app_bundles = collect_and_log_app_bundles(source_dir)
|
||||
if not app_bundles:
|
||||
return
|
||||
|
||||
dmg_filepath = get_dmg_filepath(args.dmg, app_bundles)
|
||||
volume_name = get_volume_name(args.volume_name, dmg_filepath)
|
||||
|
||||
print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)')
|
||||
|
||||
create_final_dmg(app_bundles,
|
||||
dmg_filepath,
|
||||
background_image_filepath,
|
||||
volume_name,
|
||||
applescript,
|
||||
codesign)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
@@ -1,44 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# This is a script which is used as POST-INSTALL one for regular CMake's
|
||||
# INSTALL target.
|
||||
# It is used by buildbot workers to sign every binary which is going into
|
||||
# the final buundle.
|
||||
|
||||
# On Windows Python 3 there only is python.exe, no python3.exe.
|
||||
#
|
||||
# On other platforms it is possible to have python2 and python3, and a
|
||||
# symbolic link to python to either of them. So on those platforms use
|
||||
# an explicit Python version.
|
||||
if(WIN32)
|
||||
set(PYTHON_EXECUTABLE python)
|
||||
else()
|
||||
set(PYTHON_EXECUTABLE python3)
|
||||
endif()
|
||||
|
||||
execute_process(
|
||||
COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/worker_codesign.py"
|
||||
"${CMAKE_INSTALL_PREFIX}"
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
|
||||
RESULT_VARIABLE exit_code
|
||||
)
|
||||
|
||||
if(NOT exit_code EQUAL "0")
|
||||
message(FATAL_ERROR "Non-zero exit code of codesign tool")
|
||||
endif()
|
@@ -1,74 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# Helper script which takes care of signing provided location.
|
||||
#
|
||||
# The location can either be a directory (in which case all eligible binaries
|
||||
# will be signed) or a single file (in which case a single file will be signed).
|
||||
#
|
||||
# This script takes care of all the complexity of communicating between process
|
||||
# which requests file to be signed and the code signing server.
|
||||
#
|
||||
# NOTE: Signing happens in-place.
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from codesign.simple_code_signer import SimpleCodeSigner
|
||||
|
||||
|
||||
def create_argument_parser():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('path_to_sign', type=Path)
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = create_argument_parser()
|
||||
args = parser.parse_args()
|
||||
path_to_sign = args.path_to_sign.absolute()
|
||||
|
||||
if sys.platform == 'win32':
|
||||
# When WIX packed is used to generate .msi on Windows the CPack will
|
||||
# install two different projects and install them to different
|
||||
# installation prefix:
|
||||
#
|
||||
# - C:\b\build\_CPack_Packages\WIX\Blender
|
||||
# - C:\b\build\_CPack_Packages\WIX\Unspecified
|
||||
#
|
||||
# Annoying part is: CMake's post-install script will only be run
|
||||
# once, with the install prefix which corresponds to a project which
|
||||
# was installed last. But we want to sign binaries from all projects.
|
||||
# So in order to do so we detect that we are running for a CPack's
|
||||
# project used for WIX and force parent directory (which includes both
|
||||
# projects) to be signed.
|
||||
#
|
||||
# Here we force both projects to be signed.
|
||||
if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign):
|
||||
path_to_sign = path_to_sign.parent
|
||||
|
||||
code_signer = SimpleCodeSigner()
|
||||
code_signer.sign_file_or_directory(path_to_sign)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
@@ -1,135 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import buildbot_utils
|
||||
|
||||
|
||||
def get_cmake_options(builder):
|
||||
codesign_script = os.path.join(
|
||||
builder.blender_dir, 'build_files', 'buildbot', 'worker_codesign.cmake')
|
||||
|
||||
config_file = "build_files/cmake/config/blender_release.cmake"
|
||||
options = ['-DCMAKE_BUILD_TYPE:STRING=Release',
|
||||
'-DWITH_GTESTS=ON']
|
||||
|
||||
if builder.platform == 'mac':
|
||||
options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
|
||||
options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
|
||||
elif builder.platform == 'win':
|
||||
options.extend(['-G', 'Visual Studio 16 2019', '-A', 'x64'])
|
||||
if builder.codesign:
|
||||
options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + codesign_script])
|
||||
elif builder.platform == 'linux':
|
||||
config_file = "build_files/buildbot/config/blender_linux.cmake"
|
||||
|
||||
optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK-7.1')
|
||||
options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir)
|
||||
|
||||
# Workaround to build sm_30 kernels with CUDA 10, since CUDA 11 no longer supports that architecture
|
||||
if builder.platform == 'win':
|
||||
options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1')
|
||||
options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe')
|
||||
options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1')
|
||||
options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1/bin/nvcc.exe')
|
||||
elif builder.platform == 'linux':
|
||||
options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-10.1')
|
||||
options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-10.1/bin/nvcc')
|
||||
options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-11.1')
|
||||
options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-11.1/bin/nvcc')
|
||||
|
||||
options.append("-C" + os.path.join(builder.blender_dir, config_file))
|
||||
options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir))
|
||||
|
||||
return options
|
||||
|
||||
|
||||
def update_git(builder):
|
||||
# Do extra git fetch because not all platform/git/buildbot combinations
|
||||
# update the origin remote, causing buildinfo to detect local changes.
|
||||
os.chdir(builder.blender_dir)
|
||||
|
||||
print("Fetching remotes")
|
||||
command = ['git', 'fetch', '--all']
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
|
||||
def clean_directories(builder):
|
||||
# Make sure no garbage remained from the previous run
|
||||
if os.path.isdir(builder.install_dir):
|
||||
shutil.rmtree(builder.install_dir)
|
||||
|
||||
# Make sure build directory exists and enter it
|
||||
os.makedirs(builder.build_dir, exist_ok=True)
|
||||
|
||||
# Remove buildinfo files to force buildbot to re-generate them.
|
||||
for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
|
||||
full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo)
|
||||
if os.path.exists(full_path):
|
||||
print("Removing {}" . format(buildinfo))
|
||||
os.remove(full_path)
|
||||
|
||||
|
||||
def cmake_configure(builder):
|
||||
# CMake configuration
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt')
|
||||
if os.path.exists(cmake_cache):
|
||||
print("Removing CMake cache")
|
||||
os.remove(cmake_cache)
|
||||
|
||||
print("CMake configure:")
|
||||
cmake_options = get_cmake_options(builder)
|
||||
command = ['cmake', builder.blender_dir] + cmake_options
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
|
||||
def cmake_build(builder):
|
||||
# CMake build
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
# NOTE: CPack will build an INSTALL target, which would mean that code
|
||||
# signing will happen twice when using `make install` and CPack.
|
||||
# The tricky bit here is that it is not possible to know whether INSTALL
|
||||
# target is used by CPack or by a buildbot itaself. Extra level on top of
|
||||
# this is that on Windows it is required to build INSTALL target in order
|
||||
# to have unit test binaries to run.
|
||||
# So on the one hand we do an extra unneeded code sign on Windows, but on
|
||||
# a positive side we don't add complexity and don't make build process more
|
||||
# fragile trying to avoid this. The signing process is way faster than just
|
||||
# a clean build of buildbot, especially with regression tests enabled.
|
||||
if builder.platform == 'win':
|
||||
command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release']
|
||||
else:
|
||||
command = ['make', '-s', '-j16', 'install']
|
||||
|
||||
print("CMake build:")
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
update_git(builder)
|
||||
clean_directories(builder)
|
||||
cmake_configure(builder)
|
||||
cmake_build(builder)
|
@@ -1,208 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
# Runs on buildbot worker, creating a release package using the build
|
||||
# system and zipping it into buildbot_upload.zip. This is then uploaded
|
||||
# to the master in the next buildbot step.
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import buildbot_utils
|
||||
|
||||
|
||||
def get_package_name(builder, platform=None):
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
|
||||
package_name = 'blender-' + info.full_version
|
||||
if platform:
|
||||
package_name += '-' + platform
|
||||
if not (builder.branch == 'master' or builder.is_release_branch):
|
||||
if info.is_development_build:
|
||||
package_name = builder.branch + "-" + package_name
|
||||
|
||||
return package_name
|
||||
|
||||
|
||||
def sign_file_or_directory(path):
|
||||
from codesign.simple_code_signer import SimpleCodeSigner
|
||||
code_signer = SimpleCodeSigner()
|
||||
code_signer.sign_file_or_directory(Path(path))
|
||||
|
||||
|
||||
def create_buildbot_upload_zip(builder, package_files):
|
||||
import zipfile
|
||||
|
||||
buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
|
||||
if os.path.exists(buildbot_upload_zip):
|
||||
os.remove(buildbot_upload_zip)
|
||||
|
||||
try:
|
||||
z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
|
||||
for filepath, filename in package_files:
|
||||
print("Packaged", filename)
|
||||
z.write(filepath, arcname=filename)
|
||||
z.close()
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def create_tar_xz(src, dest, package_name):
|
||||
# One extra to remove leading os.sep when cleaning root for package_root
|
||||
ln = len(src) + 1
|
||||
flist = list()
|
||||
|
||||
# Create list of tuples containing file and archive name
|
||||
for root, dirs, files in os.walk(src):
|
||||
package_root = os.path.join(package_name, root[ln:])
|
||||
flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
|
||||
|
||||
import tarfile
|
||||
|
||||
# Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
|
||||
# user compiled the package. If root then unpacks it to /usr/local/ you get
|
||||
# a security issue.
|
||||
def _fakeroot(tarinfo):
|
||||
tarinfo.gid = 0
|
||||
tarinfo.gname = "root"
|
||||
tarinfo.uid = 0
|
||||
tarinfo.uname = "root"
|
||||
return tarinfo
|
||||
|
||||
package = tarfile.open(dest, 'w:xz', preset=9)
|
||||
for entry in flist:
|
||||
package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
|
||||
package.close()
|
||||
|
||||
|
||||
def cleanup_files(dirpath, extension):
|
||||
for f in os.listdir(dirpath):
|
||||
filepath = os.path.join(dirpath, f)
|
||||
if os.path.isfile(filepath) and f.endswith(extension):
|
||||
os.remove(filepath)
|
||||
|
||||
|
||||
def pack_mac(builder):
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
|
||||
os.chdir(builder.build_dir)
|
||||
cleanup_files(builder.build_dir, '.dmg')
|
||||
|
||||
package_name = get_package_name(builder, 'macOS')
|
||||
package_filename = package_name + '.dmg'
|
||||
package_filepath = os.path.join(builder.build_dir, package_filename)
|
||||
|
||||
release_dir = os.path.join(builder.blender_dir, 'release', 'darwin')
|
||||
buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot')
|
||||
bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py')
|
||||
|
||||
command = [bundle_script]
|
||||
command += ['--dmg', package_filepath]
|
||||
if info.is_development_build:
|
||||
background_image = os.path.join(release_dir, 'buildbot', 'background.tif')
|
||||
command += ['--background-image', background_image]
|
||||
if builder.codesign:
|
||||
command += ['--codesign']
|
||||
command += [builder.install_dir]
|
||||
buildbot_utils.call(command)
|
||||
|
||||
create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
|
||||
|
||||
|
||||
def pack_win(builder):
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
|
||||
os.chdir(builder.build_dir)
|
||||
cleanup_files(builder.build_dir, '.zip')
|
||||
|
||||
# CPack will add the platform name
|
||||
cpack_name = get_package_name(builder, None)
|
||||
package_name = get_package_name(builder, 'windows' + str(builder.bits))
|
||||
|
||||
command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.']
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
command = ['cpack', '-G', 'ZIP']
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
package_filename = package_name + '.zip'
|
||||
package_filepath = os.path.join(builder.build_dir, package_filename)
|
||||
package_files = [(package_filepath, package_filename)]
|
||||
|
||||
if info.version_cycle == 'release':
|
||||
# Installer only for final release builds, otherwise will get
|
||||
# 'this product is already installed' messages.
|
||||
command = ['cpack', '-G', 'WIX']
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
package_filename = package_name + '.msi'
|
||||
package_filepath = os.path.join(builder.build_dir, package_filename)
|
||||
if builder.codesign:
|
||||
sign_file_or_directory(package_filepath)
|
||||
|
||||
package_files += [(package_filepath, package_filename)]
|
||||
|
||||
create_buildbot_upload_zip(builder, package_files)
|
||||
|
||||
|
||||
def pack_linux(builder):
|
||||
blender_executable = os.path.join(builder.install_dir, 'blender')
|
||||
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
|
||||
# Strip all unused symbols from the binaries
|
||||
print("Stripping binaries...")
|
||||
buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
|
||||
|
||||
print("Stripping python...")
|
||||
py_target = os.path.join(builder.install_dir, info.short_version)
|
||||
buildbot_utils.call(
|
||||
builder.command_prefix + [
|
||||
'find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';',
|
||||
],
|
||||
)
|
||||
|
||||
# Construct package name
|
||||
platform_name = 'linux64'
|
||||
package_name = get_package_name(builder, platform_name)
|
||||
package_filename = package_name + ".tar.xz"
|
||||
|
||||
print("Creating .tar.xz archive")
|
||||
package_filepath = builder.install_dir + '.tar.xz'
|
||||
create_tar_xz(builder.install_dir, package_filepath, package_name)
|
||||
|
||||
# Create buildbot_upload.zip
|
||||
create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
|
||||
# Make sure install directory always exists
|
||||
os.makedirs(builder.install_dir, exist_ok=True)
|
||||
|
||||
if builder.platform == 'mac':
|
||||
pack_mac(builder)
|
||||
elif builder.platform == 'win':
|
||||
pack_win(builder)
|
||||
elif builder.platform == 'linux':
|
||||
pack_linux(builder)
|
@@ -1,42 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import buildbot_utils
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def get_ctest_arguments(builder):
|
||||
args = ['--output-on-failure']
|
||||
if builder.platform == 'win':
|
||||
args += ['-C', 'Release']
|
||||
return args
|
||||
|
||||
|
||||
def test(builder):
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder)
|
||||
buildbot_utils.call(command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
test(builder)
|
@@ -1,31 +0,0 @@
|
||||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import buildbot_utils
|
||||
import os
|
||||
import sys
|
||||
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
os.chdir(builder.blender_dir)
|
||||
|
||||
# Run make update which handles all libraries and submodules.
|
||||
make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py")
|
||||
buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"])
|
8
build_files/config/README.md
Normal file
8
build_files/config/README.md
Normal file
@@ -0,0 +1,8 @@
|
||||
Pipeline Config
|
||||
===============
|
||||
|
||||
This configuration file is used by buildbot new pipeline for the `update-code` step.
|
||||
|
||||
It will soon be used by the ../utils/make_update.py script.
|
||||
|
||||
Both buildbot and developers will eventually use the same configuration file.
|
87
build_files/config/pipeline_config.json
Normal file
87
build_files/config/pipeline_config.json
Normal file
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"update-code":
|
||||
{
|
||||
"git" :
|
||||
{
|
||||
"submodules":
|
||||
[
|
||||
{ "path": "release/scripts/addons", "branch": "master", "commit_id": "HEAD" },
|
||||
{ "path": "release/scripts/addons_contrib", "branch": "master", "commit_id": "HEAD" },
|
||||
{ "path": "release/datafiles/locale", "branch": "master", "commit_id": "HEAD" },
|
||||
{ "path": "source/tools", "branch": "master", "commit_id": "HEAD" }
|
||||
]
|
||||
},
|
||||
"svn":
|
||||
{
|
||||
"tests": { "path": "lib/tests", "branch": "trunk", "commit_id": "HEAD" },
|
||||
"libraries":
|
||||
{
|
||||
"darwin-x86_64": { "path": "lib/darwin", "branch": "trunk", "commit_id": "HEAD" },
|
||||
"darwin-arm64": { "path": "lib/darwin_arm64", "branch": "trunk", "commit_id": "HEAD" },
|
||||
"linux-x86_64": { "path": "lib/linux_centos7_x86_64", "branch": "trunk", "commit_id": "HEAD" },
|
||||
"windows-amd64": { "path": "lib/win64_vc15", "branch": "trunk", "commit_id": "HEAD" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"buildbot":
|
||||
{
|
||||
"gcc":
|
||||
{
|
||||
"version": "9.0"
|
||||
},
|
||||
"sdks":
|
||||
{
|
||||
"optix":
|
||||
{
|
||||
"version": "7.1.0"
|
||||
},
|
||||
"cuda10":
|
||||
{
|
||||
"version": "10.1"
|
||||
},
|
||||
"cuda11":
|
||||
{
|
||||
"version": "11.3"
|
||||
}
|
||||
},
|
||||
"cmake":
|
||||
{
|
||||
"default":
|
||||
{
|
||||
"version": "any",
|
||||
"overrides":
|
||||
{
|
||||
|
||||
}
|
||||
},
|
||||
"darwin-x86_64":
|
||||
{
|
||||
"overrides":
|
||||
{
|
||||
|
||||
}
|
||||
},
|
||||
"darwin-arm64":
|
||||
{
|
||||
"overrides":
|
||||
{
|
||||
|
||||
}
|
||||
},
|
||||
"linux-x86_64":
|
||||
{
|
||||
"overrides":
|
||||
{
|
||||
|
||||
}
|
||||
},
|
||||
"windows-amd64":
|
||||
{
|
||||
"overrides":
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
5
build_files/utils/README.md
Normal file
5
build_files/utils/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
Make Utility Scripts
|
||||
====================
|
||||
|
||||
Scripts used only by developers for now
|
||||
|
5
release/darwin/README.md
Normal file
5
release/darwin/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
Buildbot Configuration
|
||||
======================
|
||||
|
||||
Files used by Buildbot's `package-code-binaires` step for the darwin platform.
|
||||
|
@@ -1,55 +0,0 @@
|
||||
|
||||
macOS app bundling guide
|
||||
========================
|
||||
|
||||
Install Code Signing Certificate
|
||||
--------------------------------
|
||||
|
||||
* Go to https://developer.apple.com/account/resources/certificates/list
|
||||
* Download the Developer ID Application certificate.
|
||||
* Double click the file and add to key chain (default options).
|
||||
* Delete the file from the Downloads folder.
|
||||
|
||||
* You will also need to install a .p12 public/private key file for the
|
||||
certificate. This is only available for the owner of the Blender account,
|
||||
or can be exported and copied from another system that already has code
|
||||
signing set up.
|
||||
|
||||
Find the codesigning identity by running:
|
||||
|
||||
$ security find-identity -v -p codesigning
|
||||
|
||||
"Developer ID Application: Stichting Blender Foundation" is the identity needed.
|
||||
The long code at the start of the line is used as <identity> below.
|
||||
|
||||
Setup Apple ID
|
||||
--------------
|
||||
|
||||
* The Apple ID must have two step verification enabled.
|
||||
* Create an app specific password for the code signing app (label can be anything):
|
||||
https://support.apple.com/en-us/HT204397
|
||||
* Add the app specific password to keychain:
|
||||
|
||||
$ security add-generic-password -a <apple-id> -w <app-specific-password> -s altool-password
|
||||
|
||||
When running the bundle script, there will be a popup. To avoid that either:
|
||||
* Click Always Allow in the popup
|
||||
* In the Keychain Access app, change the Access Control settings on altool-password
|
||||
|
||||
Bundle
|
||||
------
|
||||
|
||||
Then the bundle is created as follows:
|
||||
|
||||
$ ./bundle.sh --source <sourcedir> --dmg <dmg> --bundle-id <bundleid> --username <apple-id> --password "@keychain:altool-password" --codesign <identity>
|
||||
|
||||
<sourcedir> directory where built Blender.app is
|
||||
<dmg> location and name of the final disk image
|
||||
<bundleid> id on notarization, for example org.blenderfoundation.blender.release
|
||||
<apple-id> your appleid email
|
||||
<identity> codesigning identity
|
||||
|
||||
When specifying only --sourcedir and --dmg, the build will not be signed.
|
||||
|
||||
Example :
|
||||
$ ./bundle.sh --source /data/build/bin --dmg /data/Blender-2.8-alpha-macOS-10.11.dmg --bundle-id org.blenderfoundation.blender.release --username "foo@mac.com" --password "@keychain:altool-password" --codesign AE825E26F12D08B692F360133210AF46F4CF7B97
|
@@ -1,18 +0,0 @@
|
||||
tell application "Finder"
|
||||
tell disk "Blender"
|
||||
open
|
||||
set current view of container window to icon view
|
||||
set toolbar visible of container window to false
|
||||
set statusbar visible of container window to false
|
||||
set the bounds of container window to {100, 100, 640, 472}
|
||||
set theViewOptions to icon view options of container window
|
||||
set arrangement of theViewOptions to not arranged
|
||||
set icon size of theViewOptions to 128
|
||||
set background picture of theViewOptions to file ".background:background.tif"
|
||||
set position of item " " of container window to {400, 190}
|
||||
set position of item "blender.app" of container window to {135, 190}
|
||||
update without registering applications
|
||||
delay 5
|
||||
close
|
||||
end tell
|
||||
end tell
|
@@ -1,212 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Script to create a macOS dmg file for Blender builds, including code
|
||||
# signing and notarization for releases.
|
||||
|
||||
# Check that we have all needed tools.
|
||||
for i in osascript git codesign hdiutil xcrun ; do
|
||||
if [ ! -x "$(which ${i})" ]; then
|
||||
echo "Unable to execute command $i, macOS broken?"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Defaults settings.
|
||||
_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
_volume_name="Blender"
|
||||
_tmp_dir="$(mktemp -d)"
|
||||
_tmp_dmg="/tmp/blender-tmp.dmg"
|
||||
_background_image="${_script_dir}/background.tif"
|
||||
_mount_dir="/Volumes/${_volume_name}"
|
||||
_entitlements="${_script_dir}/entitlements.plist"
|
||||
|
||||
# Handle arguments.
|
||||
while [[ $# -gt 0 ]]; do
|
||||
key=$1
|
||||
case $key in
|
||||
-s|--source)
|
||||
SRC_DIR="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-d|--dmg)
|
||||
DEST_DMG="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-b|--bundle-id)
|
||||
N_BUNDLE_ID="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-u|--username)
|
||||
N_USERNAME="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-p|--password)
|
||||
N_PASSWORD="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-c|--codesign)
|
||||
C_CERT="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--background-image)
|
||||
_background_image="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage:"
|
||||
echo " $(basename "$0") --source DIR --dmg IMAGENAME "
|
||||
echo " optional arguments:"
|
||||
echo " --codesign <certname>"
|
||||
echo " --username <username>"
|
||||
echo " --password <password>"
|
||||
echo " --bundle-id <bundleid>"
|
||||
echo " Check https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow "
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -d "${SRC_DIR}/Blender.app" ]; then
|
||||
echo "use --source parameter to set source directory where Blender.app can be found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${DEST_DMG}" ]; then
|
||||
echo "use --dmg parameter to set output dmg name"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Destroy destination dmg if there is any.
|
||||
test -f "${DEST_DMG}" && rm "${DEST_DMG}"
|
||||
if [ -d "${_mount_dir}" ]; then
|
||||
echo -n "Ejecting existing blender volume.."
|
||||
DEV_FILE=$(mount | grep "${_mount_dir}" | awk '{ print $1 }')
|
||||
diskutil eject "${DEV_FILE}" || exit 1
|
||||
echo
|
||||
fi
|
||||
|
||||
# Copy dmg contents.
|
||||
echo -n "Copying Blender.app..."
|
||||
cp -r "${SRC_DIR}/Blender.app" "${_tmp_dir}/" || exit 1
|
||||
echo
|
||||
|
||||
# Create the disk image.
|
||||
_directory_size=$(du -sh ${_tmp_dir} | awk -F'[^0-9]*' '$0=$1')
|
||||
_image_size=$(echo "${_directory_size}" + 400 | bc) # extra 400 need for codesign to work (why on earth?)
|
||||
|
||||
echo
|
||||
echo -n "Creating disk image of size ${_image_size}M.."
|
||||
test -f "${_tmp_dmg}" && rm "${_tmp_dmg}"
|
||||
hdiutil create -size "${_image_size}m" -fs HFS+ -srcfolder "${_tmp_dir}" -volname "${_volume_name}" -format UDRW "${_tmp_dmg}" -mode 755
|
||||
|
||||
echo "Mounting readwrite image..."
|
||||
hdiutil attach -readwrite -noverify -noautoopen "${_tmp_dmg}"
|
||||
|
||||
echo "Setting background picture.."
|
||||
if ! test -z "${_background_image}"; then
|
||||
echo "Copying background image ..."
|
||||
test -d "${_mount_dir}/.background" || mkdir "${_mount_dir}/.background"
|
||||
_background_image_NAME=$(basename "${_background_image}")
|
||||
cp "${_background_image}" "${_mount_dir}/.background/${_background_image_NAME}"
|
||||
fi
|
||||
|
||||
echo "Creating link to /Applications ..."
|
||||
ln -s /Applications "${_mount_dir}/Applications"
|
||||
echo "Renaming Applications to empty string."
|
||||
mv ${_mount_dir}/Applications "${_mount_dir}/ "
|
||||
|
||||
echo "Running applescript to set folder looks ..."
|
||||
cat "${_script_dir}/blender.applescript" | osascript
|
||||
|
||||
echo "Waiting after applescript ..."
|
||||
sleep 5
|
||||
|
||||
if [ ! -z "${C_CERT}" ]; then
|
||||
# Codesigning requires all libs and binaries to be signed separately.
|
||||
echo -n "Codesigning Python"
|
||||
for f in $(find "${_mount_dir}/Blender.app/Contents/Resources" -name "python*"); do
|
||||
if [ -x ${f} ] && [ ! -d ${f} ]; then
|
||||
codesign --remove-signature "${f}"
|
||||
codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${f}"
|
||||
fi
|
||||
done
|
||||
echo ; echo -n "Codesigning .dylib and .so libraries"
|
||||
for f in $(find "${_mount_dir}/Blender.app" -name "*.dylib" -o -name "*.so"); do
|
||||
codesign --remove-signature "${f}"
|
||||
codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${f}"
|
||||
done
|
||||
echo ; echo -n "Codesigning Blender.app"
|
||||
codesign --remove-signature "${_mount_dir}/Blender.app"
|
||||
codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${_mount_dir}/Blender.app"
|
||||
echo
|
||||
else
|
||||
echo "No codesigning cert given, skipping..."
|
||||
fi
|
||||
|
||||
# Need to eject dev files to remove /dev files and free .dmg for converting
|
||||
echo "Unmounting rw disk image ..."
|
||||
DEV_FILE=$(mount | grep "${_mount_dir}" | awk '{ print $1 }')
|
||||
diskutil eject "${DEV_FILE}"
|
||||
|
||||
sleep 3
|
||||
|
||||
echo "Compressing disk image ..."
|
||||
hdiutil convert "${_tmp_dmg}" -format UDZO -o "${DEST_DMG}"
|
||||
|
||||
# Codesign the dmg
|
||||
if [ ! -z "${C_CERT}" ]; then
|
||||
echo -n "Codesigning dmg..."
|
||||
codesign --timestamp --force --sign "${C_CERT}" "${DEST_DMG}"
|
||||
echo
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
rm -rf "${_tmp_dir}"
|
||||
rm "${_tmp_dmg}"
|
||||
|
||||
# Notarize
|
||||
if [ ! -z "${N_USERNAME}" ] && [ ! -z "${N_PASSWORD}" ] && [ ! -z "${N_BUNDLE_ID}" ]; then
|
||||
# Send to Apple
|
||||
echo "Sending ${DEST_DMG} for notarization..."
|
||||
_tmpout=$(mktemp)
|
||||
echo xcrun altool --notarize-app --verbose -f "${DEST_DMG}" --primary-bundle-id "${N_BUNDLE_ID}" --username "${N_USERNAME}" --password "${N_PASSWORD}"
|
||||
xcrun altool --notarize-app --verbose -f "${DEST_DMG}" --primary-bundle-id "${N_BUNDLE_ID}" --username "${N_USERNAME}" --password "${N_PASSWORD}" >${_tmpout} 2>&1
|
||||
|
||||
# Parse request uuid
|
||||
_requuid=$(cat "${_tmpout}" | grep "RequestUUID" | awk '{ print $3 }')
|
||||
echo "RequestUUID: ${_requuid}"
|
||||
if [ ! -z "${_requuid}" ]; then
|
||||
# Wait for Apple to confirm notarization is complete
|
||||
echo "Waiting for notarization to be complete.."
|
||||
for c in {20..0};do
|
||||
sleep 600
|
||||
xcrun altool --notarization-info "${_requuid}" --username "${N_USERNAME}" --password "${N_PASSWORD}" >${_tmpout} 2>&1
|
||||
_status=$(cat "${_tmpout}" | grep "Status:" | awk '{ print $2 }')
|
||||
if [ "${_status}" == "invalid" ]; then
|
||||
echo "Got invalid notarization!"
|
||||
break;
|
||||
fi
|
||||
|
||||
if [ "${_status}" == "success" ]; then
|
||||
echo -n "Notarization successful! Stapling..."
|
||||
xcrun stapler staple -v "${DEST_DMG}"
|
||||
break;
|
||||
fi
|
||||
echo "Notarization in progress, waiting..."
|
||||
done
|
||||
else
|
||||
cat ${_tmpout}
|
||||
echo "Error getting RequestUUID, notarization unsuccessful"
|
||||
fi
|
||||
else
|
||||
echo "No notarization credentials supplied, skipping..."
|
||||
fi
|
||||
|
||||
echo "..done. You should have ${DEST_DMG} ready to upload"
|
17
release/freedesktop/snap/README.md
Normal file
17
release/freedesktop/snap/README.md
Normal file
@@ -0,0 +1,17 @@
|
||||
Snap Configuration
|
||||
===================
|
||||
|
||||
Files used by Buildbot's `package-code-store-snap` and `deliver-code-store-snap` steps.
|
||||
|
||||
Build pipeline snap tracks and channels
|
||||
|
||||
```
|
||||
<track>/stable
|
||||
- Latest stable release for the specified track
|
||||
<track>/candidate
|
||||
- Test builds for the upcoming stable release - *not used for now*
|
||||
<track>/beta
|
||||
- Nightly automated builds provided by a release branch
|
||||
<track>/egde/<branch>
|
||||
- Nightly or on demand builds - will also make use of branch
|
||||
```
|
@@ -1,38 +0,0 @@
|
||||
|
||||
Snap Package Instructions
|
||||
=========================
|
||||
|
||||
This folder contains the scripts for creating and uploading the snap on:
|
||||
https://snapcraft.io/blender
|
||||
|
||||
|
||||
Setup
|
||||
-----
|
||||
|
||||
This has only been tested to work on Ubuntu.
|
||||
|
||||
# Install required packages
|
||||
sudo apt install snapd snapcraft
|
||||
|
||||
|
||||
Steps
|
||||
-----
|
||||
|
||||
# Build the snap file
|
||||
python3 bundle.py --version 2.XX --url https://download.blender.org/release/Blender2.XX/blender-2.XX-x86_64.tar.bz2
|
||||
|
||||
# Install snap to test
|
||||
# --dangerous is needed since the snap has not been signed yet
|
||||
# --classic is required for installing Blender in general
|
||||
sudo snap install --dangerous --classic blender_2.XX_amd64.snap
|
||||
|
||||
# Upload
|
||||
snapcraft push --release=stable blender_2.XX_amd64.snap
|
||||
|
||||
|
||||
Release Values
|
||||
--------------
|
||||
|
||||
stable: final release
|
||||
candidate: release candidates
|
||||
|
@@ -10,12 +10,7 @@ description: |
|
||||
scientists, students, VFX experts, animators, game artists, modders, and
|
||||
the list goes on.
|
||||
|
||||
The standard snap channels are used in the following way:
|
||||
|
||||
stable - Latest stable release.
|
||||
candidate - Test builds for the upcoming stable release.
|
||||
|
||||
icon: ../icons/scalable/apps/blender.svg
|
||||
icon: @ICON_PATH@
|
||||
|
||||
passthrough:
|
||||
license: GPL-3.0
|
||||
@@ -27,13 +22,14 @@ apps:
|
||||
command: ./blender-wrapper
|
||||
desktop: ./blender.desktop
|
||||
|
||||
base: core18
|
||||
version: '@VERSION@'
|
||||
grade: @GRADE@
|
||||
|
||||
parts:
|
||||
blender:
|
||||
plugin: dump
|
||||
source: @URL@
|
||||
source: @PACKAGE_PATH@
|
||||
build-attributes: [keep-execstack, no-patchelf]
|
||||
override-build: |
|
||||
snapcraftctl build
|
||||
@@ -47,7 +43,7 @@ parts:
|
||||
- libxrender1
|
||||
- libxxf86vm1
|
||||
wrapper:
|
||||
plugin: copy
|
||||
plugin: dump
|
||||
source: .
|
||||
files:
|
||||
blender-wrapper: blender-wrapper
|
||||
stage:
|
||||
- ./blender-wrapper
|
@@ -1,21 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--grade", default="stable", choices=["stable", "devel"])
|
||||
args = parser.parse_args()
|
||||
|
||||
yaml_text = pathlib.Path("snapcraft.yaml.in").read_text()
|
||||
yaml_text = yaml_text.replace("@VERSION@", args.version)
|
||||
yaml_text = yaml_text.replace("@URL@", args.url)
|
||||
yaml_text = yaml_text.replace("@GRADE@", args.grade)
|
||||
pathlib.Path("snapcraft.yaml").write_text(yaml_text)
|
||||
|
||||
subprocess.call(["snapcraft", "clean"])
|
||||
subprocess.call(["snapcraft", "snap"])
|
@@ -1,70 +0,0 @@
|
||||
Creating Steam builds for Blender
|
||||
=================================
|
||||
|
||||
This script automates creation of the Steam files: download of the archives,
|
||||
extraction of the archives, preparation of the build scripts (VDF files), actual
|
||||
building of the Steam game files.
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* MacOS machine - Tested on Catalina 10.15.6. Extracting contents from the DMG
|
||||
archive did not work Windows nor on Linux using 7-zip. All DMG archives tested
|
||||
failed to be extracted. As such only MacOS is known to work.
|
||||
* Steam SDK downloaded from SteamWorks - The `steamcmd` is used to generate the
|
||||
Steam game files. The path to the `steamcmd` is what is actually needed.
|
||||
* SteamWorks credentials - Needed to log in using `steamcmd`.
|
||||
* Login to SteamWorks with the `steamcmd` from the command-line at least once -
|
||||
Needded to ensure the user is properly logged in. On a new machine the user
|
||||
will have to go through two-factor authentication.
|
||||
* App ID and Depot IDs - Needed to create the VDF files.
|
||||
* Python 3.x - 3.7 was tested.
|
||||
* Base URL - for downloading the archives.
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
```bash
|
||||
$ export STEAMUSER=SteamUserName
|
||||
$ export STEAMPW=SteamUserPW
|
||||
$ export BASEURL=https://download.blender.org/release/Blender2.83/
|
||||
$ export VERSION=2.83.3
|
||||
$ export APPID=appidnr
|
||||
$ export WINID=winidnr
|
||||
$ export LINID=linuxidnr
|
||||
$ export MACOSID=macosidnr
|
||||
|
||||
# log in to SteamWorks from command-line at least once
|
||||
|
||||
$ ../sdk/tools/ContentBuilder/builder_osx/steamcmd +login $STEAMUSER $STEAMPW
|
||||
|
||||
# once that has been done we can now actually start our tool
|
||||
|
||||
$ python3.7 create_steam_builds.py --baseurl $BASEURL --version $VERSION --appid $APPID --winid $WINID --linuxid $LINID --macosid $MACOSID --steamuser $STEAMUSER --steampw $STEAMPW --steamcmd ../sdk/tools/ContentBuilder/builder_osx/steamcmd
|
||||
```
|
||||
|
||||
All arguments in the above example are required.
|
||||
|
||||
At the start the tool will login using `steamcmd`. This is necessary to let the
|
||||
Steam SDK update itself if necessary.
|
||||
|
||||
There are a few optional arguments:
|
||||
|
||||
* `--dryrun`: If set building the game files will not actually happen. A set of
|
||||
log files and a preview manifest per depot will be created in the output folder.
|
||||
This can be used to double-check everything works as expected.
|
||||
* `--skipdl`: If set will skip downloading of the archives. The tool expects the
|
||||
archives to already exist in the correct content location.
|
||||
* `--skipextract`: If set will skip extraction of all archives. The tool expects
|
||||
the archives to already have been correctly extracted in the content location.
|
||||
|
||||
Run the tool with `-h` for detailed information on each argument.
|
||||
|
||||
The content and output folders are generated through appending the version
|
||||
without dots to the words `content` and `output` respectively, e.g. `content2833`
|
||||
and `output2833`. These folders are created next to the tool.
|
||||
|
||||
From all `.template` files the Steam build scripts will be generated also in the
|
||||
same directory as the tool. The files will have the extension `.vdf`.
|
||||
|
||||
In case of errors the tool will have a non-zero return code.
|
@@ -1,17 +0,0 @@
|
||||
"appbuild"
|
||||
{
|
||||
"appid" "[APPID]"
|
||||
"desc" "Blender [VERSION]" // description for this build
|
||||
"buildoutput" "./[OUTPUT]" // build output folder for .log, .csm & .csd files, relative to location of this file
|
||||
"contentroot" "./[CONTENT]" // root content folder, relative to location of this file
|
||||
"setlive" "" // branch to set live after successful build, non if empty
|
||||
"preview" "[DRYRUN]" // 1 to enable preview builds, 0 to commit build to steampipe
|
||||
"local" "" // set to flie path of local content server
|
||||
|
||||
"depots"
|
||||
{
|
||||
"[WINID]" "depot_build_win.vdf"
|
||||
"[LINUXID]" "depot_build_linux.vdf"
|
||||
"[MACOSID]" "depot_build_macos.vdf"
|
||||
}
|
||||
}
|
@@ -1,397 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Callable, Iterator, List, Tuple
|
||||
|
||||
# supported archive and platform endings, used to create actual archive names
|
||||
archive_endings = ["windows64.zip", "linux64.tar.xz", "macOS.dmg"]
|
||||
|
||||
|
||||
def add_optional_argument(option: str, help: str) -> None:
|
||||
global parser
|
||||
"""Add an optional argument
|
||||
|
||||
Args:
|
||||
option (str): Option to add
|
||||
help (str): Help description for the argument
|
||||
"""
|
||||
parser.add_argument(option, help=help, action='store_const', const=1)
|
||||
|
||||
|
||||
def blender_archives(version: str) -> Iterator[str]:
|
||||
"""Generator for Blender archives for version.
|
||||
|
||||
Yields for items in archive_endings an archive name in the form of
|
||||
blender-{version}-{ending}.
|
||||
|
||||
Args:
|
||||
version (str): Version string of the form 2.83.2
|
||||
|
||||
|
||||
Yields:
|
||||
Iterator[str]: Name in the form of blender-{version}-{ending}
|
||||
"""
|
||||
global archive_endings
|
||||
|
||||
for ending in archive_endings:
|
||||
yield f"blender-{version}-{ending}"
|
||||
|
||||
|
||||
def get_archive_type(archive_type: str, version: str) -> str:
|
||||
"""Return the archive of given type and version.
|
||||
|
||||
Args:
|
||||
archive_type (str): extension for archive type to check for
|
||||
version (str): Version string in the form 2.83.2
|
||||
|
||||
Raises:
|
||||
Exception: Execption when archive type isn't found
|
||||
|
||||
Returns:
|
||||
str: archive name for given type
|
||||
"""
|
||||
|
||||
for archive in blender_archives(version):
|
||||
if archive.endswith(archive_type):
|
||||
return archive
|
||||
raise Exception("Unknown archive type")
|
||||
|
||||
|
||||
def execute_command(cmd: List[str], name: str, errcode: int, cwd=".", capture_output=True) -> str:
|
||||
"""Execute the given command.
|
||||
|
||||
Returns the process stdout upon success if any.
|
||||
|
||||
On error print message the command with name that has failed. Print stdout
|
||||
and stderr of the process if any, and then exit with given error code.
|
||||
|
||||
Args:
|
||||
cmd (List[str]): Command in list format, each argument as their own item
|
||||
name (str): Name of command to use when printing to command-line
|
||||
errcode (int): Error code to use in case of exit()
|
||||
cwd (str, optional): Folder to use as current work directory for command
|
||||
execution. Defaults to ".".
|
||||
capture_output (bool, optional): Whether to capture command output or not.
|
||||
Defaults to True.
|
||||
|
||||
Returns:
|
||||
str: stdout if any, or empty string
|
||||
"""
|
||||
cmd_process = subprocess.run(
|
||||
cmd, capture_output=capture_output, encoding="UTF-8", cwd=cwd)
|
||||
if cmd_process.returncode == 0:
|
||||
if cmd_process.stdout:
|
||||
return cmd_process.stdout
|
||||
else:
|
||||
return ""
|
||||
else:
|
||||
print(f"ERROR: {name} failed.")
|
||||
if cmd_process.stdout:
|
||||
print(cmd_process.stdout)
|
||||
if cmd_process.stderr:
|
||||
print(cmd_process.stderr)
|
||||
exit(errcode)
|
||||
return ""
|
||||
|
||||
|
||||
def download_archives(base_url: str, archives: Callable[[str], Iterator[str]], version: str, dst_dir: pathlib.Path):
|
||||
"""Download archives from the given base_url.
|
||||
|
||||
Archives is a generator for Blender archive names based on version.
|
||||
|
||||
Archive names are appended to the base_url to load from, and appended to
|
||||
dst_dir to save to.
|
||||
|
||||
Args:
|
||||
base_url (str): Base URL to load archives from
|
||||
archives (Callable[[str], Iterator[str]]): Generator for Blender archive
|
||||
names based on version
|
||||
version (str): Version string in the form of 2.83.2
|
||||
dst_dir (pathlib.Path): Download destination
|
||||
"""
|
||||
|
||||
if base_url[-1] != '/':
|
||||
base_url = base_url + '/'
|
||||
|
||||
for archive in archives(version):
|
||||
download_url = f"{base_url}{archive}"
|
||||
target_file = dst_dir.joinpath(archive)
|
||||
download_file(download_url, target_file)
|
||||
|
||||
|
||||
def download_file(from_url: str, to_file: pathlib.Path) -> None:
|
||||
"""Download from_url as to_file.
|
||||
|
||||
Actual downloading will be skipped if --skipdl is given on the command-line.
|
||||
|
||||
Args:
|
||||
from_url (str): Full URL to resource to download
|
||||
to_file (pathlib.Path): Full path to save downloaded resource as
|
||||
"""
|
||||
global args
|
||||
|
||||
if not args.skipdl or not to_file.exists():
|
||||
print(f"Downloading {from_url}")
|
||||
with open(to_file, "wb") as download_zip:
|
||||
response = requests.get(from_url)
|
||||
if response.status_code != requests.codes.ok:
|
||||
print(f"ERROR: failed to download {from_url} (status code: {response.status_code})")
|
||||
exit(1313)
|
||||
download_zip.write(response.content)
|
||||
else:
|
||||
print(f"Downloading {from_url} skipped")
|
||||
print(" ... OK")
|
||||
|
||||
|
||||
def copy_contents_from_dmg_to_path(dmg_file: pathlib.Path, dst: pathlib.Path) -> None:
|
||||
"""Copy the contents of the given DMG file to the destination folder.
|
||||
|
||||
Args:
|
||||
dmg_file (pathlib.Path): Full path to DMG archive to extract from
|
||||
dst (pathlib.Path): Full path to destination to extract to
|
||||
"""
|
||||
hdiutil_attach = ["hdiutil",
|
||||
"attach",
|
||||
"-readonly",
|
||||
f"{dmg_file}"
|
||||
]
|
||||
attached = execute_command(hdiutil_attach, "hdiutil attach", 1)
|
||||
|
||||
# Last line of output is what we want, it is of the form
|
||||
# /dev/somedisk Apple_HFS /Volumes/Blender
|
||||
# We want to retain the mount point, and the folder the mount is
|
||||
# created on. The mounted disk we need for detaching, the folder we
|
||||
# need to be able to copy the contents to where we can use them
|
||||
attachment_items = attached.splitlines()[-1].split()
|
||||
mounted_disk = attachment_items[0]
|
||||
source_location = pathlib.Path(attachment_items[2], "Blender.app")
|
||||
|
||||
print(f"{source_location} -> {dst}")
|
||||
|
||||
shutil.copytree(source_location, dst)
|
||||
|
||||
hdiutil_detach = ["hdiutil",
|
||||
"detach",
|
||||
f"{mounted_disk}"
|
||||
]
|
||||
execute_command(hdiutil_detach, "hdiutil detach", 2)
|
||||
|
||||
|
||||
def create_build_script(template_name: str, vars: List[Tuple[str, str]]) -> pathlib.Path:
|
||||
"""
|
||||
Create the Steam build script
|
||||
|
||||
Use the given template and template variable tuple list.
|
||||
|
||||
Returns pathlib.Path to the created script.
|
||||
|
||||
Args:
|
||||
template_name (str): [description]
|
||||
vars (List[Tuple[str, str]]): [description]
|
||||
|
||||
Returns:
|
||||
pathlib.Path: Full path to the generated script
|
||||
"""
|
||||
build_script = pathlib.Path(".", template_name).read_text()
|
||||
for var in vars:
|
||||
build_script = build_script.replace(var[0], var[1])
|
||||
build_script_file = template_name.replace(".template", "")
|
||||
build_script_path = pathlib.Path(".", build_script_file)
|
||||
build_script_path.write_text(build_script)
|
||||
return build_script_path
|
||||
|
||||
|
||||
def clean_up() -> None:
|
||||
"""Remove intermediate files depending on given command-line arguments
|
||||
"""
|
||||
global content_location, args
|
||||
|
||||
if not args.leavearch and not args.leaveextracted:
|
||||
shutil.rmtree(content_location)
|
||||
|
||||
if args.leavearch and not args.leaveextracted:
|
||||
shutil.rmtree(content_location.joinpath(zip_extract_folder))
|
||||
shutil.rmtree(content_location.joinpath(tarxz_extract_folder))
|
||||
shutil.rmtree(content_location.joinpath(dmg_extract_folder))
|
||||
|
||||
if args.leaveextracted and not args.leavearch:
|
||||
import os
|
||||
os.remove(content_location.joinpath(zipped_blender))
|
||||
os.remove(content_location.joinpath(tarxz_blender))
|
||||
os.remove(content_location.joinpath(dmg_blender))
|
||||
|
||||
|
||||
def extract_archive(archive: str, extract_folder_name: str,
|
||||
cmd: List[str], errcode: int) -> None:
|
||||
"""Extract all files from archive to given folder name.
|
||||
|
||||
Will not extract if
|
||||
target folder already exists, or if --skipextract was given on the
|
||||
command-line.
|
||||
|
||||
Args:
|
||||
archive (str): Archive name to extract
|
||||
extract_folder_name (str): Folder name to extract to
|
||||
cmd (List[str]): Command with arguments to use
|
||||
errcode (int): Error code to use for exit()
|
||||
"""
|
||||
global args, content_location
|
||||
|
||||
extract_location = content_location.joinpath(extract_folder_name)
|
||||
|
||||
pre_extract = set(content_location.glob("*"))
|
||||
|
||||
if not args.skipextract or not extract_location.exists():
|
||||
print(f"Extracting files from {archive}...")
|
||||
cmd.append(content_location.joinpath(archive))
|
||||
execute_command(cmd, cmd[0], errcode, cwd=content_location)
|
||||
# in case we use a non-release archive the naming will be incorrect.
|
||||
# simply rename to expected target name
|
||||
post_extract = set(content_location.glob("*"))
|
||||
diff_extract = post_extract - pre_extract
|
||||
if not extract_location in diff_extract:
|
||||
folder_to_rename = list(diff_extract)[0]
|
||||
folder_to_rename.rename(extract_location)
|
||||
print(" OK")
|
||||
else:
|
||||
print(f"Skipping extraction {archive}!")
|
||||
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument("--baseurl", required=True,
|
||||
help="The base URL for files to download, "
|
||||
"i.e. https://download.blender.org/release/Blender2.83/")
|
||||
|
||||
parser.add_argument("--version", required=True,
|
||||
help="The Blender version to release, in the form 2.83.3")
|
||||
|
||||
parser.add_argument("--appid", required=True,
|
||||
help="The Blender App ID on Steam")
|
||||
parser.add_argument("--winid", required=True,
|
||||
help="The Windows depot ID")
|
||||
parser.add_argument("--linuxid", required=True,
|
||||
help="The Linux depot ID")
|
||||
parser.add_argument("--macosid", required=True,
|
||||
help="The MacOS depot ID")
|
||||
|
||||
parser.add_argument("--steamcmd", required=True,
|
||||
help="Path to the steamcmd")
|
||||
parser.add_argument("--steamuser", required=True,
|
||||
help="The login for the Steam builder user")
|
||||
parser.add_argument("--steampw", required=True,
|
||||
help="Login password for the Steam builder user")
|
||||
|
||||
add_optional_argument("--dryrun",
|
||||
"If set the Steam files will not be uploaded")
|
||||
add_optional_argument("--leavearch",
|
||||
help="If set don't clean up the downloaded archives")
|
||||
add_optional_argument("--leaveextracted",
|
||||
help="If set don't clean up the extraction folders")
|
||||
add_optional_argument("--skipdl",
|
||||
help="If set downloading the archives is skipped if it already exists locally.")
|
||||
add_optional_argument("--skipextract",
|
||||
help="If set skips extracting of archives. The tool assumes the archives"
|
||||
"have already been extracted to their correct locations")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
VERSIONNODOTS = args.version.replace('.', '')
|
||||
OUTPUT = f"output{VERSIONNODOTS}"
|
||||
CONTENT = f"content{VERSIONNODOTS}"
|
||||
|
||||
# ===== set up main locations
|
||||
|
||||
content_location = pathlib.Path(".", CONTENT).absolute()
|
||||
output_location = pathlib.Path(".", OUTPUT).absolute()
|
||||
|
||||
content_location.mkdir(parents=True, exist_ok=True)
|
||||
output_location.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ===== login
|
||||
|
||||
# Logging into Steam once to ensure the SDK updates itself properly. If we don't
|
||||
# do that the combined +login and +run_app_build_http at the end of the tool
|
||||
# will fail.
|
||||
steam_login = [args.steamcmd,
|
||||
"+login",
|
||||
args.steamuser,
|
||||
args.steampw,
|
||||
"+quit"
|
||||
]
|
||||
print("Logging in to Steam...")
|
||||
execute_command(steam_login, "Login to Steam", 10)
|
||||
print(" OK")
|
||||
|
||||
# ===== prepare Steam build scripts
|
||||
|
||||
template_vars = [
|
||||
("[APPID]", args.appid),
|
||||
("[OUTPUT]", OUTPUT),
|
||||
("[CONTENT]", CONTENT),
|
||||
("[VERSION]", args.version),
|
||||
("[WINID]", args.winid),
|
||||
("[LINUXID]", args.linuxid),
|
||||
("[MACOSID]", args.macosid),
|
||||
("[DRYRUN]", f"{args.dryrun}" if args.dryrun else "0")
|
||||
]
|
||||
|
||||
blender_app_build = create_build_script(
|
||||
"blender_app_build.vdf.template", template_vars)
|
||||
create_build_script("depot_build_win.vdf.template", template_vars)
|
||||
create_build_script("depot_build_linux.vdf.template", template_vars)
|
||||
create_build_script("depot_build_macos.vdf.template", template_vars)
|
||||
|
||||
# ===== download archives
|
||||
|
||||
download_archives(args.baseurl, blender_archives,
|
||||
args.version, content_location)
|
||||
|
||||
# ===== set up file and folder names
|
||||
|
||||
zipped_blender = get_archive_type("zip", args.version)
|
||||
zip_extract_folder = zipped_blender.replace(".zip", "")
|
||||
tarxz_blender = get_archive_type("tar.xz", args.version)
|
||||
tarxz_extract_folder = tarxz_blender.replace(".tar.xz", "")
|
||||
dmg_blender = get_archive_type("dmg", args.version)
|
||||
dmg_extract_folder = dmg_blender.replace(".dmg", "")
|
||||
|
||||
# ===== extract
|
||||
|
||||
unzip_cmd = ["unzip", "-q"]
|
||||
extract_archive(zipped_blender, zip_extract_folder, unzip_cmd, 3)
|
||||
|
||||
untarxz_cmd = ["tar", "-xf"]
|
||||
extract_archive(tarxz_blender, tarxz_extract_folder, untarxz_cmd, 4)
|
||||
|
||||
if not args.skipextract or not content_location.joinpath(dmg_extract_folder).exists():
|
||||
print("Extracting files from Blender MacOS archive...")
|
||||
blender_dmg = content_location.joinpath(dmg_blender)
|
||||
target_location = content_location.joinpath(
|
||||
dmg_extract_folder, "Blender.app")
|
||||
copy_contents_from_dmg_to_path(blender_dmg, target_location)
|
||||
print(" OK")
|
||||
else:
|
||||
print("Skipping extraction of .dmg!")
|
||||
|
||||
# ===== building
|
||||
|
||||
print("Build Steam game files...")
|
||||
steam_build = [args.steamcmd,
|
||||
"+login",
|
||||
args.steamuser,
|
||||
args.steampw,
|
||||
"+run_app_build_http",
|
||||
blender_app_build.absolute(),
|
||||
"+quit"
|
||||
]
|
||||
execute_command(steam_build, "Build with steamcmd", 13)
|
||||
print(" OK")
|
||||
|
||||
clean_up()
|
@@ -1,31 +0,0 @@
|
||||
"DepotBuildConfig"
|
||||
{
|
||||
// Set your assigned depot ID here
|
||||
"DepotID" "[LINUXID]"
|
||||
|
||||
// Set a root for all content.
|
||||
// All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths)
|
||||
// will be resolved relative to this root.
|
||||
// If you don't define ContentRoot, then it will be assumed to be
|
||||
// the location of this script file, which probably isn't what you want
|
||||
"ContentRoot" "./blender-[VERSION]-linux64/"
|
||||
|
||||
// include all files recursivley
|
||||
"FileMapping"
|
||||
{
|
||||
// This can be a full path, or a path relative to ContentRoot
|
||||
"LocalPath" "*"
|
||||
|
||||
// This is a path relative to the install folder of your game
|
||||
"DepotPath" "."
|
||||
|
||||
// If LocalPath contains wildcards, setting this means that all
|
||||
// matching files within subdirectories of LocalPath will also
|
||||
// be included.
|
||||
"recursive" "1"
|
||||
}
|
||||
|
||||
// but exclude all symbol files
|
||||
// This can be a full path, or a path relative to ContentRoot
|
||||
"FileExclusion" "*.pdb"
|
||||
}
|
@@ -1,30 +0,0 @@
|
||||
"DepotBuildConfig"
|
||||
{
|
||||
// Set your assigned depot ID here
|
||||
"DepotID" "[MACOSID]"
|
||||
|
||||
// Set a root for all content.
|
||||
// All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths)
|
||||
// will be resolved relative to this root.
|
||||
// If you don't define ContentRoot, then it will be assumed to be
|
||||
// the location of this script file, which probably isn't what you want
|
||||
"ContentRoot" "./blender-[VERSION]-macOS/"
|
||||
// include all files recursivley
|
||||
"FileMapping"
|
||||
{
|
||||
// This can be a full path, or a path relative to ContentRoot
|
||||
"LocalPath" "*"
|
||||
|
||||
// This is a path relative to the install folder of your game
|
||||
"DepotPath" "."
|
||||
|
||||
// If LocalPath contains wildcards, setting this means that all
|
||||
// matching files within subdirectories of LocalPath will also
|
||||
// be included.
|
||||
"recursive" "1"
|
||||
}
|
||||
|
||||
// but exclude all symbol files
|
||||
// This can be a full path, or a path relative to ContentRoot
|
||||
"FileExclusion" "*.pdb"
|
||||
}
|
@@ -1,31 +0,0 @@
|
||||
"DepotBuildConfig"
|
||||
{
|
||||
// Set your assigned depot ID here
|
||||
"DepotID" "[WINID]"
|
||||
|
||||
// Set a root for all content.
|
||||
// All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths)
|
||||
// will be resolved relative to this root.
|
||||
// If you don't define ContentRoot, then it will be assumed to be
|
||||
// the location of this script file, which probably isn't what you want
|
||||
"ContentRoot" "./blender-[VERSION]-windows64/"
|
||||
|
||||
// include all files recursivley
|
||||
"FileMapping"
|
||||
{
|
||||
// This can be a full path, or a path relative to ContentRoot
|
||||
"LocalPath" "*"
|
||||
|
||||
// This is a path relative to the install folder of your game
|
||||
"DepotPath" "."
|
||||
|
||||
// If LocalPath contains wildcards, setting this means that all
|
||||
// matching files within subdirectories of LocalPath will also
|
||||
// be included.
|
||||
"recursive" "1"
|
||||
}
|
||||
|
||||
// but exclude all symbol files
|
||||
// This can be a full path, or a path relative to ContentRoot
|
||||
"FileExclusion" "*.pdb"
|
||||
}
|
@@ -1,82 +1,4 @@
|
||||
create_msix_package
|
||||
===================
|
||||
Buildbot Configuration
|
||||
======================
|
||||
|
||||
This tool is used to create MSIX packages from a given ZiP archive. The MSIX
|
||||
package is distributed mainly through the Microsoft Store. It can also be
|
||||
installed when downloaded from blender.org. For that to work the MSIX package
|
||||
needs to be signed.
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* MakeAppX.exe - this tool is distributed with the Windows 10 SDK and is used to build the .appx package.
|
||||
* MakePri.exe - this tool is distributed with the Windows 10 SDK and is used to generate a resources file.
|
||||
* SignTool.exe - this tool is distributed with the Windows 10 SDK and is used to sign the .appx package.
|
||||
* Python 3 (3.7 or later tested) - to run the create_msix_package.py script
|
||||
* requests module - can be installed with `pip install requests`
|
||||
* PFX file (optional, but strongly recommended) - for signing the resulting MSIX
|
||||
package. **NOTE:** If the MSIX package is not signed when uploaded to the Microsoft
|
||||
store the validation and certification process can take up to three full
|
||||
business day.
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
On the command-line:
|
||||
```batch
|
||||
set VERSION=2.83.4.0
|
||||
set URL=https://download.blender.org/release/Blender2.83/blender-2.83.4-windows64.zip
|
||||
set PUBID=CN=PUBIDHERE
|
||||
set PFX=X:\path\to\cert.pfx
|
||||
set PFXPW=pwhere
|
||||
|
||||
python create_msix_package.py --version %VERSION% --url %URL% --publisher %PUBID% --pfx %PFX% --password %PFXPW%
|
||||
```
|
||||
|
||||
Result will be a MSIX package with the name `blender-2.83.4-windows64.msix`.
|
||||
With the above usage it will be signed. If the signing options are left out the
|
||||
package will not be signed.
|
||||
|
||||
Optional arguments
|
||||
==================
|
||||
|
||||
In support of testing and developing the manifest and scripts there are a few
|
||||
optional arguments:
|
||||
|
||||
* `--skipdl` : If a `blender.zip` is available already next to the tool use this
|
||||
to skip actual downloading of the archive designated by `--url`. The latter
|
||||
option is still required
|
||||
* `--overwrite` : When script fails the final clean-up may be incomplete leaving
|
||||
the `Content` folder with its structure. Specify this argument to automatically
|
||||
clean up this folder before starting to seed the `Content` folder
|
||||
* `--leavezip` : When specified leave the `blender.zip` file while cleaning up
|
||||
all other intermediate files, including the `Content` folder. This is useful
|
||||
to not have to re-download the same archive from `--url` on each usage
|
||||
|
||||
|
||||
What it does
|
||||
============
|
||||
|
||||
The tool creates in the directory it lives a subfolder called `Content`. This is
|
||||
where all necessary files are placed.
|
||||
|
||||
The `Assets` folder is copied to the `Content` folder.
|
||||
|
||||
From the application manifest template a version with necessary parts replaced as
|
||||
their actual values as specified on the command-line is realized. This manifest controls the packaging of Blender into the MSIX format.
|
||||
|
||||
Next the tool downloads the designated ZIP archive locally as blender.zip. From
|
||||
this archive the files are extracted into the `Content\Blender` folder, but skip
|
||||
the leading part of paths in the ZIP. We want to write the files to the
|
||||
content_blender_folder where blender.exe ends up as
|
||||
`Content\Blender\blender.exe`, and not
|
||||
`Content\Blender\blender-2.83.4-windows64\blender.exe`
|
||||
|
||||
Once the extraction is completed the MakeAppX tool is executed with the `Content`
|
||||
folder as input. The result will be the MSIX package with the name in the form
|
||||
`blender-X.YY.Z-windows64.msix`.
|
||||
|
||||
If the PFX file and its password are given on the command-line this MSIX package
|
||||
will be signed.
|
||||
|
||||
All intermediate files and directories will be removed.
|
||||
Files used by Buildbot's `package-code-store-windows` step.
|
||||
|
@@ -1,197 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
required=True,
|
||||
help="Version string in the form of 2.83.3.0",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
required=True,
|
||||
help="Location of the release ZIP archive to download",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--publisher",
|
||||
required=True,
|
||||
help="A string in the form of 'CN=PUBLISHER'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pfx",
|
||||
required=False,
|
||||
help="Absolute path to the PFX file used for signing the resulting MSIX package",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--password",
|
||||
required=False,
|
||||
default="blender",
|
||||
help="Password for the PFX file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lts",
|
||||
required=False,
|
||||
help="If set this MSIX is for an LTS release",
|
||||
action='store_const',
|
||||
const=1,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skipdl",
|
||||
required=False,
|
||||
help="If set skip downloading of the specified URL as blender.zip. The tool assumes blender.zip exists",
|
||||
action='store_const',
|
||||
const=1,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--leavezip",
|
||||
required=False,
|
||||
help="If set don't clean up the downloaded blender.zip",
|
||||
action='store_const',
|
||||
const=1,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overwrite",
|
||||
required=False,
|
||||
help="If set remove Content folder if it already exists",
|
||||
action='store_const',
|
||||
const=1,
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def execute_command(cmd: list, name: str, errcode: int):
|
||||
"""
|
||||
Execute given command in cmd. Output is captured. If an error
|
||||
occurs name is used to print ERROR message, along with stderr and
|
||||
stdout of the process if either was captured.
|
||||
"""
|
||||
cmd_process = subprocess.run(cmd, capture_output=True, encoding="UTF-8")
|
||||
if cmd_process.returncode != 0:
|
||||
print(f"ERROR: {name} failed.")
|
||||
if cmd_process.stdout:
|
||||
print(cmd_process.stdout)
|
||||
if cmd_process.stderr:
|
||||
print(cmd_process.stderr)
|
||||
exit(errcode)
|
||||
|
||||
|
||||
LTSORNOT = ""
|
||||
PACKAGETYPE = ""
|
||||
if args.lts:
|
||||
versionparts = args.version.split(".")
|
||||
LTSORNOT = f" {versionparts[0]}.{versionparts[1]} LTS"
|
||||
PACKAGETYPE = f"{versionparts[0]}.{versionparts[1]}LTS"
|
||||
|
||||
blender_package_msix = pathlib.Path(".", f"blender-{args.version}-windows64.msix").absolute()
|
||||
content_folder = pathlib.Path(".", "Content")
|
||||
content_blender_folder = pathlib.Path(content_folder, "Blender").absolute()
|
||||
content_assets_folder = pathlib.Path(content_folder, "Assets")
|
||||
assets_original_folder = pathlib.Path(".", "Assets")
|
||||
|
||||
pri_config_file = pathlib.Path(".", "priconfig.xml")
|
||||
pri_resources_file = pathlib.Path(content_folder, "resources.pri")
|
||||
|
||||
local_blender_zip = pathlib.Path(".", "blender.zip")
|
||||
|
||||
if args.pfx:
|
||||
pfx_path = pathlib.Path(args.pfx)
|
||||
if not pfx_path.exists():
|
||||
print("ERROR: PFX file not found. Please ensure you give the correct path to the PFX file on the command-line.")
|
||||
exit(1)
|
||||
print(f"Creating MSIX package with signing using PFX file at {pfx_path}")
|
||||
else:
|
||||
pfx_path = None
|
||||
print("Creating MSIX package without signing.")
|
||||
|
||||
pri_command = ["makepri",
|
||||
"new",
|
||||
"/pr", f"{content_folder.absolute()}",
|
||||
"/cf", f"{pri_config_file.absolute()}",
|
||||
"/of", f"{pri_resources_file.absolute()}"
|
||||
]
|
||||
|
||||
msix_command = ["makeappx",
|
||||
"pack",
|
||||
"/h", "SHA256",
|
||||
"/d", f"{content_folder.absolute()}",
|
||||
"/p", f"{blender_package_msix}"
|
||||
]
|
||||
if pfx_path:
|
||||
sign_command = ["signtool",
|
||||
"sign",
|
||||
"/fd", "sha256",
|
||||
"/a", "/f", f"{pfx_path.absolute()}",
|
||||
"/p", f"{args.password}",
|
||||
f"{blender_package_msix}"
|
||||
]
|
||||
|
||||
if args.overwrite:
|
||||
if content_folder.joinpath("Assets").exists():
|
||||
shutil.rmtree(content_folder)
|
||||
content_folder.mkdir(exist_ok=True)
|
||||
shutil.copytree(assets_original_folder, content_assets_folder)
|
||||
|
||||
manifest_text = pathlib.Path("AppxManifest.xml.template").read_text()
|
||||
manifest_text = manifest_text.replace("[VERSION]", args.version)
|
||||
manifest_text = manifest_text.replace("[PUBLISHER]", args.publisher)
|
||||
manifest_text = manifest_text.replace("[LTSORNOT]", LTSORNOT)
|
||||
manifest_text = manifest_text.replace("[PACKAGETYPE]", PACKAGETYPE)
|
||||
pathlib.Path(content_folder, "AppxManifest.xml").write_text(manifest_text)
|
||||
|
||||
if not args.skipdl:
|
||||
print(f"Downloading blender archive {args.url} to {local_blender_zip}...")
|
||||
|
||||
with open(local_blender_zip, "wb") as download_zip:
|
||||
response = requests.get(args.url)
|
||||
download_zip.write(response.content)
|
||||
|
||||
print("... download complete.")
|
||||
else:
|
||||
print("Skipping download")
|
||||
|
||||
print(f"Extracting files from ZIP to {content_blender_folder}...")
|
||||
|
||||
# Extract the files from the ZIP archive, but skip the leading part of paths
|
||||
# in the ZIP. We want to write the files to the content_blender_folder where
|
||||
# blender.exe ends up as ./Content/Blender/blender.exe, and not
|
||||
# ./Content/Blender/blender-2.83.3-windows64/blender.exe
|
||||
with zipfile.ZipFile(local_blender_zip, "r") as blender_zip:
|
||||
for entry in blender_zip.infolist():
|
||||
if entry.is_dir():
|
||||
continue
|
||||
entry_location = pathlib.Path(entry.filename)
|
||||
target_location = content_blender_folder.joinpath(*entry_location.parts[1:])
|
||||
pathlib.Path(target_location.parent).mkdir(parents=True, exist_ok=True)
|
||||
extracted_entry = blender_zip.read(entry)
|
||||
target_location.write_bytes(extracted_entry)
|
||||
|
||||
print("... extraction complete.")
|
||||
|
||||
|
||||
print(f"Generating Package Resource Index (PRI) file using command: {' '.join(pri_command)}")
|
||||
execute_command(pri_command, "MakePri", 4)
|
||||
|
||||
print(f"Creating MSIX package using command: {' '.join(msix_command)}")
|
||||
|
||||
# Remove MSIX file if it already exists. Otherwise the MakeAppX tool
|
||||
# will hang.
|
||||
if blender_package_msix.exists():
|
||||
os.remove(blender_package_msix)
|
||||
execute_command(msix_command, "MakeAppX", 2)
|
||||
|
||||
if args.pfx:
|
||||
print(f"Signing MSIX package using command: {' '.join(sign_command)}")
|
||||
execute_command(sign_command, "SignTool", 3)
|
||||
|
||||
if not args.leavezip:
|
||||
os.remove(local_blender_zip)
|
||||
shutil.rmtree(content_folder)
|
||||
|
||||
print("Done.")
|
Reference in New Issue
Block a user