From 087cc0808a342ebcba08d60a3c5dc2960b0bda14 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 30 Apr 2012 07:54:00 -0700 Subject: [PATCH] Make SQL patch management DAG-based and provide namespace support Summary: This addresses three issues with the current patch management system: # Two people developing at the same time often pick the same SQL patch number, and then have to go rename it. The system catches this, but it's silly. # Second/third-party developers can't use the same system to manage auxiliary storage they may want to add. # There's no way to build mock databases for unit tests that need to do reads. To resolve these things, you can now name your patches whatever you want and conflicts are just merge conflicts, which are less of a pain to fix than filename conflicts. Dependencies are now a DAG, with implicit dependencies created on the prior patch if no dependencies are specified. Developers can add new concrete subclasses of `PhabricatorSQLPatchList` to add storage management, and define the dependency branchpoint of their patches so they apply in the correct order (although, generally, they should not depend on the mainline patches, presumably). The commands `storage upgrade --namespace test1234` and `storage destroy --namespace test1234` will allow unit tests to build and destroy MySQL storage. A "quickstart" mode allows an upgrade from scratch in ~1200ms. Destruction takes about 200ms. These seem like fairily reasonable costs to actually use in tests. Building from scratch patch-by-patch takes about 6000ms. Test Plan: - Created new databases from scratch with and without quickstart in a separate test namespace. Pointed the webapp at the test namespaces, browsed around, everything looked good. - Compared quickstart and no-quickstart dump states, they're identical except for mysqldump timestamps and a few similar things. - Upgraded a legacy database to the new storage format. - Destroyed / dumped storage. Reviewers: edward, vrana, btrahan, jungejason Reviewed By: btrahan CC: aran, nh Maniphest Tasks: T140, T345 Differential Revision: https://secure.phabricator.com/D2323 --- bin/storage | 1 + scripts/__init_script__.php | 12 +- scripts/install/update_phabricator.sh | 2 +- scripts/sql/manage_storage.php | 123 +++ scripts/sql/upgrade_schema.php | 208 +---- src/__phutil_library_map__.php | 18 +- .../base/DatabaseConfigurationProvider.php | 6 +- .../DefaultDatabaseConfigurationProvider.php | 10 +- .../base/storage/lisk/PhabricatorLiskDAO.php | 10 +- .../configuration/configuration_guide.diviner | 55 +- src/docs/configuration/upgrade_schema.diviner | 36 - src/docs/installation_guide.diviner | 13 +- src/infrastructure/setup/PhabricatorSetup.php | 23 +- src/infrastructure/setup/__init__.php | 1 - .../setup/sql/PhabricatorSQLPatchList.php | 62 -- src/infrastructure/setup/sql/__init__.php | 5 - .../sql/base/PhabricatorSQLPatchList.php | 164 ++++ .../setup/sql/base/__init__.php | 15 + .../PhabricatorBuiltinPatchList.php | 853 ++++++++++++++++++ .../setup/sql/phabricator/__init__.php | 15 + .../PhabricatorStorageManagementAPI.php | 193 ++++ .../setup/storage/management/__init__.php | 17 + .../storage/patch/PhabricatorStoragePatch.php | 61 ++ .../setup/storage/patch/__init__.php | 10 + .../PhabricatorStorageManagementWorkflow.php | 48 + .../setup/storage/workflow/base/__init__.php | 13 + ...atorStorageManagementDatabasesWorkflow.php | 39 + .../storage/workflow/databases/__init__.php | 12 + ...icatorStorageManagementDestroyWorkflow.php | 74 ++ .../storage/workflow/destroy/__init__.php | 15 + ...abricatorStorageManagementDumpWorkflow.php | 83 ++ .../setup/storage/workflow/dump/__init__.php | 17 + ...ricatorStorageManagementStatusWorkflow.php | 67 ++ .../storage/workflow/status/__init__.php | 14 + ...icatorStorageManagementUpgradeWorkflow.php | 209 +++++ .../storage/workflow/upgrade/__init__.php | 16 + .../schema/AphrontQuerySchemaException.php | 2 +- 37 files changed, 2168 insertions(+), 354 deletions(-) create mode 120000 bin/storage create mode 100755 scripts/sql/manage_storage.php delete mode 100644 src/docs/configuration/upgrade_schema.diviner delete mode 100644 src/infrastructure/setup/sql/PhabricatorSQLPatchList.php create mode 100644 src/infrastructure/setup/sql/base/PhabricatorSQLPatchList.php create mode 100644 src/infrastructure/setup/sql/base/__init__.php create mode 100644 src/infrastructure/setup/sql/phabricator/PhabricatorBuiltinPatchList.php create mode 100644 src/infrastructure/setup/sql/phabricator/__init__.php create mode 100644 src/infrastructure/setup/storage/management/PhabricatorStorageManagementAPI.php create mode 100644 src/infrastructure/setup/storage/management/__init__.php create mode 100644 src/infrastructure/setup/storage/patch/PhabricatorStoragePatch.php create mode 100644 src/infrastructure/setup/storage/patch/__init__.php create mode 100644 src/infrastructure/setup/storage/workflow/base/PhabricatorStorageManagementWorkflow.php create mode 100644 src/infrastructure/setup/storage/workflow/base/__init__.php create mode 100644 src/infrastructure/setup/storage/workflow/databases/PhabricatorStorageManagementDatabasesWorkflow.php create mode 100644 src/infrastructure/setup/storage/workflow/databases/__init__.php create mode 100644 src/infrastructure/setup/storage/workflow/destroy/PhabricatorStorageManagementDestroyWorkflow.php create mode 100644 src/infrastructure/setup/storage/workflow/destroy/__init__.php create mode 100644 src/infrastructure/setup/storage/workflow/dump/PhabricatorStorageManagementDumpWorkflow.php create mode 100644 src/infrastructure/setup/storage/workflow/dump/__init__.php create mode 100644 src/infrastructure/setup/storage/workflow/status/PhabricatorStorageManagementStatusWorkflow.php create mode 100644 src/infrastructure/setup/storage/workflow/status/__init__.php create mode 100644 src/infrastructure/setup/storage/workflow/upgrade/PhabricatorStorageManagementUpgradeWorkflow.php create mode 100644 src/infrastructure/setup/storage/workflow/upgrade/__init__.php diff --git a/bin/storage b/bin/storage new file mode 120000 index 0000000000..788528eb8d --- /dev/null +++ b/bin/storage @@ -0,0 +1 @@ +../scripts/sql/manage_storage.php \ No newline at end of file diff --git a/scripts/__init_script__.php b/scripts/__init_script__.php index e2551149c3..ea51f78e03 100644 --- a/scripts/__init_script__.php +++ b/scripts/__init_script__.php @@ -42,7 +42,17 @@ $env = isset($_SERVER['PHABRICATOR_ENV']) ? $_SERVER['PHABRICATOR_ENV'] : getenv('PHABRICATOR_ENV'); if (!$env) { - echo "Define PHABRICATOR_ENV before running this script.\n"; + phutil_require_module('phutil', 'console'); + echo phutil_console_wrap( + phutil_console_format( + "**ERROR**: PHABRICATOR_ENV Not Set\n\n". + "Define the __PHABRICATOR_ENV__ environment variable before running ". + "this script. You can do it on the command line like this:\n\n". + " $ PHABRICATOR_ENV=__custom/myconfig__ %s ...\n\n". + "Replace __custom/myconfig__ with the path to your configuration file. ". + "For more information, see the 'Configuration Guide' in the ". + "Phabricator documentation.\n\n", + $argv[0])); exit(1); } diff --git a/scripts/install/update_phabricator.sh b/scripts/install/update_phabricator.sh index c30b6ad385..77925e9cc9 100755 --- a/scripts/install/update_phabricator.sh +++ b/scripts/install/update_phabricator.sh @@ -49,7 +49,7 @@ $ROOT/phabricator/bin/phd stop sudo /etc/init.d/httpd stop # Upgrade the database schema. -$ROOT/phabricator/scripts/sql/upgrade_schema.php -f +$ROOT/phabricator/bin/storage upgrade --force # Restart apache. sudo /etc/init.d/httpd start diff --git a/scripts/sql/manage_storage.php b/scripts/sql/manage_storage.php new file mode 100755 index 0000000000..e2ed32cc02 --- /dev/null +++ b/scripts/sql/manage_storage.php @@ -0,0 +1,123 @@ +#!/usr/bin/env php +setTagline('manage Phabricator storage and schemata'); +$args->setSynopsis(<<parseStandardArguments(); + +$conf = PhabricatorEnv::newObjectFromConfig('mysql.configuration-provider'); + +$default_user = $conf->getUser(); +$default_password = $conf->getPassword(); +$default_host = $conf->getHost(); +$default_namespace = 'phabricator'; + +try { + $args->parsePartial( + array( + array( + 'name' => 'force', + 'short' => 'f', + 'help' => 'Do not prompt before performing dangerous operations.', + ), + array( + 'name' => 'user', + 'short' => 'u', + 'param' => 'username', + 'default' => $default_user, + 'help' => "Connect with __username__ instead of the configured ". + "default ('{$default_user}').", + ), + array( + 'name' => 'password', + 'short' => 'p', + 'param' => 'password', + 'default' => $default_password, + 'help' => 'Use __password__ instead of the configured default.', + ), + array( + 'name' => 'namespace', + 'param' => 'name', + 'default' => $default_namespace, + 'help' => "Use namespace __namespace__ instead of the configured ". + "default ('{$default_namespace}'). This is an advanced ". + "feature used by unit tests; you should not normally ". + "use this flag.", + ), + array( + 'name' => 'dryrun', + 'help' => 'Do not actually change anything, just show what would be '. + 'changed.', + ), + )); +} catch (PhutilArgumentUsageException $ex) { + $args->printUsageException($ex); + exit(77); +} + +$api = new PhabricatorStorageManagementAPI(); +$api->setUser($args->getArg('user')); +$api->setHost($default_host); +$api->setPassword($args->getArg('password')); +$api->setNamespace($args->getArg('namespace')); + +try { + queryfx( + $api->getConn('meta_data', $select_database = false), + 'SELECT 1'); +} catch (AphrontQueryException $ex) { + echo phutil_console_format( + "**%s**: %s\n", + 'Unable To Connect', + $ex->getMessage()); + exit(1); +} + +$workflows = array( + new PhabricatorStorageManagementDatabasesWorkflow(), + new PhabricatorStorageManagementDestroyWorkflow(), + new PhabricatorStorageManagementDumpWorkflow(), + new PhabricatorStorageManagementStatusWorkflow(), + new PhabricatorStorageManagementUpgradeWorkflow(), +); + +$patches = PhabricatorSQLPatchList::buildAllPatches(); + +foreach ($workflows as $workflow) { + $workflow->setAPI($api); + $workflow->setPatches($patches); +} + +$workflows[] = new PhutilHelpArgumentWorkflow(); + +$args->parseWorkflows($workflows); diff --git a/scripts/sql/upgrade_schema.php b/scripts/sql/upgrade_schema.php index 1eb362a0ef..16383493f9 100755 --- a/scripts/sql/upgrade_schema.php +++ b/scripts/sql/upgrade_schema.php @@ -17,204 +17,10 @@ * limitations under the License. */ -$root = dirname(dirname(dirname(__FILE__))); -require_once $root.'/scripts/__init_script__.php'; - -phutil_require_module('phutil', 'console'); -phutil_require_module('phabricator', 'infrastructure/setup/sql'); - -define('SCHEMA_VERSION_TABLE_NAME', 'schema_version'); - -// TODO: getopt() is super terrible, move to something less terrible. -$options = getopt('fhdv:u:p:m:') + array( - 'v' => null, // Upgrade from specific version - 'u' => null, // Override MySQL User - 'p' => null, // Override MySQL Pass - 'm' => null, // Specify max version to upgrade to -); - -foreach (array('h', 'f', 'd') as $key) { - // By default, these keys are set to 'false' to indicate that the flag was - // passed. - if (array_key_exists($key, $options)) { - $options[$key] = true; - } -} - -if (!empty($options['h']) || ($options['v'] && !is_numeric($options['v'])) - || ($options['m'] && !is_numeric($options['m']))) { - usage(); -} - -if (empty($options['f']) && empty($options['d'])) { - echo phutil_console_wrap( - "Before running this script, you should take down the Phabricator web ". - "interface and stop any running Phabricator daemons."); - - if (!phutil_console_confirm('Are you ready to continue?')) { - echo "Cancelled.\n"; - exit(1); - } -} - -// Use always the version from the commandline if it is defined -$next_version = isset($options['v']) ? (int)$options['v'] : null; -$max_version = isset($options['m']) ? (int)$options['m'] : null; - -$conf = PhabricatorEnv::newObjectFromConfig('mysql.configuration-provider'); - -if ($options['u']) { - $conn_user = $options['u']; - $conn_pass = $options['p']; -} else { - $conn_user = $conf->getUser(); - $conn_pass = $conf->getPassword(); -} -$conn_host = $conf->getHost(); - -// Split out port information, since the command-line client requires a -// separate flag for the port. -$uri = new PhutilURI('mysql://'.$conn_host); -if ($uri->getPort()) { - $conn_port = $uri->getPort(); - $conn_bare_hostname = $uri->getDomain(); -} else { - $conn_port = null; - $conn_bare_hostname = $conn_host; -} - -$conn = PhabricatorEnv::newObjectFromConfig( - 'mysql.implementation', - array( - array( - 'user' => $conn_user, - 'pass' => $conn_pass, - 'host' => $conn_host, - 'database' => null, - ), - )); - -try { - - $create_sql = << $max_version) { - continue; - } - - $short_name = basename($patch['path']); - print "Applying patch {$short_name}...\n"; - - if (!empty($options['d'])) { - $patch_applied = true; - continue; - } - - if ($conn_port) { - $port = '--port='.(int)$conn_port; - } else { - $port = null; - } - - if (preg_match('/\.php$/', $patch['path'])) { - $schema_conn = $conn; - require_once $patch['path']; - } else { - list($stdout, $stderr) = execx( - "mysql --user=%s --password=%s --host=%s {$port} ". - "--default-character-set=utf8 < %s", - $conn_user, - $conn_pass, - $conn_bare_hostname, - $patch['path']); - - if ($stderr) { - print $stderr; - exit(-1); - } - } - - // Patch was successful, update the db with the latest applied patch version - // 'DELETE' and 'INSERT' instead of update, because the table might be empty - queryfx( - $conn, - 'DELETE FROM phabricator_meta_data.%T', - SCHEMA_VERSION_TABLE_NAME); - queryfx( - $conn, - 'INSERT INTO phabricator_meta_data.%T VALUES (%d)', - SCHEMA_VERSION_TABLE_NAME, - $patch['version']); - - $patch_applied = true; - } - - if (!$patch_applied) { - print "Your database is already up-to-date.\n"; - } - -} catch (AphrontQueryAccessDeniedException $ex) { - echo - "ACCESS DENIED\n". - "The user '{$conn_user}' does not have sufficient MySQL privileges to\n". - "execute the schema upgrade. Use the -u and -p flags to run as a user\n". - "with more privileges (e.g., root).". - "\n\n". - "EXCEPTION:\n". - $ex->getMessage(). - "\n\n"; - exit(1); -} - -function usage() { - echo - "usage: upgrade_schema.php [-v version] [-u user -p pass] [-f] [-h]". - "\n\n". - "Run 'upgrade_schema.php -u root -p hunter2' to override the configured ". - "default user.\n". - "Run 'upgrade_schema.php -v 12' to apply all patches starting from ". - "version 12. It is very unlikely you need to do this.\n". - "Run 'upgrade_schema.php -m 110' to apply all patches up to and ". - "including version 110 (but nothing past).\n". - "Use the -f flag to upgrade noninteractively, without prompting.\n". - "Use the -d flag to do a dry run - patches that would be applied ". - "will be listed, but not applied.\n". - "Use the -h flag to show this help.\n"; - exit(1); -} - +echo "This script has been replaced by 'phabricator/bin/storage'.\n\n". + "Run:\n\n". + " phabricator/bin $ ./storage help\n\n". + "...for help, or:\n\n". + " phabricator/bin $ ./storage upgrade\n\n". + "...to upgrade storage.\n\n"; +exit(1); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index bd9c8e03f7..7f46fddf6f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -520,6 +520,7 @@ phutil_register_library_map(array( 'PhabricatorAuditReplyHandler' => 'applications/audit/replyhandler', 'PhabricatorAuditStatusConstants' => 'applications/audit/constants/status', 'PhabricatorAuthController' => 'applications/auth/controller/base', + 'PhabricatorBuiltinPatchList' => 'infrastructure/setup/sql/phabricator', 'PhabricatorCalendarBrowseController' => 'applications/calendar/controller/browse', 'PhabricatorCalendarController' => 'applications/calendar/controller/base', 'PhabricatorChangesetResponse' => 'infrastructure/diff/response', @@ -851,7 +852,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__', 'PhabricatorRepositoryType' => 'applications/repository/constants/repositorytype', 'PhabricatorS3FileStorageEngine' => 'applications/files/engine/s3', - 'PhabricatorSQLPatchList' => 'infrastructure/setup/sql', + 'PhabricatorSQLPatchList' => 'infrastructure/setup/sql/base', 'PhabricatorScopedEnv' => 'infrastructure/env', 'PhabricatorSearchAbstractDocument' => 'applications/search/index/abstractdocument', 'PhabricatorSearchAttachController' => 'applications/search/controller/attach', @@ -893,6 +894,14 @@ phutil_register_library_map(array( 'PhabricatorSortTableExample' => 'applications/uiexample/examples/sorttable', 'PhabricatorStandardPageView' => 'view/page/standard', 'PhabricatorStatusController' => 'applications/status/base', + 'PhabricatorStorageManagementAPI' => 'infrastructure/setup/storage/management', + 'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/setup/storage/workflow/databases', + 'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/setup/storage/workflow/destroy', + 'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/setup/storage/workflow/dump', + 'PhabricatorStorageManagementStatusWorkflow' => 'infrastructure/setup/storage/workflow/status', + 'PhabricatorStorageManagementUpgradeWorkflow' => 'infrastructure/setup/storage/workflow/upgrade', + 'PhabricatorStorageManagementWorkflow' => 'infrastructure/setup/storage/workflow/base', + 'PhabricatorStoragePatch' => 'infrastructure/setup/storage/patch', 'PhabricatorSymbolNameLinter' => 'infrastructure/lint/hook/xhpastsymbolname', 'PhabricatorSyntaxHighlighter' => 'applications/markup/syntax', 'PhabricatorTaskmasterDaemon' => 'infrastructure/daemon/workers/taskmaster', @@ -1435,6 +1444,7 @@ phutil_register_library_map(array( 'PhabricatorAuditPreviewController' => 'PhabricatorAuditController', 'PhabricatorAuditReplyHandler' => 'PhabricatorMailReplyHandler', 'PhabricatorAuthController' => 'PhabricatorController', + 'PhabricatorBuiltinPatchList' => 'PhabricatorSQLPatchList', 'PhabricatorCalendarBrowseController' => 'PhabricatorCalendarController', 'PhabricatorCalendarController' => 'PhabricatorController', 'PhabricatorChangesetResponse' => 'AphrontProxyResponse', @@ -1742,6 +1752,12 @@ phutil_register_library_map(array( 'PhabricatorSortTableExample' => 'PhabricatorUIExample', 'PhabricatorStandardPageView' => 'AphrontPageView', 'PhabricatorStatusController' => 'PhabricatorController', + 'PhabricatorStorageManagementDatabasesWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementStatusWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementUpgradeWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementWorkflow' => 'PhutilArgumentWorkflow', 'PhabricatorSymbolNameLinter' => 'ArcanistXHPASTLintNamingHook', 'PhabricatorTaskmasterDaemon' => 'PhabricatorDaemon', 'PhabricatorTestCase' => 'ArcanistPhutilTestCase', diff --git a/src/applications/base/storage/configuration/base/DatabaseConfigurationProvider.php b/src/applications/base/storage/configuration/base/DatabaseConfigurationProvider.php index 44baabed2c..d90b115ce8 100644 --- a/src/applications/base/storage/configuration/base/DatabaseConfigurationProvider.php +++ b/src/applications/base/storage/configuration/base/DatabaseConfigurationProvider.php @@ -21,7 +21,11 @@ */ interface DatabaseConfigurationProvider { - public function __construct(LiskDAO $dao = null, $mode = 'r'); + public function __construct( + LiskDAO $dao = null, + $mode = 'r', + $namespace = 'phabricator'); + public function getUser(); public function getPassword(); public function getHost(); diff --git a/src/applications/base/storage/configuration/default/DefaultDatabaseConfigurationProvider.php b/src/applications/base/storage/configuration/default/DefaultDatabaseConfigurationProvider.php index a55ddfbcf3..55b8989b26 100644 --- a/src/applications/base/storage/configuration/default/DefaultDatabaseConfigurationProvider.php +++ b/src/applications/base/storage/configuration/default/DefaultDatabaseConfigurationProvider.php @@ -21,10 +21,16 @@ final class DefaultDatabaseConfigurationProvider private $dao; private $mode; + private $namespace; + + public function __construct( + LiskDAO $dao = null, + $mode = 'r', + $namespace = 'phabricator') { - public function __construct(LiskDAO $dao = null, $mode = 'r') { $this->dao = $dao; $this->mode = $mode; + $this->namespace = $namespace; } public function getUser() { @@ -43,7 +49,7 @@ final class DefaultDatabaseConfigurationProvider if (!$this->getDao()) { return null; } - return 'phabricator_'.$this->getDao()->getApplicationName(); + return $this->namespace.'_'.$this->getDao()->getApplicationName(); } final protected function getDao() { diff --git a/src/applications/base/storage/lisk/PhabricatorLiskDAO.php b/src/applications/base/storage/lisk/PhabricatorLiskDAO.php index 675531ee35..c6324fc4ff 100644 --- a/src/applications/base/storage/lisk/PhabricatorLiskDAO.php +++ b/src/applications/base/storage/lisk/PhabricatorLiskDAO.php @@ -23,6 +23,7 @@ abstract class PhabricatorLiskDAO extends LiskDAO { private $edges = array(); + private static $namespace = 'phabricator'; /* -( Managing Edges )----------------------------------------------------- */ @@ -61,6 +62,13 @@ abstract class PhabricatorLiskDAO extends LiskDAO { /* -( Configuring Storage )------------------------------------------------ */ + /** + * @task config + */ + public static function setApplicationNamespace($namespace) { + self::$namespace = $namespace; + } + /** * @task config @@ -68,7 +76,7 @@ abstract class PhabricatorLiskDAO extends LiskDAO { public function establishLiveConnection($mode) { $conf = PhabricatorEnv::newObjectFromConfig( 'mysql.configuration-provider', - array($this, $mode)); + array($this, $mode, self::$namespace)); return PhabricatorEnv::newObjectFromConfig( 'mysql.implementation', diff --git a/src/docs/configuration/configuration_guide.diviner b/src/docs/configuration/configuration_guide.diviner index 2d9e47b944..1fd98c9c48 100644 --- a/src/docs/configuration/configuration_guide.diviner +++ b/src/docs/configuration/configuration_guide.diviner @@ -8,17 +8,6 @@ This document contains basic configuration instructions for Phabricator. This document assumes you've already installed all the components you need. If you haven't, see @{article:Installation Guide}. -= Configuring MySQL = - -Get MySQL running and verify you can connect to it. Consult the MySQL -documentation for help. When MySQL works, you need to load the Phabricator -schemata into it. First, load the initial database schema. - - mysql -uroot < path/to/phabricator/resources/sql/init/initialize.sql - -After this you need to upgrade the schema (see @{article:Upgrading Schema}), -but you need to finish the rest of the configuration first. - = Configuring Phabricator = Create a new file here: @@ -59,6 +48,43 @@ For the last line, you can also use ##'development'## instead of ##'production'## if you are planning to develop Phabricator itself. This will turn on some debugging features. += PHABRICATOR_ENV Environment Variable = + +When running Phabricator scripts, they will ask you to set the `PHABRICATOR_ENV` +environment variable to point at your config. If you put your script in +`custom/myconfig.conf.php`, you can identify the config with +`custom/myconfig`, like this: + + $ PHABRICATOR_ENV=custom/myconfig ./some_phabricator_command + +NOTE: Make sure you put 'PHABRICATOR_ENV=...' at the beginning of the line, not +in the middle. The shell won't parse environmental variables declared after the +command. You can also ##export PHABRICATOR_ENV=...## in your ~/.bashrc or +~/.profile or similar, depending on which shell you use and how your system is +configured. + += Storage: Configuring MySQL = + +Get MySQL running and verify you can connect to it. Consult the MySQL +documentation for help. When MySQL works, you need to load the Phabricator +schemata into it. To do this, run: + + phabricator/ $ ./bin/storage upgrade + +If your configuration uses an unprivileged user to connect to the database, you +may have to override the default user so the schema changes can be applied with +root or some other admin user: + + phabricator/ $ ./bin/storage upgrade --user --password + +You can avoid the prompt the script issues by passing the ##--force## flag (for +example, if you are scripting the upgrade process). + + phabricator/ $ ./bin/storage upgrade --force + +NOTE: When you update Phabricator, run `storage upgrade` again to apply any +new updates. + = Webserver: Configuring Apache = Get Apache running and verify it's serving a test page. Consult the Apache @@ -145,12 +171,6 @@ For nginx, use a configuration like this: } } -= Upgrading Schema = - -Now, it's time for you to upgrade the database schema -- see -@{article:Upgrading Schema}. You'll also need to do this after you update the -code in the future. - = Setup = Now, restart your webserver and navigate to whichever subdomain you set up. You @@ -200,7 +220,6 @@ that will make upgrading Phabricator more difficult in the future. Continue by: - - upgrading the database schema with @{article:Upgrading Schema}; or - setting up your admin account and login/registration with @{article:Configuring Accounts and Registration}; or - configuring where uploaded fils and attachments will be stored with diff --git a/src/docs/configuration/upgrade_schema.diviner b/src/docs/configuration/upgrade_schema.diviner deleted file mode 100644 index 4374d8f81a..0000000000 --- a/src/docs/configuration/upgrade_schema.diviner +++ /dev/null @@ -1,36 +0,0 @@ -@title Upgrading Schema -@group config - -This document describes how to upgrade the database schema. - -= Prerequisites = - -This document assumes you've already initialized the MySQL database and -configured your Phabricator environment. If you haven't, see -@{article:Configuration Guide}. - -= Loading patches = - -To upgrade your database schema to the latest version, just run this command: - - PHABRICATOR_ENV= path/to/phabricator/scripts/sql/upgrade_schema.php - -NOTE: Make sure you put 'PHABRICATOR_ENV=...' at the beginning of the line, not -in the middle. The shell won't parse environmental variables declared after the -command. You can also ##export PHABRICATOR_ENV=...## in your ~/.bashrc or -~/.profile or similar, depending on which shell you use and how your system is -configured. - -This will install all the patches that are new since you installed, or since the -last time you ran this script. - -If your configuration uses an unprivileged user to connect to the database, you -may have to override the default user so the schema changes can be applied with -root or some other admin user: - - PHABRICATOR_ENV= path/to/phabricator/scripts/sql/upgrade_schema.php -u -p - -You can avoid the prompt the script issues by passing the ##-f## flag (for -example, if you are scripting the upgrade process). - - PHABRICATOR_ENV= path/to/phabricator/scripts/sql/upgrade_schema.php -f diff --git a/src/docs/installation_guide.diviner b/src/docs/installation_guide.diviner index ad5a178228..3e6651a09a 100644 --- a/src/docs/installation_guide.diviner +++ b/src/docs/installation_guide.diviner @@ -118,8 +118,17 @@ See for more information. = Updating Phabricator = -Since Phabricator is under active development, you should update frequently. -You can use a script similar to this one to automate the process: +Since Phabricator is under active development, you should update frequently. To +update Phabricator: + + - Stop the webserver. + - Run `git pull && git submodule update --init` in `libphutil/`, + `arcanist/` and `phabricator/`. + - Run `phabricator/bin/storage upgrade`. + - Restart the webserver. + +For more details, see @{article:Configuration Guide}. You can use a script +similar to this one to automate the process: http://www.phabricator.com/rsrc/install/update_phabricator.sh diff --git a/src/infrastructure/setup/PhabricatorSetup.php b/src/infrastructure/setup/PhabricatorSetup.php index fac8101665..2ce57a9193 100644 --- a/src/infrastructure/setup/PhabricatorSetup.php +++ b/src/infrastructure/setup/PhabricatorSetup.php @@ -519,33 +519,14 @@ final class PhabricatorSetup { if (empty($databases['phabricator_meta_data'])) { self::writeFailure(); self::write( - "Setup failure! You haven't loaded the 'initialize.sql' file into ". - "MySQL. This file initializes necessary databases. See this guide for ". - "instructions:\n"); + "Setup failure! You haven't run 'bin/storage upgrade'. See this ". + "article for instructions:\n"); self::writeDoc('article/Configuration_Guide.html'); return; } else { self::write(" okay Databases have been initialized.\n"); } - $schema_version = queryfx_one( - $conn_raw, - 'SELECT version FROM phabricator_meta_data.schema_version'); - $schema_version = idx($schema_version, 'version', 'null'); - - $expect = PhabricatorSQLPatchList::getExpectedSchemaVersion(); - if ($schema_version != $expect) { - self::writeFailure(); - self::write( - "Setup failure! You haven't upgraded your database schema to the ". - "latest version. Expected version is '{$expect}', but your local ". - "version is '{$schema_version}'. See this guide for instructions:\n"); - self::writeDoc('article/Upgrading_Schema.html'); - return; - } else { - self::write(" okay Database schema are up to date (v{$expect}).\n"); - } - $index_min_length = queryfx_one( $conn_raw, 'SHOW VARIABLES LIKE %s', diff --git a/src/infrastructure/setup/__init__.php b/src/infrastructure/setup/__init__.php index 36ade10b12..2e5332c1ed 100644 --- a/src/infrastructure/setup/__init__.php +++ b/src/infrastructure/setup/__init__.php @@ -7,7 +7,6 @@ phutil_require_module('phabricator', 'infrastructure/env'); -phutil_require_module('phabricator', 'infrastructure/setup/sql'); phutil_require_module('phabricator', 'storage/queryfx'); phutil_require_module('phutil', 'filesystem'); diff --git a/src/infrastructure/setup/sql/PhabricatorSQLPatchList.php b/src/infrastructure/setup/sql/PhabricatorSQLPatchList.php deleted file mode 100644 index 6200bf45ae..0000000000 --- a/src/infrastructure/setup/sql/PhabricatorSQLPatchList.php +++ /dev/null @@ -1,62 +0,0 @@ -find(); - - $versions = array(); - $patches = array(); - foreach ($results as $path) { - $matches = array(); - if (!preg_match('/(\d+)\..*\.(sql|php)$/', $path, $matches)) { - continue; - } - $version = (int)$matches[1]; - $patches[] = array( - 'version' => $version, - 'path' => $patches_dir.$path, - ); - if (empty($versions[$version])) { - $versions[$version] = true; - } else { - throw new Exception("Two patches have version {$version}!"); - } - } - - // Files are in some 'random' order returned by the operating system - // We need to apply them in proper order - $patches = isort($patches, 'version'); - - return $patches; - } - - public static function getExpectedSchemaVersion() { - $patches = self::getPatchList(); - $versions = ipull($patches, 'version'); - $max_version = max($versions); - return $max_version; - } - -} diff --git a/src/infrastructure/setup/sql/__init__.php b/src/infrastructure/setup/sql/__init__.php index 248a60848c..62a8e8f4a0 100644 --- a/src/infrastructure/setup/sql/__init__.php +++ b/src/infrastructure/setup/sql/__init__.php @@ -6,9 +6,4 @@ -phutil_require_module('phutil', 'filesystem/filefinder'); -phutil_require_module('phutil', 'moduleutils'); -phutil_require_module('phutil', 'utils'); - -phutil_require_source('PhabricatorSQLPatchList.php'); diff --git a/src/infrastructure/setup/sql/base/PhabricatorSQLPatchList.php b/src/infrastructure/setup/sql/base/PhabricatorSQLPatchList.php new file mode 100644 index 0000000000..224dfcc187 --- /dev/null +++ b/src/infrastructure/setup/sql/base/PhabricatorSQLPatchList.php @@ -0,0 +1,164 @@ +setAncestorClass('PhabricatorSQLPatchList') + ->setConcreteOnly(true) + ->selectAndLoadSymbols(); + + $specs = array(); + $seen_namespaces = array(); + + foreach ($patch_lists as $patch_class) { + $patch_class = $patch_class['name']; + $patch_list = newv($patch_class, array()); + + $namespace = $patch_list->getNamespace(); + if (isset($seen_namespaces[$namespace])) { + $prior = $seen_namespaces[$namespace]; + throw new Exception( + "PatchList '{$patch_class}' has the same namespace, '{$namespace}', ". + "as another patch list class, '{$prior}'. Each patch list MUST have ". + "a unique namespace."); + } + + $last_key = null; + foreach ($patch_list->getPatches() as $key => $patch) { + if (!is_array($patch)) { + throw new Exception( + "PatchList '{$patch_class}' has a patch '{$key}' which is not ". + "an array."); + } + + $valid = array( + 'type' => true, + 'name' => true, + 'after' => true, + 'legacy' => true, + ); + + foreach ($patch as $pkey => $pval) { + if (empty($valid[$pkey])) { + throw new Exception( + "PatchList '{$patch_class}' has a patch, '{$key}', with an ". + "unknown property, '{$pkey}'. Patches must have only valid ". + "keys: ".implode(', ', array_keys($valid)).'.'); + } + } + + if (is_numeric($key)) { + throw new Exception( + "PatchList '{$patch_class}' has a patch with a numeric key, ". + "'{$key}'. Patches must use string keys."); + } + + if (strpos($key, ':') !== false) { + throw new Exception( + "PatchList '{$patch_class}' has a patch with a colon in the ". + "key name, '{$key}'. Patch keys may not contain colons."); + } + + $full_key = "{$namespace}:{$key}"; + + if (isset($specs[$full_key])) { + throw new Exception( + "PatchList '{$patch_class}' has a patch '{$key}' which ". + "duplicates an existing patch key."); + } + + $patch['key'] = $key; + $patch['fullKey'] = $full_key; + + if (isset($patch['legacy'])) { + if ($namespace != 'phabricator') { + throw new Exception( + "Only patches in the 'phabricator' namespace may contain ". + "'legacy' keys."); + } + } else { + $patch['legacy'] = false; + } + + if (!array_key_exists('after', $patch)) { + if ($last_key === null) { + throw new Exception( + "Patch '{$full_key}' is missing key 'after', and is the first ". + "patch in the patch list '{$patch_class}', so its application ". + "order can not be determined implicitly. The first patch in a ". + "patch list must list the patch or patches it depends on ". + "explicitly."); + } else { + $patch['after'] = array($last_key); + } + } + $last_key = $full_key; + + foreach ($patch['after'] as $after_key => $after) { + if (strpos($after, ':') === false) { + $patch['after'][$after_key] = $namespace.':'.$after; + } + } + + $type = idx($patch, 'type'); + if (!$type) { + throw new Exception( + "Patch '{$namespace}:{$key}' is missing key 'type'. Every patch ". + "must have a type."); + } + + switch ($type) { + case 'db': + case 'sql': + case 'php': + break; + default: + throw new Exception( + "Patch '{$namespace}:{$key}' has unknown patch type '{$type}'."); + } + + $specs[$full_key] = $patch; + } + } + + foreach ($specs as $key => $patch) { + foreach ($patch['after'] as $after) { + if (empty($specs[$after])) { + throw new Exception( + "Patch '{$key}' references nonexistent dependency, '{$after}'. ". + "Patches may only depend on patches which actually exist."); + } + } + } + + $patches = array(); + foreach ($specs as $full_key => $spec) { + $patches[$full_key] = new PhabricatorStoragePatch($spec); + } + + // TODO: Detect cycles? + + return $patches; + } + +} diff --git a/src/infrastructure/setup/sql/base/__init__.php b/src/infrastructure/setup/sql/base/__init__.php new file mode 100644 index 0000000000..495c0c6b60 --- /dev/null +++ b/src/infrastructure/setup/sql/base/__init__.php @@ -0,0 +1,15 @@ + array( + 'type' => 'db', + 'name' => 'audit', + 'after' => array( /* First Patch */ ), + ), + 'db.chatlog' => array( + 'type' => 'db', + 'name' => 'chatlog', + ), + 'db.conduit' => array( + 'type' => 'db', + 'name' => 'conduit', + ), + 'db.countdown' => array( + 'type' => 'db', + 'name' => 'countdown', + ), + 'db.daemon' => array( + 'type' => 'db', + 'name' => 'daemon', + ), + 'db.differential' => array( + 'type' => 'db', + 'name' => 'differential', + ), + 'db.draft' => array( + 'type' => 'db', + 'name' => 'draft', + ), + 'db.drydock' => array( + 'type' => 'db', + 'name' => 'drydock', + ), + 'db.feed' => array( + 'type' => 'db', + 'name' => 'feed', + ), + 'db.file' => array( + 'type' => 'db', + 'name' => 'file', + ), + 'db.flag' => array( + 'type' => 'db', + 'name' => 'flag', + ), + 'db.herald' => array( + 'type' => 'db', + 'name' => 'herald', + ), + 'db.maniphest' => array( + 'type' => 'db', + 'name' => 'maniphest', + ), + 'db.meta_data' => array( + 'type' => 'db', + 'name' => 'meta_data', + ), + 'db.metamta' => array( + 'type' => 'db', + 'name' => 'metamta', + ), + 'db.oauth_server' => array( + 'type' => 'db', + 'name' => 'oauth_server', + ), + 'db.owners' => array( + 'type' => 'db', + 'name' => 'owners', + ), + 'db.pastebin' => array( + 'type' => 'db', + 'name' => 'pastebin', + ), + 'db.phame' => array( + 'type' => 'db', + 'name' => 'phame', + ), + 'db.phid' => array( + 'type' => 'db', + 'name' => 'phid', + ), + 'db.phriction' => array( + 'type' => 'db', + 'name' => 'phriction', + ), + 'db.project' => array( + 'type' => 'db', + 'name' => 'project', + ), + 'db.repository' => array( + 'type' => 'db', + 'name' => 'repository', + ), + 'db.search' => array( + 'type' => 'db', + 'name' => 'search', + ), + 'db.slowvote' => array( + 'type' => 'db', + 'name' => 'slowvote', + ), + 'db.timeline' => array( + 'type' => 'db', + 'name' => 'timeline', + ), + 'db.user' => array( + 'type' => 'db', + 'name' => 'user', + ), + 'db.worker' => array( + 'type' => 'db', + 'name' => 'worker', + ), + 'db.xhpastview' => array( + 'type' => 'db', + 'name' => 'xhpastview', + ), + '0000.legacy.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('0000.legacy.sql'), + 'legacy' => 0, + ), + '000.project.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('000.project.sql'), + 'legacy' => 0, + ), + '001.maniphest_projects.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('001.maniphest_projects.sql'), + 'legacy' => 1, + ), + '002.oauth.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('002.oauth.sql'), + 'legacy' => 2, + ), + '003.more_oauth.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('003.more_oauth.sql'), + 'legacy' => 3, + ), + '004.daemonrepos.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('004.daemonrepos.sql'), + 'legacy' => 4, + ), + '005.workers.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('005.workers.sql'), + 'legacy' => 5, + ), + '006.repository.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('006.repository.sql'), + 'legacy' => 6, + ), + '007.daemonlog.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('007.daemonlog.sql'), + 'legacy' => 7, + ), + '008.repoopt.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('008.repoopt.sql'), + 'legacy' => 8, + ), + '009.repo_summary.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('009.repo_summary.sql'), + 'legacy' => 9, + ), + '010.herald.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('010.herald.sql'), + 'legacy' => 10, + ), + '011.badcommit.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('011.badcommit.sql'), + 'legacy' => 11, + ), + '012.dropphidtype.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('012.dropphidtype.sql'), + 'legacy' => 12, + ), + '013.commitdetail.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('013.commitdetail.sql'), + 'legacy' => 13, + ), + '014.shortcuts.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('014.shortcuts.sql'), + 'legacy' => 14, + ), + '015.preferences.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('015.preferences.sql'), + 'legacy' => 15, + ), + '016.userrealnameindex.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('016.userrealnameindex.sql'), + 'legacy' => 16, + ), + '017.sessionkeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('017.sessionkeys.sql'), + 'legacy' => 17, + ), + '018.owners.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('018.owners.sql'), + 'legacy' => 18, + ), + '019.arcprojects.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('019.arcprojects.sql'), + 'legacy' => 19, + ), + '020.pathcapital.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('020.pathcapital.sql'), + 'legacy' => 20, + ), + '021.xhpastview.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('021.xhpastview.sql'), + 'legacy' => 21, + ), + '022.differentialcommit.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('022.differentialcommit.sql'), + 'legacy' => 22, + ), + '023.dxkeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('023.dxkeys.sql'), + 'legacy' => 23, + ), + '024.mlistkeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('024.mlistkeys.sql'), + 'legacy' => 24, + ), + '025.commentopt.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('025.commentopt.sql'), + 'legacy' => 25, + ), + '026.diffpropkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('026.diffpropkey.sql'), + 'legacy' => 26, + ), + '027.metamtakeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('027.metamtakeys.sql'), + 'legacy' => 27, + ), + '028.systemagent.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('028.systemagent.sql'), + 'legacy' => 28, + ), + '029.cursors.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('029.cursors.sql'), + 'legacy' => 29, + ), + '030.imagemacro.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('030.imagemacro.sql'), + 'legacy' => 30, + ), + '031.workerrace.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('031.workerrace.sql'), + 'legacy' => 31, + ), + '032.viewtime.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('032.viewtime.sql'), + 'legacy' => 32, + ), + '033.privtest.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('033.privtest.sql'), + 'legacy' => 33, + ), + '034.savedheader.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('034.savedheader.sql'), + 'legacy' => 34, + ), + '035.proxyimage.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('035.proxyimage.sql'), + 'legacy' => 35, + ), + '036.mailkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('036.mailkey.sql'), + 'legacy' => 36, + ), + '037.setuptest.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('037.setuptest.sql'), + 'legacy' => 37, + ), + '038.admin.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('038.admin.sql'), + 'legacy' => 38, + ), + '039.userlog.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('039.userlog.sql'), + 'legacy' => 39, + ), + '040.transform.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('040.transform.sql'), + 'legacy' => 40, + ), + '041.heraldrepetition.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('041.heraldrepetition.sql'), + 'legacy' => 41, + ), + '042.commentmetadata.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('042.commentmetadata.sql'), + 'legacy' => 42, + ), + '043.pastebin.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('043.pastebin.sql'), + 'legacy' => 43, + ), + '044.countdown.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('044.countdown.sql'), + 'legacy' => 44, + ), + '045.timezone.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('045.timezone.sql'), + 'legacy' => 45, + ), + '046.conduittoken.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('046.conduittoken.sql'), + 'legacy' => 46, + ), + '047.projectstatus.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('047.projectstatus.sql'), + 'legacy' => 47, + ), + '048.relationshipkeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('048.relationshipkeys.sql'), + 'legacy' => 48, + ), + '049.projectowner.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('049.projectowner.sql'), + 'legacy' => 49, + ), + '050.taskdenormal.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('050.taskdenormal.sql'), + 'legacy' => 50, + ), + '051.projectfilter.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('051.projectfilter.sql'), + 'legacy' => 51, + ), + '052.pastelanguage.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('052.pastelanguage.sql'), + 'legacy' => 52, + ), + '053.feed.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('053.feed.sql'), + 'legacy' => 53, + ), + '054.subscribers.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('054.subscribers.sql'), + 'legacy' => 54, + ), + '055.add_author_to_files.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('055.add_author_to_files.sql'), + 'legacy' => 55, + ), + '056.slowvote.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('056.slowvote.sql'), + 'legacy' => 56, + ), + '057.parsecache.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('057.parsecache.sql'), + 'legacy' => 57, + ), + '058.missingkeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('058.missingkeys.sql'), + 'legacy' => 58, + ), + '059.engines.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('059.engines.php'), + 'legacy' => 59, + ), + '060.phriction.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('060.phriction.sql'), + 'legacy' => 60, + ), + '061.phrictioncontent.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('061.phrictioncontent.sql'), + 'legacy' => 61, + ), + '062.phrictionmenu.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('062.phrictionmenu.sql'), + 'legacy' => 62, + ), + '063.pasteforks.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('063.pasteforks.sql'), + 'legacy' => 63, + ), + '064.subprojects.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('064.subprojects.sql'), + 'legacy' => 64, + ), + '065.sshkeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('065.sshkeys.sql'), + 'legacy' => 65, + ), + '066.phrictioncontent.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('066.phrictioncontent.sql'), + 'legacy' => 66, + ), + '067.preferences.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('067.preferences.sql'), + 'legacy' => 67, + ), + '068.maniphestauxiliarystorage.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('068.maniphestauxiliarystorage.sql'), + 'legacy' => 68, + ), + '069.heraldxscript.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('069.heraldxscript.sql'), + 'legacy' => 69, + ), + '070.differentialaux.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('070.differentialaux.sql'), + 'legacy' => 70, + ), + '071.contentsource.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('071.contentsource.sql'), + 'legacy' => 71, + ), + '072.blamerevert.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('072.blamerevert.sql'), + 'legacy' => 72, + ), + '073.reposymbols.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('073.reposymbols.sql'), + 'legacy' => 73, + ), + '074.affectedpath.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('074.affectedpath.sql'), + 'legacy' => 74, + ), + '075.revisionhash.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('075.revisionhash.sql'), + 'legacy' => 75, + ), + '076.indexedlanguages.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('076.indexedlanguages.sql'), + 'legacy' => 76, + ), + '077.originalemail.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('077.originalemail.sql'), + 'legacy' => 77, + ), + '078.nametoken.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('078.nametoken.sql'), + 'legacy' => 78, + ), + '079.nametokenindex.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('079.nametokenindex.php'), + 'legacy' => 79, + ), + '080.filekeys.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('080.filekeys.sql'), + 'legacy' => 80, + ), + '081.filekeys.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('081.filekeys.php'), + 'legacy' => 81, + ), + '082.xactionkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('082.xactionkey.sql'), + 'legacy' => 82, + ), + '083.dxviewtime.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('083.dxviewtime.sql'), + 'legacy' => 83, + ), + '084.pasteauthorkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('084.pasteauthorkey.sql'), + 'legacy' => 84, + ), + '085.packagecommitrelationship.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('085.packagecommitrelationship.sql'), + 'legacy' => 85, + ), + '086.formeraffil.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('086.formeraffil.sql'), + 'legacy' => 86, + ), + '087.phrictiondelete.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('087.phrictiondelete.sql'), + 'legacy' => 87, + ), + '088.audit.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('088.audit.sql'), + 'legacy' => 88, + ), + '089.projectwiki.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('089.projectwiki.sql'), + 'legacy' => 89, + ), + '090.forceuniqueprojectnames.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('090.forceuniqueprojectnames.php'), + 'legacy' => 90, + ), + '091.uniqueslugkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('091.uniqueslugkey.sql'), + 'legacy' => 91, + ), + '092.dropgithubnotification.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('092.dropgithubnotification.sql'), + 'legacy' => 92, + ), + '093.gitremotes.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('093.gitremotes.php'), + 'legacy' => 93, + ), + '094.phrictioncolumn.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('094.phrictioncolumn.sql'), + 'legacy' => 94, + ), + '095.directory.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('095.directory.sql'), + 'legacy' => 95, + ), + '096.filename.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('096.filename.sql'), + 'legacy' => 96, + ), + '097.heraldruletypes.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('097.heraldruletypes.sql'), + 'legacy' => 97, + ), + '098.heraldruletypemigration.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('098.heraldruletypemigration.php'), + 'legacy' => 98, + ), + '099.drydock.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('099.drydock.sql'), + 'legacy' => 99, + ), + '100.projectxaction.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('100.projectxaction.sql'), + 'legacy' => 100, + ), + '101.heraldruleapplied.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('101.heraldruleapplied.sql'), + 'legacy' => 101, + ), + '102.heraldcleanup.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('102.heraldcleanup.php'), + 'legacy' => 102, + ), + '103.heraldedithistory.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('103.heraldedithistory.sql'), + 'legacy' => 103, + ), + '104.searchkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('104.searchkey.sql'), + 'legacy' => 104, + ), + '105.mimetype.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('105.mimetype.sql'), + 'legacy' => 105, + ), + '106.chatlog.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('106.chatlog.sql'), + 'legacy' => 106, + ), + '107.oauthserver.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('107.oauthserver.sql'), + 'legacy' => 107, + ), + '108.oauthscope.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('108.oauthscope.sql'), + 'legacy' => 108, + ), + '109.oauthclientphidkey.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('109.oauthclientphidkey.sql'), + 'legacy' => 109, + ), + '110.commitaudit.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('110.commitaudit.sql'), + 'legacy' => 110, + ), + '111.commitauditmigration.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('111.commitauditmigration.php'), + 'legacy' => 111, + ), + '112.oauthaccesscoderedirecturi.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('112.oauthaccesscoderedirecturi.sql'), + 'legacy' => 112, + ), + '113.lastreviewer.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('113.lastreviewer.sql'), + 'legacy' => 113, + ), + '114.auditrequest.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('114.auditrequest.sql'), + 'legacy' => 114, + ), + '115.prepareutf8.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('115.prepareutf8.sql'), + 'legacy' => 115, + ), + '116.utf8-backup-first-expect-wait.sql' => array( + 'type' => 'sql', + 'name' => + $this->getPatchPath('116.utf8-backup-first-expect-wait.sql'), + 'legacy' => 116, + ), + '117.repositorydescription.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('117.repositorydescription.php'), + 'legacy' => 117, + ), + '118.auditinline.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('118.auditinline.sql'), + 'legacy' => 118, + ), + '119.filehash.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('119.filehash.sql'), + 'legacy' => 119, + ), + '120.noop.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('120.noop.sql'), + 'legacy' => 120, + ), + '121.drydocklog.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('121.drydocklog.sql'), + 'legacy' => 121, + ), + '122.flag.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('122.flag.sql'), + 'legacy' => 122, + ), + '123.heraldrulelog.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('123.heraldrulelog.sql'), + 'legacy' => 123, + ), + '124.subpriority.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('124.subpriority.sql'), + 'legacy' => 124, + ), + '125.ipv6.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('125.ipv6.sql'), + 'legacy' => 125, + ), + '126.edges.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('126.edges.sql'), + 'legacy' => 126, + ), + '127.userkeybody.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('127.userkeybody.sql'), + 'legacy' => 127, + ), + '128.phabricatorcom.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('128.phabricatorcom.sql'), + 'legacy' => 128, + ), + '129.savedquery.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('129.savedquery.sql'), + 'legacy' => 129, + ), + '130.denormalrevisionquery.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('130.denormalrevisionquery.sql'), + 'legacy' => 130, + ), + '131.migraterevisionquery.php' => array( + 'type' => 'php', + 'name' => $this->getPatchPath('131.migraterevisionquery.php'), + 'legacy' => 131, + ), + '132.phame.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('132.phame.sql'), + 'legacy' => 132, + ), + '133.imagemacro.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('133.imagemacro.sql'), + 'legacy' => 133, + ), + '134.emptysearch.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('134.emptysearch.sql'), + 'legacy' => 134, + ), + '135.datecommitted.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('135.datecommitted.sql'), + 'legacy' => 135, + ), + '136.sex.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('136.sex.sql'), + 'legacy' => 136, + ), + '137.auditmetadata.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('137.auditmetadata.sql'), + 'legacy' => 137, + ), + ); + } + +} diff --git a/src/infrastructure/setup/sql/phabricator/__init__.php b/src/infrastructure/setup/sql/phabricator/__init__.php new file mode 100644 index 0000000000..092ff2773d --- /dev/null +++ b/src/infrastructure/setup/sql/phabricator/__init__.php @@ -0,0 +1,15 @@ +namespace = $namespace; + PhabricatorLiskDAO::setApplicationNamespace($namespace); + return $this; + } + + public function getNamespace() { + return $this->namespace; + } + + public function setUser($user) { + $this->user = $user; + return $this; + } + + public function getUser() { + return $this->user; + } + + public function setPassword($password) { + $this->password = $password; + return $this; + } + + public function getPassword() { + return $this->password; + } + + public function setHost($host) { + $this->host = $host; + return $this; + } + + public function getHost() { + return $this->host; + } + + public function getDatabaseName($fragment) { + return $this->namespace.'_'.$fragment; + } + + public function getDatabaseList(array $patches) { + assert_instances_of($patches, 'PhabricatorStoragePatch'); + + $list = array(); + + foreach ($patches as $patch) { + if ($patch->getType() == 'db') { + $list[] = $this->getDatabaseName($patch->getName()); + } + } + + return $list; + } + + public function getConn($fragment, $select_database = true) { + return PhabricatorEnv::newObjectFromConfig( + 'mysql.implementation', + array( + array( + 'user' => $this->user, + 'pass' => $this->password, + 'host' => $this->host, + 'database' => $select_database + ? $this->getDatabaseName($fragment) + : null, + ), + )); + } + + public function getAppliedPatches() { + try { + $applied = queryfx_all( + $this->getConn('meta_data'), + 'SELECT patch FROM patch_status'); + return ipull($applied, 'patch'); + } catch (AphrontQueryException $ex) { + return null; + } + } + + public function createDatabase($fragment) { + queryfx( + $this->getConn($fragment, $select_database = false), + 'CREATE DATABASE IF NOT EXISTS %T COLLATE utf8_general_ci', + $this->getDatabaseName($fragment)); + } + + public function createTable($fragment, $table, array $cols) { + queryfx( + $this->getConn($fragment), + 'CREATE TABLE IF NOT EXISTS %T.%T (%Q) '. + 'ENGINE=InnoDB, COLLATE utf8_general_ci', + $this->getDatabaseName($fragment), + $table, + implode(', ', $cols)); + } + + public function getLegacyPatches(array $patches) { + assert_instances_of($patches, 'PhabricatorStoragePatch'); + + try { + $row = queryfx_one( + $this->getConn('meta_data'), + 'SELECT version FROM %T', + 'schema_version'); + $version = $row['version']; + } catch (AphrontQueryException $ex) { + return array(); + } + + $legacy = array(); + foreach ($patches as $key => $patch) { + if ($patch->getLegacy() !== false && $patch->getLegacy() <= $version) { + $legacy[] = $key; + } + } + + return $legacy; + } + + public function markPatchApplied($patch) { + queryfx( + $this->getConn('meta_data'), + 'INSERT INTO %T (patch, applied) VALUES (%s, %d)', + 'patch_status', + $patch, + time()); + } + + public function applyPatch(PhabricatorStoragePatch $patch) { + $type = $patch->getType(); + $name = $patch->getName(); + switch ($type) { + case 'db': + $this->createDatabase($name); + break; + case 'sql': + $this->applyPatchSQL($name); + break; + case 'php': + $this->applyPatchPHP($name); + break; + default: + throw new Exception("Unable to apply patch of type '{$type}'."); + } + } + + public function applyPatchSQL($sql) { + $sql = Filesystem::readFile($sql); + $queries = preg_split('/;\s+/', $sql); + $queries = array_filter($queries); + + foreach ($queries as $query) { + $query = str_replace('{$NAMESPACE}', $this->namespace, $query); + queryfx( + $this->getConn('meta_data', $select_database = false), + '%Q', + $query); + } + } + + public function applyPatchPHP($script) { + $schema_conn = $this->getConn('meta_data', $select_database = false); + require_once $script; + } + +} diff --git a/src/infrastructure/setup/storage/management/__init__.php b/src/infrastructure/setup/storage/management/__init__.php new file mode 100644 index 0000000000..15e71f43d4 --- /dev/null +++ b/src/infrastructure/setup/storage/management/__init__.php @@ -0,0 +1,17 @@ +key = $dict['key']; + $this->type = $dict['type']; + $this->fullKey = $dict['fullKey']; + $this->legacy = $dict['legacy']; + $this->name = $dict['name']; + $this->after = $dict['after']; + } + + public function getLegacy() { + return $this->legacy; + } + + public function getAfter() { + return $this->after; + } + + public function getType() { + return $this->type; + } + + public function getName() { + return $this->name; + } + + public function getFullKey() { + return $this->fullKey; + } + + public function getKey() { + return $this->key; + } + +} diff --git a/src/infrastructure/setup/storage/patch/__init__.php b/src/infrastructure/setup/storage/patch/__init__.php new file mode 100644 index 0000000000..e4524d1884 --- /dev/null +++ b/src/infrastructure/setup/storage/patch/__init__.php @@ -0,0 +1,10 @@ +patches = $patches; + return $this; + } + + public function getPatches() { + return $this->patches; + } + + final public function setAPI(PhabricatorStorageManagementAPI $api) { + $this->api = $api; + return $this; + } + + final public function getAPI() { + return $this->api; + } + + public function isExecutable() { + return true; + } + +} diff --git a/src/infrastructure/setup/storage/workflow/base/__init__.php b/src/infrastructure/setup/storage/workflow/base/__init__.php new file mode 100644 index 0000000000..894c23d92e --- /dev/null +++ b/src/infrastructure/setup/storage/workflow/base/__init__.php @@ -0,0 +1,13 @@ +setName('databases') + ->setExamples('**databases** [__options__]') + ->setSynopsis('List Phabricator databases.'); + } + + public function execute(PhutilArgumentParser $args) { + $api = $this->getAPI(); + $patches = $this->getPatches(); + + $databases = $api->getDatabaseList($patches); + echo implode("\n", $databases)."\n"; + + return 0; + } + +} diff --git a/src/infrastructure/setup/storage/workflow/databases/__init__.php b/src/infrastructure/setup/storage/workflow/databases/__init__.php new file mode 100644 index 0000000000..8269d2ee90 --- /dev/null +++ b/src/infrastructure/setup/storage/workflow/databases/__init__.php @@ -0,0 +1,12 @@ +setName('destroy') + ->setExamples('**destroy** [__options__]') + ->setSynopsis('Permanently destroy all storage and data.'); + } + + public function execute(PhutilArgumentParser $args) { + $is_dry = $args->getArg('dryrun'); + $is_force = $args->getArg('force'); + + if (!$is_dry && !$is_force) { + echo phutil_console_wrap( + "Are you completely sure you really want to permanently destroy all ". + "storage for Phabricator data? This operation can not be undone and ". + "your data will not be recoverable if you proceed."); + + if (!phutil_console_confirm('Permanently destroy all data?')) { + echo "Cancelled.\n"; + exit(1); + } + + if (!phutil_console_confirm('Really destroy all data forever?')) { + echo "Cancelled.\n"; + exit(1); + } + } + + $api = $this->getAPI(); + $patches = $this->getPatches(); + + $databases = $api->getDatabaseList($patches); + $databases[] = $api->getDatabaseName('meta_data'); + foreach ($databases as $database) { + if ($is_dry) { + echo "DRYRUN: Would drop database '{$database}'.\n"; + } else { + echo "Dropping database '{$database}'...\n"; + queryfx( + $api->getConn('meta_data', $select_database = false), + 'DROP DATABASE IF EXISTS %T', + $database); + } + } + + if (!$is_dry) { + echo "Storage was destroyed.\n"; + } + + return 0; + } + +} diff --git a/src/infrastructure/setup/storage/workflow/destroy/__init__.php b/src/infrastructure/setup/storage/workflow/destroy/__init__.php new file mode 100644 index 0000000000..d81769095a --- /dev/null +++ b/src/infrastructure/setup/storage/workflow/destroy/__init__.php @@ -0,0 +1,15 @@ +setName('dump') + ->setExamples('**dump** [__options__]') + ->setSynopsis('Dump all data in storage to stdout.'); + } + + public function execute(PhutilArgumentParser $args) { + $api = $this->getAPI(); + $patches = $this->getPatches(); + + $applied = $api->getAppliedPatches(); + if ($applied === null) { + $namespace = $api->getNamespace(); + echo phutil_console_wrap( + phutil_console_format( + "**No Storage**: There is no database storage initialized in this ". + "storage namespace ('{$namespace}'). Use '**storage upgrade**' to ". + "initialize storage.\n")); + return 1; + } + + $databases = $api->getDatabaseList($patches); + + list($host, $port) = $this->getBareHostAndPort($api->getHost()); + + $flag_password = $api->getPassword() + ? csprintf('-p %s', $api->getPassword()) + : ''; + + $flag_port = $port + ? csprintf('--port %d', $port) + : ''; + + return phutil_passthru( + + 'mysqldump --default-character-set=utf8 '. + '-u %s %C -h %s %C --databases %Ls', + + $api->getUser(), + $flag_password, + $host, + $flag_port, + $databases); + } + + private function getBareHostAndPort($host) { + // Split out port information, since the command-line client requires a + // separate flag for the port. + $uri = new PhutilURI('mysql://'.$host); + if ($uri->getPort()) { + $port = $uri->getPort(); + $bare_hostname = $uri->getDomain(); + } else { + $port = null; + $bare_hostname = $host; + } + + return array($bare_hostname, $port); + } + +} diff --git a/src/infrastructure/setup/storage/workflow/dump/__init__.php b/src/infrastructure/setup/storage/workflow/dump/__init__.php new file mode 100644 index 0000000000..2fccc76c8a --- /dev/null +++ b/src/infrastructure/setup/storage/workflow/dump/__init__.php @@ -0,0 +1,17 @@ +setName('status') + ->setExamples('**status** [__options__]') + ->setSynopsis('Show patch application status.'); + } + + public function execute(PhutilArgumentParser $args) { + $api = $this->getAPI(); + $patches = $this->getPatches(); + + $applied = $api->getAppliedPatches(); + + if ($applied === null) { + echo phutil_console_format( + "**Database Not Initialized**: Run **storage upgrade** to ". + "initialize.\n"); + + return 1; + } + + $len = 0; + foreach ($patches as $patch) { + $len = max($len, strlen($patch->getFullKey())); + } + + foreach ($patches as $patch) { + printf( + + "% -".($len + 2)."s ". + "%-".strlen("Not Applied")."s ". + "%-4s ". + "%s\n", + + $patch->getFullKey(), + in_array($patch->getFullKey(), $applied) + ? 'Applied' + : 'Not Applied', + $patch->getType(), + $patch->getName()); + } + + return 0; + } + +} diff --git a/src/infrastructure/setup/storage/workflow/status/__init__.php b/src/infrastructure/setup/storage/workflow/status/__init__.php new file mode 100644 index 0000000000..ec7209d0f6 --- /dev/null +++ b/src/infrastructure/setup/storage/workflow/status/__init__.php @@ -0,0 +1,14 @@ +setName('upgrade') + ->setExamples('**upgrade** [__options__]') + ->setSynopsis("Upgrade database schemata.") + ->setArguments( + array( + array( + 'name' => 'apply', + 'param' => 'patch', + 'help' => 'Apply __patch__ explicitly. This is an advanced '. + 'feature for development and debugging; you should '. + 'not normally use this flag.', + ), + array( + 'name' => 'no-quickstart', + 'help' => 'Build storage patch-by-patch from scatch, even if it '. + 'could be loaded from the quickstart template.', + ), + array( + 'name' => 'init-only', + 'help' => 'Initialize storage only; do not apply patches.', + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $is_dry = $args->getArg('dryrun'); + $is_force = $args->getArg('force'); + + $api = $this->getAPI(); + $patches = $this->getPatches(); + + if (!$is_dry && !$is_force) { + echo phutil_console_wrap( + "Before running storage upgrades, you should take down the ". + "Phabricator web interface and stop any running Phabricator ". + "daemons (you can disable this warning with --force)."); + + if (!phutil_console_confirm('Are you ready to continue?')) { + echo "Cancelled.\n"; + return 1; + } + } + + $apply_only = $args->getArg('apply'); + if ($apply_only) { + if (empty($patches[$apply_only])) { + throw new PhutilArgumentUsageException( + "--apply argument '{$apply_only}' is not a valid patch. Use ". + "'storage status' to show patch status."); + } + } + + $no_quickstart = $args->getArg('no-quickstart'); + $init_only = $args->getArg('init-only'); + + $applied = $api->getAppliedPatches(); + if ($applied === null) { + + if ($is_dry) { + echo "DRYRUN: Patch metadata storage doesn't exist yet, it would ". + "be created.\n"; + return 0; + } + + if ($apply_only) { + throw new PhutilArgumentUsageException( + "Storage has not been initialized yet, you must initialize storage ". + "before selectively applying patches."); + return 1; + } + + $legacy = $api->getLegacyPatches($patches); + if ($legacy || $no_quickstart || $init_only) { + + // If we have legacy patches, we can't quickstart. + + $api->createDatabase('meta_data'); + $api->createTable( + 'meta_data', + 'patch_status', + array( + 'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci', + 'applied INT UNSIGNED NOT NULL', + )); + + foreach ($legacy as $patch) { + $api->markPatchApplied($patch); + } + } else { + echo "Loading quickstart template...\n"; + $root = dirname(phutil_get_library_root('phabricator')); + $sql = $root.'/resources/sql/quickstart.sql'; + $api->applyPatchSQL($sql); + } + } + + if ($init_only) { + echo "Storage initialized.\n"; + return 0; + } + + $applied = $api->getAppliedPatches(); + $applied = array_fill_keys($applied, true); + + $skip_mark = false; + if ($apply_only) { + if (isset($applied[$apply_only])) { + + unset($applied[$apply_only]); + $skip_mark = true; + + if (!$is_force && !$is_dry) { + echo phutil_console_wrap( + "Patch '{$apply_only}' has already been applied. Are you sure ". + "you want to apply it again? This may put your storage in a state ". + "that the upgrade scripts can not automatically manage."); + if (!phutil_console_confirm('Apply patch again?')) { + echo "Cancelled.\n"; + return 1; + } + } + } + } + + while (true) { + $applied_something = false; + foreach ($patches as $key => $patch) { + if (isset($applied[$key])) { + unset($patches[$key]); + continue; + } + + if ($apply_only && $apply_only != $key) { + unset($patches[$key]); + continue; + } + + $can_apply = true; + foreach ($patch->getAfter() as $after) { + if (empty($applied[$after])) { + if ($apply_only) { + echo "Unable to apply patch '{$apply_only}' because it depends ". + "on patch '{$after}', which has not been applied.\n"; + return 1; + } + $can_apply = false; + break; + } + } + + if (!$can_apply) { + continue; + } + + $applied_something = true; + + if ($is_dry) { + echo "DRYRUN: Would apply patch '{$key}'.\n"; + } else { + echo "Applying patch '{$key}'...\n"; + $api->applyPatch($patch); + if (!$skip_mark) { + $api->markPatchApplied($key); + } + } + + unset($patches[$key]); + $applied[$key] = true; + } + + if (!$applied_something) { + if (count($patches)) { + throw new Exception( + "Some patches could not be applied: ". + implode(', ', array_keys($patches))); + } else if (!$is_dry && !$apply_only) { + echo "Storage is up to date. Use 'storage status' for details.\n"; + } + break; + } + } + + return 0; + } + +} diff --git a/src/infrastructure/setup/storage/workflow/upgrade/__init__.php b/src/infrastructure/setup/storage/workflow/upgrade/__init__.php new file mode 100644 index 0000000000..778aca94d0 --- /dev/null +++ b/src/infrastructure/setup/storage/workflow/upgrade/__init__.php @@ -0,0 +1,16 @@ +