diff --git a/scripts/daemon/phabricator_daemon_launcher.php b/scripts/daemon/phabricator_daemon_launcher.php index 7a5776fa5d..b0d3239f81 100755 --- a/scripts/daemon/phabricator_daemon_launcher.php +++ b/scripts/daemon/phabricator_daemon_launcher.php @@ -52,13 +52,20 @@ switch (isset($argv[1]) ? $argv[1] : 'help') { $need_launch = phd_load_tracked_repositories(); if (!$need_launch) { echo "There are no repositories with tracking enabled.\n"; - exit(0); + exit(1); } will_launch($control); + + echo "Launching PullLocal daemon in readonly mode...\n"; + $control->launchDaemon( 'PhabricatorRepositoryPullLocalDaemon', - array()); + array( + '--no-discovery', + )); + + echo "Done.\n"; break; case 'repository-launch-master': @@ -66,55 +73,24 @@ switch (isset($argv[1]) ? $argv[1] : 'help') { if (!$need_launch) { echo "There are no repositories with tracking enabled.\n"; exit(1); - } else { - will_launch($control); - - $control->launchDaemon( - 'PhabricatorRepositoryPullLocalDaemon', - array()); - - foreach ($need_launch as $repository) { - $name = $repository->getName(); - $callsign = $repository->getCallsign(); - $desc = "'{$name}' ({$callsign})"; - $phid = $repository->getPHID(); - - switch ($repository->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - echo "Launching discovery daemon on the {$desc} repository...\n"; - $control->launchDaemon( - 'PhabricatorRepositoryGitCommitDiscoveryDaemon', - array( - $phid, - )); - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - echo "Launching discovery daemon on the {$desc} repository...\n"; - $control->launchDaemon( - 'PhabricatorRepositorySvnCommitDiscoveryDaemon', - array( - $phid, - )); - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - echo "Launching discovery daemon on the {$desc} repository...\n"; - $control->launchDaemon( - 'PhabricatorRepositoryMercurialCommitDiscoveryDaemon', - array( - $phid, - )); - break; - - } - } - - echo "Launching CommitTask daemon...\n"; - $control->launchDaemon( - 'PhabricatorRepositoryCommitTaskDaemon', - array()); - - echo "Done.\n"; } + + will_launch($control); + + echo "Launching PullLocal daemon in master mode...\n"; + $control->launchDaemon( + 'PhabricatorRepositoryPullLocalDaemon', + array()); + + echo "Launching CommitTask daemon...\n"; + $control->launchDaemon( + 'PhabricatorRepositoryCommitTaskDaemon', + array()); + + echo "NOTE: Make sure you run some taskmaster daemons too, e.g. ". + "with 'phd launch 4 taskmaster'.\n"; + + echo "Done.\n"; break; case 'launch': diff --git a/scripts/repository/discover.php b/scripts/repository/discover.php new file mode 100755 index 0000000000..51a02f9618 --- /dev/null +++ b/scripts/repository/discover.php @@ -0,0 +1,51 @@ +#!/usr/bin/env php +setTagline('manually discover working copies'); +$args->setSynopsis(<<parseStandardArguments(); +$args->parse( + array( + array( + 'name' => 'repositories', + 'wildcard' => true, + ), + )); + +$repo_names = $args->getArg('repositories'); +if (!$repo_names) { + echo "Specify one or more repositories to pull, by callsign or PHID.\n"; + exit(1); +} + +$repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names); +foreach ($repos as $repo) { + $callsign = $repo->getCallsign(); + echo "Discovering '{$callsign}'...\n"; + PhabricatorRepositoryPullLocalDaemon::discoverRepository($repo); +} +echo "Done.\n"; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 3515566b4c..dfd635ce42 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -840,7 +840,6 @@ phutil_register_library_map(array( 'PhabricatorRepositoryCommit' => 'applications/repository/storage/commit', 'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/base', 'PhabricatorRepositoryCommitData' => 'applications/repository/storage/commitdata', - 'PhabricatorRepositoryCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/base', 'PhabricatorRepositoryCommitHeraldWorker' => 'applications/repository/worker/herald', 'PhabricatorRepositoryCommitMessageDetailParser' => 'applications/repository/parser/base', 'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base', @@ -850,22 +849,18 @@ phutil_register_library_map(array( 'PhabricatorRepositoryController' => 'applications/repository/controller/base', 'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create', 'PhabricatorRepositoryDAO' => 'applications/repository/storage/base', - 'PhabricatorRepositoryDaemon' => 'applications/repository/daemon/base', 'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'applications/repository/parser/default', 'PhabricatorRepositoryDeleteController' => 'applications/repository/controller/delete', 'PhabricatorRepositoryEditController' => 'applications/repository/controller/edit', 'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/git', - 'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/git', - 'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'applications/repository/daemon/commitdiscovery/git/__tests__', 'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git', 'PhabricatorRepositoryListController' => 'applications/repository/controller/list', 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial', - 'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial', 'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial', 'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal', + 'PhabricatorRepositoryPullLocalDaemonTestCase' => 'applications/repository/daemon/pulllocal/__tests__', 'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut', 'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn', - 'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn', 'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn', 'PhabricatorRepositorySymbol' => 'applications/repository/storage/symbol', 'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__', @@ -1735,31 +1730,26 @@ phutil_register_library_map(array( 'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker', 'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO', - 'PhabricatorRepositoryCommitDiscoveryDaemon' => 'PhabricatorRepositoryDaemon', 'PhabricatorRepositoryCommitHeraldWorker' => 'PhabricatorRepositoryCommitParserWorker', 'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker', 'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker', 'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker', - 'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorRepositoryDaemon', + 'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorDaemon', 'PhabricatorRepositoryController' => 'PhabricatorController', 'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController', 'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO', - 'PhabricatorRepositoryDaemon' => 'PhabricatorDaemon', 'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'PhabricatorRepositoryCommitMessageDetailParser', 'PhabricatorRepositoryDeleteController' => 'PhabricatorRepositoryController', 'PhabricatorRepositoryEditController' => 'PhabricatorRepositoryController', 'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', - 'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon', - 'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'PhabricatorTestCase', 'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker', 'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController', 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', - 'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon', 'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker', 'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon', + 'PhabricatorRepositoryPullLocalDaemonTestCase' => 'PhabricatorTestCase', 'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', - 'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon', 'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker', 'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase', diff --git a/src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php b/src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php index 8605470518..ee2c7573c3 100644 --- a/src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php +++ b/src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php @@ -1,7 +1,7 @@ needDirectChanges) { diff --git a/src/applications/repository/controller/base/PhabricatorRepositoryController.php b/src/applications/repository/controller/base/PhabricatorRepositoryController.php index ca865a9cbe..2a96aeaa5d 100644 --- a/src/applications/repository/controller/base/PhabricatorRepositoryController.php +++ b/src/applications/repository/controller/base/PhabricatorRepositoryController.php @@ -1,7 +1,7 @@ setGlyph("rX"); $page->appendChild($view); + $response = new AphrontWebpageResponse(); return $response->setContent($page->render()); } + private function isPullDaemonRunningOnThisMachine() { + + // This is sort of hacky, but should probably work. + + list($stdout) = execx('ps auxwww'); + return preg_match('/PhabricatorRepositoryPullLocalDaemon/', $stdout); + } + + protected function renderDaemonNotice() { + $daemon_running = $this->isPullDaemonRunningOnThisMachine(); + if ($daemon_running) { + return null; + } + + $documentation = phutil_render_tag( + 'a', + array( + 'href' => PhabricatorEnv::getDoclink( + 'article/Diffusion_User_Guide.html'), + ), + 'Diffusion User Guide'); + + $view = new AphrontErrorView(); + $view->setSeverity(AphrontErrorView::SEVERITY_WARNING); + $view->setTitle('Repository Daemon Not Running'); + $view->appendChild( + "

The repository daemon is not running on this machine. Without this ". + "daemon, Phabricator will not be able to import or update repositories. ". + "For instructions on starting the daemon, see ". + "{$documentation}.

"); + + return $view; + } + } diff --git a/src/applications/repository/controller/base/__init__.php b/src/applications/repository/controller/base/__init__.php index 4f185531d2..64e8804251 100644 --- a/src/applications/repository/controller/base/__init__.php +++ b/src/applications/repository/controller/base/__init__.php @@ -8,7 +8,11 @@ phutil_require_module('phabricator', 'aphront/response/webpage'); phutil_require_module('phabricator', 'applications/base/controller/base'); +phutil_require_module('phabricator', 'infrastructure/env'); +phutil_require_module('phabricator', 'view/form/error'); +phutil_require_module('phutil', 'future/exec'); +phutil_require_module('phutil', 'markup'); phutil_require_module('phutil', 'utils'); diff --git a/src/applications/repository/controller/edit/PhabricatorRepositoryEditController.php b/src/applications/repository/controller/edit/PhabricatorRepositoryEditController.php index d0c9276ea3..5e91d74170 100644 --- a/src/applications/repository/controller/edit/PhabricatorRepositoryEditController.php +++ b/src/applications/repository/controller/edit/PhabricatorRepositoryEditController.php @@ -63,6 +63,8 @@ final class PhabricatorRepositoryEditController phutil_escape_html($name))); } + $nav->appendChild($this->renderDaemonNotice()); + $this->sideNav = $nav; switch ($this->view) { @@ -345,9 +347,7 @@ final class PhabricatorRepositoryEditController $error_view = new AphrontErrorView(); $error_view->setSeverity(AphrontErrorView::SEVERITY_NOTICE); $error_view->setTitle('Changes Saved'); - $error_view->appendChild( - 'Tracking changes were saved. You may need to restart the daemon '. - 'before changes will take effect.'); + $error_view->appendChild('Tracking changes were saved.'); } else if (!$repository->isTracked()) { $error_view = new AphrontErrorView(); $error_view->setSeverity(AphrontErrorView::SEVERITY_WARNING); diff --git a/src/applications/repository/controller/list/PhabricatorRepositoryListController.php b/src/applications/repository/controller/list/PhabricatorRepositoryListController.php index e2c2bf1121..e2ed76284f 100644 --- a/src/applications/repository/controller/list/PhabricatorRepositoryListController.php +++ b/src/applications/repository/controller/list/PhabricatorRepositoryListController.php @@ -157,6 +157,7 @@ final class PhabricatorRepositoryListController return $this->buildStandardPageResponse( array( + $this->renderDaemonNotice(), $panel, $project_panel, ), diff --git a/src/applications/repository/daemon/base/PhabricatorRepositoryDaemon.php b/src/applications/repository/daemon/base/PhabricatorRepositoryDaemon.php deleted file mode 100644 index e9f2a3531a..0000000000 --- a/src/applications/repository/daemon/base/PhabricatorRepositoryDaemon.php +++ /dev/null @@ -1,38 +0,0 @@ -getArgv(); - if (count($argv) !== 1) { - throw new Exception("No repository PHID provided!"); - } - - $repository = id(new PhabricatorRepository())->loadOneWhere( - 'phid = %s', - $argv[0]); - - if (!$repository) { - throw new Exception("No such repository exists!"); - } - - return $repository; - } - -} diff --git a/src/applications/repository/daemon/base/__init__.php b/src/applications/repository/daemon/base/__init__.php deleted file mode 100644 index 97230a3be2..0000000000 --- a/src/applications/repository/daemon/base/__init__.php +++ /dev/null @@ -1,15 +0,0 @@ -repository; - } - - final public function run() { - while (true) { - // Reload the repository every time to pick up changes from the web - // console. - $this->repository = $this->loadRepository(); - $this->discoverCommits(); - - $sleep = max(2, $this->getRepository()->getDetail('pull-frequency')); - $this->sleep($sleep); - } - } - - final public function runOnce() { - $this->repository = $this->loadRepository(); - $this->discoverCommits(); - } - - protected function isKnownCommit($target) { - if (isset($this->commitCache[$target])) { - return true; - } - - $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( - 'repositoryID = %s AND commitIdentifier = %s', - $this->getRepository()->getID(), - $target); - - if (!$commit) { - return false; - } - - $this->commitCache[$target] = true; - while (count($this->commitCache) > 64) { - array_shift($this->commitCache); - } - - return true; - } - - protected function recordCommit($commit_identifier, $epoch) { - $repository = $this->getRepository(); - - $commit = new PhabricatorRepositoryCommit(); - $commit->setRepositoryID($repository->getID()); - $commit->setCommitIdentifier($commit_identifier); - $commit->setEpoch($epoch); - - try { - $commit->save(); - $event = new PhabricatorTimelineEvent( - 'cmit', - array( - 'id' => $commit->getID(), - )); - $event->recordEvent(); - - queryfx( - $repository->establishConnection('w'), - 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) - VALUES (%d, 1, %d, %d) - ON DUPLICATE KEY UPDATE - size = size + 1, - lastCommitID = - IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID), - epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)', - PhabricatorRepository::TABLE_SUMMARY, - $repository->getID(), - $commit->getID(), - $epoch); - - $this->commitCache[$commit_identifier] = true; - } catch (AphrontQueryDuplicateKeyException $ex) { - // Ignore. This can happen because we discover the same new commit - // more than once when looking at history, or because of races or - // data inconsistency or cosmic radiation; in any case, we're still - // in a good state if we ignore the failure. - $this->commitCache[$commit_identifier] = true; - } - - $this->stillWorking(); - } - - abstract protected function discoverCommits(); - -} diff --git a/src/applications/repository/daemon/commitdiscovery/base/__init__.php b/src/applications/repository/daemon/commitdiscovery/base/__init__.php deleted file mode 100644 index d26ee49b18..0000000000 --- a/src/applications/repository/daemon/commitdiscovery/base/__init__.php +++ /dev/null @@ -1,18 +0,0 @@ -getRepository(); - - $vcs = $repository->getVersionControlSystem(); - if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_GIT) { - throw new Exception("Repository is not a git repository."); - } - - list($remotes) = $repository->execxLocalCommand( - 'remote show -n origin'); - - $matches = null; - if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) { - throw new Exception( - "Expected 'Fetch URL' in 'git remote show -n origin'."); - } - - self::verifySameGitOrigin( - $matches[1], - $repository->getRemoteURI(), - $repository->getLocalPath()); - - list($stdout) = $repository->execxLocalCommand( - 'branch -r --verbose --no-abbrev'); - - $branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput( - $stdout, - $only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE); - - $got_something = false; - $tracked_something = false; - foreach ($branches as $name => $commit) { - if (!$repository->shouldTrackBranch($name)) { - continue; - } - - $tracked_something = true; - - if ($this->isKnownCommit($commit)) { - continue; - } else { - $this->discoverCommit($commit); - $got_something = true; - } - } - - if (!$tracked_something) { - $repo_name = $repository->getName(); - $repo_callsign = $repository->getCallsign(); - throw new Exception( - "Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ". - "Verify that your branch filtering settings are correct."); - } - - return $got_something; - } - - private function discoverCommit($commit) { - $discover = array(); - $insert = array(); - - $repository = $this->getRepository(); - - $discover[] = $commit; - $insert[] = $commit; - - $seen_parent = array(); - - while (true) { - $target = array_pop($discover); - list($parents) = $repository->execxLocalCommand( - 'log -n1 --pretty="%%P" %s', - $target); - $parents = array_filter(explode(' ', trim($parents))); - foreach ($parents as $parent) { - if (isset($seen_parent[$parent])) { - // We end up in a loop here somehow when we parse Arcanist if we - // don't do this. TODO: Figure out why and draw a pretty diagram - // since it's not evident how parsing a DAG with this causes the - // loop to stop terminating. - continue; - } - $seen_parent[$parent] = true; - if (!$this->isKnownCommit($parent)) { - $discover[] = $parent; - $insert[] = $parent; - } - } - if (empty($discover)) { - break; - } - $this->stillWorking(); - } - - while (true) { - $target = array_pop($insert); - list($epoch) = $repository->execxLocalCommand( - 'log -n1 --pretty="%%ct" %s', - $target); - $epoch = trim($epoch); - - $this->recordCommit($target, $epoch); - - if (empty($insert)) { - break; - } - } - } - - public static function verifySameGitOrigin($remote, $expect, $where) { - $remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote); - $expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect); - - $remote_path = $remote_uri->getPath(); - $expect_path = $expect_uri->getPath(); - - $remote_match = self::normalizeGitPath($remote_path); - $expect_match = self::normalizeGitPath($expect_path); - - if ($remote_match != $expect_match) { - throw new Exception( - "Working copy at '{$where}' has a mismatched origin URL. It has ". - "origin URL '{$remote}' (with remote path '{$remote_path}'), but the ". - "configured URL '{$expect}' (with remote path '{$expect_path}') is ". - "expected. Refusing to proceed because this may indicate that the ". - "working copy is actually some other repository."); - } - } - - private static function normalizeGitPath($path) { - // Strip away trailing "/" and ".git", so similar paths correctly match. - - $path = rtrim($path, '/'); - $path = preg_replace('/\.git$/', '', $path); - return $path; - } - -} diff --git a/src/applications/repository/daemon/commitdiscovery/git/__init__.php b/src/applications/repository/daemon/commitdiscovery/git/__init__.php deleted file mode 100644 index f06c780f94..0000000000 --- a/src/applications/repository/daemon/commitdiscovery/git/__init__.php +++ /dev/null @@ -1,16 +0,0 @@ -getRepository(); - - $vcs = $repository->getVersionControlSystem(); - if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL) { - throw new Exception("Repository is not a Mercurial repository."); - } - - $repository_phid = $repository->getPHID(); - - list($stdout) = $repository->execxLocalCommand('branches'); - - $branches = ArcanistMercurialParser::parseMercurialBranches($stdout); - $got_something = false; - foreach ($branches as $name => $branch) { - $commit = $branch['rev']; - $commit = $this->getFullHash($commit); - if ($this->isKnownCommit($commit)) { - continue; - } else { - $this->discoverCommit($commit); - $got_something = true; - } - } - - return $got_something; - } - - private function getFullHash($commit) { - - // NOTE: Mercurial shortens hashes to 12 characters by default. This - // implies collisions with as few as a few million commits. The - // documentation sensibly advises "Do not use short-form IDs for - // long-lived representations". It then continues "You can use the - // --debug option to display the full changeset ID". What?! Yes, this - // is in fact the only way to turn on full hashes, and the hg source - // code is littered with "hexfn = ui.debugflag and hex or short" and - // similar. There is no more-selective flag or config option. - // - // Unfortunately, "hg --debug" turns on tons of other extra output, - // including full commit messages in "hg log" and "hg parents" (which - // ignore --style); this renders them unparseable. So we have to use - // "hg id" to convert short hashes into full hashes. See: - // - // - // - // Of course, this means that if there are collisions we will break here - // (the short commit identifier won't be unambiguous) but maybe Mercurial - // will have a --full-hashes flag or something by then and we can fix it - // properly. Until we run into that, this allows us to store data in the - // right format so when we eventually encounter this we won't have to - // reparse every Mercurial repository. - - $repository = $this->getRepository(); - list($stdout) = $repository->execxLocalCommand( - 'id --debug -i --rev %s', - $commit); - return trim($stdout); - } - - private function discoverCommit($commit) { - $discover = array(); - $insert = array(); - - $repository = $this->getRepository(); - - $discover[] = $commit; - $insert[] = $commit; - - $seen_parent = array(); - - // For all the new commits at the branch heads, walk backward until we find - // only commits we've aleady seen. - while (true) { - $target = array_pop($discover); - list($stdout) = $repository->execxLocalCommand( - 'parents --style default --rev %s', - $target); - $parents = ArcanistMercurialParser::parseMercurialLog($stdout); - if ($parents) { - foreach ($parents as $parent) { - $parent_commit = $parent['rev']; - $parent_commit = $this->getFullHash($parent_commit); - if (isset($seen_parent[$parent_commit])) { - continue; - } - $seen_parent[$parent_commit] = true; - if (!$this->isKnownCommit($parent_commit)) { - $discover[] = $parent_commit; - $insert[] = $parent_commit; - } - } - } - if (empty($discover)) { - break; - } - $this->stillWorking(); - } - - while (true) { - $target = array_pop($insert); - list($stdout) = $repository->execxLocalCommand( - 'log --rev %s --template %s', - $target, - '{date|rfc822date}'); - $epoch = strtotime($stdout); - - $this->recordCommit($target, $epoch); - - if (empty($insert)) { - break; - } - } - } - -} diff --git a/src/applications/repository/daemon/commitdiscovery/mercurial/__init__.php b/src/applications/repository/daemon/commitdiscovery/mercurial/__init__.php deleted file mode 100644 index e18b3c3941..0000000000 --- a/src/applications/repository/daemon/commitdiscovery/mercurial/__init__.php +++ /dev/null @@ -1,15 +0,0 @@ -getRepository(); - - $vcs = $repository->getVersionControlSystem(); - if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) { - throw new Exception("Repository is not a svn repository."); - } - - $uri = $this->getBaseSVNLogURI(); - list($xml) = $repository->execxRemoteCommand( - 'log --xml --quiet --limit 1 %s@HEAD', - $uri); - - $results = $this->parseSVNLogXML($xml); - $commit = head_key($results); - $epoch = head($results); - - if ($this->isKnownCommit($commit)) { - return false; - } - - $this->discoverCommit($commit, $epoch); - - return true; - } - - private function discoverCommit($commit, $epoch) { - $uri = $this->getBaseSVNLogURI(); - $repository = $this->getRepository(); - - $discover = array( - $commit => $epoch, - ); - $upper_bound = $commit; - - $limit = 1; - while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound)) { - // Find all the unknown commits on this path. Note that we permit - // importing an SVN subdirectory rather than the entire repository, so - // commits may be nonsequential. - list($err, $xml, $stderr) = $repository->execRemoteCommand( - ' log --xml --quiet --limit %d %s@%d', - $limit, - $uri, - $upper_bound - 1); - if ($err) { - if (preg_match('/(path|File) not found/', $stderr)) { - // We've gone all the way back through history and this path was not - // affected by earlier commits. - break; - } else { - throw new Exception("svn log error #{$err}: {$stderr}"); - } - } - $discover += $this->parseSVNLogXML($xml); - - $upper_bound = min(array_keys($discover)); - - // Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially - // import large repositories fairly quickly, while pulling only as much - // data as we need in the common case (when we've already imported the - // repository and are just grabbing one commit at a time). - $limit = min($limit * 2, 256); - } - - // NOTE: We do writes only after discovering all the commits so that we're - // never left in a state where we've missed commits -- if the discovery - // script terminates it can always resume and restore the import to a good - // state. This is also why we sort the discovered commits so we can do - // writes forward from the smallest one. - - ksort($discover); - foreach ($discover as $commit => $epoch) { - $this->recordCommit($commit, $epoch); - } - } - - private function parseSVNLogXML($xml) { - $xml = phutil_utf8ize($xml); - - $result = array(); - - $log = new SimpleXMLElement($xml); - foreach ($log->logentry as $entry) { - $commit = (int)$entry['revision']; - $epoch = (int)strtotime((string)$entry->date[0]); - $result[$commit] = $epoch; - } - - return $result; - } - - - private function getBaseSVNLogURI() { - $repository = $this->getRepository(); - - $uri = $repository->getDetail('remote-uri'); - $subpath = $repository->getDetail('svn-subpath'); - - return $uri.$subpath; - } -} diff --git a/src/applications/repository/daemon/commitdiscovery/svn/__init__.php b/src/applications/repository/daemon/commitdiscovery/svn/__init__.php deleted file mode 100644 index e8b0dd6f29..0000000000 --- a/src/applications/repository/daemon/commitdiscovery/svn/__init__.php +++ /dev/null @@ -1,15 +0,0 @@ -getArgv(); + array_unshift($argv, __CLASS__); + $args = new PhutilArgumentParser($argv); + $args->parse( + array( + array( + 'name' => 'no-discovery', + 'help' => 'Pull only, without discovering commits.', + ), + array( + 'name' => 'not', + 'param' => 'repository', + 'repeat' => true, + 'help' => 'Do not pull __repository__.', + ), + array( + 'name' => 'repositories', + 'wildcard' => true, + 'help' => 'Pull specific __repositories__ instead of all.', + ), + )); + + $no_discovery = $args->getArg('no-discovery'); + $repo_names = $args->getArg('repositories', array()); + $exclude_names = $args->getArg('not', array()); // Each repository has an individual pull frequency; after we pull it, // wait that long to pull it again. When we start up, try to pull everything // serially. $retry_after = array(); + $min_sleep = 15; + while (true) { - $repositories = $this->loadRepositories(); + $repositories = $this->loadRepositories($repo_names); + if ($exclude_names) { + $exclude = $this->loadRepositories($exclude_names); + $repositories = array_diff_key($repositories, $exclude); + } // Shuffle the repositories, then re-key the array since shuffle() // discards keys. This is mostly for startup, we'll use soft priorities @@ -79,22 +117,41 @@ final class PhabricatorRepositoryPullLocalDaemon foreach ($repositories as $id => $repository) { $after = idx($retry_after, $id, 0); - if ($after >= time()) { + if ($after > time()) { + continue; + } + + $tracked = $repository->isTracked(); + if (!$tracked) { continue; } try { self::pullRepository($repository); - $sleep_for = $repository->getDetail('pull-frequency', 15); + + if (!$no_discovery) { + // TODO: It would be nice to discover only if we pulled something, + // but this isn't totally trivial. + self::discoverRepository($repository); + } + + $sleep_for = $repository->getDetail('pull-frequency', $min_sleep); $retry_after[$id] = time() + $sleep_for; } catch (Exception $ex) { - $retry_after[$id] = time() + 15; + $retry_after[$id] = time() + $min_sleep; phlog($ex); } + + $this->stillWorking(); } - $sleep_until = max(min($retry_after), time() + 15); - sleep($sleep_until - time()); + if ($retry_after) { + $sleep_until = max(min($retry_after), time() + $min_sleep); + } else { + $sleep_until = time() + $min_sleep; + } + + $this->sleep($sleep_until - time()); } } @@ -102,12 +159,11 @@ final class PhabricatorRepositoryPullLocalDaemon /** * @task pull */ - protected function loadRepositories() { - $argv = $this->getArgv(); - if (!count($argv)) { + protected function loadRepositories(array $names) { + if (!count($names)) { return id(new PhabricatorRepository())->loadAll(); } else { - return PhabricatorRepository::loadAllByPHIDOrCallsign($argv); + return PhabricatorRepository::loadAllByPHIDOrCallsign($names); } } @@ -116,11 +172,6 @@ final class PhabricatorRepositoryPullLocalDaemon * @task pull */ public static function pullRepository(PhabricatorRepository $repository) { - $tracked = $repository->isTracked(); - if (!$tracked) { - return; - } - $vcs = $repository->getVersionControlSystem(); $is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN); @@ -153,19 +204,126 @@ final class PhabricatorRepositoryPullLocalDaemon } if ($is_git) { - self::executeGitCreate($repository, $local_path); + return self::executeGitCreate($repository, $local_path); } else if ($is_hg) { - self::executeHgCreate($repository, $local_path); + return self::executeHgCreate($repository, $local_path); } } else { if ($is_git) { - self::executeGitUpdate($repository, $local_path); + return self::executeGitUpdate($repository, $local_path); } else if ($is_hg) { - self::executeHgUpdate($repository, $local_path); + return self::executeHgUpdate($repository, $local_path); } } } + public static function discoverRepository(PhabricatorRepository $repository) { + $vcs = $repository->getVersionControlSystem(); + switch ($vcs) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + return self::executeGitDiscover($repository); + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + return self::executeSvnDiscover($repository); + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: + return self::executeHgDiscover($repository); + default: + throw new Exception("Unknown VCS '{$vcs}'!"); + } + } + + + private static function isKnownCommit( + PhabricatorRepository $repository, + $target) { + + if (self::getCache($repository, $target)) { + return true; + } + + $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( + 'repositoryID = %s AND commitIdentifier = %s', + $repository->getID(), + $target); + + if (!$commit) { + return false; + } + + self::setCache($repository, $target); + while (count(self::$commitCache) > 2048) { + array_shift(self::$commitCache); + } + + return true; + } + + private static function recordCommit( + PhabricatorRepository $repository, + $commit_identifier, + $epoch) { + + $commit = new PhabricatorRepositoryCommit(); + $commit->setRepositoryID($repository->getID()); + $commit->setCommitIdentifier($commit_identifier); + $commit->setEpoch($epoch); + + try { + $commit->save(); + $event = new PhabricatorTimelineEvent( + 'cmit', + array( + 'id' => $commit->getID(), + )); + $event->recordEvent(); + + queryfx( + $repository->establishConnection('w'), + 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) + VALUES (%d, 1, %d, %d) + ON DUPLICATE KEY UPDATE + size = size + 1, + lastCommitID = + IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID), + epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)', + PhabricatorRepository::TABLE_SUMMARY, + $repository->getID(), + $commit->getID(), + $epoch); + + self::setCache($repository, $commit_identifier); + } catch (AphrontQueryDuplicateKeyException $ex) { + // Ignore. This can happen because we discover the same new commit + // more than once when looking at history, or because of races or + // data inconsistency or cosmic radiation; in any case, we're still + // in a good state if we ignore the failure. + self::setCache($repository, $commit_identifier); + } + } + + private static function setCache( + PhabricatorRepository $repository, + $commit_identifier) { + + $key = self::getCacheKey($repository, $commit_identifier); + self::$commitCache[$key] = true; + } + + private static function getCache( + PhabricatorRepository $repository, + $commit_identifier) { + + $key = self::getCacheKey($repository, $commit_identifier); + return idx(self::$commitCache, $key, false); + } + + private static function getCacheKey( + PhabricatorRepository $repository, + $commit_identifier) { + + return $repository->getID().':'.$commit_identifier; + } + + /* -( Git Implementation )------------------------------------------------- */ @@ -249,6 +407,147 @@ final class PhabricatorRepositoryPullLocalDaemon } + /** + * @task git + */ + private static function executeGitDiscover( + PhabricatorRepository $repository) { + + list($remotes) = $repository->execxLocalCommand( + 'remote show -n origin'); + + $matches = null; + if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) { + throw new Exception( + "Expected 'Fetch URL' in 'git remote show -n origin'."); + } + + self::executeGitverifySameOrigin( + $matches[1], + $repository->getRemoteURI(), + $repository->getLocalPath()); + + list($stdout) = $repository->execxLocalCommand( + 'branch -r --verbose --no-abbrev'); + + $branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput( + $stdout, + $only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE); + + $tracked_something = false; + foreach ($branches as $name => $commit) { + if (!$repository->shouldTrackBranch($name)) { + continue; + } + + $tracked_something = true; + + if (self::isKnownCommit($repository, $commit)) { + continue; + } else { + self::executeGitDiscoverCommit($repository, $commit); + } + } + + if (!$tracked_something) { + $repo_name = $repository->getName(); + $repo_callsign = $repository->getCallsign(); + throw new Exception( + "Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ". + "Verify that your branch filtering settings are correct."); + } + } + + + /** + * @task git + */ + private static function executeGitDiscoverCommit( + PhabricatorRepository $repository, + $commit) { + + $discover = array($commit); + $insert = array($commit); + + $seen_parent = array(); + + while (true) { + $target = array_pop($discover); + list($parents) = $repository->execxLocalCommand( + 'log -n1 --pretty="%%P" %s', + $target); + $parents = array_filter(explode(' ', trim($parents))); + foreach ($parents as $parent) { + if (isset($seen_parent[$parent])) { + // We end up in a loop here somehow when we parse Arcanist if we + // don't do this. TODO: Figure out why and draw a pretty diagram + // since it's not evident how parsing a DAG with this causes the + // loop to stop terminating. + continue; + } + $seen_parent[$parent] = true; + if (!self::isKnownCommit($repository, $parent)) { + $discover[] = $parent; + $insert[] = $parent; + } + } + if (empty($discover)) { + break; + } + } + + while (true) { + $target = array_pop($insert); + list($epoch) = $repository->execxLocalCommand( + 'log -n1 --pretty="%%ct" %s', + $target); + $epoch = trim($epoch); + + self::recordCommit($repository, $target, $epoch); + + if (empty($insert)) { + break; + } + } + } + + + /** + * @task git + */ + public static function executeGitVerifySameOrigin($remote, $expect, $where) { + $remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote); + $expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect); + + $remote_path = $remote_uri->getPath(); + $expect_path = $expect_uri->getPath(); + + $remote_match = self::executeGitNormalizePath($remote_path); + $expect_match = self::executeGitNormalizePath($expect_path); + + if ($remote_match != $expect_match) { + throw new Exception( + "Working copy at '{$where}' has a mismatched origin URL. It has ". + "origin URL '{$remote}' (with remote path '{$remote_path}'), but the ". + "configured URL '{$expect}' (with remote path '{$expect_path}') is ". + "expected. Refusing to proceed because this may indicate that the ". + "working copy is actually some other repository."); + } + } + + + /** + * @task git + */ + private static function executeGitNormalizePath($path) { + // Strip away trailing "/" and ".git", so similar paths correctly match. + + $path = rtrim($path, '/'); + $path = preg_replace('/\.git$/', '', $path); + return $path; + } + + /* -( Mercurial Implementation )------------------------------------------- */ @@ -306,4 +605,177 @@ final class PhabricatorRepositoryPullLocalDaemon } } + private static function executeHgDiscover(PhabricatorRepository $repository) { + // NOTE: "--debug" gives us 40-character hashes. + list($stdout) = $repository->execxLocalCommand('--debug branches'); + + $branches = ArcanistMercurialParser::parseMercurialBranches($stdout); + $got_something = false; + foreach ($branches as $name => $branch) { + $commit = $branch['rev']; + if (self::isKnownCommit($repository, $commit)) { + continue; + } else { + self::executeHgDiscoverCommit($repository, $commit); + $got_something = true; + } + } + + return $got_something; + } + + private static function executeHgDiscoverCommit( + PhabricatorRepository $repository, + $commit) { + + $discover = array($commit); + $insert = array($commit); + + $seen_parent = array(); + + // For all the new commits at the branch heads, walk backward until we find + // only commits we've aleady seen. + while (true) { + $target = array_pop($discover); + list($stdout) = $repository->execxLocalCommand( + 'parents --rev %s --template %s', + $target, + '{node}\n'); + $parents = array_filter(explode("\n", trim($stdout))); + foreach ($parents as $parent) { + if (isset($seen_parent[$parent])) { + continue; + } + $seen_parent[$parent] = true; + if (!self::isKnownCommit($repository, $parent)) { + $discover[] = $parent; + $insert[] = $parent; + } + } + if (empty($discover)) { + break; + } + } + + while (true) { + $target = array_pop($insert); + list($stdout) = $repository->execxLocalCommand( + 'log --rev %s --template %s', + $target, + '{date|rfc822date}'); + $epoch = strtotime($stdout); + + self::recordCommit($repository, $target, $epoch); + + if (empty($insert)) { + break; + } + } + } + + +/* -( Subversion Implementation )------------------------------------------ */ + + + private static function executeSvnDiscover( + PhabricatorRepository $repository) { + + $uri = self::executeSvnGetBaseSVNLogURI($repository); + + list($xml) = $repository->execxRemoteCommand( + 'log --xml --quiet --limit 1 %s@HEAD', + $uri); + + $results = self::executeSvnParseLogXML($xml); + $commit = head_key($results); + $epoch = head($results); + + if (self::isKnownCommit($repository, $commit)) { + return false; + } + + self::executeSvnDiscoverCommit($repository, $commit, $epoch); + return true; + } + + private static function executeSvnDiscoverCommit( + PhabricatorRepository $repository, + $commit, + $epoch) { + + $uri = self::executeSvnGetBaseSVNLogURI($repository); + + $discover = array( + $commit => $epoch, + ); + $upper_bound = $commit; + + $limit = 1; + while ($upper_bound > 1 && + !self::isKnownCommit($repository, $upper_bound)) { + // Find all the unknown commits on this path. Note that we permit + // importing an SVN subdirectory rather than the entire repository, so + // commits may be nonsequential. + list($err, $xml, $stderr) = $repository->execRemoteCommand( + ' log --xml --quiet --limit %d %s@%d', + $limit, + $uri, + $upper_bound - 1); + if ($err) { + if (preg_match('/(path|File) not found/', $stderr)) { + // We've gone all the way back through history and this path was not + // affected by earlier commits. + break; + } else { + throw new Exception("svn log error #{$err}: {$stderr}"); + } + } + $discover += self::executeSvnParseLogXML($xml); + + $upper_bound = min(array_keys($discover)); + + // Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially + // import large repositories fairly quickly, while pulling only as much + // data as we need in the common case (when we've already imported the + // repository and are just grabbing one commit at a time). + $limit = min($limit * 2, 256); + } + + // NOTE: We do writes only after discovering all the commits so that we're + // never left in a state where we've missed commits -- if the discovery + // script terminates it can always resume and restore the import to a good + // state. This is also why we sort the discovered commits so we can do + // writes forward from the smallest one. + + ksort($discover); + foreach ($discover as $commit => $epoch) { + self::recordCommit($repository, $commit, $epoch); + } + } + + private static function executeSvnParseLogXML($xml) { + $xml = phutil_utf8ize($xml); + + $result = array(); + + $log = new SimpleXMLElement($xml); + foreach ($log->logentry as $entry) { + $commit = (int)$entry['revision']; + $epoch = (int)strtotime((string)$entry->date[0]); + $result[$commit] = $epoch; + } + + return $result; + } + + + private static function executeSvnGetBaseSVNLogURI( + PhabricatorRepository $repository) { + + $uri = $repository->getDetail('remote-uri'); + $subpath = $repository->getDetail('svn-subpath'); + + return $uri.$subpath; + } + } diff --git a/src/applications/repository/daemon/pulllocal/__init__.php b/src/applications/repository/daemon/pulllocal/__init__.php index cb9709df52..c0c306a6bd 100644 --- a/src/applications/repository/daemon/pulllocal/__init__.php +++ b/src/applications/repository/daemon/pulllocal/__init__.php @@ -6,12 +6,20 @@ +phutil_require_module('arcanist', 'repository/parser/mercurial'); + +phutil_require_module('phabricator', 'applications/diffusion/data/branch'); +phutil_require_module('phabricator', 'applications/diffusion/query/branch/git'); phutil_require_module('phabricator', 'applications/repository/constants/repositorytype'); +phutil_require_module('phabricator', 'applications/repository/storage/commit'); phutil_require_module('phabricator', 'applications/repository/storage/repository'); phutil_require_module('phabricator', 'infrastructure/daemon/base'); +phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event'); +phutil_require_module('phabricator', 'storage/queryfx'); phutil_require_module('phutil', 'error'); phutil_require_module('phutil', 'filesystem'); +phutil_require_module('phutil', 'parser/argument/parser'); phutil_require_module('phutil', 'utils'); diff --git a/src/applications/repository/daemon/commitdiscovery/git/__tests__/PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php b/src/applications/repository/daemon/pulllocal/__tests__/PhabricatorRepositoryPullLocalDaemonTestCase.php similarity index 94% rename from src/applications/repository/daemon/commitdiscovery/git/__tests__/PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php rename to src/applications/repository/daemon/pulllocal/__tests__/PhabricatorRepositoryPullLocalDaemonTestCase.php index 1bef77e5c1..43130bd6b9 100644 --- a/src/applications/repository/daemon/commitdiscovery/git/__tests__/PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php +++ b/src/applications/repository/daemon/pulllocal/__tests__/PhabricatorRepositoryPullLocalDaemonTestCase.php @@ -16,10 +16,10 @@ * limitations under the License. */ -final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase +final class PhabricatorRepositoryPullLocalDaemonTestCase extends PhabricatorTestCase { - public function testVerifySameGitOrigin() { + public function testExecuteGitVerifySameOrigin() { $cases = array( array( 'ssh://user@domain.com/path.git', @@ -94,7 +94,7 @@ final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase $ex = null; try { - PhabricatorRepositoryGitCommitDiscoveryDaemon::verifySameGitOrigin( + PhabricatorRepositoryPullLocalDaemon::executeGitverifySameOrigin( $remote, $config, '(a test case)'); diff --git a/src/applications/repository/daemon/commitdiscovery/git/__tests__/__init__.php b/src/applications/repository/daemon/pulllocal/__tests__/__init__.php similarity index 68% rename from src/applications/repository/daemon/commitdiscovery/git/__tests__/__init__.php rename to src/applications/repository/daemon/pulllocal/__tests__/__init__.php index d72145acfd..92606421a2 100644 --- a/src/applications/repository/daemon/commitdiscovery/git/__tests__/__init__.php +++ b/src/applications/repository/daemon/pulllocal/__tests__/__init__.php @@ -6,8 +6,8 @@ -phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/git'); +phutil_require_module('phabricator', 'applications/repository/daemon/pulllocal'); phutil_require_module('phabricator', 'infrastructure/testing/testcase'); -phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php'); +phutil_require_source('PhabricatorRepositoryPullLocalDaemonTestCase.php');