archive/phabricator
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Introduce ref cursors for repository parsing

Browse Source 
Summary:
Ref T4327. I want to make change parsing testable; one thing which is blocking this is that the Git discovery process is still part of `PullLocal` daemon instead of being part of `DiscoveryEngine`. The unit test stuff which I want to use for change parsing relies on `DiscoveryEngine` to discover repositories during unit tests.

The major reason git discovery isn't part of `DiscoveryEngine` is that it relies on the messy "autoclose" logic, which we never implemented for Mercurial. Generally, I don't like how autoclose was implemented: it's complicated and gross and too hard to figure out and extend.

Instead, I want to do something more similar to what we do for pushes, which is cleaner overall. Basically this means remembering the old branch heads from the last time we parsed a repository, and figuring out what's new by comparing the old and new branch heads. This should give us several advantages:

  - It should be simpler to understand than the autoclose stuff, which is pretty mind-numbing, at least for me.
  - It will let us satisfy branch and tag queries cheaply (from the database) instead of having to go to the repository. We could also satisfy some ref-resolve queries from the database.
  - It should be easier to extend to Mercurial.

This implements the basics -- pretty much a table to store the cursors, which we update only for Git for now.

Test Plan:
  - Ran migration.
  - Ran `bin/repository discover X --trace --verbose` on various repositories with branches and tags, before and after modifying pushes.
  - Pushed commits to a git repo.
  - Looked at database tables.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T4327

Differential Revision: https://secure.phabricator.com/D7982
This commit is contained in:
epriestley
2014-01-16 12:05:30 -08:00
parent 0ac58d7db6
commit f4b9efe256
12 changed files with 496 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php Normal file
View File

@@ -0,0 +1,223 @@
<?php
/**
* Update the ref cursors for a repository, which track the positions of
* branches, bookmarks, and tags.
*/
final class PhabricatorRepositoryRefEngine
extends PhabricatorRepositoryEngine {
private $newRefs = array();
private $deadRefs = array();
public function updateRefs() {
$this->newRefs = array();
$this->deadRefs = array();
$repository = $this->getRepository();
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
// No meaningful refs of any type in Subversion.
$branches = array();
$bookmarks = array();
$tags = array();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$branches = $this->loadMercurialBranchPositions($repository);
$bookmarks = $this->loadMercurialBookmarkPositions($repository);
$tags = array();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$branches = $this->loadGitBranchPositions($repository);
$bookmarks = array();
$tags = $this->loadGitTagPositions($repository);
break;
default:
throw new Exception(pht('Unknown VCS "%s"!', $vcs));
}
$maps = array(
PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
PhabricatorRepositoryRefCursor::TYPE_TAG => $tags,
PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
);
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withRepositoryPHIDs(array($repository->getPHID()))
->execute();
$cursor_groups = mgroup($all_cursors, 'getRefType');
foreach ($maps as $type => $refs) {
$cursor_group = idx($cursor_groups, $type, array());
$this->updateCursors($cursor_group, $refs, $type);
}
if ($this->newRefs || $this->deadRefs) {
$repository->openTransaction();
foreach ($this->newRefs as $ref) {
$ref->save();
}
foreach ($this->deadRefs as $ref) {
$ref->delete();
}
$repository->saveTransaction();
$this->newRefs = array();
$this->deadRefs = array();
}
}
private function markRefNew(PhabricatorRepositoryRefCursor $cursor) {
$this->newRefs[] = $cursor;
return $this;
}
private function markRefDead(PhabricatorRepositoryRefCursor $cursor) {
$this->deadRefs[] = $cursor;
return $this;
}
private function updateCursors(
array $cursors,
array $new_refs,
$ref_type) {
$repository = $this->getRepository();
// NOTE: Mercurial branches may have multiple branch heads; this logic
// is complex primarily to account for that.
// Group all the cursors by their ref name, like "master". Since Mercurial
// branches may have multiple heads, there could be several cursors with
// the same name.
$cursor_groups = mgroup($cursors, 'getRefNameRaw');
// Group all the new ref values by their name. As above, these groups may
// have multiple members in Mercurial.
$ref_groups = mgroup($new_refs, 'getShortName');
foreach ($ref_groups as $name => $refs) {
$new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');
$ref_cursors = idx($cursor_groups, $name, array());
$old_commits = mpull($ref_cursors, null, 'getCommitIdentifier');
// We're going to delete all the cursors pointing at commits which are
// no longer associated with the refs. This primarily makes the Mercurial
// multiple head case easier, and means that when we update a ref we
// delete the old one and write a new one.
foreach ($ref_cursors as $cursor) {
if (isset($new_commits[$cursor->getCommitIdentifier()])) {
// This ref previously pointed at this commit, and still does.
$this->log(
pht(
'Ref %s "%s" still points at %s.',
$ref_type,
$name,
$cursor->getCommitIdentifier()));
} else {
// This ref previously pointed at this commit, but no longer does.
$this->log(
pht(
'Ref %s "%s" no longer points at %s.',
$ref_type,
$name,
$cursor->getCommitIdentifier()));
// Nuke the obsolete cursor.
$this->markRefDead($cursor);
}
}
// Now, we're going to insert new cursors for all the commits which are
// associated with this ref that don't currently have cursors.
$added_commits = array_diff_key($new_commits, $old_commits);
foreach ($added_commits as $identifier) {
$this->log(
pht(
'Ref %s "%s" now points at %s.',
$ref_type,
$name,
$identifier));
$this->markRefNew(
id(new PhabricatorRepositoryRefCursor())
->setRepositoryPHID($repository->getPHID())
->setRefType($ref_type)
->setRefName($name)
->setCommitIdentifier($identifier));
}
foreach ($added_commits as $identifier) {
// TODO: Do autoclose stuff here.
}
}
// Find any cursors for refs which no longer exist. This happens when a
// branch, tag or bookmark is deleted.
foreach ($cursor_groups as $name => $cursor_group) {
if (idx($ref_groups, $name) === null) {
$this->log(
pht(
'Ref %s "%s" no longer exists.',
$cursor->getRefType(),
$cursor->getRefName()));
foreach ($cursor_group as $cursor) {
$this->markRefDead($cursor);
}
}
}
}
/* -( Updating Git Refs )-------------------------------------------------- */
/**
* @task git
*/
private function loadGitBranchPositions(PhabricatorRepository $repository) {
return id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
->withIsOriginBranch(true)
->execute();
}
/**
* @task git
*/
private function loadGitTagPositions(PhabricatorRepository $repository) {
return id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
->withIsTag(true)
->execute();
}
/* -( Updating Mercurial Refs )-------------------------------------------- */
/**
* @task hg
*/
private function loadMercurialBranchPositions(
PhabricatorRepository $repository) {
return id(new DiffusionLowLevelMercurialBranchesQuery())
->setRepository($repository)
->execute();
}
/**
* @task hg
*/
private function loadMercurialBookmarkPositions(
PhabricatorRepository $repository) {
// TODO: Implement support for Mercurial bookmarks.
return array();
}
}