Use one daemon to discover commits in all repositories, not one per repository

Summary:
See D2418. This merges the commit discovery daemon into the same single daemon, and applies all the same rules to it.

There are relatively few implementation changes, but a few things did change:

  - I simplified/improved Mercurial importing, by finding full branch tip hashes with "--debug branches" and using "parents --template {node}" so we don't need to do separate "--debug id" calls.
  - Added a new "--not" flag to exclude repositories, since I switched to real arg parsing anyway.
  - I removed a web UI notification that you need to restart the daemons, this is no longer true.
  - I added a web UI notification that no pull daemon is running on the machine.

NOTE: @makinde, this doesn't change anything from your perspective, but it something breaks this is the likely cause.

This implicitly resolves T792, because discovery no longer runs before pulling.

Test Plan:

  - Swapped databases to a fresh install.
  - Ran "pulllocal" in debug mode. Verified it correctly does nothing (fixed a minor issue with min() on empty array).
  - Added an SVN repository. Verified it cloned and discovered correctly.
  - Added a Mercurial repository. Verified it cloned and discovered correctly.
  - Added a Git repository. Verified it cloned and discovered correctly.
  - Ran with arguments to verify behaviors: "--not MTEST --not STEST", "P --no-discovery", "P".

Reviewers: btrahan, csilvers, Makinde

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T792

Differential Revision: https://secure.phabricator.com/D2430

scripts/daemon/phabricator_daemon_launcher.php

@@ -52,13 +52,20 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
     $need_launch = phd_load_tracked_repositories();
     if (!$need_launch) {
       echo "There are no repositories with tracking enabled.\n";
-      exit(0);
+      exit(1);
     }
 
     will_launch($control);
+
+    echo "Launching PullLocal daemon in readonly mode...\n";
+
     $control->launchDaemon(
       'PhabricatorRepositoryPullLocalDaemon',
-      array());
+      array(
+        '--no-discovery',
+      ));
+
+    echo "Done.\n";
     break;
 
   case 'repository-launch-master':
@@ -66,55 +73,24 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
     if (!$need_launch) {
       echo "There are no repositories with tracking enabled.\n";
       exit(1);
-    } else {
-      will_launch($control);
-
-      $control->launchDaemon(
-        'PhabricatorRepositoryPullLocalDaemon',
-        array());
-
-      foreach ($need_launch as $repository) {
-        $name = $repository->getName();
-        $callsign = $repository->getCallsign();
-        $desc = "'{$name}' ({$callsign})";
-        $phid = $repository->getPHID();
-
-        switch ($repository->getVersionControlSystem()) {
-          case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-            echo "Launching discovery daemon on the {$desc} repository...\n";
-            $control->launchDaemon(
-              'PhabricatorRepositoryGitCommitDiscoveryDaemon',
-              array(
-                $phid,
-              ));
-            break;
-          case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
-            echo "Launching discovery daemon on the {$desc} repository...\n";
-            $control->launchDaemon(
-              'PhabricatorRepositorySvnCommitDiscoveryDaemon',
-              array(
-                $phid,
-              ));
-            break;
-          case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
-            echo "Launching discovery daemon on the {$desc} repository...\n";
-            $control->launchDaemon(
-              'PhabricatorRepositoryMercurialCommitDiscoveryDaemon',
-              array(
-                $phid,
-              ));
-            break;
-
-        }
-      }
-
-      echo "Launching CommitTask daemon...\n";
-      $control->launchDaemon(
-        'PhabricatorRepositoryCommitTaskDaemon',
-        array());
-
-      echo "Done.\n";
     }
+
+    will_launch($control);
+
+    echo "Launching PullLocal daemon in master mode...\n";
+    $control->launchDaemon(
+      'PhabricatorRepositoryPullLocalDaemon',
+      array());
+
+    echo "Launching CommitTask daemon...\n";
+    $control->launchDaemon(
+      'PhabricatorRepositoryCommitTaskDaemon',
+      array());
+
+    echo "NOTE: Make sure you run some taskmaster daemons too, e.g. ".
+         "with 'phd launch 4 taskmaster'.\n";
+
+    echo "Done.\n";
     break;
 
   case 'launch':

scripts/repository/discover.php

@@ -0,0 +1,51 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline('manually discover working copies');
+$args->setSynopsis(<<<EOHELP
+**discover.php** [__options__] __repository-callsign-or-phid ...__
+    Manually discover commits in working copies for the named repositories.
+EOHELP
+);
+$args->parseStandardArguments();
+$args->parse(
+  array(
+    array(
+      'name'      => 'repositories',
+      'wildcard'  => true,
+    ),
+  ));
+
+$repo_names = $args->getArg('repositories');
+if (!$repo_names) {
+  echo "Specify one or more repositories to pull, by callsign or PHID.\n";
+  exit(1);
+}
+
+$repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
+foreach ($repos as $repo) {
+  $callsign = $repo->getCallsign();
+  echo "Discovering '{$callsign}'...\n";
+  PhabricatorRepositoryPullLocalDaemon::discoverRepository($repo);
+}
+echo "Done.\n";

src/__phutil_library_map__.php

@@ -840,7 +840,6 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryCommit' => 'applications/repository/storage/commit',
     'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/base',
     'PhabricatorRepositoryCommitData' => 'applications/repository/storage/commitdata',
-    'PhabricatorRepositoryCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/base',
     'PhabricatorRepositoryCommitHeraldWorker' => 'applications/repository/worker/herald',
     'PhabricatorRepositoryCommitMessageDetailParser' => 'applications/repository/parser/base',
     'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base',
@@ -850,22 +849,18 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryController' => 'applications/repository/controller/base',
     'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create',
     'PhabricatorRepositoryDAO' => 'applications/repository/storage/base',
-    'PhabricatorRepositoryDaemon' => 'applications/repository/daemon/base',
     'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'applications/repository/parser/default',
     'PhabricatorRepositoryDeleteController' => 'applications/repository/controller/delete',
     'PhabricatorRepositoryEditController' => 'applications/repository/controller/edit',
     'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/git',
-    'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/git',
-    'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'applications/repository/daemon/commitdiscovery/git/__tests__',
     'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git',
     'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
     'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
-    'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
     'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal',
+    'PhabricatorRepositoryPullLocalDaemonTestCase' => 'applications/repository/daemon/pulllocal/__tests__',
     'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut',
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn',
-    'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
     'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
     'PhabricatorRepositorySymbol' => 'applications/repository/storage/symbol',
     'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__',
@@ -1735,31 +1730,26 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO',
-    'PhabricatorRepositoryCommitDiscoveryDaemon' => 'PhabricatorRepositoryDaemon',
     'PhabricatorRepositoryCommitHeraldWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
     'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
-    'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorRepositoryDaemon',
+    'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorDaemon',
     'PhabricatorRepositoryController' => 'PhabricatorController',
     'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
-    'PhabricatorRepositoryDaemon' => 'PhabricatorDaemon',
     'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'PhabricatorRepositoryCommitMessageDetailParser',
     'PhabricatorRepositoryDeleteController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryEditController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
-    'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
-    'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'PhabricatorTestCase',
     'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
     'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
-    'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
     'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
+    'PhabricatorRepositoryPullLocalDaemonTestCase' => 'PhabricatorTestCase',
     'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
-    'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
     'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
     'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase',

src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php

@@ -1,7 +1,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,7 +33,11 @@ final class DiffusionSvnHistoryQuery extends DiffusionHistoryQuery {
       PhabricatorRepository::TABLE_PATH,
       array(md5('/'.trim($path, '/'))));
     $paths = ipull($paths, 'id', 'path');
-    $path_id = $paths['/'.trim($path, '/')];
+    $path_id = idx($paths, '/'.trim($path, '/'));
+
+    if (!$path_id) {
+      return array();
+    }
 
     $filter_query = '';
     if ($this->needDirectChanges) {

src/applications/repository/controller/base/PhabricatorRepositoryController.php

@@ -1,7 +1,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,8 +32,43 @@ abstract class PhabricatorRepositoryController extends PhabricatorController {
     $page->setGlyph("rX");
     $page->appendChild($view);
 
+
     $response = new AphrontWebpageResponse();
     return $response->setContent($page->render());
   }
 
+  private function isPullDaemonRunningOnThisMachine() {
+
+    // This is sort of hacky, but should probably work.
+
+    list($stdout) = execx('ps auxwww');
+    return preg_match('/PhabricatorRepositoryPullLocalDaemon/', $stdout);
+  }
+
+  protected function renderDaemonNotice() {
+    $daemon_running = $this->isPullDaemonRunningOnThisMachine();
+    if ($daemon_running) {
+      return null;
+    }
+
+    $documentation = phutil_render_tag(
+      'a',
+      array(
+        'href' => PhabricatorEnv::getDoclink(
+          'article/Diffusion_User_Guide.html'),
+      ),
+      'Diffusion User Guide');
+
+    $view = new AphrontErrorView();
+    $view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
+    $view->setTitle('Repository Daemon Not Running');
+    $view->appendChild(
+      "<p>The repository daemon is not running on this machine. Without this ".
+      "daemon, Phabricator will not be able to import or update repositories. ".
+      "For instructions on starting the daemon, see ".
+      "<strong>{$documentation}</strong>.</p>");
+
+    return $view;
+  }
+
 }

src/applications/repository/controller/base/__init__.php

@@ -8,7 +8,11 @@
 
 phutil_require_module('phabricator', 'aphront/response/webpage');
 phutil_require_module('phabricator', 'applications/base/controller/base');
+phutil_require_module('phabricator', 'infrastructure/env');
+phutil_require_module('phabricator', 'view/form/error');
 
+phutil_require_module('phutil', 'future/exec');
+phutil_require_module('phutil', 'markup');
 phutil_require_module('phutil', 'utils');
 
 

src/applications/repository/controller/edit/PhabricatorRepositoryEditController.php

@@ -63,6 +63,8 @@ final class PhabricatorRepositoryEditController
           phutil_escape_html($name)));
     }
 
+    $nav->appendChild($this->renderDaemonNotice());
+
     $this->sideNav = $nav;
 
     switch ($this->view) {
@@ -345,9 +347,7 @@ final class PhabricatorRepositoryEditController
       $error_view = new AphrontErrorView();
       $error_view->setSeverity(AphrontErrorView::SEVERITY_NOTICE);
       $error_view->setTitle('Changes Saved');
-      $error_view->appendChild(
+      $error_view->appendChild('Tracking changes were saved.');
-        'Tracking changes were saved. You may need to restart the daemon '.
-        'before changes will take effect.');
     } else if (!$repository->isTracked()) {
       $error_view = new AphrontErrorView();
       $error_view->setSeverity(AphrontErrorView::SEVERITY_WARNING);

src/applications/repository/controller/list/PhabricatorRepositoryListController.php

@@ -157,6 +157,7 @@ final class PhabricatorRepositoryListController
 
     return $this->buildStandardPageResponse(
       array(
+        $this->renderDaemonNotice(),
         $panel,
         $project_panel,
       ),

src/applications/repository/daemon/base/PhabricatorRepositoryDaemon.php

@@ -1,38 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-abstract class PhabricatorRepositoryDaemon extends PhabricatorDaemon {
-
-  protected function loadRepository() {
-    $argv = $this->getArgv();
-    if (count($argv) !== 1) {
-      throw new Exception("No repository PHID provided!");
-    }
-
-    $repository = id(new PhabricatorRepository())->loadOneWhere(
-      'phid = %s',
-      $argv[0]);
-
-    if (!$repository) {
-      throw new Exception("No such repository exists!");
-    }
-
-    return $repository;
-  }
-
-}

src/applications/repository/daemon/base/__init__.php

@@ -1,15 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('phabricator', 'applications/repository/storage/repository');
-phutil_require_module('phabricator', 'infrastructure/daemon/base');
-
-phutil_require_module('phutil', 'utils');
-
-
-phutil_require_source('PhabricatorRepositoryDaemon.php');

src/applications/repository/daemon/commitdiscovery/base/PhabricatorRepositoryCommitDiscoveryDaemon.php

@@ -1,113 +0,0 @@
-<?php
-
-/*
- * Copyright 2012 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-abstract class PhabricatorRepositoryCommitDiscoveryDaemon
-  extends PhabricatorRepositoryDaemon {
-
-  private $repository;
-  private $commitCache = array();
-
-  final protected function getRepository() {
-    return $this->repository;
-  }
-
-  final public function run() {
-    while (true) {
-      // Reload the repository every time to pick up changes from the web
-      // console.
-      $this->repository = $this->loadRepository();
-      $this->discoverCommits();
-
-      $sleep = max(2, $this->getRepository()->getDetail('pull-frequency'));
-      $this->sleep($sleep);
-    }
-  }
-
-  final public function runOnce() {
-    $this->repository = $this->loadRepository();
-    $this->discoverCommits();
-  }
-
-  protected function isKnownCommit($target) {
-    if (isset($this->commitCache[$target])) {
-      return true;
-    }
-
-    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
-      'repositoryID = %s AND commitIdentifier = %s',
-      $this->getRepository()->getID(),
-      $target);
-
-    if (!$commit) {
-      return false;
-    }
-
-    $this->commitCache[$target] = true;
-    while (count($this->commitCache) > 64) {
-      array_shift($this->commitCache);
-    }
-
-    return true;
-  }
-
-  protected function recordCommit($commit_identifier, $epoch) {
-    $repository = $this->getRepository();
-
-    $commit = new PhabricatorRepositoryCommit();
-    $commit->setRepositoryID($repository->getID());
-    $commit->setCommitIdentifier($commit_identifier);
-    $commit->setEpoch($epoch);
-
-    try {
-      $commit->save();
-      $event = new PhabricatorTimelineEvent(
-        'cmit',
-        array(
-          'id' => $commit->getID(),
-        ));
-      $event->recordEvent();
-
-      queryfx(
-        $repository->establishConnection('w'),
-        'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
-          VALUES (%d, 1, %d, %d)
-          ON DUPLICATE KEY UPDATE
-            size = size + 1,
-            lastCommitID =
-              IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
-            epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
-        PhabricatorRepository::TABLE_SUMMARY,
-        $repository->getID(),
-        $commit->getID(),
-        $epoch);
-
-      $this->commitCache[$commit_identifier] = true;
-    } catch (AphrontQueryDuplicateKeyException $ex) {
-      // Ignore. This can happen because we discover the same new commit
-      // more than once when looking at history, or because of races or
-      // data inconsistency or cosmic radiation; in any case, we're still
-      // in a good state if we ignore the failure.
-      $this->commitCache[$commit_identifier] = true;
-    }
-
-    $this->stillWorking();
-  }
-
-  abstract protected function discoverCommits();
-
-}

src/applications/repository/daemon/commitdiscovery/base/__init__.php

@@ -1,18 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('phabricator', 'applications/repository/daemon/base');
-phutil_require_module('phabricator', 'applications/repository/storage/commit');
-phutil_require_module('phabricator', 'applications/repository/storage/repository');
-phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
-phutil_require_module('phabricator', 'storage/queryfx');
-
-phutil_require_module('phutil', 'utils');
-
-
-phutil_require_source('PhabricatorRepositoryCommitDiscoveryDaemon.php');

src/applications/repository/daemon/commitdiscovery/git/PhabricatorRepositoryGitCommitDiscoveryDaemon.php

@@ -1,162 +0,0 @@
-<?php
-
-/*
- * Copyright 2012 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-final class PhabricatorRepositoryGitCommitDiscoveryDaemon
-  extends PhabricatorRepositoryCommitDiscoveryDaemon {
-
-  protected function discoverCommits() {
-    // NOTE: PhabricatorRepositoryGitFetchDaemon does the actual pulls, this
-    // just parses HEAD.
-
-    $repository = $this->getRepository();
-
-    $vcs = $repository->getVersionControlSystem();
-    if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_GIT) {
-      throw new Exception("Repository is not a git repository.");
-    }
-
-    list($remotes) = $repository->execxLocalCommand(
-      'remote show -n origin');
-
-    $matches = null;
-    if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
-      throw new Exception(
-        "Expected 'Fetch URL' in 'git remote show -n origin'.");
-    }
-
-    self::verifySameGitOrigin(
-      $matches[1],
-      $repository->getRemoteURI(),
-      $repository->getLocalPath());
-
-    list($stdout) = $repository->execxLocalCommand(
-      'branch -r --verbose --no-abbrev');
-
-    $branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
-      $stdout,
-      $only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
-
-    $got_something = false;
-    $tracked_something = false;
-    foreach ($branches as $name => $commit) {
-      if (!$repository->shouldTrackBranch($name)) {
-        continue;
-      }
-
-      $tracked_something = true;
-
-      if ($this->isKnownCommit($commit)) {
-        continue;
-      } else {
-        $this->discoverCommit($commit);
-        $got_something = true;
-      }
-    }
-
-    if (!$tracked_something) {
-      $repo_name = $repository->getName();
-      $repo_callsign = $repository->getCallsign();
-      throw new Exception(
-        "Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
-        "Verify that your branch filtering settings are correct.");
-    }
-
-    return $got_something;
-  }
-
-  private function discoverCommit($commit) {
-    $discover = array();
-    $insert = array();
-
-    $repository = $this->getRepository();
-
-    $discover[] = $commit;
-    $insert[] = $commit;
-
-    $seen_parent = array();
-
-    while (true) {
-      $target = array_pop($discover);
-      list($parents) = $repository->execxLocalCommand(
-        'log -n1 --pretty="%%P" %s',
-        $target);
-      $parents = array_filter(explode(' ', trim($parents)));
-      foreach ($parents as $parent) {
-        if (isset($seen_parent[$parent])) {
-          // We end up in a loop here somehow when we parse Arcanist if we
-          // don't do this. TODO: Figure out why and draw a pretty diagram
-          // since it's not evident how parsing a DAG with this causes the
-          // loop to stop terminating.
-          continue;
-        }
-        $seen_parent[$parent] = true;
-        if (!$this->isKnownCommit($parent)) {
-          $discover[] = $parent;
-          $insert[] = $parent;
-        }
-      }
-      if (empty($discover)) {
-        break;
-      }
-      $this->stillWorking();
-    }
-
-    while (true) {
-      $target = array_pop($insert);
-      list($epoch) = $repository->execxLocalCommand(
-        'log -n1 --pretty="%%ct" %s',
-        $target);
-      $epoch = trim($epoch);
-
-      $this->recordCommit($target, $epoch);
-
-      if (empty($insert)) {
-        break;
-      }
-    }
-  }
-
-  public static function verifySameGitOrigin($remote, $expect, $where) {
-    $remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
-    $expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
-
-    $remote_path = $remote_uri->getPath();
-    $expect_path = $expect_uri->getPath();
-
-    $remote_match = self::normalizeGitPath($remote_path);
-    $expect_match = self::normalizeGitPath($expect_path);
-
-    if ($remote_match != $expect_match) {
-      throw new Exception(
-        "Working copy at '{$where}' has a mismatched origin URL. It has ".
-        "origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
-        "configured URL '{$expect}' (with remote path '{$expect_path}') is ".
-        "expected. Refusing to proceed because this may indicate that the ".
-        "working copy is actually some other repository.");
-    }
-  }
-
-  private static function normalizeGitPath($path) {
-    // Strip away trailing "/" and ".git", so similar paths correctly match.
-
-    $path = rtrim($path, '/');
-    $path = preg_replace('/\.git$/', '', $path);
-    return $path;
-  }
-
-}

src/applications/repository/daemon/commitdiscovery/git/__init__.php

@@ -1,16 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('phabricator', 'applications/diffusion/data/branch');
-phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
-phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
-phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
-phutil_require_module('phabricator', 'applications/repository/storage/repository');
-
-
-phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemon.php');

src/applications/repository/daemon/commitdiscovery/mercurial/PhabricatorRepositoryMercurialCommitDiscoveryDaemon.php

@@ -1,137 +0,0 @@
-<?php
-
-/*
- * Copyright 2012 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-final class PhabricatorRepositoryMercurialCommitDiscoveryDaemon
-  extends PhabricatorRepositoryCommitDiscoveryDaemon {
-
-  protected function discoverCommits() {
-    $repository = $this->getRepository();
-
-    $vcs = $repository->getVersionControlSystem();
-    if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL) {
-      throw new Exception("Repository is not a Mercurial repository.");
-    }
-
-    $repository_phid = $repository->getPHID();
-
-    list($stdout) = $repository->execxLocalCommand('branches');
-
-    $branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
-    $got_something = false;
-    foreach ($branches as $name => $branch) {
-      $commit = $branch['rev'];
-      $commit = $this->getFullHash($commit);
-      if ($this->isKnownCommit($commit)) {
-        continue;
-      } else {
-        $this->discoverCommit($commit);
-        $got_something = true;
-      }
-    }
-
-    return $got_something;
-  }
-
-  private function getFullHash($commit) {
-
-    // NOTE: Mercurial shortens hashes to 12 characters by default. This
-    // implies collisions with as few as a few million commits. The
-    // documentation sensibly advises "Do not use short-form IDs for
-    // long-lived representations". It then continues "You can use the
-    // --debug option to display the full changeset ID". What?! Yes, this
-    // is in fact the only way to turn on full hashes, and the hg source
-    // code is littered with "hexfn = ui.debugflag and hex or short" and
-    // similar. There is no more-selective flag or config option.
-    //
-    // Unfortunately, "hg --debug" turns on tons of other extra output,
-    // including full commit messages in "hg log" and "hg parents" (which
-    // ignore --style); this renders them unparseable. So we have to use
-    // "hg id" to convert short hashes into full hashes. See:
-    //
-    // <http://mercurial.selenic.com/wiki/ChangeSetID>
-    //
-    // Of course, this means that if there are collisions we will break here
-    // (the short commit identifier won't be unambiguous) but maybe Mercurial
-    // will have a --full-hashes flag or something by then and we can fix it
-    // properly. Until we run into that, this allows us to store data in the
-    // right format so when we eventually encounter this we won't have to
-    // reparse every Mercurial repository.
-
-    $repository = $this->getRepository();
-    list($stdout) = $repository->execxLocalCommand(
-      'id --debug -i --rev %s',
-      $commit);
-    return trim($stdout);
-  }
-
-  private function discoverCommit($commit) {
-    $discover = array();
-    $insert = array();
-
-    $repository = $this->getRepository();
-
-    $discover[] = $commit;
-    $insert[] = $commit;
-
-    $seen_parent = array();
-
-    // For all the new commits at the branch heads, walk backward until we find
-    // only commits we've aleady seen.
-    while (true) {
-      $target = array_pop($discover);
-      list($stdout) = $repository->execxLocalCommand(
-        'parents --style default --rev %s',
-        $target);
-      $parents = ArcanistMercurialParser::parseMercurialLog($stdout);
-      if ($parents) {
-        foreach ($parents as $parent) {
-          $parent_commit = $parent['rev'];
-          $parent_commit = $this->getFullHash($parent_commit);
-          if (isset($seen_parent[$parent_commit])) {
-            continue;
-          }
-          $seen_parent[$parent_commit] = true;
-          if (!$this->isKnownCommit($parent_commit)) {
-            $discover[] = $parent_commit;
-            $insert[] = $parent_commit;
-          }
-        }
-      }
-      if (empty($discover)) {
-        break;
-      }
-      $this->stillWorking();
-    }
-
-    while (true) {
-      $target = array_pop($insert);
-      list($stdout) = $repository->execxLocalCommand(
-        'log --rev %s --template %s',
-        $target,
-        '{date|rfc822date}');
-      $epoch = strtotime($stdout);
-
-      $this->recordCommit($target, $epoch);
-
-      if (empty($insert)) {
-        break;
-      }
-    }
-  }
-
-}

src/applications/repository/daemon/commitdiscovery/mercurial/__init__.php

@@ -1,15 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('arcanist', 'repository/parser/mercurial');
-
-phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
-phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
-
-
-phutil_require_source('PhabricatorRepositoryMercurialCommitDiscoveryDaemon.php');

src/applications/repository/daemon/commitdiscovery/svn/PhabricatorRepositorySvnCommitDiscoveryDaemon.php

@@ -1,123 +0,0 @@
-<?php
-
-/*
- * Copyright 2012 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-final class PhabricatorRepositorySvnCommitDiscoveryDaemon
-  extends PhabricatorRepositoryCommitDiscoveryDaemon {
-
-  protected function discoverCommits() {
-    $repository = $this->getRepository();
-
-    $vcs = $repository->getVersionControlSystem();
-    if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) {
-      throw new Exception("Repository is not a svn repository.");
-    }
-
-    $uri = $this->getBaseSVNLogURI();
-    list($xml) = $repository->execxRemoteCommand(
-      'log --xml --quiet --limit 1 %s@HEAD',
-      $uri);
-
-    $results = $this->parseSVNLogXML($xml);
-    $commit = head_key($results);
-    $epoch  = head($results);
-
-    if ($this->isKnownCommit($commit)) {
-      return false;
-    }
-
-    $this->discoverCommit($commit, $epoch);
-
-    return true;
-  }
-
-  private function discoverCommit($commit, $epoch) {
-    $uri = $this->getBaseSVNLogURI();
-    $repository = $this->getRepository();
-
-    $discover = array(
-      $commit => $epoch,
-    );
-    $upper_bound = $commit;
-
-    $limit = 1;
-    while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound)) {
-      // Find all the unknown commits on this path. Note that we permit
-      // importing an SVN subdirectory rather than the entire repository, so
-      // commits may be nonsequential.
-      list($err, $xml, $stderr) = $repository->execRemoteCommand(
-        ' log --xml --quiet --limit %d %s@%d',
-        $limit,
-        $uri,
-        $upper_bound - 1);
-      if ($err) {
-        if (preg_match('/(path|File) not found/', $stderr)) {
-          // We've gone all the way back through history and this path was not
-          // affected by earlier commits.
-          break;
-        } else {
-          throw new Exception("svn log error #{$err}: {$stderr}");
-        }
-      }
-      $discover += $this->parseSVNLogXML($xml);
-
-      $upper_bound = min(array_keys($discover));
-
-      // Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
-      // import large repositories fairly quickly, while pulling only as much
-      // data as we need in the common case (when we've already imported the
-      // repository and are just grabbing one commit at a time).
-      $limit = min($limit * 2, 256);
-    }
-
-    // NOTE: We do writes only after discovering all the commits so that we're
-    // never left in a state where we've missed commits -- if the discovery
-    // script terminates it can always resume and restore the import to a good
-    // state. This is also why we sort the discovered commits so we can do
-    // writes forward from the smallest one.
-
-    ksort($discover);
-    foreach ($discover as $commit => $epoch) {
-      $this->recordCommit($commit, $epoch);
-    }
-  }
-
-  private function parseSVNLogXML($xml) {
-    $xml = phutil_utf8ize($xml);
-
-    $result = array();
-
-    $log = new SimpleXMLElement($xml);
-    foreach ($log->logentry as $entry) {
-      $commit = (int)$entry['revision'];
-      $epoch  = (int)strtotime((string)$entry->date[0]);
-      $result[$commit] = $epoch;
-    }
-
-    return $result;
-  }
-
-
-  private function getBaseSVNLogURI() {
-    $repository = $this->getRepository();
-
-    $uri = $repository->getDetail('remote-uri');
-    $subpath = $repository->getDetail('svn-subpath');
-
-    return $uri.$subpath;
-  }
-}

src/applications/repository/daemon/commitdiscovery/svn/__init__.php

@@ -1,15 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
-phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
-
-phutil_require_module('phutil', 'utils');
-
-
-phutil_require_source('PhabricatorRepositorySvnCommitDiscoveryDaemon.php');

src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php

@@ -17,7 +17,7 @@
  */
 
 final class PhabricatorRepositoryCommitTaskDaemon
-  extends PhabricatorRepositoryDaemon {
+  extends PhabricatorDaemon {
 
   final public function run() {
     do {

src/applications/repository/daemon/committask/__init__.php

@@ -7,9 +7,9 @@
 
 
 phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
-phutil_require_module('phabricator', 'applications/repository/daemon/base');
 phutil_require_module('phabricator', 'applications/repository/storage/commit');
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
+phutil_require_module('phabricator', 'infrastructure/daemon/base');
 phutil_require_module('phabricator', 'infrastructure/daemon/timeline/cursor/iterator');
 phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
 

src/applications/repository/daemon/pulllocal/PhabricatorRepositoryPullLocalDaemon.php

@@ -24,7 +24,12 @@
  * responsible for only some repositories, you can launch it with a list of
  * PHIDs or callsigns:
  *
- *   ./phd launch repositorypulllocal X Q Z
+ *   ./phd launch repositorypulllocal -- X Q Z
+ *
+ * You can also launch a daemon which is responsible for all //but// one or
+ * more repositories:
+ *
+ *   ./phd launch repositorypulllocal -- --not A --not B
  *
  * If you have a very large number of repositories and some aren't being pulled
  * as frequently as you'd like, you can either change the pull frequency of
@@ -40,6 +45,8 @@
 final class PhabricatorRepositoryPullLocalDaemon
   extends PhabricatorDaemon {
 
+  private static $commitCache = array();
+
 
 /* -(  Pulling Repositories  )----------------------------------------------- */
 
@@ -48,14 +55,45 @@ final class PhabricatorRepositoryPullLocalDaemon
    * @task pull
    */
   public function run() {
+    $argv = $this->getArgv();
+    array_unshift($argv, __CLASS__);
+    $args = new PhutilArgumentParser($argv);
+    $args->parse(
+      array(
+        array(
+          'name'      => 'no-discovery',
+          'help'      => 'Pull only, without discovering commits.',
+        ),
+        array(
+          'name'      => 'not',
+          'param'     => 'repository',
+          'repeat'    => true,
+          'help'      => 'Do not pull __repository__.',
+        ),
+        array(
+          'name'      => 'repositories',
+          'wildcard'  => true,
+          'help'      => 'Pull specific __repositories__ instead of all.',
+        ),
+      ));
+
+    $no_discovery   = $args->getArg('no-discovery');
+    $repo_names     = $args->getArg('repositories', array());
+    $exclude_names  = $args->getArg('not', array());
 
     // Each repository has an individual pull frequency; after we pull it,
     // wait that long to pull it again. When we start up, try to pull everything
     // serially.
     $retry_after = array();
 
+    $min_sleep = 15;
+
     while (true) {
-      $repositories = $this->loadRepositories();
+      $repositories = $this->loadRepositories($repo_names);
+      if ($exclude_names) {
+        $exclude = $this->loadRepositories($exclude_names);
+        $repositories = array_diff_key($repositories, $exclude);
+      }
 
       // Shuffle the repositories, then re-key the array since shuffle()
       // discards keys. This is mostly for startup, we'll use soft priorities
@@ -79,22 +117,41 @@ final class PhabricatorRepositoryPullLocalDaemon
 
       foreach ($repositories as $id => $repository) {
         $after = idx($retry_after, $id, 0);
-        if ($after >= time()) {
+        if ($after > time()) {
+          continue;
+        }
+
+        $tracked = $repository->isTracked();
+        if (!$tracked) {
           continue;
         }
 
         try {
           self::pullRepository($repository);
-          $sleep_for = $repository->getDetail('pull-frequency', 15);
+
+          if (!$no_discovery) {
+            // TODO: It would be nice to discover only if we pulled something,
+            // but this isn't totally trivial.
+            self::discoverRepository($repository);
+          }
+
+          $sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
           $retry_after[$id] = time() + $sleep_for;
         } catch (Exception $ex) {
-          $retry_after[$id] = time() + 15;
+          $retry_after[$id] = time() + $min_sleep;
           phlog($ex);
         }
+
+        $this->stillWorking();
       }
 
-      $sleep_until = max(min($retry_after), time() + 15);
+      if ($retry_after) {
-      sleep($sleep_until - time());
+        $sleep_until = max(min($retry_after), time() + $min_sleep);
+      } else {
+        $sleep_until = time() + $min_sleep;
+      }
+
+      $this->sleep($sleep_until - time());
     }
   }
 
@@ -102,12 +159,11 @@ final class PhabricatorRepositoryPullLocalDaemon
   /**
    * @task pull
    */
-  protected function loadRepositories() {
+  protected function loadRepositories(array $names) {
-    $argv = $this->getArgv();
+    if (!count($names)) {
-    if (!count($argv)) {
       return id(new PhabricatorRepository())->loadAll();
     } else {
-      return PhabricatorRepository::loadAllByPHIDOrCallsign($argv);
+      return PhabricatorRepository::loadAllByPHIDOrCallsign($names);
     }
   }
 
@@ -116,11 +172,6 @@ final class PhabricatorRepositoryPullLocalDaemon
    * @task pull
    */
   public static function pullRepository(PhabricatorRepository $repository) {
-    $tracked = $repository->isTracked();
-    if (!$tracked) {
-      return;
-    }
-
     $vcs = $repository->getVersionControlSystem();
 
     $is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
@@ -153,19 +204,126 @@ final class PhabricatorRepositoryPullLocalDaemon
       }
 
       if ($is_git) {
-        self::executeGitCreate($repository, $local_path);
+        return self::executeGitCreate($repository, $local_path);
       } else if ($is_hg) {
-        self::executeHgCreate($repository, $local_path);
+        return self::executeHgCreate($repository, $local_path);
       }
     } else {
       if ($is_git) {
-        self::executeGitUpdate($repository, $local_path);
+        return self::executeGitUpdate($repository, $local_path);
       } else if ($is_hg) {
-        self::executeHgUpdate($repository, $local_path);
+        return self::executeHgUpdate($repository, $local_path);
       }
     }
   }
 
+  public static function discoverRepository(PhabricatorRepository $repository) {
+    $vcs = $repository->getVersionControlSystem();
+    switch ($vcs) {
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+        return self::executeGitDiscover($repository);
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+        return self::executeSvnDiscover($repository);
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        return self::executeHgDiscover($repository);
+      default:
+        throw new Exception("Unknown VCS '{$vcs}'!");
+    }
+  }
+
+
+  private static function isKnownCommit(
+    PhabricatorRepository $repository,
+    $target) {
+
+    if (self::getCache($repository, $target)) {
+      return true;
+    }
+
+    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+      'repositoryID = %s AND commitIdentifier = %s',
+      $repository->getID(),
+      $target);
+
+    if (!$commit) {
+      return false;
+    }
+
+    self::setCache($repository, $target);
+    while (count(self::$commitCache) > 2048) {
+      array_shift(self::$commitCache);
+    }
+
+    return true;
+  }
+
+  private static function recordCommit(
+    PhabricatorRepository $repository,
+    $commit_identifier,
+    $epoch) {
+
+    $commit = new PhabricatorRepositoryCommit();
+    $commit->setRepositoryID($repository->getID());
+    $commit->setCommitIdentifier($commit_identifier);
+    $commit->setEpoch($epoch);
+
+    try {
+      $commit->save();
+      $event = new PhabricatorTimelineEvent(
+        'cmit',
+        array(
+          'id' => $commit->getID(),
+        ));
+      $event->recordEvent();
+
+      queryfx(
+        $repository->establishConnection('w'),
+        'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
+          VALUES (%d, 1, %d, %d)
+          ON DUPLICATE KEY UPDATE
+            size = size + 1,
+            lastCommitID =
+              IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
+            epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
+        PhabricatorRepository::TABLE_SUMMARY,
+        $repository->getID(),
+        $commit->getID(),
+        $epoch);
+
+      self::setCache($repository, $commit_identifier);
+    } catch (AphrontQueryDuplicateKeyException $ex) {
+      // Ignore. This can happen because we discover the same new commit
+      // more than once when looking at history, or because of races or
+      // data inconsistency or cosmic radiation; in any case, we're still
+      // in a good state if we ignore the failure.
+      self::setCache($repository, $commit_identifier);
+    }
+  }
+
+  private static function setCache(
+    PhabricatorRepository $repository,
+    $commit_identifier) {
+
+    $key = self::getCacheKey($repository, $commit_identifier);
+    self::$commitCache[$key] = true;
+  }
+
+  private static function getCache(
+    PhabricatorRepository $repository,
+    $commit_identifier) {
+
+    $key = self::getCacheKey($repository, $commit_identifier);
+    return idx(self::$commitCache, $key, false);
+  }
+
+  private static function getCacheKey(
+    PhabricatorRepository $repository,
+    $commit_identifier) {
+
+    return $repository->getID().':'.$commit_identifier;
+  }
+
+
 
 /* -(  Git Implementation  )------------------------------------------------- */
 
@@ -249,6 +407,147 @@ final class PhabricatorRepositoryPullLocalDaemon
   }
 
 
+  /**
+   * @task git
+   */
+  private static function executeGitDiscover(
+    PhabricatorRepository $repository) {
+
+    list($remotes) = $repository->execxLocalCommand(
+      'remote show -n origin');
+
+    $matches = null;
+    if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
+      throw new Exception(
+        "Expected 'Fetch URL' in 'git remote show -n origin'.");
+    }
+
+    self::executeGitverifySameOrigin(
+      $matches[1],
+      $repository->getRemoteURI(),
+      $repository->getLocalPath());
+
+    list($stdout) = $repository->execxLocalCommand(
+      'branch -r --verbose --no-abbrev');
+
+    $branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
+      $stdout,
+      $only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
+
+    $tracked_something = false;
+    foreach ($branches as $name => $commit) {
+      if (!$repository->shouldTrackBranch($name)) {
+        continue;
+      }
+
+      $tracked_something = true;
+
+      if (self::isKnownCommit($repository, $commit)) {
+        continue;
+      } else {
+        self::executeGitDiscoverCommit($repository, $commit);
+      }
+    }
+
+    if (!$tracked_something) {
+      $repo_name = $repository->getName();
+      $repo_callsign = $repository->getCallsign();
+      throw new Exception(
+        "Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
+        "Verify that your branch filtering settings are correct.");
+    }
+  }
+
+
+  /**
+   * @task git
+   */
+  private static function executeGitDiscoverCommit(
+    PhabricatorRepository $repository,
+    $commit) {
+
+    $discover = array($commit);
+    $insert = array($commit);
+
+    $seen_parent = array();
+
+    while (true) {
+      $target = array_pop($discover);
+      list($parents) = $repository->execxLocalCommand(
+        'log -n1 --pretty="%%P" %s',
+        $target);
+      $parents = array_filter(explode(' ', trim($parents)));
+      foreach ($parents as $parent) {
+        if (isset($seen_parent[$parent])) {
+          // We end up in a loop here somehow when we parse Arcanist if we
+          // don't do this. TODO: Figure out why and draw a pretty diagram
+          // since it's not evident how parsing a DAG with this causes the
+          // loop to stop terminating.
+          continue;
+        }
+        $seen_parent[$parent] = true;
+        if (!self::isKnownCommit($repository, $parent)) {
+          $discover[] = $parent;
+          $insert[] = $parent;
+        }
+      }
+      if (empty($discover)) {
+        break;
+      }
+    }
+
+    while (true) {
+      $target = array_pop($insert);
+      list($epoch) = $repository->execxLocalCommand(
+        'log -n1 --pretty="%%ct" %s',
+        $target);
+      $epoch = trim($epoch);
+
+      self::recordCommit($repository, $target, $epoch);
+
+      if (empty($insert)) {
+        break;
+      }
+    }
+  }
+
+
+  /**
+   * @task git
+   */
+  public static function executeGitVerifySameOrigin($remote, $expect, $where) {
+    $remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
+    $expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
+
+    $remote_path = $remote_uri->getPath();
+    $expect_path = $expect_uri->getPath();
+
+    $remote_match = self::executeGitNormalizePath($remote_path);
+    $expect_match = self::executeGitNormalizePath($expect_path);
+
+    if ($remote_match != $expect_match) {
+      throw new Exception(
+        "Working copy at '{$where}' has a mismatched origin URL. It has ".
+        "origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
+        "configured URL '{$expect}' (with remote path '{$expect_path}') is ".
+        "expected. Refusing to proceed because this may indicate that the ".
+        "working copy is actually some other repository.");
+    }
+  }
+
+
+  /**
+   * @task git
+   */
+  private static function executeGitNormalizePath($path) {
+    // Strip away trailing "/" and ".git", so similar paths correctly match.
+
+    $path = rtrim($path, '/');
+    $path = preg_replace('/\.git$/', '', $path);
+    return $path;
+  }
+
+
 /* -(  Mercurial Implementation  )------------------------------------------- */
 
 
@@ -306,4 +605,177 @@ final class PhabricatorRepositoryPullLocalDaemon
     }
   }
 
+  private static function executeHgDiscover(PhabricatorRepository $repository) {
+    // NOTE: "--debug" gives us 40-character hashes.
+    list($stdout) = $repository->execxLocalCommand('--debug branches');
+
+    $branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
+    $got_something = false;
+    foreach ($branches as $name => $branch) {
+      $commit = $branch['rev'];
+      if (self::isKnownCommit($repository, $commit)) {
+        continue;
+      } else {
+        self::executeHgDiscoverCommit($repository, $commit);
+        $got_something = true;
+      }
+    }
+
+    return $got_something;
+  }
+
+  private static function executeHgDiscoverCommit(
+    PhabricatorRepository $repository,
+    $commit) {
+
+    $discover = array($commit);
+    $insert = array($commit);
+
+    $seen_parent = array();
+
+    // For all the new commits at the branch heads, walk backward until we find
+    // only commits we've aleady seen.
+    while (true) {
+      $target = array_pop($discover);
+      list($stdout) = $repository->execxLocalCommand(
+        'parents --rev %s --template %s',
+        $target,
+        '{node}\n');
+      $parents = array_filter(explode("\n", trim($stdout)));
+      foreach ($parents as $parent) {
+        if (isset($seen_parent[$parent])) {
+          continue;
+        }
+        $seen_parent[$parent] = true;
+        if (!self::isKnownCommit($repository, $parent)) {
+          $discover[] = $parent;
+          $insert[] = $parent;
+        }
+      }
+      if (empty($discover)) {
+        break;
+      }
+    }
+
+    while (true) {
+      $target = array_pop($insert);
+      list($stdout) = $repository->execxLocalCommand(
+        'log --rev %s --template %s',
+        $target,
+        '{date|rfc822date}');
+      $epoch = strtotime($stdout);
+
+      self::recordCommit($repository, $target, $epoch);
+
+      if (empty($insert)) {
+        break;
+      }
+    }
+  }
+
+
+/* -(  Subversion Implementation  )------------------------------------------ */
+
+
+  private static function executeSvnDiscover(
+    PhabricatorRepository $repository) {
+
+    $uri = self::executeSvnGetBaseSVNLogURI($repository);
+
+    list($xml) = $repository->execxRemoteCommand(
+      'log --xml --quiet --limit 1 %s@HEAD',
+      $uri);
+
+    $results = self::executeSvnParseLogXML($xml);
+    $commit = head_key($results);
+    $epoch  = head($results);
+
+    if (self::isKnownCommit($repository, $commit)) {
+      return false;
+    }
+
+    self::executeSvnDiscoverCommit($repository, $commit, $epoch);
+    return true;
+  }
+
+  private static function executeSvnDiscoverCommit(
+    PhabricatorRepository $repository,
+    $commit,
+    $epoch) {
+
+    $uri = self::executeSvnGetBaseSVNLogURI($repository);
+
+    $discover = array(
+      $commit => $epoch,
+    );
+    $upper_bound = $commit;
+
+    $limit = 1;
+    while ($upper_bound > 1 &&
+           !self::isKnownCommit($repository, $upper_bound)) {
+      // Find all the unknown commits on this path. Note that we permit
+      // importing an SVN subdirectory rather than the entire repository, so
+      // commits may be nonsequential.
+      list($err, $xml, $stderr) = $repository->execRemoteCommand(
+        ' log --xml --quiet --limit %d %s@%d',
+        $limit,
+        $uri,
+        $upper_bound - 1);
+      if ($err) {
+        if (preg_match('/(path|File) not found/', $stderr)) {
+          // We've gone all the way back through history and this path was not
+          // affected by earlier commits.
+          break;
+        } else {
+          throw new Exception("svn log error #{$err}: {$stderr}");
+        }
+      }
+      $discover += self::executeSvnParseLogXML($xml);
+
+      $upper_bound = min(array_keys($discover));
+
+      // Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
+      // import large repositories fairly quickly, while pulling only as much
+      // data as we need in the common case (when we've already imported the
+      // repository and are just grabbing one commit at a time).
+      $limit = min($limit * 2, 256);
+    }
+
+    // NOTE: We do writes only after discovering all the commits so that we're
+    // never left in a state where we've missed commits -- if the discovery
+    // script terminates it can always resume and restore the import to a good
+    // state. This is also why we sort the discovered commits so we can do
+    // writes forward from the smallest one.
+
+    ksort($discover);
+    foreach ($discover as $commit => $epoch) {
+      self::recordCommit($repository, $commit, $epoch);
+    }
+  }
+
+  private static function executeSvnParseLogXML($xml) {
+    $xml = phutil_utf8ize($xml);
+
+    $result = array();
+
+    $log = new SimpleXMLElement($xml);
+    foreach ($log->logentry as $entry) {
+      $commit = (int)$entry['revision'];
+      $epoch  = (int)strtotime((string)$entry->date[0]);
+      $result[$commit] = $epoch;
+    }
+
+    return $result;
+  }
+
+
+  private static function executeSvnGetBaseSVNLogURI(
+    PhabricatorRepository $repository) {
+
+    $uri = $repository->getDetail('remote-uri');
+    $subpath = $repository->getDetail('svn-subpath');
+
+    return $uri.$subpath;
+  }
+
 }

src/applications/repository/daemon/pulllocal/__init__.php

@@ -6,12 +6,20 @@
 
 
 
+phutil_require_module('arcanist', 'repository/parser/mercurial');
+
+phutil_require_module('phabricator', 'applications/diffusion/data/branch');
+phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
 phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
+phutil_require_module('phabricator', 'applications/repository/storage/commit');
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
 phutil_require_module('phabricator', 'infrastructure/daemon/base');
+phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
+phutil_require_module('phabricator', 'storage/queryfx');
 
 phutil_require_module('phutil', 'error');
 phutil_require_module('phutil', 'filesystem');
+phutil_require_module('phutil', 'parser/argument/parser');
 phutil_require_module('phutil', 'utils');
 
 

src/applications/repository/daemon/pulllocal/__tests__/PhabricatorRepositoryPullLocalDaemonTestCase.php

@@ -16,10 +16,10 @@
  * limitations under the License.
  */
 
-final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
+final class PhabricatorRepositoryPullLocalDaemonTestCase
   extends PhabricatorTestCase {
 
-  public function testVerifySameGitOrigin() {
+  public function testExecuteGitVerifySameOrigin() {
     $cases = array(
       array(
         'ssh://user@domain.com/path.git',
@@ -94,7 +94,7 @@ final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
 
       $ex = null;
       try {
-        PhabricatorRepositoryGitCommitDiscoveryDaemon::verifySameGitOrigin(
+        PhabricatorRepositoryPullLocalDaemon::executeGitverifySameOrigin(
           $remote,
           $config,
           '(a test case)');

src/applications/repository/daemon/pulllocal/__tests__/__init__.php

@@ -6,8 +6,8 @@
 
 
 
-phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/git');
+phutil_require_module('phabricator', 'applications/repository/daemon/pulllocal');
 phutil_require_module('phabricator', 'infrastructure/testing/testcase');
 
 
-phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php');
+phutil_require_source('PhabricatorRepositoryPullLocalDaemonTestCase.php');