Improve some diffusioney things.

Summary:

Test Plan:

Reviewers:

CC:

resources/sql/patches/006.repository.sql

@@ -41,4 +41,14 @@ create table phabricator_repository.repository_filesystem (
   primary key (repositoryID, parentID, svnCommit, pathID)
 );
 
-alter table repository_filesystem add key (repositoryID, svnCommit);
+alter table repository_filesystem add key (repositoryID, svnCommit);
+
+truncate phabricator_repository.repository_commit;
+alter table phabricator_repository.repository_commit
+  change repositoryPHID repositoryID int unsigned not null;
+alter table phabricator_repository.repository_commit drop key repositoryPHID;
+alter table phabricator_repository.repository_commit add unique key
+  (repositoryID, commitIdentifier(16));
+alter table phabricator_repository.repository_commit add key
+  (repositoryID, epoch);
+

src/applications/diffusion/controller/home/DiffusionHomeController.php

@@ -27,24 +27,49 @@ class DiffusionHomeController extends DiffusionController {
     $commit = new PhabricatorRepositoryCommit();
     $conn_r = $commit->establishConnection('r');
 
-    // TODO: Both these queries are basically bogus and have total trash for
+    // TODO: These queries are pretty bogus.
-    // query plans, and don't return the right results. Build a cache instead.
-    // These are just pulling data with approximately the right look to it.
-    $commits = $commit->loadAllWhere(
-      '1 = 1 GROUP BY repositoryPHID');
-    $commits = mpull($commits, null, 'getRepositoryPHID');
 
-    $commit_counts = queryfx_all(
+    $commits = array();
-      $conn_r,
+    $commit_counts = array();
-      'SELECT repositoryPHID, count(*) N FROM %T
+
-        GROUP BY repositoryPHID',
+    $max_epoch = queryfx_all(
+      $commit->establishConnection('r'),
+      'SELECT repositoryID, MAX(epoch) maxEpoch FROM %T GROUP BY repositoryID',
       $commit->getTableName());
-    $commit_counts = ipull($commit_counts, 'N', 'repositoryPHID');
+
+    if ($max_epoch) {
+      $sql = array();
+      foreach ($max_epoch as $head) {
+        $sql[] = '('.(int)$head['repositoryID'].', '.(int)$head['maxEpoch'].')';
+      }
+
+      // NOTE: It's possible we'll pull multiple commits for some repository
+      // here but it reduces query cost around 3x to unique them in PHP rather
+      // than apply GROUP BY in MySQL.
+      $commits = $commit->loadAllWhere(
+        '(repositoryID, epoch) IN (%Q)',
+        implode(', ', $sql));
+      $commits = mpull($commits, null, 'getRepositoryID');
+
+      $commit_counts = queryfx_all(
+        $conn_r,
+        'SELECT repositoryID, count(*) N FROM %T
+          GROUP BY repositoryID',
+        $commit->getTableName());
+      $commit_counts = ipull($commit_counts, 'N', 'repositoryID');
+    }
 
     $rows = array();
     foreach ($repositories as $repository) {
-      $phid = $repository->getPHID();
+      $id = $repository->getID();
-      $commit = idx($commits, $phid);
+      $commit = idx($commits, $id);
+      $date = null;
+      $time = null;
+      if ($commit) {
+        $date = date('M j, Y', $commit->getEpoch());
+        $time = date('g:i A', $commit->getEpoch());
+      }
+
       $rows[] = array(
         phutil_render_tag(
           'a',
@@ -53,13 +78,14 @@ class DiffusionHomeController extends DiffusionController {
           ),
           phutil_escape_html($repository->getName())),
         $repository->getVersionControlSystem(),
-        idx($commit_counts, $phid, 0),
+        idx($commit_counts, $id, 0),
         $commit
-          ? $commit->getCommitIdentifier()
+          ? DiffusionView::linkCommit(
-          : null, // TODO: Link/format
+              $repository,
-        $commit
+              $commit->getCommitIdentifier())
-          ? phabricator_format_timestamp($commit->getEpoch())
           : null,
+        $date,
+        $time,
       );
     }
 
@@ -70,11 +96,17 @@ class DiffusionHomeController extends DiffusionController {
         'VCS',
         'Size',
         'Last',
-        'Committed',
+        'Date',
+        'Time',
       ));
     $table->setColumnClasses(
       array(
         'wide',
+        '',
+        'n',
+        'n',
+        '',
+        'right',
       ));
 
     $panel = new AphrontPanelView();

src/applications/diffusion/controller/repository/DiffusionRepositoryController.php

@@ -35,8 +35,16 @@ class DiffusionRepositoryController extends DiffusionController {
     $history_table->setDiffusionRequest($drequest);
     $history_table->setHistory($history);
 
+    $callsign = $drequest->getRepository()->getCallsign();
+    $all = phutil_render_tag(
+      'a',
+      array(
+        'href' => "/diffusion/{$callsign}/history/",
+      ),
+      'View Full Commit History');
+
     $panel = new AphrontPanelView();
-    $panel->setHeader('Recent Commits');
+    $panel->setHeader("Recent Commits · {$all}");
     $panel->appendChild($history_table);
 
     $content[] = $panel;

src/applications/diffusion/data/pathchange/DiffusionPathChange.php

@@ -19,6 +19,8 @@
 final class DiffusionPathChange {
 
   private $commitIdentifier;
+  private $commit;
+  private $commitData;
 
   final public function setCommitIdentifier($commit) {
     $this->commitIdentifier = $commit;
@@ -29,5 +31,49 @@ final class DiffusionPathChange {
     return $this->commitIdentifier;
   }
 
+  final public function setCommit($commit) {
+    $this->commit = $commit;
+    return $this;
+  }
+
+  final public function getCommit() {
+    return $this->commit;
+  }
+
+  final public function setCommitData($commit_data) {
+    $this->commitData = $commit_data;
+    return $this;
+  }
+
+  final public function getCommitData() {
+    return $this->commitData;
+  }
+
+
+  final public function getEpoch() {
+    if ($this->getCommit()) {
+      return $this->getCommit()->getEpoch();
+    }
+    return null;
+  }
+
+  final public function getAuthorName() {
+    if ($this->getCommitData()) {
+      return $this->getCommitData()->getAuthorName();
+    }
+    return null;
+  }
+
+  final public function getSummary() {
+    if (!$this->getCommitData()) {
+      return null;
+    }
+    $message = $this->getCommitData()->getCommitMessage();
+    $first = idx(explode("\n", $message), 0);
+    return substr($first, 0, 80);
+  }
+
+
+
 
 }

src/applications/diffusion/data/pathchange/__init__.php

@@ -6,5 +6,7 @@
 
 
 
+phutil_require_module('phutil', 'utils');
+
 
 phutil_require_source('DiffusionPathChange.php');

src/applications/diffusion/query/browse/svn/DiffusionSvnBrowseQuery.php

@@ -37,6 +37,12 @@ final class DiffusionSvnBrowseQuery extends DiffusionBrowseQuery {
     $paths = ipull($paths, 'id', 'path');
     $path_id = $paths[$path_normal];
 
+    if ($commit) {
+      $slice_clause = 'AND svnCommit <= '.(int)$commit;
+    } else {
+      $slice_clause = '';
+    }
+
     $index = queryfx_all(
       $conn_r,
       'SELECT pathID, max(svnCommit) maxCommit FROM %T WHERE
@@ -45,7 +51,7 @@ final class DiffusionSvnBrowseQuery extends DiffusionBrowseQuery {
       PhabricatorRepository::TABLE_FILESYSTEM,
       $repository->getID(),
       $path_id,
-      '');
+      $slice_clause);
 
     if (!$index) {
       // TODO: !

src/applications/diffusion/query/history/base/DiffusionHistoryQuery.php

@@ -20,6 +20,7 @@ abstract class DiffusionHistoryQuery {
 
   private $request;
   private $limit = 100;
+  private $offset = 0;
 
   final private function __construct() {
     // <private>
@@ -66,5 +67,14 @@ abstract class DiffusionHistoryQuery {
     return $this->limit;
   }
 
+  final public function setOffset($offset) {
+    $this->offset = $offset;
+    return $this;
+  }
+
+  final public function getOffset() {
+    return $this->offset;
+  }
+
   abstract protected function executeQuery();
 }

src/applications/diffusion/query/history/git/DiffusionGitHistoryQuery.php

@@ -23,7 +23,7 @@ final class DiffusionGitHistoryQuery extends DiffusionHistoryQuery {
 
     $repository = $drequest->getRepository();
     $path = $drequest->getPath();
-    $commit = $drequest->getCommit();
+    $commit_hash = $drequest->getCommit();
 
     $local_path = $repository->getDetail('local-path');
     $git = $drequest->getPathToGitBinary();
@@ -32,31 +32,49 @@ final class DiffusionGitHistoryQuery extends DiffusionHistoryQuery {
       '(cd %s && %s log '.
         '--skip=%d '.
         '-n %d '.
-        '-M '.
-        '-C '.
-        '-B '.
-        '--find-copies-harder '.
-        '--raw '.
         '-t '.
         '--abbrev=40 '.
-        '--pretty=format:%%x1c%%H%%x1d '.
+        '--pretty=format:%%H '.
         '%s -- %s)',
       $local_path,
       $git,
-      $offset = 0,
+      $this->getOffset(),
       $this->getLimit(),
-      $commit,
+      $commit_hash,
       $path);
 
-    $commits = explode("\x1c", $stdout);
+    $hashes = explode("\n", $stdout);
-    array_shift($commits); // \x1c character is first, remove empty record
+    $hashes = array_filter($hashes);
+
+    $commits = array();
+    $commit_data = array();
+
+    if ($hashes) {
+      $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
+        'repositoryID = %d AND commitIdentifier IN (%Ls)',
+          $repository->getID(),
+        $hashes);
+      $commits = mpull($commits, null, 'getCommitIdentifier');
+      if ($commits) {
+        $commit_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere(
+          'commitID in (%Ld)',
+          mpull($commits, 'getID'));
+        $commit_data = mpull($commit_data, null, 'getCommitID');
+      }
+    }
 
     $history = array();
-    foreach ($commits as $commit) {
+    foreach ($hashes as $hash) {
-      list($hash, $raw) = explode("\x1d", $commit);
-
       $item = new DiffusionPathChange();
       $item->setCommitIdentifier($hash);
+      $commit = idx($commits, $hash);
+      if ($commit) {
+        $item->setCommit($commit);
+        $data = idx($commit_data, $commit->getID());
+        if ($data) {
+          $item->setCommitData($data);
+        }
+      }
       $history[] = $item;
     }
 

src/applications/diffusion/query/history/git/__init__.php

@@ -8,8 +8,11 @@
 
 phutil_require_module('phabricator', 'applications/diffusion/data/pathchange');
 phutil_require_module('phabricator', 'applications/diffusion/query/history/base');
+phutil_require_module('phabricator', 'applications/repository/storage/commit');
+phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
 
 phutil_require_module('phutil', 'future/exec');
+phutil_require_module('phutil', 'utils');
 
 
 phutil_require_source('DiffusionGitHistoryQuery.php');

src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php

@@ -47,10 +47,36 @@ final class DiffusionSvnHistoryQuery extends DiffusionHistoryQuery {
       $commit ? $commit : 0x7FFFFFFF,
       $this->getLimit());
 
+    $commits = array();
+    $commit_data = array();
+
+    $commit_ids = ipull($history_data, 'commitID');
+    if ($commit_ids) {
+      $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
+        'id IN (%Ld)',
+        $commit_ids);
+      if ($commits) {
+        $commit_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere(
+          'commitID in (%Ld)',
+          $commit_ids);
+        $commit_data = mpull($commit_data, null, 'getCommitID');
+      }
+    }
+
+
+
     $history = array();
     foreach ($history_data as $row) {
       $item = new DiffusionPathChange();
-      $item->setCommitIdentifier($row['commitID']);
+      $commit = idx($commits, $row['commitID']);
+      if ($commit) {
+        $item->setCommit($commit);
+        $item->setCommitIdentifier($commit->getCommitIdentifier());
+        $data = idx($commit_data, $commit->getID());
+        if ($data) {
+          $item->setCommitData($data);
+        }
+      }
       $history[] = $item;
     }
 

src/applications/diffusion/view/historytable/DiffusionHistoryTableView.php

@@ -30,6 +30,16 @@ final class DiffusionHistoryTableView extends DiffusionView {
 
     $rows = array();
     foreach ($this->history as $history) {
+      $epoch = $history->getEpoch();
+
+      if ($epoch) {
+        $date = date('M j, Y', $epoch);
+        $time = date('g:i A', $epoch);
+      } else {
+        $date = null;
+        $time = null;
+      }
+
       $rows[] = array(
         $this->linkBrowse(
           $drequest->getPath(),
@@ -39,10 +49,11 @@ final class DiffusionHistoryTableView extends DiffusionView {
         self::linkCommit(
           $drequest->getRepository(),
           $history->getCommitIdentifier()),
-        '?',
+        '-',
-        '?',
+        $date,
-        '',
+        $time,
-        '',
+        phutil_escape_html($history->getAuthorName()),
+        phutil_escape_html($history->getSummary()),
         // TODO: etc etc
       );
     }
@@ -54,6 +65,7 @@ final class DiffusionHistoryTableView extends DiffusionView {
         'Commit',
         'Change',
         'Date',
+        'Time',
         'Author',
         'Details',
       ));
@@ -63,6 +75,7 @@ final class DiffusionHistoryTableView extends DiffusionView {
         'n',
         '',
         '',
+        'right',
         '',
         'wide wrap',
       ));

src/applications/diffusion/view/historytable/__init__.php

@@ -8,6 +8,9 @@
 
 phutil_require_module('phabricator', 'applications/diffusion/view/base');
 phutil_require_module('phabricator', 'view/control/table');
+phutil_require_module('phabricator', 'view/utils');
+
+phutil_require_module('phutil', 'markup');
 
 
 phutil_require_source('DiffusionHistoryTableView.php');

src/applications/repository/daemon/commitdiscovery/base/PhabricatorRepositoryCommitDiscoveryDaemon.php

@@ -47,8 +47,8 @@ abstract class PhabricatorRepositoryCommitDiscoveryDaemon
     }
 
     $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
-      'repositoryPHID = %s AND commitIdentifier = %s',
+      'repositoryID = %s AND commitIdentifier = %s',
-      $this->getRepository()->getPHID(),
+      $this->getRepository()->getID(),
       $target);
 
     if (!$commit) {
@@ -67,7 +67,7 @@ abstract class PhabricatorRepositoryCommitDiscoveryDaemon
     $repository = $this->getRepository();
 
     $commit = new PhabricatorRepositoryCommit();
-    $commit->setRepositoryPHID($repository->getPHID());
+    $commit->setRepositoryID($repository->getID());
     $commit->setCommitIdentifier($commit_identifier);
     $commit->setEpoch($epoch);
 

src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php

@@ -38,9 +38,8 @@ class PhabricatorRepositoryCommitTaskDaemon
         }
 
         // TODO: Cache these.
-        $repository = id(new PhabricatorRepository())->loadOneWhere(
+        $repository = id(new PhabricatorRepository())->load(
-          'phid = %s',
+          $commit->getRepositoryID());
-          $commit->getRepositoryPHID());
 
         $vcs = $repository->getVersionControlSystem();
         switch ($vcs) {

src/applications/repository/storage/commit/PhabricatorRepositoryCommit.php

@@ -18,7 +18,7 @@
 
 class PhabricatorRepositoryCommit extends PhabricatorRepositoryDAO {
 
-  protected $repositoryPHID;
+  protected $repositoryID;
   protected $phid;
   protected $commitIdentifier;
   protected $epoch;

src/applications/repository/worker/base/PhabricatorRepositoryCommitParserWorker.php

@@ -36,9 +36,8 @@ abstract class PhabricatorRepositoryCommitParserWorker
 
     $this->commit = $commit;
 
-    $repository = id(new PhabricatorRepository())->loadOneWhere(
+    $repository = id(new PhabricatorRepository())->load(
-      'phid = %s',
+      $commit->getRepositoryID());
-      $commit->getRepositoryPHID());
 
     if (!$repository) {
       return;

src/applications/repository/worker/commitchangeparser/base/PhabricatorRepositoryCommitChangeParserWorker.php

@@ -19,6 +19,12 @@
 abstract class PhabricatorRepositoryCommitChangeParserWorker
   extends PhabricatorRepositoryCommitParserWorker {
 
+  public function getRequiredLeaseTime() {
+    // It can take a very long time to parse commits; some commits in the
+    // Facebook repository affect many millions of paths. Acquire 24h leases.
+    return 60 * 60 * 24;
+  }
+
   protected function lookupOrCreatePaths(array $paths) {
     $repository = new PhabricatorRepository();
     $conn_w = $repository->establishConnection('w');
@@ -37,7 +43,7 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
         }
         queryfx(
           $conn_w,
-          'INSERT INTO %T (path) VALUES %Q',
+          'INSERT IGNORE INTO %T (path) VALUES %Q',
           PhabricatorRepository::TABLE_PATH,
           implode(', ', $sql));
       }