Add a publish cache for the Diviner static publisher

Summary: Keep track of what we've written to disk, and regenerate only new documents.

Test Plan: Changed a small number of files, saw that number of files get regenerated. Ran with "--clean" and saw everything regenerate.

Reviewers: btrahan, vrana, chad

Reviewed By: chad

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D4897

src/__phutil_library_map__.php

@@ -468,9 +468,11 @@ phutil_register_library_map(array(
     'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
     'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
     'DivinerDefaultRenderer' => 'applications/diviner/renderer/DivinerDefaultRenderer.php',
+    'DivinerDiskCache' => 'applications/diviner/cache/DivinerDiskCache.php',
     'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php',
     'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php',
     'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
+    'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php',
     'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php',
     'DivinerRenderer' => 'applications/diviner/renderer/DivinerRenderer.php',
     'DivinerStaticPublisher' => 'applications/diviner/publisher/DivinerStaticPublisher.php',
@@ -1965,11 +1967,13 @@ phutil_register_library_map(array(
     'DiffusionURITestCase' => 'ArcanistPhutilTestCase',
     'DiffusionView' => 'AphrontView',
     'DivinerArticleAtomizer' => 'DivinerAtomizer',
+    'DivinerAtomCache' => 'DivinerDiskCache',
     'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
     'DivinerDefaultRenderer' => 'DivinerRenderer',
     'DivinerFileAtomizer' => 'DivinerAtomizer',
     'DivinerGenerateWorkflow' => 'DivinerWorkflow',
     'DivinerListController' => 'PhabricatorController',
+    'DivinerPublishCache' => 'DivinerDiskCache',
     'DivinerStaticPublisher' => 'DivinerPublisher',
     'DivinerWorkflow' => 'PhutilArgumentWorkflow',
     'DrydockAllocatorWorker' => 'PhabricatorWorker',

src/applications/diviner/cache/DivinerAtomCache.php

@@ -1,8 +1,6 @@
 <?php
 
-final class DivinerAtomCache {
+final class DivinerAtomCache extends DivinerDiskCache {
-
-  private $cache;
 
   private $fileHashMap;
   private $atomMap;
@@ -15,20 +13,11 @@ final class DivinerAtomCache {
   private $writeAtoms = array();
 
   public function __construct($cache_directory) {
-    $dir_cache = id(new PhutilKeyValueCacheDirectory())
+    return parent::__construct($cache_directory, 'diviner-atom-cache');
-      ->setCacheDirectory($cache_directory);
-    $profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))
-      ->setProfiler(PhutilServiceProfiler::getInstance())
-      ->setName('diviner-atom-cache');
-    $this->cache = $profiled_cache;
-  }
-
-  private function getCache() {
-    return $this->cache;
   }
 
   public function delete() {
-    $this->getCache()->destroyCache();
+    parent::delete();
     $this->fileHashMap = null;
     $this->atomMap = null;
     $this->atoms = array();
@@ -36,24 +25,6 @@ final class DivinerAtomCache {
     return $this;
   }
 
-  /**
-   * Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into
-   * a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with
-   * @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories
-   * inside .divinercache instead of a million hash files with huge names at
-   * top level.
-   */
-  private function getHashKey($hash) {
-    return implode(
-      '/',
-      array(
-        substr($hash, 0, 2),
-        substr($hash, 2, 2),
-        substr($hash, 4, 8),
-      ));
-  }
-
-
 /* -(  File Hash Map  )------------------------------------------------------ */
 
 

src/applications/diviner/cache/DivinerDiskCache.php

@@ -0,0 +1,42 @@
+<?php
+
+abstract class DivinerDiskCache {
+
+  private $cache;
+
+  public function __construct($cache_directory, $name) {
+    $dir_cache = id(new PhutilKeyValueCacheDirectory())
+      ->setCacheDirectory($cache_directory);
+    $profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))
+      ->setProfiler(PhutilServiceProfiler::getInstance())
+      ->setName($name);
+    $this->cache = $profiled_cache;
+  }
+
+  protected function getCache() {
+    return $this->cache;
+  }
+
+  public function delete() {
+    $this->getCache()->destroyCache();
+    return $this;
+  }
+
+  /**
+   * Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into
+   * a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with
+   * @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories
+   * inside .divinercache instead of a million hash files with huge names at
+   * top level.
+   */
+  protected function getHashKey($hash) {
+    return implode(
+      '/',
+      array(
+        substr($hash, 0, 2),
+        substr($hash, 2, 2),
+        substr($hash, 4, 8),
+      ));
+  }
+
+}

src/applications/diviner/cache/DivinerPublishCache.php

@@ -0,0 +1,45 @@
+<?php
+
+final class DivinerPublishCache extends DivinerDiskCache {
+
+  private $pathMap;
+
+  public function __construct($cache_directory) {
+    return parent::__construct($cache_directory, 'diviner-publish-cache');
+  }
+
+
+/* -(  Path Map  )----------------------------------------------------------- */
+
+
+  public function getPathMap() {
+    if ($this->pathMap === null) {
+      $this->pathMap = $this->getCache()->getKey('path', array());
+    }
+    return $this->pathMap;
+  }
+
+  public function writePathMap() {
+    $this->getCache()->setKey('path', $this->getPathMap());
+  }
+
+  public function getAtomPathsFromCache($hash) {
+    return idx($this->getPathMap(), $hash, array());
+  }
+
+  public function removeAtomPathsFromCache($hash) {
+    $map = $this->getPathMap();
+    unset($map[$hash]);
+    $this->pathMap = $map;
+    return $this;
+  }
+
+  public function addAtomPathsToCache($hash, array $paths) {
+    $map = $this->getPathMap();
+    $map[$hash] = $paths;
+    $this->pathMap = $map;
+    return $this;
+  }
+
+
+}

src/applications/diviner/publisher/DivinerPublisher.php

@@ -113,8 +113,11 @@ abstract class DivinerPublisher {
     $deleted = array_diff_key($existing_map, $hashes_map);
     $created = array_diff_key($hashes_map, $existing_map);
 
-    $this->createDocumentsByHash(array_keys($created));
+    echo pht('Deleting %d documents.', count($deleted))."\n";
     $this->deleteDocumentsByHash(array_keys($deleted));
+
+    echo pht('Creating %d documents.', count($created))."\n";
+    $this->createDocumentsByHash(array_keys($created));
   }
 
   protected function shouldGenerateDocumentForAtom(DivinerAtom $atom) {

src/applications/diviner/publisher/DivinerStaticPublisher.php

@@ -2,25 +2,86 @@
 
 final class DivinerStaticPublisher extends DivinerPublisher {
 
+  private $publishCache;
+
+  private function getPublishCache() {
+    if (!$this->publishCache) {
+      $dir = implode(
+        DIRECTORY_SEPARATOR,
+        array(
+          $this->getConfig('root'),
+          '.divinercache',
+          $this->getConfig('name'),
+          'static',
+        ));
+      $this->publishCache = new DivinerPublishCache($dir);
+    }
+
+    return $this->publishCache;
+  }
+
   protected function loadAllPublishedHashes() {
-    return array();
+    return array_keys($this->getPublishCache()->getPathMap());
   }
 
   protected function deleteDocumentsByHash(array $hashes) {
-    return;
+    $root = $this->getConfig('root');
+    $cache = $this->getPublishCache();
+
+    foreach ($hashes as $hash) {
+      $paths = $cache->getAtomPathsFromCache($hash);
+      foreach ($paths as $path) {
+        $abs = $root.DIRECTORY_SEPARATOR.$path;
+        Filesystem::remove($abs);
+
+        // If the parent directory is now empty, clean it up.
+        $dir = dirname($abs);
+        while (true) {
+          if (!Filesystem::isDescendant($dir, $root)) {
+            // Directory is outside of the root.
+            break;
+          }
+          if (Filesystem::listDirectory($dir)) {
+            // Directory is not empty.
+            break;
+          }
+
+          Filesystem::remove($dir);
+          $dir = dirname($dir);
+        }
+      }
+
+      $cache->removeAtomPathsFromCache($hash);
+      $cache->deleteRenderCache($hash);
+    }
+
+    $cache->writePathMap();
   }
 
   protected function createDocumentsByHash(array $hashes) {
+    $indexes = array();
+
     foreach ($hashes as $hash) {
       $atom = $this->getAtomFromGraphHash($hash);
 
-      if (!$this->shouldGenerateDocumentForAtom($atom)) {
+      $paths = array();
-        continue;
+      if ($this->shouldGenerateDocumentForAtom($atom)) {
+        $content = $this->getRenderer()->renderAtom($atom);
+
+        $this->writeDocument($atom, $content);
+
+        $paths[] = $this->getAtomRelativePath($atom);
+        if ($this->getAtomSimilarIndex($atom) !== null) {
+          $index = dirname($this->getAtomRelativePath($atom)).'index.html';
+          $indexes[$index] = $atom;
+          $paths[] = $index;
+        }
       }
 
-      $content = $this->getRenderer()->renderAtom($atom);
+      $this->getPublishCache()->addAtomPathsToCache($hash, $paths);
-      $this->writeDocument($atom, $content);
     }
+
+    $this->getPublishCache()->writePathMap();
   }
 
   private function writeDocument(DivinerAtom $atom, $content) {
@@ -32,6 +93,8 @@ final class DivinerStaticPublisher extends DivinerPublisher {
     }
 
     Filesystem::writeFile($path.'index.html', $content);
+
+    return $this;
   }
 
   private function getAtomRelativePath(DivinerAtom $atom) {