Tweaks to baker to use the final directory structure

scripts/bake/bake_differential.php

@@ -59,7 +59,7 @@ $conduit = id(new ConduitClient('https://developer.blender.org/api/'))
   ->setConduitToken(trim(file_get_contents($CONDUIT_TOKEN_FILE)));
 
 /////////////////////////////////////////////////////////////////////////////////
-// Baking logic and loop.
+// Baking utilities.
 
 function EnsureDirectoryOrDie($dir) {
   if (!is_dir($dir)) {
@@ -70,11 +70,79 @@ function EnsureDirectoryOrDie($dir) {
   }
 }
 
-function EnsureOutputDirectoryOrDie() {
+/////////////////////////////////////////////////////////////////////////////////
+// Revision baking.
+
+class RevisionInfo {
+  public $title = '';
+  public $page_title = '';
+
+  public $last_diff_id = 0;
+
+  public $hash = '';
+
+  public static function ReadFromFile($file_name) {
+    $info = new RevisionInfo();
+
+    if (!file_exists($file_name)) {
+      return $info;
+    }
+
+    $json = json_decode(file_get_contents($file_name));
+    foreach ($json as $key => $value) {
+      $info->{$key} = $value;
+    }
+
+    return $info;
+  }
+
+  public function SaveToFile($file_name) {
+    $json_str = json_encode(get_object_vars($this));
+    file_put_contents($file_name, $json_str);
+  }
+};
+
+function GetRevisionOutputDirectory($revision) {
   global $OUTPUT_DIR;
-  EnsureDirectoryOrDie($OUTPUT_DIR);
+
+  $id = $revision->getID();
+
+  return PathJoin(array($OUTPUT_DIR, 'differential', SubpathFromId($id)));
 }
 
+function EnsureRevisionOutputDirectory($revision) {
+  $dir = GetRevisionOutputDirectory($revision);
+
+  EnsureDirectoryOrDie($dir);
+  EnsureDirectoryOrDie(PathJoin(array($dir, 'raw_diff')));
+
+  return $dir;
+}
+
+function GetRevisionInfoFilename($revision) {
+  $revision_dir = GetRevisionOutputDirectory($revision);
+  return PathJoin(array($revision_dir, 'info.json'));
+}
+
+function GetRevisionHTMLFilename($revision, $diff_id) {
+  $revision_id = $revision->getID();
+  $revision_dir = GetRevisionOutputDirectory($revision);
+
+  $file_name = "D{$revision_id}.id{$diff_id}.html";
+
+  return PathJoin(array($revision_dir, $file_name));
+}
+
+function GetRevisionRawDiffFilename($revision, $diff_id) {
+  $revision_id = $revision->getID();
+  $revision_dir = GetRevisionOutputDirectory($revision);
+
+  $file_name = "D{$revision_id}.id{$diff_id}.diff";
+
+  return PathJoin(array($revision_dir, 'raw_diff', $file_name));
+}
+
+/*
 function MakeGenericOutputDirectory($subject) {
   global $OUTPUT_DIR;
 
@@ -105,27 +173,9 @@ function MakeRawDiffOutputFileName($revision, $diff_id) {
   return MakeRevisionBaseOutputFileName(
       $revision, $diff_id, 'raw_diff') . '.diff';
 }
+*/
 
-function MakePasteOutputFileNameFromHandle($handle) {
-  return MakeGenericOutputDirectory('paste') .
-      DIRECTORY_SEPARATOR .
-      $handle->getName() . '.txt';
-}
-
-
-function MakeFileOutputFileNameFromFile($file) {
-  $dir = MakeGenericOutputDirectory('file');
-
-  $id = $file->getID();
-
-  EnsureDirectoryOrDie($dir . DIRECTORY_SEPARATOR . 'F' . $id);
-
-  return  $dir.
-      DIRECTORY_SEPARATOR .
-      $file->getSafeFileNameForBake();
-}
-
-function BakeDifferentialToResponse($revision, $diff_id = NULL) {
+function RenderDifferentialToResponse($revision, $diff_id) {
   global $application_configuration;
   global $revision_controller;
   global $routing_map;
@@ -133,12 +183,6 @@ function BakeDifferentialToResponse($revision, $diff_id = NULL) {
 
   $revision_id = $revision->getID();
 
-  if (is_null($diff_id)) {
-    printf('Baking D' . $revision_id . ' ...' . "\n");
-  } else {
-    printf('Baking D' . $revision_id . ', diff ' . $diff_id . ' ...' . "\n");
-  }
-
   $path = '/D' . $revision_id;
 
   $route_result = $routing_map->routePath($path);
@@ -165,7 +209,7 @@ function BakeDifferentialToResponse($revision, $diff_id = NULL) {
   return $response;
 }
 
-function BakeResponseToHTML($response) {
+function RenderResponseToHTML($response) {
   $html = '';
   foreach ($response->renderChildren() as $child) {
     $html .= $child->render();
@@ -174,71 +218,6 @@ function BakeResponseToHTML($response) {
   return $html;
 }
 
-function StorePasteHandle($handle) {
-  global $viewer;
-
-  $file_name = MakePasteOutputFileNameFromHandle($handle);
-  if (file_exists($file_name)) {
-    return;
-  }
-
-  $paste = id(new PhabricatorPasteQuery())
-    ->setViewer($viewer)
-    ->withPHIDs(array($handle->getPHID()))
-    ->needRawContent(true)
-    ->executeOne();
-
-  file_put_contents($file_name, $paste->getRawContent());
-}
-
-function StoreFile($file) {
-  print("Loading and storing file F{$file->getID()}\n");
-  $file_name = MakeFileOutputFileNameFromFile($file);
-  file_put_contents($file_name, $file->loadFileData());
-}
-
-function StoreFileHandle($handle) {
-  global $viewer;
-
-  $file = id(new PhabricatorFileQuery())
-    ->setViewer($viewer)
-    ->withPHIDs(array($handle->getPHID()))
-    ->needTransforms(
-      array(
-        PhabricatorFileThumbnailTransform::TRANSFORM_PREVIEW,
-    ))
-    ->executeOne();
-
-  $file_name = MakeFileOutputFileNameFromFile($file);
-  if (file_exists($file_name)) {
-    return;
-  }
-
-  StoreFile($file);
-
-  $preview_xform = $file->getTransform(PhabricatorFileThumbnailTransform::TRANSFORM_PREVIEW);
-  if ($preview_xform) {
-    StoreFile($preview_xform);
-  }
-}
-
-function StoreResponseAttachments($response) {
-  $user = $response->getRequest()->getUser();
-  $handlePool = $user->getHandlePool();
-  $handles = $handlePool->getHandles();
-
-  foreach ($handles as $phid => $handle) {
-    $type = $handle->getType();
-    if ($type == PhabricatorPastePastePHIDType::TYPECONST) {
-      StorePasteHandle($handle);
-    } else if ($type == PhabricatorFileFilePHIDType::TYPECONST) { 
-      StoreFileHandle($handle);
-    } else {
-      // print("$type\n");
-    }
-  }
-}
-
 function StoreRawDiff($revision, $diff_id) {
   if (is_null($diff_id)) {
     return;
@@ -247,13 +226,6 @@ function StoreRawDiff($revision, $diff_id) {
   global $viewer;
   global $conduit;
 
-  $output_file = MakeRawDiffOutputFileName($revision, $diff_id);
-  if (file_exists($output_file)) {
-    return;
-  }
-
-  print("Requesting and storing the raw diff ...\n");
-
   $diff = id(new DifferentialDiffQuery())
     ->setViewer($viewer)
     ->withIDs(array($diff_id))
@@ -271,24 +243,84 @@ function StoreRawDiff($revision, $diff_id) {
   $bundle->setConduit($conduit);
   $raw_diff = $bundle->toGitPatch();
 
+  $output_file = GetRevisionRawDiffFilename($revision, $diff_id);
   file_put_contents($output_file, $raw_diff);
 }
 
-function BakeDifferentialToFile($revision, $diff_id = NULL) {
-  $response = BakeDifferentialToResponse($revision, $diff_id);
+function GetRevisionHash($revision) {
+  global $viewer;
 
-  $html = BakeResponseToHTML($response);
+  $xaction = id(new DifferentialTransactionQuery())
+    ->setViewer($viewer)
+    ->withObjectPHIDs(array($revision->getPHID()))
+    ->setOrder('newest')
+    ->setLimit(1)
+    ->executeOne();
 
-  $output_file = MakeRevisionHTMLOutputFileName($revision, $diff_id);
-  file_put_contents($output_file, $html);
+  $digest = '';
 
-  file_put_contents($output_file . '.title', $response->getTitle());
+  $digest .= $revision->getDateModified();
+  $digest .= '_' . $xaction->getPHID();
+  $digest .= '_' . $xaction->getDateModified();
 
-  StoreResponseAttachments($response);
-  StoreRawDiff($revision, $diff_id);
+  return $digest;
 }
 
-EnsureOutputDirectoryOrDie();
+function NeedBakeRevision($info, $revision, $diff_id) {
+  if (!file_exists(GetRevisionHTMLFilename($revision, $diff_id))) {
+    return true;
+  }
+  if (!file_exists(GetRevisionRawDiffFilename($revision, $diff_id))) {
+    return true;
+  }
+
+  if ($info->last_diff_id < $diff_id) {
+    return true;
+  }
+
+  if ($revision->getTitle() != $info->title) {
+    return true;
+  }
+
+  if (GetRevisionHash($revision) != $info->hash) {
+    return true;
+  }
+
+  return false;
+}
+
+function BakeRevision($revision, $diff_id) {
+  $revision_id = $revision->getID();
+
+  printf('Baking D' . $revision_id . '?id=' . $diff_id . ' ...' . "\n");
+
+  $info_file_name = GetRevisionInfoFilename($revision);
+
+  $info = RevisionInfo::ReadFromFile($info_file_name);
+
+  if (!NeedBakeRevision($info, $revision, $diff_id)) {
+    print('  ... ignoring: up to date' . "\n");
+    return;
+  }
+
+  $response = RenderDifferentialToResponse($revision, $diff_id);
+
+  $html = RenderResponseToHTML($response);
+  $html_file_name = GetRevisionHTMLFIlename($revision, $diff_id);
+  file_put_contents($html_file_name, $html);
+
+  StoreRawDiff($revision, $diff_id);
+
+  $info->title = $revision->getTitle();
+  $info->page_title = $response->getTitle();
+  $info->hash = GetRevisionHash($revision);
+  $info->last_diff_id = max($info->last_diff_id, $diff_id);
+
+  $info->SaveToFile($info_file_name);
+}
+
+/////////////////////////////////////////////////////////////////////////////////
+// Baking main loop.
 
 print('Querying differential revisions from the database ... ' . "\n");
 $revisions = id(new DifferentialRevisionQuery())
@@ -298,10 +330,10 @@ $revisions = id(new DifferentialRevisionQuery())
   ->execute();
 
 foreach ($revisions as $revision_id => $revision) {
-  BakeDifferentialToFile($revision);
+  EnsureRevisionOutputDirectory($revision);
 
   foreach ($revision->getDiffIDs() as $diff_id) {
-    BakeDifferentialToFile($revision, $diff_id);
+    BakeRevision($revision, $diff_id);
   }
 }
 

src/__phutil_library_map__.php

@@ -5958,6 +5958,9 @@ phutil_register_library_map(array(
     'UserWhoAmIConduitAPIMethod' => 'applications/people/conduit/UserWhoAmIConduitAPIMethod.php',
   ),
   'function' => array(
+    'MakeFilemameSafe' => 'infrastructure/util/PhabricatorID.php',
+    'PathJoin' => 'infrastructure/util/PhabricatorID.php',
+    'SubpathFromId' => 'infrastructure/util/PhabricatorID.php',
     'celerity_generate_unique_node_id' => 'applications/celerity/api.php',
     'celerity_get_resource_uri' => 'applications/celerity/api.php',
     'hsprintf' => 'infrastructure/markup/render.php',

src/applications/differential/storage/DifferentialRevision.php

@@ -147,7 +147,8 @@ final class DifferentialRevision extends DifferentialDAO
   }
 
   public function getURI() {
-    return $this->getMonogram() . '.html';
+    $subpath = SubpathFromId($this->getID());
+    return '../../../differential/' . $subpath . '/';
   }
 
   public function getCommitPHIDs() {

src/applications/differential/view/DifferentialRevisionUpdateHistoryView.php

@@ -178,7 +178,7 @@ final class DifferentialRevisionUpdateHistoryView extends AphrontView {
         $id_link = phutil_tag(
           'a',
           array(
-            'href' => 'D'.$revision_id.'.'.$id.'.html',
+            'href' => 'D'.$revision_id.'.id'.$id.'.html',
           ),
           $id);
       } else {

src/applications/files/storage/PhabricatorFile.php

@@ -874,19 +874,10 @@ final class PhabricatorFile extends PhabricatorFileDAO
     }
   }
 
-  public static function makeFileNameForURI($id, $name) {
-    // $safe_name = preg_replace('/[\\/\\\\]/', '_', $name);
-    $safe_name = preg_replace('/[^A-Za-z0-9\\-_\\+\\.]/', '_', $name);
-    return 'F' . $id . DIRECTORY_SEPARATOR . $safe_name;
-  }
-
-  public function getSafeFileNameForBake() {
-    return PhabricatorFile::makeFileNameForURI(
-        $this->getID(), $this->getName());
-  }
-
   public function getInfoURI() {
-    return 'file/'.$this->getSafeFileNameForBake();
+    $safe_name = MakeFilemameSafe($this->getName());
+    $subpath = SubpathFromId($this->getID());
+    return '../../../file/' . $subpath . '/' . $safe_name;
   }
 
   public function getBestURI() {

src/applications/paste/phid/PhabricatorPastePastePHIDType.php

@@ -37,7 +37,8 @@ final class PhabricatorPastePastePHIDType extends PhabricatorPHIDType {
 
       $handle->setName("P{$id}");
       $handle->setFullName($name);
-      $handle->setURI("paste/P{$id}.txt");
+
+      $handle->setURI($paste->getURI());
     }
   }
 

src/applications/paste/query/PhabricatorPasteSearchEngine.php

@@ -176,7 +176,7 @@ final class PhabricatorPasteSearchEngine
       $item = id(new PHUIObjectItemView())
         ->setObjectName('P'.$paste->getID())
         ->setHeader($title)
-        ->setHref('paste/P'.$paste->getID().'.txt')
+        ->setHref($paste->getURI())
         ->setObject($paste)
         ->addByline(pht('Author: %s', $author))
         ->addIcon('none', $created)

src/applications/paste/storage/PhabricatorPaste.php

@@ -60,7 +60,8 @@ final class PhabricatorPaste extends PhabricatorPasteDAO
   }
 
   public function getURI() {
-    return 'paste/'.$this->getMonogram();
+    $subpath = SubpathFromId($this->getID());
+    return '../../../paste/' . $subpath . '/P' . $this->getID() . '.txt';
   }
 
   public function getMonogram() {

src/applications/paste/view/PasteEmbedView.php

@@ -38,7 +38,7 @@ final class PasteEmbedView extends AphrontView {
     $link = phutil_tag(
       'a',
       array(
-        'href' => 'paste/P'.$this->paste->getID().'.txt',
+        'href' => $this->paste->getURI(),
       ),
       $this->handle->getFullName());
 

src/infrastructure/util/PhabricatorID.php

@@ -0,0 +1,21 @@
+<?php
+
+function MakeFilemameSafe($file_name) {
+  return preg_replace('/[^\w\-_\.]/', '_', $file_name);
+}
+
+function PathJoin($parts) {
+  return join(DIRECTORY_SEPARATOR, $parts);
+}
+
+function SubpathFromId($id) {
+  $padded_id = str_pad($id, 7, '0', STR_PAD_LEFT);
+
+  $length = strlen($padded_id);
+
+  $bin = substr($padded_id, 0, $length - 3);
+
+  return PathJoin(array($bin, $padded_id));
+}
+
+?>