Support HTML5 / Javascript chunked file uploads

Summary:
Ref T7149. This adds chunking support to drag-and-drop uploads. It never activates right now unless you hack things up, since the chunk engine is still hard-coded as disabled.

The overall approach is the same as `arc upload` in D12061, with some slight changes to the API return values to avoid a few extra HTTP calls.

Test Plan:
  - Enabled chunk engine.
  - Uploaded some READMEs in a bunch of tiny 32 byte chunks.
  - Worked out of the box in Safari, Chrome, Firefox.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12066

src/applications/files/conduit/FileAllocateConduitAPIMethod.php

@@ -121,10 +121,20 @@ final class FileAllocateConduitAPIMethod
     }
 
     // None of the storage engines can accept this file.
+    if (PhabricatorFileStorageEngine::loadWritableEngines()) {
+      $error = pht(
+        'Unable to upload file: this file is too large for any '.
+        'configured storage engine.');
+    } else {
+      $error = pht(
+        'Unable to upload file: the server is not configured with any '.
+        'writable storage engines.');
+    }
 
     return array(
       'upload' => false,
       'filePHID' => null,
+      'error' => $error,
     );
   }
 

src/applications/files/controller/PhabricatorFileDropUploadController.php

@@ -13,18 +13,82 @@ final class PhabricatorFileDropUploadController
     // NOTE: Throws if valid CSRF token is not present in the request.
     $request->validateCSRF();
 
-    $data = PhabricatorStartup::getRawInput();
     $name = $request->getStr('name');
-
+    $file_phid = $request->getStr('phid');
     // If there's no explicit view policy, make it very restrictive by default.
     // This is the correct policy for files dropped onto objects during
     // creation, comment and edit flows.
-
     $view_policy = $request->getStr('viewPolicy');
     if (!$view_policy) {
       $view_policy = $viewer->getPHID();
     }
 
+    $is_chunks = $request->getBool('querychunks');
+    if ($is_chunks) {
+      $params = array(
+        'filePHID' => $file_phid,
+      );
+
+      $result = id(new ConduitCall('file.querychunks', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
+    $is_allocate = $request->getBool('allocate');
+    if ($is_allocate) {
+      $params = array(
+        'name' => $name,
+        'contentLength' => $request->getInt('length'),
+        'viewPolicy' => $view_policy,
+
+        // TODO: Remove.
+        // 'forceChunking' => true,
+      );
+
+      $result = id(new ConduitCall('file.allocate', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      $file_phid = $result['filePHID'];
+      if ($file_phid) {
+        $file = $this->loadFile($file_phid);
+        $result += $this->getFileDictionary($file);
+      }
+
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
+    // Read the raw request data. We're either doing a chunk upload or a
+    // vanilla upload, so we need it.
+    $data = PhabricatorStartup::getRawInput();
+
+
+    $is_chunk_upload = $request->getBool('uploadchunk');
+    if ($is_chunk_upload) {
+      $params = array(
+        'filePHID' => $file_phid,
+        'byteStart' => $request->getInt('byteStart'),
+        'data' => $data,
+      );
+
+      $result = id(new ConduitCall('file.uploadchunk', $params))
+        ->setUser($viewer)
+        ->execute();
+
+      $file = $this->loadFile($file_phid);
+      if ($file->getIsPartial()) {
+        $result = array();
+      } else {
+        $result = array(
+          'complete' => true,
+        ) + $this->getFileDictionary($file);
+      }
+
+      return id(new AphrontAjaxResponse())->setContent($result);
+    }
+
     $file = PhabricatorFile::newFromXHRUpload(
       $data,
       array(
@@ -34,12 +98,30 @@ final class PhabricatorFileDropUploadController
         'isExplicitUpload' => true,
       ));
 
-    return id(new AphrontAjaxResponse())->setContent(
-      array(
-        'id'   => $file->getID(),
-        'phid' => $file->getPHID(),
-        'uri'  => $file->getBestURI(),
-      ));
+    $result = $this->getFileDictionary($file);
+    return id(new AphrontAjaxResponse())->setContent($result);
+  }
+
+  private function getFileDictionary(PhabricatorFile $file) {
+    return array(
+      'id'   => $file->getID(),
+      'phid' => $file->getPHID(),
+      'uri'  => $file->getBestURI(),
+    );
+  }
+
+  private function loadFile($file_phid) {
+    $viewer = $this->getViewer();
+
+    $file = id(new PhabricatorFileQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($file_phid))
+      ->executeOne();
+    if (!$file) {
+      throw new Exception(pht('Failed to load file.'));
+    }
+
+    return $file;
   }
 
 }

src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php

@@ -162,7 +162,7 @@ final class PhabricatorChunkedFileStorageEngine
     return false;
   }
 
-  private function getChunkSize() {
+  public function getChunkSize() {
     // TODO: This is an artificially small size to make it easier to
     // test chunking.
     return 32;

src/applications/files/engine/PhabricatorFileStorageEngine.php

@@ -255,4 +255,40 @@ abstract class PhabricatorFileStorageEngine {
     return $writable;
   }
 
+
+  /**
+   * Return the largest file size which can be uploaded without chunking.
+   *
+   * Files smaller than this will always upload in one request, so clients
+   * can safely skip the allocation step.
+   *
+   * @return int|null Byte size, or `null` if there is no chunk support.
+   */
+  public static function getChunkThreshold() {
+    $engines = self::loadWritableEngines();
+
+    $min = null;
+    foreach ($engines as $engine) {
+      if (!$engine->isChunkEngine()) {
+        continue;
+      }
+
+      if (!$min) {
+        $min = $engine;
+        continue;
+      }
+
+      if ($min->getChunkSize() > $engine->getChunkSize()) {
+        $min = $engine->getChunkSize();
+      }
+    }
+
+    if (!$min) {
+      return null;
+    }
+
+    return $engine->getChunkSize();
+  }
+
+
 }

src/applications/files/view/PhabricatorGlobalUploadTargetView.php

@@ -24,11 +24,12 @@ final class PhabricatorGlobalUploadTargetView extends AphrontView {
     require_celerity_resource('global-drag-and-drop-css');
 
     Javelin::initBehavior('global-drag-and-drop', array(
-      'ifSupported'   => $this->showIfSupportedID,
-      'instructions'  => $instructions_id,
-      'uploadURI'     => '/file/dropupload/',
-      'browseURI'     => '/file/query/authored/',
-      'viewPolicy'    => PhabricatorPolicies::getMostOpenPolicy(),
+      'ifSupported' => $this->showIfSupportedID,
+      'instructions' => $instructions_id,
+      'uploadURI' => '/file/dropupload/',
+      'browseURI' => '/file/query/authored/',
+      'viewPolicy' => PhabricatorPolicies::getMostOpenPolicy(),
+      'chunkThreshold' => PhabricatorFileStorageEngine::getChunkThreshold(),
     ));
 
     return phutil_tag(

src/view/form/control/PhabricatorRemarkupControl.php

@@ -35,9 +35,10 @@ final class PhabricatorRemarkupControl extends AphrontFormTextAreaControl {
     Javelin::initBehavior(
       'aphront-drag-and-drop-textarea',
       array(
-        'target'          => $id,
-        'activatedClass'  => 'aphront-textarea-drag-and-drop',
-        'uri'             => '/file/dropupload/',
+        'target' => $id,
+        'activatedClass' => 'aphront-textarea-drag-and-drop',
+        'uri' => '/file/dropupload/',
+        'chunkThreshold' => PhabricatorFileStorageEngine::getChunkThreshold(),
       ));
 
     Javelin::initBehavior(