From 8b8a3142b3102fdf8199ee549ff245d90483f50b Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Sun, 28 Jan 2018 18:47:42 -0800
Subject: [PATCH] Support export of data in files larger than 8MB

Summary:
Depends on D18952. Ref T13049. For files larger than 8MB, we need to engage the chunk storage engine. `PhabricatorFile::newFromFileData()` always writes a single chunk, and can't handle files larger than the mandatory chunk threshold (8MB).

Use `IteratorUploadSource`, which can, and "stream" the data into it. This should raise the limit from 8MB to 2GB (maximum size of a string in PHP).

If we need to go above 2GB we could stream CSV and text pretty easily, and JSON without too much trouble, but Excel might be trickier. Hopefully no one is trying to export 2GB+ datafiles, though.

Test Plan:
  - Changed the JSON exporter to just export 8MB of the letter "q": `return str_repeat('q', 1024 * 1024 * 9);`.
  - Before change: fatal, "no storage engine can store this file".
  - After change: export works cleanly.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13049

Differential Revision: https://secure.phabricator.com/D18953
---
 .../files/storage/PhabricatorFile.php         |  8 +++--
 .../PhabricatorFileUploadSource.php           | 30 +++++++++++++++++++
 ...PhabricatorApplicationSearchController.php | 23 ++++++++------
 3 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php
index d3dc5208d2..2b8624b9c0 100644
--- a/src/applications/files/storage/PhabricatorFile.php
+++ b/src/applications/files/storage/PhabricatorFile.php
@@ -272,8 +272,12 @@ final class PhabricatorFile extends PhabricatorFileDAO
     $file->setByteSize($length);
 
     // NOTE: Once we receive the first chunk, we'll detect its MIME type and
-    // update the parent file. This matters for large media files like video.
-    $file->setMimeType('application/octet-stream');
+    // update the parent file if a MIME type hasn't been provided. This matters
+    // for large media files like video.
+    $mime_type = idx($params, 'mime-type');
+    if (!strlen($mime_type)) {
+      $file->setMimeType('application/octet-stream');
+    }
 
     $chunked_hash = idx($params, 'chunkedHash');
 
diff --git a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php
index bf213a417e..87a4486869 100644
--- a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php
+++ b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php
@@ -6,6 +6,8 @@ abstract class PhabricatorFileUploadSource
   private $name;
   private $relativeTTL;
   private $viewPolicy;
+  private $mimeType;
+  private $authorPHID;
 
   private $rope;
   private $data;
@@ -51,6 +53,24 @@ abstract class PhabricatorFileUploadSource
     return $this->byteLimit;
   }
 
+  public function setMIMEType($mime_type) {
+    $this->mimeType = $mime_type;
+    return $this;
+  }
+
+  public function getMIMEType() {
+    return $this->mimeType;
+  }
+
+  public function setAuthorPHID($author_phid) {
+    $this->authorPHID = $author_phid;
+    return $this;
+  }
+
+  public function getAuthorPHID() {
+    return $this->authorPHID;
+  }
+
   public function uploadFile() {
     if (!$this->shouldChunkFile()) {
       return $this->writeSingleFile();
@@ -245,6 +265,16 @@ abstract class PhabricatorFileUploadSource
       $parameters['ttl.relative'] = $ttl;
     }
 
+    $mime_type = $this->getMimeType();
+    if ($mime_type !== null) {
+      $parameters['mime-type'] = $mime_type;
+    }
+
+    $author_phid = $this->getAuthorPHID();
+    if ($author_phid !== null) {
+      $parameters['authorPHID'] = $author_phid;
+    }
+
     return $parameters;
   }
 
diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php
index 0402102e2b..e980498c42 100644
--- a/src/applications/search/controller/PhabricatorApplicationSearchController.php
+++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php
@@ -461,15 +461,20 @@ final class PhabricatorApplicationSearchController
 
         $export_result = $format->newFileData();
 
-        $file = PhabricatorFile::newFromFileData(
-          $export_result,
-          array(
-            'name' => $filename,
-            'authorPHID' => $viewer->getPHID(),
-            'ttl.relative' => phutil_units('15 minutes in seconds'),
-            'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
-            'mime-type' => $mime_type,
-          ));
+        // We have all the data in one big string and aren't actually
+        // streaming it, but pretending that we are allows us to actviate
+        // the chunk engine and store large files.
+        $iterator = new ArrayIterator(array($export_result));
+
+        $source = id(new PhabricatorIteratorFileUploadSource())
+          ->setName($filename)
+          ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
+          ->setMIMEType($mime_type)
+          ->setRelativeTTL(phutil_units('60 minutes in seconds'))
+          ->setAuthorPHID($viewer->getPHID())
+          ->setIterator($iterator);
+
+        $file = $source->uploadFile();
 
         return $this->newDialog()
           ->setTitle(pht('Download Results'))