Allow diffusion.filecontentquery to load data for arbitrarily large files

Summary:
Fixes T10186. After D14970, `diffusion.filecontentquery` puts the content in a file and returns the file PHID.

However, it does this in a way that doesn't go through the chunking engine, so it will fail for files larger than the chunk threshold (generally, 8MB).

Instead, stream the file from the underlying command directly into chunked storage.

Test Plan:
  - Made a commit including a really big file: 4dcd4c492b
  - Used `diffusion.filecontentquery` to load file content.
  - Parsed/imported commit locally.
  - Used `diffusion.filecontentquery` to load content for smaller files (README, etc).

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10186

Differential Revision: https://secure.phabricator.com/D15072
This commit is contained in:
epriestley
2016-01-21 04:25:38 -08:00
parent e0fef74be7
commit b51a859636
6 changed files with 309 additions and 43 deletions

View File

@@ -26,11 +26,15 @@ final class PhabricatorFilesApplicationStorageEnginePanel
$rows = array();
$rowc = array();
foreach ($engines as $key => $engine) {
$limited = $no;
if ($engine->isTestEngine()) {
continue;
}
$limit = null;
if ($engine->hasFilesizeLimit()) {
$limited = $yes;
$limit = phutil_format_bytes($engine->getFilesizeLimit());
} else {
$limit = pht('Unlimited');
}
if ($engine->canWriteFiles()) {
@@ -39,12 +43,6 @@ final class PhabricatorFilesApplicationStorageEnginePanel
$writable = $no;
}
if ($engine->isTestEngine()) {
$test = $yes;
} else {
$test = $no;
}
if (isset($writable_engines[$key]) || isset($chunk_engines[$key])) {
$rowc[] = 'highlighted';
} else {
@@ -54,9 +52,7 @@ final class PhabricatorFilesApplicationStorageEnginePanel
$rows[] = array(
$key,
get_class($engine),
$test,
$writable,
$limited,
$limit,
);
}
@@ -67,9 +63,7 @@ final class PhabricatorFilesApplicationStorageEnginePanel
array(
pht('Key'),
pht('Class'),
pht('Unit Test'),
pht('Writable'),
pht('Has Limit'),
pht('Limit'),
))
->setRowClasses($rowc)
@@ -78,8 +72,6 @@ final class PhabricatorFilesApplicationStorageEnginePanel
'',
'wide',
'',
'',
'',
'n',
));