From c0b8e4784bfad4ec07c8e5e34f0071f7acf65792 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 23 Jan 2018 09:57:41 -0800 Subject: [PATCH] Add a basic, general-purpose export workflow for all objects with SearchEngine support Summary: Depends on D18918. Ref T13046. Ref T5954. Pull logs can currently be browsed in the web UI, but this isn't very powerful, especially if you have thousands of them. Allow SearchEngine implementations to define exportable fields so that users can "Use Results > Export Data" on any query. In particular, they can use this workflow to download a file with pull logs. In the future, this can replace the existing "Export to Excel" feature in Maniphest. For now, we hard-code JSON as the only supported datatype and don't actually make any effort to format the data properly, but this leaves room to add more exporters (CSV, Excel) and data type awareness (integer casting, date formatting, etc) in the future. For sufficiently large result sets, this will probably time out. At some point, I'll make this use the job queue (like bulk editing) when the export is "large" (affects more than 1K rows?). Test Plan: Downloaded pull logs in JSON format. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13046, T5954 Differential Revision: https://secure.phabricator.com/D18919 --- src/__phutil_library_map__.php | 10 ++ .../base/PhabricatorApplication.php | 2 +- .../query/DiffusionPullLogSearchEngine.php | 81 ++++++++++++ ...PhabricatorApplicationSearchController.php | 123 +++++++++++++++++- .../PhabricatorApplicationSearchEngine.php | 17 +++ .../export/PhabricatorEpochExportField.php | 4 + .../export/PhabricatorExportField.php | 27 ++++ .../export/PhabricatorIDExportField.php | 4 + .../export/PhabricatorPHIDExportField.php | 4 + .../export/PhabricatorStringExportField.php | 4 + 10 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 src/infrastructure/export/PhabricatorEpochExportField.php create mode 100644 src/infrastructure/export/PhabricatorExportField.php create mode 100644 src/infrastructure/export/PhabricatorIDExportField.php create mode 100644 src/infrastructure/export/PhabricatorPHIDExportField.php create mode 100644 src/infrastructure/export/PhabricatorStringExportField.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index a134df5e1e..311d643fa5 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2836,12 +2836,14 @@ phutil_register_library_map(array( 'PhabricatorEnv' => 'infrastructure/env/PhabricatorEnv.php', 'PhabricatorEnvTestCase' => 'infrastructure/env/__tests__/PhabricatorEnvTestCase.php', 'PhabricatorEpochEditField' => 'applications/transactions/editfield/PhabricatorEpochEditField.php', + 'PhabricatorEpochExportField' => 'infrastructure/export/PhabricatorEpochExportField.php', 'PhabricatorEvent' => 'infrastructure/events/PhabricatorEvent.php', 'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php', 'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', + 'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php', 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', 'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php', 'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php', @@ -3061,6 +3063,7 @@ phutil_register_library_map(array( 'PhabricatorHomeProfileMenuItem' => 'applications/home/menuitem/PhabricatorHomeProfileMenuItem.php', 'PhabricatorHovercardEngineExtension' => 'applications/search/engineextension/PhabricatorHovercardEngineExtension.php', 'PhabricatorHovercardEngineExtensionModule' => 'applications/search/engineextension/PhabricatorHovercardEngineExtensionModule.php', + 'PhabricatorIDExportField' => 'infrastructure/export/PhabricatorIDExportField.php', 'PhabricatorIDsSearchEngineExtension' => 'applications/search/engineextension/PhabricatorIDsSearchEngineExtension.php', 'PhabricatorIDsSearchField' => 'applications/search/field/PhabricatorIDsSearchField.php', 'PhabricatorIconDatasource' => 'applications/files/typeahead/PhabricatorIconDatasource.php', @@ -3412,6 +3415,7 @@ phutil_register_library_map(array( 'PhabricatorPHDConfigOptions' => 'applications/config/option/PhabricatorPHDConfigOptions.php', 'PhabricatorPHID' => 'applications/phid/storage/PhabricatorPHID.php', 'PhabricatorPHIDConstants' => 'applications/phid/PhabricatorPHIDConstants.php', + 'PhabricatorPHIDExportField' => 'infrastructure/export/PhabricatorPHIDExportField.php', 'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php', 'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php', 'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php', @@ -4177,6 +4181,7 @@ phutil_register_library_map(array( 'PhabricatorStorageSchemaSpec' => 'infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php', 'PhabricatorStorageSetupCheck' => 'applications/config/check/PhabricatorStorageSetupCheck.php', 'PhabricatorStringConfigType' => 'applications/config/type/PhabricatorStringConfigType.php', + 'PhabricatorStringExportField' => 'infrastructure/export/PhabricatorStringExportField.php', 'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php', 'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php', 'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php', @@ -8255,12 +8260,14 @@ phutil_register_library_map(array( 'PhabricatorEnv' => 'Phobject', 'PhabricatorEnvTestCase' => 'PhabricatorTestCase', 'PhabricatorEpochEditField' => 'PhabricatorEditField', + 'PhabricatorEpochExportField' => 'PhabricatorExportField', 'PhabricatorEvent' => 'PhutilEvent', 'PhabricatorEventEngine' => 'Phobject', 'PhabricatorEventListener' => 'PhutilEventListener', 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', + 'PhabricatorExportField' => 'Phobject', 'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorExternalAccount' => array( @@ -8521,6 +8528,7 @@ phutil_register_library_map(array( 'PhabricatorHomeProfileMenuItem' => 'PhabricatorProfileMenuItem', 'PhabricatorHovercardEngineExtension' => 'Phobject', 'PhabricatorHovercardEngineExtensionModule' => 'PhabricatorConfigModule', + 'PhabricatorIDExportField' => 'PhabricatorExportField', 'PhabricatorIDsSearchEngineExtension' => 'PhabricatorSearchEngineExtension', 'PhabricatorIDsSearchField' => 'PhabricatorSearchField', 'PhabricatorIconDatasource' => 'PhabricatorTypeaheadDatasource', @@ -8911,6 +8919,7 @@ phutil_register_library_map(array( 'PhabricatorPHDConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorPHID' => 'Phobject', 'PhabricatorPHIDConstants' => 'Phobject', + 'PhabricatorPHIDExportField' => 'PhabricatorExportField', 'PhabricatorPHIDListEditField' => 'PhabricatorEditField', 'PhabricatorPHIDListEditType' => 'PhabricatorEditType', 'PhabricatorPHIDResolver' => 'Phobject', @@ -9850,6 +9859,7 @@ phutil_register_library_map(array( 'PhabricatorStorageSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'PhabricatorStorageSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorStringConfigType' => 'PhabricatorTextConfigType', + 'PhabricatorStringExportField' => 'PhabricatorExportField', 'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType', 'PhabricatorStringListEditField' => 'PhabricatorEditField', 'PhabricatorStringSetting' => 'PhabricatorSetting', diff --git a/src/applications/base/PhabricatorApplication.php b/src/applications/base/PhabricatorApplication.php index 0ae8e75354..4062e4e2fe 100644 --- a/src/applications/base/PhabricatorApplication.php +++ b/src/applications/base/PhabricatorApplication.php @@ -623,7 +623,7 @@ abstract class PhabricatorApplication } protected function getQueryRoutePattern($base = null) { - return $base.'(?:query/(?P[^/]+)/)?'; + return $base.'(?:query/(?P[^/]+)/(?:(?P[^/]+)/))?'; } protected function getProfileMenuRouting($controller) { diff --git a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php index 85a540f6f5..a0fdc06dfb 100644 --- a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php +++ b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php @@ -47,6 +47,87 @@ final class DiffusionPullLogSearchEngine ); } + protected function newExportFields() { + return array( + id(new PhabricatorIDExportField()) + ->setKey('id') + ->setLabel(pht('ID')), + id(new PhabricatorPHIDExportField()) + ->setKey('phid') + ->setLabel(pht('PHID')), + id(new PhabricatorPHIDExportField()) + ->setKey('repositoryPHID') + ->setLabel(pht('Repository PHID')), + id(new PhabricatorStringExportField()) + ->setKey('repository') + ->setLabel(pht('Repository')), + id(new PhabricatorPHIDExportField()) + ->setKey('pullerPHID') + ->setLabel(pht('Puller PHID')), + id(new PhabricatorStringExportField()) + ->setKey('puller') + ->setLabel(pht('Puller')), + id(new PhabricatorStringExportField()) + ->setKey('protocol') + ->setLabel(pht('Protocol')), + id(new PhabricatorStringExportField()) + ->setKey('result') + ->setLabel(pht('Result')), + id(new PhabricatorStringExportField()) + ->setKey('code') + ->setLabel(pht('Code')), + id(new PhabricatorEpochExportField()) + ->setKey('date') + ->setLabel(pht('Date')), + ); + } + + public function newExport(array $events) { + $viewer = $this->requireViewer(); + + $phids = array(); + foreach ($events as $event) { + if ($event->getPullerPHID()) { + $phids[] = $event->getPullerPHID(); + } + } + $handles = $viewer->loadHandles($phids); + + $export = array(); + foreach ($events as $event) { + $repository = $event->getRepository(); + if ($repository) { + $repository_phid = $repository->getPHID(); + $repository_name = $repository->getDisplayName(); + } else { + $repository_phid = null; + $repository_name = null; + } + + $puller_phid = $event->getPullerPHID(); + if ($puller_phid) { + $puller_name = $handles[$puller_phid]->getName(); + } else { + $puller_name = null; + } + + $export[] = array( + 'id' => $event->getID(), + 'phid' => $event->getPHID(), + 'repositoryPHID' => $repository_phid, + 'repository' => $repository_name, + 'pullerPHID' => $puller_phid, + 'puller' => $puller_name, + 'protocol' => $event->getRemoteProtocol(), + 'result' => $event->getResultType(), + 'code' => $event->getResultCode(), + 'date' => $event->getEpoch(), + ); + } + + return $export; + } + protected function getURI($path) { return '/diffusion/pulllog/'.$path; } diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index f781e20380..d8f718069c 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -66,6 +66,11 @@ final class PhabricatorApplicationSearchController public function processRequest() { $this->validateDelegatingController(); + $query_action = $this->getRequest()->getURIData('queryAction'); + if ($query_action == 'export') { + return $this->processExportRequest(); + } + $key = $this->getQueryKey(); if ($key == 'edit') { return $this->processEditRequest(); @@ -374,6 +379,96 @@ final class PhabricatorApplicationSearchController ->appendChild($body); } + private function processExportRequest() { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $request = $this->getRequest(); + + if (!$this->canExport()) { + return new Aphront404Response(); + } + + $query_key = $this->getQueryKey(); + if ($engine->isBuiltinQuery($query_key)) { + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); + } else if ($query_key) { + $saved_query = id(new PhabricatorSavedQueryQuery()) + ->setViewer($viewer) + ->withQueryKeys(array($query_key)) + ->executeOne(); + if (!$saved_query) { + return new Aphront404Response(); + } + } + + $cancel_uri = $engine->getQueryResultsPageURI($query_key); + + $named_query = idx($engine->loadEnabledNamedQueries(), $query_key); + + if ($named_query) { + $filename = $named_query->getQueryName(); + } else { + $filename = $engine->getResultTypeDescription(); + } + $filename = phutil_utf8_strtolower($filename); + $filename = PhabricatorFile::normalizeFileName($filename); + + if ($request->isFormPost()) { + $query = $engine->buildQueryFromSavedQuery($saved_query); + + // NOTE: We aren't reading the pager from the request. Exports always + // affect the entire result set. + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize(0x7FFFFFFF); + + $objects = $engine->executeQuery($query, $pager); + + $extension = 'json'; + $mime_type = 'application/json'; + $filename = $filename.'.'.$extension; + + $result = $engine->newExport($objects); + $result = id(new PhutilJSON()) + ->encodeAsList($result); + + $file = PhabricatorFile::newFromFileData( + $result, + array( + 'name' => $filename, + 'authorPHID' => $viewer->getPHID(), + 'ttl.relative' => phutil_units('15 minutes in seconds'), + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + 'mime-type' => $mime_type, + )); + + return $this->newDialog() + ->setTitle(pht('Download Results')) + ->appendParagraph( + pht('Click the download button to download the exported data.')) + ->addCancelButton($cancel_uri, pht('Done')) + ->setSubmitURI($file->getDownloadURI()) + ->setDisableWorkflowOnSubmit(true) + ->addSubmitButton(pht('Download Results')); + } + + $export_form = id(new AphrontFormView()) + ->setViewer($viewer) + ->appendControl( + id(new AphrontFormSelectControl()) + ->setName('format') + ->setLabel(pht('Format')) + ->setOptions( + array( + 'json' => 'JSON', + ))); + + return $this->newDialog() + ->setTitle(pht('Export Results')) + ->appendForm($export_form) + ->addCancelButton($cancel_uri) + ->addSubmitButton(pht('Continue')); + } + private function processEditRequest() { $parent = $this->getDelegatingController(); $request = $this->getRequest(); @@ -720,7 +815,6 @@ final class PhabricatorApplicationSearchController $viewer); if ($can_use && $is_installed) { - $dashboard_uri = '/dashboard/install/'; $actions[] = id(new PhabricatorActionView()) ->setIcon('fa-dashboard') ->setName(pht('Add to Dashboard')) @@ -728,6 +822,15 @@ final class PhabricatorApplicationSearchController ->setHref("/dashboard/panel/install/{$engine_class}/{$query_key}/"); } + if ($this->canExport()) { + $export_uri = $engine->getExportURI($query_key); + $actions[] = id(new PhabricatorActionView()) + ->setIcon('fa-download') + ->setName(pht('Export Results')) + ->setWorkflow(true) + ->setHref($export_uri); + } + if ($is_dev) { $engine = $this->getSearchEngine(); $nux_uri = $engine->getQueryBaseURI(); @@ -753,4 +856,22 @@ final class PhabricatorApplicationSearchController return $actions; } + private function canExport() { + $engine = $this->getSearchEngine(); + if (!$engine->canExport()) { + return false; + } + + // Don't allow logged-out users to perform exports. There's no technical + // or policy reason they can't, but we don't normally give them access + // to write files or jobs. For now, just err on the side of caution. + + $viewer = $this->getViewer(); + if (!$viewer->getPHID()) { + return false; + } + + return true; + } + } diff --git a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php index c0497a0c03..c0fb6ce976 100644 --- a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php +++ b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php @@ -413,6 +413,10 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { return $this->getURI(''); } + public function getExportURI($query_key) { + return $this->getURI('query/'.$query_key.'/export/'); + } + /** * Return the URI to a path within the application. Used to construct default @@ -1441,4 +1445,17 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { return array(); } + +/* -( Export )------------------------------------------------------------- */ + + + public function canExport() { + $fields = $this->newExportFields(); + return (bool)$fields; + } + + protected function newExportFields() { + return array(); + } + } diff --git a/src/infrastructure/export/PhabricatorEpochExportField.php b/src/infrastructure/export/PhabricatorEpochExportField.php new file mode 100644 index 0000000000..e212dd5d8c --- /dev/null +++ b/src/infrastructure/export/PhabricatorEpochExportField.php @@ -0,0 +1,4 @@ +key = $key; + return $this; + } + + public function getKey() { + return $this->key; + } + + public function setLabel($label) { + $this->label = $label; + return $this; + } + + public function getLabel() { + return $this->label; + } + +} diff --git a/src/infrastructure/export/PhabricatorIDExportField.php b/src/infrastructure/export/PhabricatorIDExportField.php new file mode 100644 index 0000000000..bf59e3567c --- /dev/null +++ b/src/infrastructure/export/PhabricatorIDExportField.php @@ -0,0 +1,4 @@ +