From de7f836f03e62d08697cb54540d190a715358604 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 27 Jan 2018 05:22:09 -0800 Subject: [PATCH 01/89] Wrap edge transaction readers in a translation layer Summary: Ref T13051. This puts a translation layer between the raw edge data in the transaction table and the UI that uses it. The intent is to start writing new, more compact data soon. This class give us a consistent API for interacting with either the new or old data format, so we don't have to migrate everything upfront. Test Plan: Browsed around, saw existing edge transactions render properly in transactions and feed. Added and removed subscribers and projects, saw good transaction rendering. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13051 Differential Revision: https://secure.phabricator.com/D18946 --- src/__phutil_library_map__.php | 2 + .../storage/DifferentialTransaction.php | 13 ---- .../PhabricatorApplicationTransaction.php | 25 ++++--- .../util/PhabricatorEdgeChangeRecord.php | 68 +++++++++++++++++++ 4 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 44d67c6259..c40f79b052 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2747,6 +2747,7 @@ phutil_register_library_map(array( 'PhabricatorDraftEngine' => 'applications/transactions/draft/PhabricatorDraftEngine.php', 'PhabricatorDraftInterface' => 'applications/transactions/draft/PhabricatorDraftInterface.php', 'PhabricatorDrydockApplication' => 'applications/drydock/application/PhabricatorDrydockApplication.php', + 'PhabricatorEdgeChangeRecord' => 'infrastructure/edges/util/PhabricatorEdgeChangeRecord.php', 'PhabricatorEdgeConfig' => 'infrastructure/edges/constants/PhabricatorEdgeConfig.php', 'PhabricatorEdgeConstants' => 'infrastructure/edges/constants/PhabricatorEdgeConstants.php', 'PhabricatorEdgeCycleException' => 'infrastructure/edges/exception/PhabricatorEdgeCycleException.php', @@ -8170,6 +8171,7 @@ phutil_register_library_map(array( 'PhabricatorDraftDAO' => 'PhabricatorLiskDAO', 'PhabricatorDraftEngine' => 'Phobject', 'PhabricatorDrydockApplication' => 'PhabricatorApplication', + 'PhabricatorEdgeChangeRecord' => 'Phobject', 'PhabricatorEdgeConfig' => 'PhabricatorEdgeConstants', 'PhabricatorEdgeConstants' => 'Phobject', 'PhabricatorEdgeCycleException' => 'Exception', diff --git a/src/applications/differential/storage/DifferentialTransaction.php b/src/applications/differential/storage/DifferentialTransaction.php index 667341cef4..f7dcf29860 100644 --- a/src/applications/differential/storage/DifferentialTransaction.php +++ b/src/applications/differential/storage/DifferentialTransaction.php @@ -87,19 +87,6 @@ final class DifferentialTransaction } break; - case PhabricatorTransactions::TYPE_EDGE: - $add = array_diff_key($new, $old); - $rem = array_diff_key($old, $new); - - // Hide metadata-only edge transactions. These correspond to users - // accepting or rejecting revisions, but the change is always explicit - // because of the TYPE_ACTION transaction. Rendering these transactions - // just creates clutter. - - if (!$add && !$rem) { - return true; - } - break; case DifferentialRevisionRequestReviewTransaction::TRANSACTIONTYPE: // Don't hide the initial "X requested review: ..." transaction from // mail or feed even when it occurs during creation. We need this diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php index d49c3bf675..97b57009ec 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php @@ -302,8 +302,8 @@ abstract class PhabricatorApplicationTransaction $phids[] = $new; break; case PhabricatorTransactions::TYPE_EDGE: - $phids[] = ipull($old, 'dst'); - $phids[] = ipull($new, 'dst'); + $record = PhabricatorEdgeChangeRecord::newFromTransaction($this); + $phids[] = $record->getChangedPHIDs(); break; case PhabricatorTransactions::TYPE_COLUMNS: foreach ($new as $move) { @@ -632,9 +632,8 @@ abstract class PhabricatorApplicationTransaction return true; break; case PhabricatorObjectMentionedByObjectEdgeType::EDGECONST: - $new = ipull($this->getNewValue(), 'dst'); - $old = ipull($this->getOldValue(), 'dst'); - $add = array_diff($new, $old); + $record = PhabricatorEdgeChangeRecord::newFromTransaction($this); + $add = $record->getAddedPHIDs(); $add_value = reset($add); $add_handle = $this->getHandle($add_value); if ($add_handle->getPolicyFiltered()) { @@ -933,10 +932,10 @@ abstract class PhabricatorApplicationTransaction } break; case PhabricatorTransactions::TYPE_EDGE: - $new = ipull($new, 'dst'); - $old = ipull($old, 'dst'); - $add = array_diff($new, $old); - $rem = array_diff($old, $new); + $record = PhabricatorEdgeChangeRecord::newFromTransaction($this); + $add = $record->getAddedPHIDs(); + $rem = $record->getRemovedPHIDs(); + $type = $this->getMetadata('edge:type'); $type = head($type); @@ -1172,10 +1171,10 @@ abstract class PhabricatorApplicationTransaction $this->renderHandleLink($new)); } case PhabricatorTransactions::TYPE_EDGE: - $new = ipull($new, 'dst'); - $old = ipull($old, 'dst'); - $add = array_diff($new, $old); - $rem = array_diff($old, $new); + $record = PhabricatorEdgeChangeRecord::newFromTransaction($this); + $add = $record->getAddedPHIDs(); + $rem = $record->getRemovedPHIDs(); + $type = $this->getMetadata('edge:type'); $type = head($type); diff --git a/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php b/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php new file mode 100644 index 0000000000..6e0cb1d3e0 --- /dev/null +++ b/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php @@ -0,0 +1,68 @@ +xaction = $xaction; + return $record; + } + + public function getChangedPHIDs() { + $add = $this->getAddedPHIDs(); + $rem = $this->getRemovedPHIDs(); + + $add = array_fuse($add); + $rem = array_fuse($rem); + + return array_keys($add + $rem); + } + + public function getAddedPHIDs() { + $old = $this->getOldDestinationPHIDs(); + $new = $this->getNewDestinationPHIDs(); + + $old = array_fuse($old); + $new = array_fuse($new); + + $add = array_diff_key($new, $old); + return array_keys($add); + } + + public function getRemovedPHIDs() { + $old = $this->getOldDestinationPHIDs(); + $new = $this->getNewDestinationPHIDs(); + + $old = array_fuse($old); + $new = array_fuse($new); + + $rem = array_diff_key($old, $new); + return array_keys($rem); + } + + private function getOldDestinationPHIDs() { + if ($this->xaction) { + $old = $this->xaction->getOldValue(); + return ipull($old, 'dst'); + } + + throw new Exception( + pht('Edge change record is not configured with any change data.')); + } + + private function getNewDestinationPHIDs() { + if ($this->xaction) { + $new = $this->xaction->getNewValue(); + return ipull($new, 'dst'); + } + + throw new Exception( + pht('Edge change record is not configured with any change data.')); + } + + +} From e5639a8ed90fad70cc3e61cf01c2c9cfc185ac66 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 27 Jan 2018 05:54:21 -0800 Subject: [PATCH 02/89] Write edge transactions in a more compact way Summary: Depends on D18946. Ref T13051. Begins writing edge transactions as just a list of changed PHIDs. Test Plan: Added, edited, and removed projects. Reviewed transaction record and database. Saw no user-facing changes but a far more compact database representation. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13051 Differential Revision: https://secure.phabricator.com/D18947 --- ...habricatorApplicationTransactionEditor.php | 26 +++++++++++++++++- .../util/PhabricatorEdgeChangeRecord.php | 27 +++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 7dbd41f1e4..155592fc4e 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -999,7 +999,31 @@ abstract class PhabricatorApplicationTransactionEditor $xaction->setPHID($xaction->generatePHID()); $comment_editor->applyEdit($xaction, $xaction->getComment()); } else { - $xaction->save(); + + // TODO: This is a transitional hack to let us migrate edge + // transactions to a more efficient storage format. For now, we're + // going to write a new slim format to the database but keep the old + // bulky format on the objects so we don't have to upgrade all the + // edit logic to the new format yet. See T13051. + + $edge_type = PhabricatorTransactions::TYPE_EDGE; + if ($xaction->getTransactionType() == $edge_type) { + $bulky_old = $xaction->getOldValue(); + $bulky_new = $xaction->getNewValue(); + + $record = PhabricatorEdgeChangeRecord::newFromTransaction($xaction); + $slim_old = $record->getModernOldEdgeTransactionData(); + $slim_new = $record->getModernNewEdgeTransactionData(); + + $xaction->setOldValue($slim_old); + $xaction->setNewValue($slim_new); + $xaction->save(); + + $xaction->setOldValue($bulky_old); + $xaction->setNewValue($bulky_new); + } else { + $xaction->save(); + } } } diff --git a/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php b/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php index 6e0cb1d3e0..38557dc09f 100644 --- a/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php +++ b/src/infrastructure/edges/util/PhabricatorEdgeChangeRecord.php @@ -44,10 +44,18 @@ final class PhabricatorEdgeChangeRecord return array_keys($rem); } + public function getModernOldEdgeTransactionData() { + return $this->getRemovedPHIDs(); + } + + public function getModernNewEdgeTransactionData() { + return $this->getAddedPHIDs(); + } + private function getOldDestinationPHIDs() { if ($this->xaction) { $old = $this->xaction->getOldValue(); - return ipull($old, 'dst'); + return $this->getPHIDsFromTransactionValue($old); } throw new Exception( @@ -57,12 +65,27 @@ final class PhabricatorEdgeChangeRecord private function getNewDestinationPHIDs() { if ($this->xaction) { $new = $this->xaction->getNewValue(); - return ipull($new, 'dst'); + return $this->getPHIDsFromTransactionValue($new); } throw new Exception( pht('Edge change record is not configured with any change data.')); } + private function getPHIDsFromTransactionValue($value) { + if (!$value) { + return array(); + } + + // If the list items are arrays, this is an older-style map of + // dictionaries. + $head = head($value); + if (is_array($head)) { + return ipull($value, 'dst'); + } + + // If the list items are not arrays, this is a newer-style list of PHIDs. + return $value; + } } From 6d2d1d3a9770adea6f165a82d42ab9ed613b2876 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 27 Jan 2018 06:12:03 -0800 Subject: [PATCH 03/89] Add `bin/garbage compact-edges` to compact edges into the new format Summary: Depends on D18947. Ref T13051. This goes through transaction tables and compacts the edge storage into the slim format. I put this on `bin/garbage` instead of `bin/storage` because `bin/storage` has a lot of weird stuff about how it manages databases so that it can run before configuration (e.g., all the `--user`, `--password` type flags for configuring DB connections). Test Plan: Loaded an object with a bunch of transactions. Ran migration. Spot checked table for sanity. Loaded another copy of the object in the web UI, compared the two pages, saw no user-visible changes. Here's a concrete example of the migration effect -- old row: ``` *************************** 44. row *************************** id: 757 phid: PHID-XACT-PSTE-5gnaaway2vnyen5 authorPHID: PHID-USER-cvfydnwadpdj7vdon36z objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7 viewPolicy: public editPolicy: PHID-USER-cvfydnwadpdj7vdon36z commentPHID: NULL commentVersion: 0 transactionType: core:edge oldValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]}} newValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]},"PHID-PROJ-tbowhnwinujwhb346q36":{"dst":"PHID-PROJ-tbowhnwinujwhb346q36","type":41,"data":[]},"PHID-PROJ-izrto7uflimduo6uw2tp":{"dst":"PHID-PROJ-izrto7uflimduo6uw2tp","type":41,"data":[]}} contentSource: {"source":"web","params":[]} metadata: {"edge:type":41} dateCreated: 1450197571 dateModified: 1450197571 ``` New row: ``` *************************** 44. row *************************** id: 757 phid: PHID-XACT-PSTE-5gnaaway2vnyen5 authorPHID: PHID-USER-cvfydnwadpdj7vdon36z objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7 viewPolicy: public editPolicy: PHID-USER-cvfydnwadpdj7vdon36z commentPHID: NULL commentVersion: 0 transactionType: core:edge oldValue: [] newValue: ["PHID-PROJ-tbowhnwinujwhb346q36","PHID-PROJ-izrto7uflimduo6uw2tp"] contentSource: {"source":"web","params":[]} metadata: {"edge:type":41} dateCreated: 1450197571 dateModified: 1450197571 ``` Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13051 Differential Revision: https://secure.phabricator.com/D18948 --- src/__phutil_library_map__.php | 2 + ...ollectorManagementCompactEdgesWorkflow.php | 103 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c40f79b052..eaaa4d9e66 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3017,6 +3017,7 @@ phutil_register_library_map(array( 'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php', 'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php', 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php', + 'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php', 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php', 'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php', 'PhabricatorGeneralCachePurger' => 'applications/cache/purger/PhabricatorGeneralCachePurger.php', @@ -8484,6 +8485,7 @@ phutil_register_library_map(array( 'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorGarbageCollector' => 'Phobject', 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', + 'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorGeneralCachePurger' => 'PhabricatorCachePurger', diff --git a/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php new file mode 100644 index 0000000000..c47d159850 --- /dev/null +++ b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php @@ -0,0 +1,103 @@ +setName('compact-edges') + ->setExamples('**compact-edges**') + ->setSynopsis( + pht( + 'Rebuild old edge transactions storage to use a more compact '. + 'format.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $tables = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorApplicationTransaction') + ->execute(); + + foreach ($tables as $table) { + $this->compactEdges($table); + } + + return 0; + } + + private function compactEdges(PhabricatorApplicationTransaction $table) { + $conn = $table->establishConnection('w'); + $class = get_class($table); + + echo tsprintf( + "%s\n", + pht( + 'Rebuilding transactions for "%s"...', + $class)); + + $cursor = 0; + $updated = 0; + while (true) { + $rows = $table->loadAllWhere( + 'transactionType = %s + AND id > %d + AND (oldValue LIKE %> OR newValue LIKE %>) + ORDER BY id ASC LIMIT 100', + PhabricatorTransactions::TYPE_EDGE, + $cursor, + // We're looking for transactions with JSON objects in their value + // fields: the new style transactions have JSON lists instead and + // start with "[" rather than "{". + '{', + '{'); + + if (!$rows) { + break; + } + + foreach ($rows as $row) { + $id = $row->getID(); + + $old = $row->getOldValue(); + $new = $row->getNewValue(); + + if (!is_array($old) || !is_array($new)) { + echo tsprintf( + "%s\n", + pht( + 'Transaction %s (of type %s) has unexpected data, skipping.', + $id, + $class)); + } + + $record = PhabricatorEdgeChangeRecord::newFromTransaction($row); + + $old_data = $record->getModernOldEdgeTransactionData(); + $old_json = phutil_json_encode($old_data); + + $new_data = $record->getModernNewEdgeTransactionData(); + $new_json = phutil_json_encode($new_data); + + queryfx( + $conn, + 'UPDATE %T SET oldValue = %s, newValue = %s WHERE id = %d', + $table->getTableName(), + $old_json, + $new_json, + $id); + + $updated++; + + $cursor = $row->getID(); + } + } + + echo tsprintf( + "%s\n", + pht( + 'Done, compacted %s edge transactions.', + new PhutilNumber($updated))); + } + +} From 98402b885b62b945499c12efc4e2bb6dc1e59d49 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 27 Jan 2018 07:30:26 -0800 Subject: [PATCH 04/89] Add a bit of test coverage for bulky vs compact edge data representations Summary: Depends on D18948. Ref T13051. The actual logic ended up so simple that this doesn't really feel terribly valuable, but maybe it'll catch something later on. Test Plan: Ran test. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13051 Differential Revision: https://secure.phabricator.com/D18949 --- src/__phutil_library_map__.php | 2 + .../PhabricatorEdgeChangeRecordTestCase.php | 158 ++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 src/infrastructure/edges/__tests__/PhabricatorEdgeChangeRecordTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index eaaa4d9e66..1ad3565809 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2748,6 +2748,7 @@ phutil_register_library_map(array( 'PhabricatorDraftInterface' => 'applications/transactions/draft/PhabricatorDraftInterface.php', 'PhabricatorDrydockApplication' => 'applications/drydock/application/PhabricatorDrydockApplication.php', 'PhabricatorEdgeChangeRecord' => 'infrastructure/edges/util/PhabricatorEdgeChangeRecord.php', + 'PhabricatorEdgeChangeRecordTestCase' => 'infrastructure/edges/__tests__/PhabricatorEdgeChangeRecordTestCase.php', 'PhabricatorEdgeConfig' => 'infrastructure/edges/constants/PhabricatorEdgeConfig.php', 'PhabricatorEdgeConstants' => 'infrastructure/edges/constants/PhabricatorEdgeConstants.php', 'PhabricatorEdgeCycleException' => 'infrastructure/edges/exception/PhabricatorEdgeCycleException.php', @@ -8173,6 +8174,7 @@ phutil_register_library_map(array( 'PhabricatorDraftEngine' => 'Phobject', 'PhabricatorDrydockApplication' => 'PhabricatorApplication', 'PhabricatorEdgeChangeRecord' => 'Phobject', + 'PhabricatorEdgeChangeRecordTestCase' => 'PhabricatorTestCase', 'PhabricatorEdgeConfig' => 'PhabricatorEdgeConstants', 'PhabricatorEdgeConstants' => 'Phobject', 'PhabricatorEdgeCycleException' => 'Exception', diff --git a/src/infrastructure/edges/__tests__/PhabricatorEdgeChangeRecordTestCase.php b/src/infrastructure/edges/__tests__/PhabricatorEdgeChangeRecordTestCase.php new file mode 100644 index 0000000000..cf46c00d4d --- /dev/null +++ b/src/infrastructure/edges/__tests__/PhabricatorEdgeChangeRecordTestCase.php @@ -0,0 +1,158 @@ +setOldValue($old_bulky); + $bulky_xaction->setNewValue($new_bulky); + + $slim_xaction = new ManiphestTransaction(); + $slim_xaction->setOldValue($old_slim); + $slim_xaction->setNewValue($new_slim); + + $bulky_record = PhabricatorEdgeChangeRecord::newFromTransaction( + $bulky_xaction); + + $slim_record = PhabricatorEdgeChangeRecord::newFromTransaction( + $slim_xaction); + + $this->assertEqual( + $bulky_record->getAddedPHIDs(), + $slim_record->getAddedPHIDs()); + + $this->assertEqual( + $bulky_record->getRemovedPHIDs(), + $slim_record->getRemovedPHIDs()); + } + +} From d8f51dff6ec3d374e13a462e59c63b194570ded7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 28 Jan 2018 17:49:30 -0800 Subject: [PATCH 05/89] Use the configured viewer more consistently in the Herald commit adapter Summary: See PHI276. Ref T13048. The fix in D18933 got one callsite, but missed the one in the `callConduit()` method, so the issue isn't fully fixed in production. Convert this adapter to use a real viewer (if one is available) more thoroughly. Test Plan: Ran rules in test console, saw field values. Will test in production again. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13048 Differential Revision: https://secure.phabricator.com/D18950 --- .../diffusion/herald/HeraldCommitAdapter.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/applications/diffusion/herald/HeraldCommitAdapter.php b/src/applications/diffusion/herald/HeraldCommitAdapter.php index e0e15352b6..fc3c8c4f5b 100644 --- a/src/applications/diffusion/herald/HeraldCommitAdapter.php +++ b/src/applications/diffusion/herald/HeraldCommitAdapter.php @@ -135,13 +135,16 @@ final class HeraldCommitAdapter } public function loadAffectedPaths() { + $viewer = $this->getViewer(); + if ($this->affectedPaths === null) { $result = PhabricatorOwnerPathQuery::loadAffectedPaths( $this->getRepository(), $this->commit, - PhabricatorUser::getOmnipotentUser()); + $viewer); $this->affectedPaths = $result; } + return $this->affectedPaths; } @@ -172,6 +175,8 @@ final class HeraldCommitAdapter } public function loadDifferentialRevision() { + $viewer = $this->getViewer(); + if ($this->affectedRevision === null) { $this->affectedRevision = false; @@ -189,7 +194,7 @@ final class HeraldCommitAdapter $revision = id(new DifferentialRevisionQuery()) ->withIDs(array($revision_id)) - ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->setViewer($viewer) ->needReviewers(true) ->executeOne(); if ($revision) { @@ -197,6 +202,7 @@ final class HeraldCommitAdapter } } } + return $this->affectedRevision; } @@ -323,7 +329,7 @@ final class HeraldCommitAdapter } private function callConduit($method, array $params) { - $viewer = PhabricatorUser::getOmnipotentUser(); + $viewer = $this->getViewer(); $drequest = DiffusionRequest::newFromDictionary( array( From 40e9806e3c88500a67d382a5ec07594325f1b000 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 12:54:04 -0800 Subject: [PATCH 06/89] Remove the caret dropdown from transaction lists when no actions are available Summary: See PHI325. When a transaction group in Differential (or Pholio) only has an inline comment, it renders with a "V" caret but no actual dropdown menu. This caret renders in a "disabled" color, but the color is "kinda grey". The "active" color is "kinda grey with a dab of blue". Here's what they look like today: {F5401581} Just remove it. Test Plan: Viewed one of these, no longer saw the inactive caret. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18963 --- resources/celerity/map.php | 6 +++--- src/view/phui/PHUITimelineEventView.php | 8 +++----- webroot/rsrc/css/phui/phui-timeline-view.css | 4 ---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index fece1e10ee..661ac58c15 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -9,7 +9,7 @@ return array( 'names' => array( 'conpherence.pkg.css' => 'e68cf1fa', 'conpherence.pkg.js' => '15191c65', - 'core.pkg.css' => '075f9867', + 'core.pkg.css' => '51debec3', 'core.pkg.js' => '4c79d74f', 'darkconsole.pkg.js' => '1f9a31bc', 'differential.pkg.css' => '45951e9e', @@ -176,7 +176,7 @@ return array( 'rsrc/css/phui/phui-spacing.css' => '042804d6', 'rsrc/css/phui/phui-status.css' => 'd5263e49', 'rsrc/css/phui/phui-tag-view.css' => 'b4719c50', - 'rsrc/css/phui/phui-timeline-view.css' => 'e2ef62b1', + 'rsrc/css/phui/phui-timeline-view.css' => '6ddf8126', 'rsrc/css/phui/phui-two-column-view.css' => '44ec4951', 'rsrc/css/phui/workboards/phui-workboard-color.css' => '783cdff5', 'rsrc/css/phui/workboards/phui-workboard.css' => '3bc85455', @@ -873,7 +873,7 @@ return array( 'phui-status-list-view-css' => 'd5263e49', 'phui-tag-view-css' => 'b4719c50', 'phui-theme-css' => '9f261c6b', - 'phui-timeline-view-css' => 'e2ef62b1', + 'phui-timeline-view-css' => '6ddf8126', 'phui-two-column-view-css' => '44ec4951', 'phui-workboard-color-css' => '783cdff5', 'phui-workboard-view-css' => '3bc85455', diff --git a/src/view/phui/PHUITimelineEventView.php b/src/view/phui/PHUITimelineEventView.php index 161dd0c944..e64b9b06b2 100644 --- a/src/view/phui/PHUITimelineEventView.php +++ b/src/view/phui/PHUITimelineEventView.php @@ -301,18 +301,14 @@ final class PHUITimelineEventView extends AphrontView { $menu = null; $items = array(); - $has_menu = false; if (!$this->getIsPreview() && !$this->getHideCommentOptions()) { foreach ($this->getEventGroup() as $event) { $items[] = $event->getMenuItems($this->anchor); - if ($event->hasChildren()) { - $has_menu = true; - } } $items = array_mergev($items); } - if ($items || $has_menu) { + if ($items) { $icon = id(new PHUIIconView()) ->setIcon('fa-caret-down'); $aural = javelin_tag( @@ -351,6 +347,8 @@ final class PHUITimelineEventView extends AphrontView { )); $has_menu = true; + } else { + $has_menu = false; } // Render "extra" information (timestamp, etc). diff --git a/webroot/rsrc/css/phui/phui-timeline-view.css b/webroot/rsrc/css/phui/phui-timeline-view.css index b0ae9c0044..6fae6802be 100644 --- a/webroot/rsrc/css/phui/phui-timeline-view.css +++ b/webroot/rsrc/css/phui/phui-timeline-view.css @@ -390,10 +390,6 @@ outline: none; } -.phui-timeline-menu .phui-icon-view { - color: {$lightgreytext}; -} - a.phui-timeline-menu .phui-icon-view { color: {$bluetext}; } From 1db281bcd1bbe141817fe6bc14a9d5dae3e612c5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 13:16:46 -0800 Subject: [PATCH 07/89] Fix a possible `count(null)` in PHUIInfoView Summary: See . PHP7.2 raises a warning about `count(scalar)` (GREAT!) and we have one here if the caller doesn't `setErrors(...)`. Test Plan: Sanity-checked usage of `$this->errors`. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18964 --- src/view/phui/PHUIInfoView.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/view/phui/PHUIInfoView.php b/src/view/phui/PHUIInfoView.php index 161e49108b..69d0549299 100644 --- a/src/view/phui/PHUIInfoView.php +++ b/src/view/phui/PHUIInfoView.php @@ -10,7 +10,7 @@ final class PHUIInfoView extends AphrontTagView { const SEVERITY_PLAIN = 'plain'; private $title; - private $errors; + private $errors = array(); private $severity = null; private $id; private $buttons = array(); From 213eb8e93de567fb4577fa8688873b69f1b01593 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 28 Jan 2018 18:24:42 -0800 Subject: [PATCH 08/89] Define common ID and PHID export fields in SearchEngine Summary: Ref T13049. All exportable objects should always have these fields, so make them builtins. This also sets things up for extensions (like custom fields). Test Plan: Exported user data, got the same export as before. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18951 --- .../query/DiffusionPullLogSearchEngine.php | 10 +-- .../query/PhabricatorPeopleSearchEngine.php | 10 +-- ...PhabricatorApplicationSearchController.php | 11 ---- .../PhabricatorApplicationSearchEngine.php | 65 ++++++++++++++++++- 4 files changed, 66 insertions(+), 30 deletions(-) diff --git a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php index 8d6102d4eb..fc7cee52ce 100644 --- a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php +++ b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php @@ -49,12 +49,6 @@ final class DiffusionPullLogSearchEngine protected function newExportFields() { return array( - id(new PhabricatorIDExportField()) - ->setKey('id') - ->setLabel(pht('ID')), - id(new PhabricatorPHIDExportField()) - ->setKey('phid') - ->setLabel(pht('PHID')), id(new PhabricatorPHIDExportField()) ->setKey('repositoryPHID') ->setLabel(pht('Repository PHID')), @@ -82,7 +76,7 @@ final class DiffusionPullLogSearchEngine ); } - public function newExport(array $events) { + protected function newExportData(array $events) { $viewer = $this->requireViewer(); $phids = array(); @@ -112,8 +106,6 @@ final class DiffusionPullLogSearchEngine } $export[] = array( - 'id' => $event->getID(), - 'phid' => $event->getPHID(), 'repositoryPHID' => $repository_phid, 'repository' => $repository_name, 'pullerPHID' => $puller_phid, diff --git a/src/applications/people/query/PhabricatorPeopleSearchEngine.php b/src/applications/people/query/PhabricatorPeopleSearchEngine.php index db2256a8b8..e0e1b5070e 100644 --- a/src/applications/people/query/PhabricatorPeopleSearchEngine.php +++ b/src/applications/people/query/PhabricatorPeopleSearchEngine.php @@ -322,12 +322,6 @@ final class PhabricatorPeopleSearchEngine protected function newExportFields() { return array( - id(new PhabricatorIDExportField()) - ->setKey('id') - ->setLabel(pht('ID')), - id(new PhabricatorPHIDExportField()) - ->setKey('phid') - ->setLabel(pht('PHID')), id(new PhabricatorStringExportField()) ->setKey('username') ->setLabel(pht('Username')), @@ -340,14 +334,12 @@ final class PhabricatorPeopleSearchEngine ); } - public function newExport(array $users) { + protected function newExportData(array $users) { $viewer = $this->requireViewer(); $export = array(); foreach ($users as $user) { $export[] = array( - 'id' => $user->getID(), - 'phid' => $user->getPHID(), 'username' => $user->getUsername(), 'realName' => $user->getRealName(), 'created' => $user->getDateCreated(), diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index d3b05d1e3e..037a626f26 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -449,18 +449,7 @@ final class PhabricatorApplicationSearchController $format->setViewer($viewer); $export_data = $engine->newExport($objects); - - if (count($export_data) !== count($objects)) { - throw new Exception( - pht( - 'Search engine exported the wrong number of objects, expected '. - '%s but got %s.', - phutil_count($objects), - phutil_count($export_data))); - } - $objects = array_values($objects); - $export_data = array_values($export_data); $field_list = $engine->newExportFieldList(); $field_list = mpull($field_list, null, 'getKey'); diff --git a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php index 33efd890bb..e77c06a62b 100644 --- a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php +++ b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php @@ -1455,11 +1455,74 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { } final public function newExportFieldList() { - return $this->newExportFields(); + $builtin_fields = array( + id(new PhabricatorIDExportField()) + ->setKey('id') + ->setLabel(pht('ID')), + id(new PhabricatorPHIDExportField()) + ->setKey('phid') + ->setLabel(pht('PHID')), + ); + + $fields = mpull($builtin_fields, null, 'getKey'); + + $export_fields = $this->newExportFields(); + foreach ($export_fields as $export_field) { + $key = $export_field->getKey(); + + if (isset($fields[$key])) { + throw new Exception( + pht( + 'Search engine ("%s") defines an export field with a key ("%s") '. + 'that collides with another field. Each field must have a '. + 'unique key.', + get_class($this), + $key)); + } + + $fields[$key] = $export_field; + } + + return $fields; + } + + final public function newExport(array $objects) { + $objects = array_values($objects); + $n = count($objects); + + $maps = array(); + foreach ($objects as $object) { + $maps[] = array( + 'id' => $object->getID(), + 'phid' => $object->getPHID(), + ); + } + + $export_data = $this->newExportData($objects); + $export_data = array_values($export_data); + if (count($export_data) !== count($objects)) { + throw new Exception( + pht( + 'Search engine ("%s") exported the wrong number of objects, '. + 'expected %s but got %s.', + get_class($this), + phutil_count($objects), + phutil_count($export_data))); + } + + for ($ii = 0; $ii < $n; $ii++) { + $maps[$ii] += $export_data[$ii]; + } + + return $maps; } protected function newExportFields() { return array(); } + protected function newExportData(array $objects) { + throw new PhutilMethodNotImplementedException(); + } + } From 0de6210808d7de78b0af0bea4e45f8537435be85 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 28 Jan 2018 18:32:22 -0800 Subject: [PATCH 09/89] Give data exporters a header row Summary: Depends on D18951. Ref T13049. When we export to CSV or plain text, add a header row in the first line of the file to explain what each column means. This often isn't obvious with PHIDs, etc. JSON has keys and is essentially self-labeling, so don't do anything special. Test Plan: Exported CSV and text, saw new headers. Exported JSON, no changes. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18952 --- .../PhabricatorApplicationSearchController.php | 1 + .../export/PhabricatorCSVExportFormat.php | 17 +++++++++++++++-- .../export/PhabricatorExportFormat.php | 4 ++++ .../export/PhabricatorTextExportFormat.php | 18 +++++++++++++++--- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index 037a626f26..0402102e2b 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -454,6 +454,7 @@ final class PhabricatorApplicationSearchController $field_list = $engine->newExportFieldList(); $field_list = mpull($field_list, null, 'getKey'); + $format->addHeaders($field_list); for ($ii = 0; $ii < count($objects); $ii++) { $format->addObject($objects[$ii], $field_list, $export_data[$ii]); } diff --git a/src/infrastructure/export/PhabricatorCSVExportFormat.php b/src/infrastructure/export/PhabricatorCSVExportFormat.php index d9662467ec..67f0a4963a 100644 --- a/src/infrastructure/export/PhabricatorCSVExportFormat.php +++ b/src/infrastructure/export/PhabricatorCSVExportFormat.php @@ -23,21 +23,34 @@ final class PhabricatorCSVExportFormat return 'text/csv'; } + public function addHeaders(array $fields) { + $headers = mpull($fields, 'getLabel'); + $this->addRow($headers); + } + public function addObject($object, array $fields, array $map) { $values = array(); foreach ($fields as $key => $field) { $value = $map[$key]; $value = $field->getTextValue($value); + $values[] = $value; + } + $this->addRow($values); + } + + private function addRow(array $values) { + $row = array(); + foreach ($values as $value) { if (preg_match('/\s|,|\"/', $value)) { $value = str_replace('"', '""', $value); $value = '"'.$value.'"'; } - $values[] = $value; + $row[] = $value; } - $this->rows[] = implode(',', $values); + $this->rows[] = implode(',', $row); } public function newFileData() { diff --git a/src/infrastructure/export/PhabricatorExportFormat.php b/src/infrastructure/export/PhabricatorExportFormat.php index a1da4e90d8..9a8e035c58 100644 --- a/src/infrastructure/export/PhabricatorExportFormat.php +++ b/src/infrastructure/export/PhabricatorExportFormat.php @@ -22,6 +22,10 @@ abstract class PhabricatorExportFormat abstract public function getMIMEContentType(); abstract public function getFileExtension(); + public function addHeaders(array $fields) { + return; + } + abstract public function addObject($object, array $fields, array $map); abstract public function newFileData(); diff --git a/src/infrastructure/export/PhabricatorTextExportFormat.php b/src/infrastructure/export/PhabricatorTextExportFormat.php index ec308f2eb5..d51e199f91 100644 --- a/src/infrastructure/export/PhabricatorTextExportFormat.php +++ b/src/infrastructure/export/PhabricatorTextExportFormat.php @@ -23,17 +23,29 @@ final class PhabricatorTextExportFormat return 'text/plain'; } + public function addHeaders(array $fields) { + $headers = mpull($fields, 'getLabel'); + $this->addRow($headers); + } + public function addObject($object, array $fields, array $map) { $values = array(); foreach ($fields as $key => $field) { $value = $map[$key]; $value = $field->getTextValue($value); - $value = addcslashes($value, "\0..\37\\\177..\377"); - $values[] = $value; } - $this->rows[] = implode("\t", $values); + $this->addRow($values); + } + + private function addRow(array $values) { + $row = array(); + foreach ($values as $value) { + $row[] = addcslashes($value, "\0..\37\\\177..\377"); + } + + $this->rows[] = implode("\t", $row); } public function newFileData() { From 8b8a3142b3102fdf8199ee549ff245d90483f50b Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 28 Jan 2018 18:47:42 -0800 Subject: [PATCH 10/89] Support export of data in files larger than 8MB Summary: Depends on D18952. Ref T13049. For files larger than 8MB, we need to engage the chunk storage engine. `PhabricatorFile::newFromFileData()` always writes a single chunk, and can't handle files larger than the mandatory chunk threshold (8MB). Use `IteratorUploadSource`, which can, and "stream" the data into it. This should raise the limit from 8MB to 2GB (maximum size of a string in PHP). If we need to go above 2GB we could stream CSV and text pretty easily, and JSON without too much trouble, but Excel might be trickier. Hopefully no one is trying to export 2GB+ datafiles, though. Test Plan: - Changed the JSON exporter to just export 8MB of the letter "q": `return str_repeat('q', 1024 * 1024 * 9);`. - Before change: fatal, "no storage engine can store this file". - After change: export works cleanly. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18953 --- .../files/storage/PhabricatorFile.php | 8 +++-- .../PhabricatorFileUploadSource.php | 30 +++++++++++++++++++ ...PhabricatorApplicationSearchController.php | 23 ++++++++------ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php index d3dc5208d2..2b8624b9c0 100644 --- a/src/applications/files/storage/PhabricatorFile.php +++ b/src/applications/files/storage/PhabricatorFile.php @@ -272,8 +272,12 @@ final class PhabricatorFile extends PhabricatorFileDAO $file->setByteSize($length); // NOTE: Once we receive the first chunk, we'll detect its MIME type and - // update the parent file. This matters for large media files like video. - $file->setMimeType('application/octet-stream'); + // update the parent file if a MIME type hasn't been provided. This matters + // for large media files like video. + $mime_type = idx($params, 'mime-type'); + if (!strlen($mime_type)) { + $file->setMimeType('application/octet-stream'); + } $chunked_hash = idx($params, 'chunkedHash'); diff --git a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php index bf213a417e..87a4486869 100644 --- a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php +++ b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php @@ -6,6 +6,8 @@ abstract class PhabricatorFileUploadSource private $name; private $relativeTTL; private $viewPolicy; + private $mimeType; + private $authorPHID; private $rope; private $data; @@ -51,6 +53,24 @@ abstract class PhabricatorFileUploadSource return $this->byteLimit; } + public function setMIMEType($mime_type) { + $this->mimeType = $mime_type; + return $this; + } + + public function getMIMEType() { + return $this->mimeType; + } + + public function setAuthorPHID($author_phid) { + $this->authorPHID = $author_phid; + return $this; + } + + public function getAuthorPHID() { + return $this->authorPHID; + } + public function uploadFile() { if (!$this->shouldChunkFile()) { return $this->writeSingleFile(); @@ -245,6 +265,16 @@ abstract class PhabricatorFileUploadSource $parameters['ttl.relative'] = $ttl; } + $mime_type = $this->getMimeType(); + if ($mime_type !== null) { + $parameters['mime-type'] = $mime_type; + } + + $author_phid = $this->getAuthorPHID(); + if ($author_phid !== null) { + $parameters['authorPHID'] = $author_phid; + } + return $parameters; } diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index 0402102e2b..e980498c42 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -461,15 +461,20 @@ final class PhabricatorApplicationSearchController $export_result = $format->newFileData(); - $file = PhabricatorFile::newFromFileData( - $export_result, - array( - 'name' => $filename, - 'authorPHID' => $viewer->getPHID(), - 'ttl.relative' => phutil_units('15 minutes in seconds'), - 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, - 'mime-type' => $mime_type, - )); + // We have all the data in one big string and aren't actually + // streaming it, but pretending that we are allows us to actviate + // the chunk engine and store large files. + $iterator = new ArrayIterator(array($export_result)); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName($filename) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType($mime_type) + ->setRelativeTTL(phutil_units('60 minutes in seconds')) + ->setAuthorPHID($viewer->getPHID()) + ->setIterator($iterator); + + $file = $source->uploadFile(); return $this->newDialog() ->setTitle(pht('Download Results')) From a067f64ebb326f65078322aabb0790a7ddb34d66 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 28 Jan 2018 19:30:41 -0800 Subject: [PATCH 11/89] Support export engine extensions and implement an extension for custom fields Summary: Depends on D18953. Ref T13049. Allow applications and infrastructure to supplement exportable fields for objects. Then, implement an extension for custom fields. Only a couple field types (int, string) are supported for now. Test Plan: Added some custom fields to Users, populated them, exported users. Saw custom fields in the export. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18954 --- src/__phutil_library_map__.php | 4 + .../PhabricatorApplicationSearchEngine.php | 58 +++++++++++++ .../field/PhabricatorCustomField.php | 43 ++++++++++ .../PhabricatorStandardCustomField.php | 3 + .../PhabricatorStandardCustomFieldInt.php | 4 + .../PhabricatorStandardCustomFieldText.php | 4 + ...icatorCustomFieldExportEngineExtension.php | 86 +++++++++++++++++++ .../PhabricatorExportEngineExtension.php | 31 +++++++ 8 files changed, 233 insertions(+) create mode 100644 src/infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php create mode 100644 src/infrastructure/export/PhabricatorExportEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 1ad3565809..03ac41fe7f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2583,6 +2583,7 @@ phutil_register_library_map(array( 'PhabricatorCustomFieldEditEngineExtension' => 'infrastructure/customfield/engineextension/PhabricatorCustomFieldEditEngineExtension.php', 'PhabricatorCustomFieldEditField' => 'infrastructure/customfield/editor/PhabricatorCustomFieldEditField.php', 'PhabricatorCustomFieldEditType' => 'infrastructure/customfield/editor/PhabricatorCustomFieldEditType.php', + 'PhabricatorCustomFieldExportEngineExtension' => 'infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php', 'PhabricatorCustomFieldFulltextEngineExtension' => 'infrastructure/customfield/engineextension/PhabricatorCustomFieldFulltextEngineExtension.php', 'PhabricatorCustomFieldHeraldAction' => 'infrastructure/customfield/herald/PhabricatorCustomFieldHeraldAction.php', 'PhabricatorCustomFieldHeraldActionGroup' => 'infrastructure/customfield/herald/PhabricatorCustomFieldHeraldActionGroup.php', @@ -2847,6 +2848,7 @@ phutil_register_library_map(array( 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', + 'PhabricatorExportEngineExtension' => 'infrastructure/export/PhabricatorExportEngineExtension.php', 'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php', 'PhabricatorExportFormat' => 'infrastructure/export/PhabricatorExportFormat.php', 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', @@ -7991,6 +7993,7 @@ phutil_register_library_map(array( 'PhabricatorCustomFieldEditEngineExtension' => 'PhabricatorEditEngineExtension', 'PhabricatorCustomFieldEditField' => 'PhabricatorEditField', 'PhabricatorCustomFieldEditType' => 'PhabricatorEditType', + 'PhabricatorCustomFieldExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorCustomFieldFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', 'PhabricatorCustomFieldHeraldAction' => 'HeraldAction', 'PhabricatorCustomFieldHeraldActionGroup' => 'HeraldActionGroup', @@ -8280,6 +8283,7 @@ phutil_register_library_map(array( 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', + 'PhabricatorExportEngineExtension' => 'Phobject', 'PhabricatorExportField' => 'Phobject', 'PhabricatorExportFormat' => 'Phobject', 'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions', diff --git a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php index e77c06a62b..3de7b9c4b9 100644 --- a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php +++ b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php @@ -1483,6 +1483,26 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { $fields[$key] = $export_field; } + $extensions = $this->newExportExtensions(); + foreach ($extensions as $extension) { + $extension_fields = $extension->newExportFields(); + foreach ($extension_fields as $extension_field) { + $key = $extension_field->getKey(); + + if (isset($fields[$key])) { + throw new Exception( + pht( + 'Export engine extension ("%s") defines an export field with '. + 'a key ("%s") that collides with another field. Each field '. + 'must have a unique key.', + get_class($extension_field), + $key)); + } + + $fields[$key] = $extension_field; + } + } + return $fields; } @@ -1514,6 +1534,25 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { $maps[$ii] += $export_data[$ii]; } + $extensions = $this->newExportExtensions(); + foreach ($extensions as $extension) { + $extension_data = $extension->newExportData($objects); + $extension_data = array_values($extension_data); + if (count($export_data) !== count($objects)) { + throw new Exception( + pht( + 'Export engine extension ("%s") exported the wrong number of '. + 'objects, expected %s but got %s.', + get_class($extension), + phutil_count($objects), + phutil_count($export_data))); + } + + for ($ii = 0; $ii < $n; $ii++) { + $maps[$ii] += $extension_data[$ii]; + } + } + return $maps; } @@ -1525,4 +1564,23 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { throw new PhutilMethodNotImplementedException(); } + private function newExportExtensions() { + $object = $this->newResultObject(); + $viewer = $this->requireViewer(); + + $extensions = PhabricatorExportEngineExtension::getAllExtensions(); + + $supported = array(); + foreach ($extensions as $extension) { + $extension = clone $extension; + $extension->setViewer($viewer); + + if ($extension->supportsObject($object)) { + $supported[] = $extension; + } + } + + return $supported; + } + } diff --git a/src/infrastructure/customfield/field/PhabricatorCustomField.php b/src/infrastructure/customfield/field/PhabricatorCustomField.php index c96f09f369..818bf119ff 100644 --- a/src/infrastructure/customfield/field/PhabricatorCustomField.php +++ b/src/infrastructure/customfield/field/PhabricatorCustomField.php @@ -35,6 +35,7 @@ abstract class PhabricatorCustomField extends Phobject { const ROLE_HERALD = 'herald'; const ROLE_EDITENGINE = 'EditEngine'; const ROLE_HERALDACTION = 'herald.action'; + const ROLE_EXPORT = 'export'; /* -( Building Applications with Custom Fields )--------------------------- */ @@ -299,6 +300,8 @@ abstract class PhabricatorCustomField extends Phobject { case self::ROLE_EDITENGINE: return $this->shouldAppearInEditView() || $this->shouldAppearInEditEngine(); + case self::ROLE_EXPORT: + return $this->shouldAppearInDataExport(); case self::ROLE_DEFAULT: return true; default: @@ -1362,6 +1365,46 @@ abstract class PhabricatorCustomField extends Phobject { } +/* -( Data Export )-------------------------------------------------------- */ + + + public function shouldAppearInDataExport() { + if ($this->proxy) { + return $this->proxy->shouldAppearInDataExport(); + } + + try { + $this->newExportFieldType(); + return true; + } catch (PhabricatorCustomFieldImplementationIncompleteException $ex) { + return false; + } + } + + public function newExportField() { + if ($this->proxy) { + return $this->proxy->newExportField(); + } + + return $this->newExportFieldType() + ->setLabel($this->getFieldName()); + } + + public function newExportData() { + if ($this->proxy) { + return $this->proxy->newExportData(); + } + throw new PhabricatorCustomFieldImplementationIncompleteException($this); + } + + protected function newExportFieldType() { + if ($this->proxy) { + return $this->proxy->newExportFieldType(); + } + throw new PhabricatorCustomFieldImplementationIncompleteException($this); + } + + /* -( Conduit )------------------------------------------------------------ */ diff --git a/src/infrastructure/customfield/standard/PhabricatorStandardCustomField.php b/src/infrastructure/customfield/standard/PhabricatorStandardCustomField.php index f617d3a849..5bd6256b73 100644 --- a/src/infrastructure/customfield/standard/PhabricatorStandardCustomField.php +++ b/src/infrastructure/customfield/standard/PhabricatorStandardCustomField.php @@ -496,5 +496,8 @@ abstract class PhabricatorStandardCustomField return $this->getFieldValue(); } + public function newExportData() { + return $this->getFieldValue(); + } } diff --git a/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldInt.php b/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldInt.php index 4a3e45d757..f06c30d482 100644 --- a/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldInt.php +++ b/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldInt.php @@ -124,4 +124,8 @@ final class PhabricatorStandardCustomFieldInt return new ConduitIntParameterType(); } + protected function newExportFieldType() { + return new PhabricatorIntExportField(); + } + } diff --git a/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldText.php b/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldText.php index 8242c477fd..56164bb7b5 100644 --- a/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldText.php +++ b/src/infrastructure/customfield/standard/PhabricatorStandardCustomFieldText.php @@ -76,4 +76,8 @@ final class PhabricatorStandardCustomFieldText return new ConduitStringParameterType(); } + protected function newExportFieldType() { + return new PhabricatorStringExportField(); + } + } diff --git a/src/infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php b/src/infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php new file mode 100644 index 0000000000..c2a9eddae2 --- /dev/null +++ b/src/infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php @@ -0,0 +1,86 @@ +object = $object; + return ($object instanceof PhabricatorCustomFieldInterface); + } + + public function newExportFields() { + $prototype = $this->object; + + $fields = $this->newCustomFields($prototype); + + $results = array(); + foreach ($fields as $field) { + $field_key = $field->getModernFieldKey(); + + $results[] = $field->newExportField() + ->setKey($field_key); + } + + return $results; + } + + public function newExportData(array $objects) { + $viewer = $this->getViewer(); + + $field_map = array(); + foreach ($objects as $object) { + $object_phid = $object->getPHID(); + + $fields = PhabricatorCustomField::getObjectFields( + $object, + PhabricatorCustomField::ROLE_EXPORT); + + $fields + ->setViewer($viewer) + ->readFieldsFromObject($object); + + $field_map[$object_phid] = $fields; + } + + $all_fields = array(); + foreach ($field_map as $field_list) { + foreach ($field_list->getFields() as $field) { + $all_fields[] = $field; + } + } + + id(new PhabricatorCustomFieldStorageQuery()) + ->addFields($all_fields) + ->execute(); + + $results = array(); + foreach ($objects as $object) { + $object_phid = $object->getPHID(); + $object_fields = $field_map[$object_phid]; + + $map = array(); + foreach ($object_fields->getFields() as $field) { + $key = $field->getModernFieldKey(); + $map[$key] = $field->newExportData(); + } + + $results[] = $map; + } + + return $results; + } + + private function newCustomFields($object) { + $fields = PhabricatorCustomField::getObjectFields( + $object, + PhabricatorCustomField::ROLE_EXPORT); + $fields->setViewer($this->getViewer()); + + return $fields->getFields(); + } + +} diff --git a/src/infrastructure/export/PhabricatorExportEngineExtension.php b/src/infrastructure/export/PhabricatorExportEngineExtension.php new file mode 100644 index 0000000000..01d4471ef2 --- /dev/null +++ b/src/infrastructure/export/PhabricatorExportEngineExtension.php @@ -0,0 +1,31 @@ +getPhobjectClassConstant('EXTENSIONKEY'); + } + + final public function setViewer($viewer) { + $this->viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + abstract public function supportsObject($object); + abstract public function newExportFields(); + abstract public function newExportData(array $objects); + + final public static function getAllExtensions() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getExtensionKey') + ->execute(); + } + +} From 0409279595464924d1dc728a8c6830228d75dc5f Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 06:50:11 -0800 Subject: [PATCH 12/89] Support Excel as a data export format Summary: Depends on D18954. Ref T13049. This brings over the existing Maniphest Excel export pipeline in a generic way. The `ExportField` classes know directly that `PHPExcel` exists, which is a little sketchy, but writing an Excel indirection layer sounds like a lot of work and I don't anticipate us changing Excel backends anytime soon, so trying to abstract this feels YAGNI. This doesn't bring over the install instructions for PHPExcel or the detection of whether or not it exists. I'll bring that over in a future change. Test Plan: Exported users as Excel, opened them up, got a sensible-looking Excel sheet. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18955 --- src/__phutil_library_map__.php | 2 + ...PhabricatorApplicationSearchController.php | 7 +- .../export/PhabricatorEpochExportField.php | 20 +++ .../export/PhabricatorExcelExportFormat.php | 145 ++++++++++++++++++ .../export/PhabricatorExportField.php | 15 ++ .../export/PhabricatorExportFormat.php | 10 ++ .../export/PhabricatorIDExportField.php | 4 + .../export/PhabricatorIntExportField.php | 15 ++ .../export/PhabricatorPHIDExportField.php | 8 +- 9 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 src/infrastructure/export/PhabricatorExcelExportFormat.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 03ac41fe7f..3ef9f4ec3b 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2847,6 +2847,7 @@ phutil_register_library_map(array( 'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', + 'PhabricatorExcelExportFormat' => 'infrastructure/export/PhabricatorExcelExportFormat.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', 'PhabricatorExportEngineExtension' => 'infrastructure/export/PhabricatorExportEngineExtension.php', 'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php', @@ -8282,6 +8283,7 @@ phutil_register_library_map(array( 'PhabricatorEventListener' => 'PhutilEventListener', 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', + 'PhabricatorExcelExportFormat' => 'PhabricatorExportFormat', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', 'PhabricatorExportEngineExtension' => 'Phobject', 'PhabricatorExportField' => 'Phobject', diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index e980498c42..88cfd3c137 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -410,8 +410,10 @@ final class PhabricatorApplicationSearchController if ($named_query) { $filename = $named_query->getQueryName(); + $sheet_title = $named_query->getQueryName(); } else { $filename = $engine->getResultTypeDescription(); + $sheet_title = $engine->getResultTypeDescription(); } $filename = phutil_utf8_strtolower($filename); $filename = PhabricatorFile::normalizeFileName($filename); @@ -445,8 +447,9 @@ final class PhabricatorApplicationSearchController $mime_type = $format->getMIMEContentType(); $filename = $filename.'.'.$extension; - $format = clone $format; - $format->setViewer($viewer); + $format = id(clone $format) + ->setViewer($viewer) + ->setTitle($sheet_title); $export_data = $engine->newExport($objects); $objects = array_values($objects); diff --git a/src/infrastructure/export/PhabricatorEpochExportField.php b/src/infrastructure/export/PhabricatorEpochExportField.php index a19e60b50e..4dffde5aa8 100644 --- a/src/infrastructure/export/PhabricatorEpochExportField.php +++ b/src/infrastructure/export/PhabricatorEpochExportField.php @@ -24,4 +24,24 @@ final class PhabricatorEpochExportField return (int)$value; } + public function getPHPExcelValue($value) { + $epoch = $this->getNaturalValue($value); + + $seconds_per_day = phutil_units('1 day in seconds'); + $offset = ($seconds_per_day * 25569); + + return ($epoch + $offset) / $seconds_per_day; + } + + /** + * @phutil-external-symbol class PHPExcel_Style_NumberFormat + */ + public function formatPHPExcelCell($cell, $style) { + $code = PHPExcel_Style_NumberFormat::FORMAT_DATE_YYYYMMDD2; + + $style + ->getNumberFormat() + ->setFormatCode($code); + } + } diff --git a/src/infrastructure/export/PhabricatorExcelExportFormat.php b/src/infrastructure/export/PhabricatorExcelExportFormat.php new file mode 100644 index 0000000000..633d98fa53 --- /dev/null +++ b/src/infrastructure/export/PhabricatorExcelExportFormat.php @@ -0,0 +1,145 @@ +getSheet(); + + $header_format = array( + 'font' => array( + 'bold' => true, + ), + ); + + $row = 1; + $col = 0; + foreach ($fields as $field) { + $cell_value = $field->getLabel(); + + $cell_name = $this->getCellName($col, $row); + + $cell = $sheet->setCellValue( + $cell_name, + $cell_value, + $return_cell = true); + + $sheet->getStyle($cell_name)->applyFromArray($header_format); + $cell->setDataType(PHPExcel_Cell_DataType::TYPE_STRING); + + $width = $field->getCharacterWidth(); + if ($width !== null) { + $col_name = $this->getCellName($col); + $sheet->getColumnDimension($col_name) + ->setWidth($width); + } + + $col++; + } + } + + public function addObject($object, array $fields, array $map) { + $sheet = $this->getSheet(); + + $col = 0; + foreach ($fields as $key => $field) { + $cell_value = $map[$key]; + $cell_value = $field->getPHPExcelValue($cell_value); + + $cell_name = $this->getCellName($col, $this->rowCursor); + + $cell = $sheet->setCellValue( + $cell_name, + $cell_value, + $return_cell = true); + + $style = $sheet->getStyle($cell_name); + $field->formatPHPExcelCell($cell, $style); + + $col++; + } + + $this->rowCursor++; + } + + /** + * @phutil-external-symbol class PHPExcel_IOFactory + */ + public function newFileData() { + $workbook = $this->getWorkbook(); + $writer = PHPExcel_IOFactory::createWriter($workbook, 'Excel2007'); + + ob_start(); + $writer->save('php://output'); + $data = ob_get_clean(); + + return $data; + } + + private function getWorkbook() { + if (!$this->workbook) { + $this->workbook = $this->newWorkbook(); + } + return $this->workbook; + } + + /** + * @phutil-external-symbol class PHPExcel + */ + private function newWorkbook() { + include_once 'PHPExcel.php'; + return new PHPExcel(); + } + + private function getSheet() { + if (!$this->sheet) { + $workbook = $this->getWorkbook(); + + $sheet = $workbook->setActiveSheetIndex(0); + $sheet->setTitle($this->getTitle()); + + $this->sheet = $sheet; + + // The row cursor starts on the second row, after the header row. + $this->rowCursor = 2; + } + + return $this->sheet; + } + + private function getCellName($col, $row = null) { + $col_name = chr(ord('A') + $col); + + if ($row === null) { + return $col_name; + } + + return $col_name.$row; + } + +} diff --git a/src/infrastructure/export/PhabricatorExportField.php b/src/infrastructure/export/PhabricatorExportField.php index 3efb7a8b9a..85e21b3e37 100644 --- a/src/infrastructure/export/PhabricatorExportField.php +++ b/src/infrastructure/export/PhabricatorExportField.php @@ -32,4 +32,19 @@ abstract class PhabricatorExportField return $value; } + public function getPHPExcelValue($value) { + return $this->getTextValue($value); + } + + /** + * @phutil-external-symbol class PHPExcel_Cell_DataType + */ + public function formatPHPExcelCell($cell, $style) { + $cell->setDataType(PHPExcel_Cell_DataType::TYPE_STRING); + } + + public function getCharacterWidth() { + return 24; + } + } diff --git a/src/infrastructure/export/PhabricatorExportFormat.php b/src/infrastructure/export/PhabricatorExportFormat.php index 9a8e035c58..7e174f5197 100644 --- a/src/infrastructure/export/PhabricatorExportFormat.php +++ b/src/infrastructure/export/PhabricatorExportFormat.php @@ -4,6 +4,7 @@ abstract class PhabricatorExportFormat extends Phobject { private $viewer; + private $title; final public function getExportFormatKey() { return $this->getPhobjectClassConstant('EXPORTKEY'); @@ -18,6 +19,15 @@ abstract class PhabricatorExportFormat return $this->viewer; } + final public function setTitle($title) { + $this->title = $title; + return $this; + } + + final public function getTitle() { + return $this->title; + } + abstract public function getExportFormatName(); abstract public function getMIMEContentType(); abstract public function getFileExtension(); diff --git a/src/infrastructure/export/PhabricatorIDExportField.php b/src/infrastructure/export/PhabricatorIDExportField.php index 5b29fdb21d..1ef3d53370 100644 --- a/src/infrastructure/export/PhabricatorIDExportField.php +++ b/src/infrastructure/export/PhabricatorIDExportField.php @@ -7,4 +7,8 @@ final class PhabricatorIDExportField return (int)$value; } + public function getCharacterWidth() { + return 12; + } + } diff --git a/src/infrastructure/export/PhabricatorIntExportField.php b/src/infrastructure/export/PhabricatorIntExportField.php index 3363f9b5d5..57f7e0ab29 100644 --- a/src/infrastructure/export/PhabricatorIntExportField.php +++ b/src/infrastructure/export/PhabricatorIntExportField.php @@ -4,7 +4,22 @@ final class PhabricatorIntExportField extends PhabricatorExportField { public function getNaturalValue($value) { + if ($value === null) { + return $value; + } + return (int)$value; } + /** + * @phutil-external-symbol class PHPExcel_Cell_DataType + */ + public function formatPHPExcelCell($cell, $style) { + $cell->setDataType(PHPExcel_Cell_DataType::TYPE_NUMERIC); + } + + public function getCharacterWidth() { + return 8; + } + } diff --git a/src/infrastructure/export/PhabricatorPHIDExportField.php b/src/infrastructure/export/PhabricatorPHIDExportField.php index 7c08ae0226..052c73fbd6 100644 --- a/src/infrastructure/export/PhabricatorPHIDExportField.php +++ b/src/infrastructure/export/PhabricatorPHIDExportField.php @@ -1,4 +1,10 @@ Date: Mon, 29 Jan 2018 07:24:32 -0800 Subject: [PATCH 13/89] Organize the export code into subdirectories Summary: Depends on D18955. Ref T13049. This directory was getting a little cluttered with different kinds of code. Put the formats (csv, json, ...), the field types (int, string, epoch, ...) and the engine-related stuff in subdirectories. Test Plan: wow so aesthetic Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18956 --- src/__phutil_library_map__.php | 26 +++++++++---------- ...icatorCustomFieldExportEngineExtension.php | 0 .../PhabricatorExportEngineExtension.php | 0 .../PhabricatorEpochExportField.php | 0 .../{ => field}/PhabricatorExportField.php | 0 .../{ => field}/PhabricatorIDExportField.php | 0 .../{ => field}/PhabricatorIntExportField.php | 0 .../PhabricatorPHIDExportField.php | 0 .../PhabricatorStringExportField.php | 0 .../PhabricatorCSVExportFormat.php | 0 .../PhabricatorExcelExportFormat.php | 0 .../{ => format}/PhabricatorExportFormat.php | 0 .../PhabricatorJSONExportFormat.php | 0 .../PhabricatorTextExportFormat.php | 0 14 files changed, 13 insertions(+), 13 deletions(-) rename src/infrastructure/export/{ => engine}/PhabricatorCustomFieldExportEngineExtension.php (100%) rename src/infrastructure/export/{ => engine}/PhabricatorExportEngineExtension.php (100%) rename src/infrastructure/export/{ => field}/PhabricatorEpochExportField.php (100%) rename src/infrastructure/export/{ => field}/PhabricatorExportField.php (100%) rename src/infrastructure/export/{ => field}/PhabricatorIDExportField.php (100%) rename src/infrastructure/export/{ => field}/PhabricatorIntExportField.php (100%) rename src/infrastructure/export/{ => field}/PhabricatorPHIDExportField.php (100%) rename src/infrastructure/export/{ => field}/PhabricatorStringExportField.php (100%) rename src/infrastructure/export/{ => format}/PhabricatorCSVExportFormat.php (100%) rename src/infrastructure/export/{ => format}/PhabricatorExcelExportFormat.php (100%) rename src/infrastructure/export/{ => format}/PhabricatorExportFormat.php (100%) rename src/infrastructure/export/{ => format}/PhabricatorJSONExportFormat.php (100%) rename src/infrastructure/export/{ => format}/PhabricatorTextExportFormat.php (100%) diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 3ef9f4ec3b..8ada6932ae 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2232,7 +2232,7 @@ phutil_register_library_map(array( 'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php', 'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php', - 'PhabricatorCSVExportFormat' => 'infrastructure/export/PhabricatorCSVExportFormat.php', + 'PhabricatorCSVExportFormat' => 'infrastructure/export/format/PhabricatorCSVExportFormat.php', 'PhabricatorCacheDAO' => 'applications/cache/storage/PhabricatorCacheDAO.php', 'PhabricatorCacheEngine' => 'applications/system/engine/PhabricatorCacheEngine.php', 'PhabricatorCacheEngineExtension' => 'applications/system/engine/PhabricatorCacheEngineExtension.php', @@ -2583,7 +2583,7 @@ phutil_register_library_map(array( 'PhabricatorCustomFieldEditEngineExtension' => 'infrastructure/customfield/engineextension/PhabricatorCustomFieldEditEngineExtension.php', 'PhabricatorCustomFieldEditField' => 'infrastructure/customfield/editor/PhabricatorCustomFieldEditField.php', 'PhabricatorCustomFieldEditType' => 'infrastructure/customfield/editor/PhabricatorCustomFieldEditType.php', - 'PhabricatorCustomFieldExportEngineExtension' => 'infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php', + 'PhabricatorCustomFieldExportEngineExtension' => 'infrastructure/export/engine/PhabricatorCustomFieldExportEngineExtension.php', 'PhabricatorCustomFieldFulltextEngineExtension' => 'infrastructure/customfield/engineextension/PhabricatorCustomFieldFulltextEngineExtension.php', 'PhabricatorCustomFieldHeraldAction' => 'infrastructure/customfield/herald/PhabricatorCustomFieldHeraldAction.php', 'PhabricatorCustomFieldHeraldActionGroup' => 'infrastructure/customfield/herald/PhabricatorCustomFieldHeraldActionGroup.php', @@ -2841,17 +2841,17 @@ phutil_register_library_map(array( 'PhabricatorEnv' => 'infrastructure/env/PhabricatorEnv.php', 'PhabricatorEnvTestCase' => 'infrastructure/env/__tests__/PhabricatorEnvTestCase.php', 'PhabricatorEpochEditField' => 'applications/transactions/editfield/PhabricatorEpochEditField.php', - 'PhabricatorEpochExportField' => 'infrastructure/export/PhabricatorEpochExportField.php', + 'PhabricatorEpochExportField' => 'infrastructure/export/field/PhabricatorEpochExportField.php', 'PhabricatorEvent' => 'infrastructure/events/PhabricatorEvent.php', 'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php', 'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', - 'PhabricatorExcelExportFormat' => 'infrastructure/export/PhabricatorExcelExportFormat.php', + 'PhabricatorExcelExportFormat' => 'infrastructure/export/format/PhabricatorExcelExportFormat.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', - 'PhabricatorExportEngineExtension' => 'infrastructure/export/PhabricatorExportEngineExtension.php', - 'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php', - 'PhabricatorExportFormat' => 'infrastructure/export/PhabricatorExportFormat.php', + 'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php', + 'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php', + 'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php', 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', 'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php', 'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php', @@ -3072,7 +3072,7 @@ phutil_register_library_map(array( 'PhabricatorHomeProfileMenuItem' => 'applications/home/menuitem/PhabricatorHomeProfileMenuItem.php', 'PhabricatorHovercardEngineExtension' => 'applications/search/engineextension/PhabricatorHovercardEngineExtension.php', 'PhabricatorHovercardEngineExtensionModule' => 'applications/search/engineextension/PhabricatorHovercardEngineExtensionModule.php', - 'PhabricatorIDExportField' => 'infrastructure/export/PhabricatorIDExportField.php', + 'PhabricatorIDExportField' => 'infrastructure/export/field/PhabricatorIDExportField.php', 'PhabricatorIDsSearchEngineExtension' => 'applications/search/engineextension/PhabricatorIDsSearchEngineExtension.php', 'PhabricatorIDsSearchField' => 'applications/search/field/PhabricatorIDsSearchField.php', 'PhabricatorIconDatasource' => 'applications/files/typeahead/PhabricatorIconDatasource.php', @@ -3096,7 +3096,7 @@ phutil_register_library_map(array( 'PhabricatorInlineSummaryView' => 'infrastructure/diff/view/PhabricatorInlineSummaryView.php', 'PhabricatorInstructionsEditField' => 'applications/transactions/editfield/PhabricatorInstructionsEditField.php', 'PhabricatorIntConfigType' => 'applications/config/type/PhabricatorIntConfigType.php', - 'PhabricatorIntExportField' => 'infrastructure/export/PhabricatorIntExportField.php', + 'PhabricatorIntExportField' => 'infrastructure/export/field/PhabricatorIntExportField.php', 'PhabricatorInternalSetting' => 'applications/settings/setting/PhabricatorInternalSetting.php', 'PhabricatorInternationalizationManagementExtractWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php', 'PhabricatorInternationalizationManagementWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementWorkflow.php', @@ -3106,7 +3106,7 @@ phutil_register_library_map(array( 'PhabricatorIteratorFileUploadSource' => 'applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php', 'PhabricatorJIRAAuthProvider' => 'applications/auth/provider/PhabricatorJIRAAuthProvider.php', 'PhabricatorJSONConfigType' => 'applications/config/type/PhabricatorJSONConfigType.php', - 'PhabricatorJSONExportFormat' => 'infrastructure/export/PhabricatorJSONExportFormat.php', + 'PhabricatorJSONExportFormat' => 'infrastructure/export/format/PhabricatorJSONExportFormat.php', 'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php', 'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php', 'PhabricatorJumpNavHandler' => 'applications/search/engine/PhabricatorJumpNavHandler.php', @@ -3426,7 +3426,7 @@ phutil_register_library_map(array( 'PhabricatorPHDConfigOptions' => 'applications/config/option/PhabricatorPHDConfigOptions.php', 'PhabricatorPHID' => 'applications/phid/storage/PhabricatorPHID.php', 'PhabricatorPHIDConstants' => 'applications/phid/PhabricatorPHIDConstants.php', - 'PhabricatorPHIDExportField' => 'infrastructure/export/PhabricatorPHIDExportField.php', + 'PhabricatorPHIDExportField' => 'infrastructure/export/field/PhabricatorPHIDExportField.php', 'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php', 'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php', 'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php', @@ -4192,7 +4192,7 @@ phutil_register_library_map(array( 'PhabricatorStorageSchemaSpec' => 'infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php', 'PhabricatorStorageSetupCheck' => 'applications/config/check/PhabricatorStorageSetupCheck.php', 'PhabricatorStringConfigType' => 'applications/config/type/PhabricatorStringConfigType.php', - 'PhabricatorStringExportField' => 'infrastructure/export/PhabricatorStringExportField.php', + 'PhabricatorStringExportField' => 'infrastructure/export/field/PhabricatorStringExportField.php', 'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php', 'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php', 'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php', @@ -4256,7 +4256,7 @@ phutil_register_library_map(array( 'PhabricatorTextAreaEditField' => 'applications/transactions/editfield/PhabricatorTextAreaEditField.php', 'PhabricatorTextConfigType' => 'applications/config/type/PhabricatorTextConfigType.php', 'PhabricatorTextEditField' => 'applications/transactions/editfield/PhabricatorTextEditField.php', - 'PhabricatorTextExportFormat' => 'infrastructure/export/PhabricatorTextExportFormat.php', + 'PhabricatorTextExportFormat' => 'infrastructure/export/format/PhabricatorTextExportFormat.php', 'PhabricatorTextListConfigType' => 'applications/config/type/PhabricatorTextListConfigType.php', 'PhabricatorTime' => 'infrastructure/time/PhabricatorTime.php', 'PhabricatorTimeFormatSetting' => 'applications/settings/setting/PhabricatorTimeFormatSetting.php', diff --git a/src/infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php b/src/infrastructure/export/engine/PhabricatorCustomFieldExportEngineExtension.php similarity index 100% rename from src/infrastructure/export/PhabricatorCustomFieldExportEngineExtension.php rename to src/infrastructure/export/engine/PhabricatorCustomFieldExportEngineExtension.php diff --git a/src/infrastructure/export/PhabricatorExportEngineExtension.php b/src/infrastructure/export/engine/PhabricatorExportEngineExtension.php similarity index 100% rename from src/infrastructure/export/PhabricatorExportEngineExtension.php rename to src/infrastructure/export/engine/PhabricatorExportEngineExtension.php diff --git a/src/infrastructure/export/PhabricatorEpochExportField.php b/src/infrastructure/export/field/PhabricatorEpochExportField.php similarity index 100% rename from src/infrastructure/export/PhabricatorEpochExportField.php rename to src/infrastructure/export/field/PhabricatorEpochExportField.php diff --git a/src/infrastructure/export/PhabricatorExportField.php b/src/infrastructure/export/field/PhabricatorExportField.php similarity index 100% rename from src/infrastructure/export/PhabricatorExportField.php rename to src/infrastructure/export/field/PhabricatorExportField.php diff --git a/src/infrastructure/export/PhabricatorIDExportField.php b/src/infrastructure/export/field/PhabricatorIDExportField.php similarity index 100% rename from src/infrastructure/export/PhabricatorIDExportField.php rename to src/infrastructure/export/field/PhabricatorIDExportField.php diff --git a/src/infrastructure/export/PhabricatorIntExportField.php b/src/infrastructure/export/field/PhabricatorIntExportField.php similarity index 100% rename from src/infrastructure/export/PhabricatorIntExportField.php rename to src/infrastructure/export/field/PhabricatorIntExportField.php diff --git a/src/infrastructure/export/PhabricatorPHIDExportField.php b/src/infrastructure/export/field/PhabricatorPHIDExportField.php similarity index 100% rename from src/infrastructure/export/PhabricatorPHIDExportField.php rename to src/infrastructure/export/field/PhabricatorPHIDExportField.php diff --git a/src/infrastructure/export/PhabricatorStringExportField.php b/src/infrastructure/export/field/PhabricatorStringExportField.php similarity index 100% rename from src/infrastructure/export/PhabricatorStringExportField.php rename to src/infrastructure/export/field/PhabricatorStringExportField.php diff --git a/src/infrastructure/export/PhabricatorCSVExportFormat.php b/src/infrastructure/export/format/PhabricatorCSVExportFormat.php similarity index 100% rename from src/infrastructure/export/PhabricatorCSVExportFormat.php rename to src/infrastructure/export/format/PhabricatorCSVExportFormat.php diff --git a/src/infrastructure/export/PhabricatorExcelExportFormat.php b/src/infrastructure/export/format/PhabricatorExcelExportFormat.php similarity index 100% rename from src/infrastructure/export/PhabricatorExcelExportFormat.php rename to src/infrastructure/export/format/PhabricatorExcelExportFormat.php diff --git a/src/infrastructure/export/PhabricatorExportFormat.php b/src/infrastructure/export/format/PhabricatorExportFormat.php similarity index 100% rename from src/infrastructure/export/PhabricatorExportFormat.php rename to src/infrastructure/export/format/PhabricatorExportFormat.php diff --git a/src/infrastructure/export/PhabricatorJSONExportFormat.php b/src/infrastructure/export/format/PhabricatorJSONExportFormat.php similarity index 100% rename from src/infrastructure/export/PhabricatorJSONExportFormat.php rename to src/infrastructure/export/format/PhabricatorJSONExportFormat.php diff --git a/src/infrastructure/export/PhabricatorTextExportFormat.php b/src/infrastructure/export/format/PhabricatorTextExportFormat.php similarity index 100% rename from src/infrastructure/export/PhabricatorTextExportFormat.php rename to src/infrastructure/export/format/PhabricatorTextExportFormat.php From 61b8c12970be2280b38a2842313c4ca9a6d76a42 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 07:33:04 -0800 Subject: [PATCH 14/89] Make the data export format selector remember your last setting Summary: Depends on D18956. Ref T13049. Make the "Export Format" selector sticky. This is partly selfish, since it makes testing format changes a bit easier. It also seems like it's probably a good behavior in general: if you export to Excel once, that's probably what you're going to pick next time. Test Plan: Exported to excel. Exported again, got excel as the default option. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18957 --- src/__phutil_library_map__.php | 2 + ...PhabricatorApplicationSearchController.php | 38 +++++++++++++++++++ .../engine/PhabricatorExportFormatSetting.php | 16 ++++++++ 3 files changed, 56 insertions(+) create mode 100644 src/infrastructure/export/engine/PhabricatorExportFormatSetting.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 8ada6932ae..4d4aeda7aa 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2852,6 +2852,7 @@ phutil_register_library_map(array( 'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php', 'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php', 'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php', + 'PhabricatorExportFormatSetting' => 'infrastructure/export/engine/PhabricatorExportFormatSetting.php', 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', 'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php', 'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php', @@ -8288,6 +8289,7 @@ phutil_register_library_map(array( 'PhabricatorExportEngineExtension' => 'Phobject', 'PhabricatorExportField' => 'Phobject', 'PhabricatorExportFormat' => 'Phobject', + 'PhabricatorExportFormatSetting' => 'PhabricatorInternalSetting', 'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorExternalAccount' => array( diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index 88cfd3c137..a5c58aac7f 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -421,6 +421,13 @@ final class PhabricatorApplicationSearchController $formats = PhabricatorExportFormat::getAllEnabledExportFormats(); $format_options = mpull($formats, 'getExportFormatName'); + // Try to default to the format the user used last time. If you just + // exported to Excel, you probably want to export to Excel again. + $format_key = $this->readExportFormatPreference(); + if (!isset($formats[$format_key])) { + $format_key = head_key($format_options); + } + $errors = array(); $e_format = null; @@ -434,6 +441,8 @@ final class PhabricatorApplicationSearchController } if (!$errors) { + $this->writeExportFormatPreference($format_key); + $query = $engine->buildQueryFromSavedQuery($saved_query); // NOTE: We aren't reading the pager from the request. Exports always @@ -497,6 +506,7 @@ final class PhabricatorApplicationSearchController ->setName('format') ->setLabel(pht('Format')) ->setError($e_format) + ->setValue($format_key) ->setOptions($format_options)); return $this->newDialog() @@ -912,4 +922,32 @@ final class PhabricatorApplicationSearchController return true; } + private function readExportFormatPreference() { + $viewer = $this->getViewer(); + $export_key = PhabricatorPolicyFavoritesSetting::SETTINGKEY; + return $viewer->getUserSetting($export_key); + } + + private function writeExportFormatPreference($value) { + $viewer = $this->getViewer(); + $request = $this->getRequest(); + + if (!$viewer->isLoggedIn()) { + return; + } + + $export_key = PhabricatorPolicyFavoritesSetting::SETTINGKEY; + $preferences = PhabricatorUserPreferences::loadUserPreferences($viewer); + + $editor = id(new PhabricatorUserPreferencesEditor()) + ->setActor($viewer) + ->setContentSourceFromRequest($request) + ->setContinueOnNoEffect(true) + ->setContinueOnMissingFields(true); + + $xactions = array(); + $xactions[] = $preferences->newTransaction($export_key, $value); + $editor->applyTransactions($preferences, $xactions); + } + } diff --git a/src/infrastructure/export/engine/PhabricatorExportFormatSetting.php b/src/infrastructure/export/engine/PhabricatorExportFormatSetting.php new file mode 100644 index 0000000000..625a90719b --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportFormatSetting.php @@ -0,0 +1,16 @@ + Date: Mon, 29 Jan 2018 07:53:25 -0800 Subject: [PATCH 15/89] When PHPExcel is not installed, detect it and provide install instructions Summary: Depends on D18957. Ref T13049. To do Excel exports, PHPExcel needs to be installed on the system somewhere. This library is enormous (1K files, ~100K SLOC), which is why we don't just include it in `externals/`. This install process is a little weird and we could improve it, but users don't seem to have too much difficulty with it. This shouldn't be worse than the existing workflow in Maniphest, and I tried to make it at least slightly more clear. Test Plan: Uninstalled PHPExcel, got it marked "Unavailable" and got reasonably-helpful-ish guidance on how to get it to work. Reinstalled, exported, got a sheet. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18958 --- ...PhabricatorApplicationSearchController.php | 36 +++++++++++++++++-- .../format/PhabricatorExcelExportFormat.php | 25 ++++++++++++- .../export/format/PhabricatorExportFormat.php | 12 ------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index a5c58aac7f..f0bf9bb365 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -418,8 +418,24 @@ final class PhabricatorApplicationSearchController $filename = phutil_utf8_strtolower($filename); $filename = PhabricatorFile::normalizeFileName($filename); - $formats = PhabricatorExportFormat::getAllEnabledExportFormats(); - $format_options = mpull($formats, 'getExportFormatName'); + $all_formats = PhabricatorExportFormat::getAllExportFormats(); + + $available_options = array(); + $unavailable_options = array(); + $formats = array(); + $unavailable_formats = array(); + foreach ($all_formats as $key => $format) { + if ($format->isExportFormatEnabled()) { + $available_options[$key] = $format->getExportFormatName(); + $formats[$key] = $format; + } else { + $unavailable_options[$key] = pht( + '%s (Not Available)', + $format->getExportFormatName()); + $unavailable_formats[$key] = $format; + } + } + $format_options = $available_options + $unavailable_options; // Try to default to the format the user used last time. If you just // exported to Excel, you probably want to export to Excel again. @@ -433,6 +449,22 @@ final class PhabricatorApplicationSearchController $e_format = null; if ($request->isFormPost()) { $format_key = $request->getStr('format'); + + if (isset($unavailable_formats[$format_key])) { + $unavailable = $unavailable_formats[$format_key]; + $instructions = $unavailable->getInstallInstructions(); + + $markup = id(new PHUIRemarkupView($viewer, $instructions)) + ->setRemarkupOption( + PHUIRemarkupView::OPTION_PRESERVE_LINEBREAKS, + false); + + return $this->newDialog() + ->setTitle(pht('Export Format Not Available')) + ->appendChild($markup) + ->addCancelButton($cancel_uri, pht('Done')); + } + $format = idx($formats, $format_key); if (!$format) { diff --git a/src/infrastructure/export/format/PhabricatorExcelExportFormat.php b/src/infrastructure/export/format/PhabricatorExcelExportFormat.php index 633d98fa53..2b0c787884 100644 --- a/src/infrastructure/export/format/PhabricatorExcelExportFormat.php +++ b/src/infrastructure/export/format/PhabricatorExcelExportFormat.php @@ -14,7 +14,30 @@ final class PhabricatorExcelExportFormat } public function isExportFormatEnabled() { - return true; + // TODO: PHPExcel has a dependency on the PHP zip extension. We should test + // for that here, since it fatals if we don't have the ZipArchive class. + return @include_once 'PHPExcel.php'; + } + + public function getInstallInstructions() { + return pht(<< https://github.com/PHPOffice/PHPExcel + +Briefly: + + - Clone that repository somewhere on the sever + (like `/path/to/example/PHPExcel`). + - Update your PHP `%s` setting (in `php.ini`) to include the PHPExcel + `Classes` directory (like `/path/to/example/PHPExcel/Classes`). +EOHELP + , + 'include_path'); } public function getFileExtension() { diff --git a/src/infrastructure/export/format/PhabricatorExportFormat.php b/src/infrastructure/export/format/PhabricatorExportFormat.php index 7e174f5197..4566814b9d 100644 --- a/src/infrastructure/export/format/PhabricatorExportFormat.php +++ b/src/infrastructure/export/format/PhabricatorExportFormat.php @@ -50,16 +50,4 @@ abstract class PhabricatorExportFormat ->execute(); } - final public static function getAllEnabledExportFormats() { - $formats = self::getAllExportFormats(); - - foreach ($formats as $key => $format) { - if (!$format->isExportFormatEnabled()) { - unset($formats[$key]); - } - } - - return $formats; - } - } From 2ac4e1991b4d64972366c5a1e7c7e129eeea0f02 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 08:12:25 -0800 Subject: [PATCH 16/89] Support new data export infrastructure in Maniphest Summary: Depends on D18958. Ref T13049. Support the new stuff. There are a couple more fields this needs to strictly improve on the old export, but I'll add them as extensions shortly. Test Plan: Exported tasks to Excel, saw reasonble-looking data in the export. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18959 --- src/__phutil_library_map__.php | 2 + .../PhabricatorManiphestApplication.php | 2 +- .../query/ManiphestTaskSearchEngine.php | 107 ++++++++++++++++++ .../field/PhabricatorURIExportField.php | 4 + 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 src/infrastructure/export/field/PhabricatorURIExportField.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 4d4aeda7aa..8e7b09a959 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -4322,6 +4322,7 @@ phutil_register_library_map(array( 'PhabricatorUIExample' => 'applications/uiexample/examples/PhabricatorUIExample.php', 'PhabricatorUIExampleRenderController' => 'applications/uiexample/controller/PhabricatorUIExampleRenderController.php', 'PhabricatorUIExamplesApplication' => 'applications/uiexample/application/PhabricatorUIExamplesApplication.php', + 'PhabricatorURIExportField' => 'infrastructure/export/field/PhabricatorURIExportField.php', 'PhabricatorUSEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php', 'PhabricatorUnifiedDiffsSetting' => 'applications/settings/setting/PhabricatorUnifiedDiffsSetting.php', 'PhabricatorUnitTestContentSource' => 'infrastructure/contentsource/PhabricatorUnitTestContentSource.php', @@ -10023,6 +10024,7 @@ phutil_register_library_map(array( 'PhabricatorUIExample' => 'Phobject', 'PhabricatorUIExampleRenderController' => 'PhabricatorController', 'PhabricatorUIExamplesApplication' => 'PhabricatorApplication', + 'PhabricatorURIExportField' => 'PhabricatorExportField', 'PhabricatorUSEnglishTranslation' => 'PhutilTranslation', 'PhabricatorUnifiedDiffsSetting' => 'PhabricatorSelectSetting', 'PhabricatorUnitTestContentSource' => 'PhabricatorContentSource', diff --git a/src/applications/maniphest/application/PhabricatorManiphestApplication.php b/src/applications/maniphest/application/PhabricatorManiphestApplication.php index 6e4ac0a8f6..4376eb3f32 100644 --- a/src/applications/maniphest/application/PhabricatorManiphestApplication.php +++ b/src/applications/maniphest/application/PhabricatorManiphestApplication.php @@ -50,7 +50,7 @@ final class PhabricatorManiphestApplication extends PhabricatorApplication { return array( '/T(?P[1-9]\d*)' => 'ManiphestTaskDetailController', '/maniphest/' => array( - '(?:query/(?P[^/]+)/)?' => 'ManiphestTaskListController', + $this->getQueryRoutePattern() => 'ManiphestTaskListController', 'report/(?:(?P\w+)/)?' => 'ManiphestReportController', $this->getBulkRoutePattern('bulk/') => 'ManiphestBulkEditController', 'task/' => array( diff --git a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php index ec1956bd8e..caf6eb30f5 100644 --- a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php +++ b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php @@ -432,4 +432,111 @@ final class ManiphestTaskSearchEngine return $view; } + + protected function newExportFields() { + $fields = array( + id(new PhabricatorStringExportField()) + ->setKey('monogram') + ->setLabel(pht('Monogram')), + id(new PhabricatorPHIDExportField()) + ->setKey('authorPHID') + ->setLabel(pht('Author PHID')), + id(new PhabricatorStringExportField()) + ->setKey('author') + ->setLabel(pht('Author')), + id(new PhabricatorPHIDExportField()) + ->setKey('ownerPHID') + ->setLabel(pht('Owner PHID')), + id(new PhabricatorStringExportField()) + ->setKey('owner') + ->setLabel(pht('Owner')), + id(new PhabricatorStringExportField()) + ->setKey('status') + ->setLabel(pht('Status')), + id(new PhabricatorStringExportField()) + ->setKey('statusName') + ->setLabel(pht('Status Name')), + id(new PhabricatorStringExportField()) + ->setKey('priority') + ->setLabel(pht('Priority')), + id(new PhabricatorStringExportField()) + ->setKey('priorityName') + ->setLabel(pht('Priority Name')), + id(new PhabricatorStringExportField()) + ->setKey('subtype') + ->setLabel('string'), + id(new PhabricatorURIExportField()) + ->setKey('uri') + ->setLabel(pht('URI')), + id(new PhabricatorStringExportField()) + ->setKey('title') + ->setLabel(pht('Title')), + id(new PhabricatorStringExportField()) + ->setKey('description') + ->setLabel(pht('Description')), + ); + + if (ManiphestTaskPoints::getIsEnabled()) { + $fields[] = id(new PhabricatorIntExportField()) + ->setKey('points') + ->setLabel('Points'); + } + + return $fields; + } + + protected function newExportData(array $tasks) { + $viewer = $this->requireViewer(); + + $phids = array(); + foreach ($tasks as $task) { + $phids[] = $task->getAuthorPHID(); + $phids[] = $task->getOwnerPHID(); + } + $handles = $viewer->loadHandles($phids); + + $export = array(); + foreach ($tasks as $task) { + + $author_phid = $task->getAuthorPHID(); + if ($author_phid) { + $author_name = $handles[$author_phid]->getName(); + } else { + $author_name = null; + } + + $owner_phid = $task->getOwnerPHID(); + if ($owner_phid) { + $owner_name = $handles[$owner_phid]->getName(); + } else { + $owner_name = null; + } + + $status_value = $task->getStatus(); + $status_name = ManiphestTaskStatus::getTaskStatusName($status_value); + + $priority_value = $task->getPriority(); + $priority_name = ManiphestTaskPriority::getTaskPriorityName( + $priority_value); + + $export[] = array( + 'monogram' => $task->getMonogram(), + 'authorPHID' => $author_phid, + 'author' => $author_name, + 'ownerPHID' => $owner_phid, + 'owner' => $owner_name, + 'status' => $status_value, + 'statusName' => $status_name, + 'priority' => $priority_value, + 'priorityName' => $priority_name, + 'points' => $task->getPoints(), + 'subtype' => $task->getSubtype(), + 'title' => $task->getTitle(), + 'uri' => PhabricatorEnv::getProductionURI($task->getURI()), + 'description' => $task->getDescription(), + ); + } + + return $export; + } } diff --git a/src/infrastructure/export/field/PhabricatorURIExportField.php b/src/infrastructure/export/field/PhabricatorURIExportField.php new file mode 100644 index 0000000000..9ba2cd4b75 --- /dev/null +++ b/src/infrastructure/export/field/PhabricatorURIExportField.php @@ -0,0 +1,4 @@ + Date: Mon, 29 Jan 2018 08:37:38 -0800 Subject: [PATCH 17/89] Implement common infrastructure fields as export extensions Summary: Depends on D18959. Ref T13049. Provide tags, subscribers, spaces, and created/modified as automatic extensions for all objects which support them. (Also, for JSON export, be a little more consistent about exporting `null` instead of empty string when there's no value in a text field.) Test Plan: Exported users and tasks, saw relevant fields in the export. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18960 --- src/__phutil_library_map__.php | 14 +++++ .../query/ManiphestTaskSearchEngine.php | 2 +- .../query/PhabricatorPeopleSearchEngine.php | 4 -- .../PhabricatorLiskExportEngineExtension.php | 42 +++++++++++++ ...abricatorProjectsExportEngineExtension.php | 60 +++++++++++++++++++ ...PhabricatorSpacesExportEngineExtension.php | 53 ++++++++++++++++ ...atorSubscriptionsExportEngineExtension.php | 60 +++++++++++++++++++ .../export/field/PhabricatorExportField.php | 8 ++- .../field/PhabricatorListExportField.php | 10 ++++ .../field/PhabricatorPHIDListExportField.php | 10 ++++ .../field/PhabricatorStringExportField.php | 16 ++++- .../PhabricatorStringListExportField.php | 4 ++ 12 files changed, 276 insertions(+), 7 deletions(-) create mode 100644 src/infrastructure/export/engine/PhabricatorLiskExportEngineExtension.php create mode 100644 src/infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php create mode 100644 src/infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php create mode 100644 src/infrastructure/export/engine/PhabricatorSubscriptionsExportEngineExtension.php create mode 100644 src/infrastructure/export/field/PhabricatorListExportField.php create mode 100644 src/infrastructure/export/field/PhabricatorPHIDListExportField.php create mode 100644 src/infrastructure/export/field/PhabricatorStringListExportField.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 8e7b09a959..132c112879 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3130,9 +3130,11 @@ phutil_register_library_map(array( 'PhabricatorLipsumManagementWorkflow' => 'applications/lipsum/management/PhabricatorLipsumManagementWorkflow.php', 'PhabricatorLipsumMondrianArtist' => 'applications/lipsum/image/PhabricatorLipsumMondrianArtist.php', 'PhabricatorLiskDAO' => 'infrastructure/storage/lisk/PhabricatorLiskDAO.php', + 'PhabricatorLiskExportEngineExtension' => 'infrastructure/export/engine/PhabricatorLiskExportEngineExtension.php', 'PhabricatorLiskFulltextEngineExtension' => 'applications/search/engineextension/PhabricatorLiskFulltextEngineExtension.php', 'PhabricatorLiskSearchEngineExtension' => 'applications/search/engineextension/PhabricatorLiskSearchEngineExtension.php', 'PhabricatorLiskSerializer' => 'infrastructure/storage/lisk/PhabricatorLiskSerializer.php', + 'PhabricatorListExportField' => 'infrastructure/export/field/PhabricatorListExportField.php', 'PhabricatorLocalDiskFileStorageEngine' => 'applications/files/engine/PhabricatorLocalDiskFileStorageEngine.php', 'PhabricatorLocalTimeTestCase' => 'view/__tests__/PhabricatorLocalTimeTestCase.php', 'PhabricatorLocaleScopeGuard' => 'infrastructure/internationalization/scope/PhabricatorLocaleScopeGuard.php', @@ -3431,6 +3433,7 @@ phutil_register_library_map(array( 'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php', 'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php', 'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php', + 'PhabricatorPHIDListExportField' => 'infrastructure/export/field/PhabricatorPHIDListExportField.php', 'PhabricatorPHIDResolver' => 'applications/phid/resolver/PhabricatorPHIDResolver.php', 'PhabricatorPHIDType' => 'applications/phid/type/PhabricatorPHIDType.php', 'PhabricatorPHIDTypeTestCase' => 'applications/phid/type/__tests__/PhabricatorPHIDTypeTestCase.php', @@ -3839,6 +3842,7 @@ phutil_register_library_map(array( 'PhabricatorProjectsCurtainExtension' => 'applications/project/engineextension/PhabricatorProjectsCurtainExtension.php', 'PhabricatorProjectsEditEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsEditEngineExtension.php', 'PhabricatorProjectsEditField' => 'applications/transactions/editfield/PhabricatorProjectsEditField.php', + 'PhabricatorProjectsExportEngineExtension' => 'infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php', 'PhabricatorProjectsFulltextEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsFulltextEngineExtension.php', 'PhabricatorProjectsMembersSearchEngineAttachment' => 'applications/project/engineextension/PhabricatorProjectsMembersSearchEngineAttachment.php', 'PhabricatorProjectsMembershipIndexEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsMembershipIndexEngineExtension.php', @@ -4130,6 +4134,7 @@ phutil_register_library_map(array( 'PhabricatorSpacesController' => 'applications/spaces/controller/PhabricatorSpacesController.php', 'PhabricatorSpacesDAO' => 'applications/spaces/storage/PhabricatorSpacesDAO.php', 'PhabricatorSpacesEditController' => 'applications/spaces/controller/PhabricatorSpacesEditController.php', + 'PhabricatorSpacesExportEngineExtension' => 'infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php', 'PhabricatorSpacesInterface' => 'applications/spaces/interface/PhabricatorSpacesInterface.php', 'PhabricatorSpacesListController' => 'applications/spaces/controller/PhabricatorSpacesListController.php', 'PhabricatorSpacesNamespace' => 'applications/spaces/storage/PhabricatorSpacesNamespace.php', @@ -4196,6 +4201,7 @@ phutil_register_library_map(array( 'PhabricatorStringExportField' => 'infrastructure/export/field/PhabricatorStringExportField.php', 'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php', 'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php', + 'PhabricatorStringListExportField' => 'infrastructure/export/field/PhabricatorStringListExportField.php', 'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php', 'PhabricatorSubmitEditField' => 'applications/transactions/editfield/PhabricatorSubmitEditField.php', 'PhabricatorSubscribableInterface' => 'applications/subscriptions/interface/PhabricatorSubscribableInterface.php', @@ -4210,6 +4216,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsEditController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsEditController.php', 'PhabricatorSubscriptionsEditEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsEditEngineExtension.php', 'PhabricatorSubscriptionsEditor' => 'applications/subscriptions/editor/PhabricatorSubscriptionsEditor.php', + 'PhabricatorSubscriptionsExportEngineExtension' => 'infrastructure/export/engine/PhabricatorSubscriptionsExportEngineExtension.php', 'PhabricatorSubscriptionsFulltextEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsFulltextEngineExtension.php', 'PhabricatorSubscriptionsHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsHeraldAction.php', 'PhabricatorSubscriptionsListController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsListController.php', @@ -8608,9 +8615,11 @@ phutil_register_library_map(array( 'PhabricatorLipsumManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorLipsumMondrianArtist' => 'PhabricatorLipsumArtist', 'PhabricatorLiskDAO' => 'LiskDAO', + 'PhabricatorLiskExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorLiskFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', 'PhabricatorLiskSearchEngineExtension' => 'PhabricatorSearchEngineExtension', 'PhabricatorLiskSerializer' => 'Phobject', + 'PhabricatorListExportField' => 'PhabricatorExportField', 'PhabricatorLocalDiskFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorLocalTimeTestCase' => 'PhabricatorTestCase', 'PhabricatorLocaleScopeGuard' => 'Phobject', @@ -8948,6 +8957,7 @@ phutil_register_library_map(array( 'PhabricatorPHIDExportField' => 'PhabricatorExportField', 'PhabricatorPHIDListEditField' => 'PhabricatorEditField', 'PhabricatorPHIDListEditType' => 'PhabricatorEditType', + 'PhabricatorPHIDListExportField' => 'PhabricatorListExportField', 'PhabricatorPHIDResolver' => 'Phobject', 'PhabricatorPHIDType' => 'Phobject', 'PhabricatorPHIDTypeTestCase' => 'PhutilTestCase', @@ -9448,6 +9458,7 @@ phutil_register_library_map(array( 'PhabricatorProjectsCurtainExtension' => 'PHUICurtainExtension', 'PhabricatorProjectsEditEngineExtension' => 'PhabricatorEditEngineExtension', 'PhabricatorProjectsEditField' => 'PhabricatorTokenizerEditField', + 'PhabricatorProjectsExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorProjectsFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', 'PhabricatorProjectsMembersSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', 'PhabricatorProjectsMembershipIndexEngineExtension' => 'PhabricatorIndexEngineExtension', @@ -9815,6 +9826,7 @@ phutil_register_library_map(array( 'PhabricatorSpacesController' => 'PhabricatorController', 'PhabricatorSpacesDAO' => 'PhabricatorLiskDAO', 'PhabricatorSpacesEditController' => 'PhabricatorSpacesController', + 'PhabricatorSpacesExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorSpacesInterface' => 'PhabricatorPHIDInterface', 'PhabricatorSpacesListController' => 'PhabricatorSpacesController', 'PhabricatorSpacesNamespace' => array( @@ -9888,6 +9900,7 @@ phutil_register_library_map(array( 'PhabricatorStringExportField' => 'PhabricatorExportField', 'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType', 'PhabricatorStringListEditField' => 'PhabricatorEditField', + 'PhabricatorStringListExportField' => 'PhabricatorListExportField', 'PhabricatorStringSetting' => 'PhabricatorSetting', 'PhabricatorSubmitEditField' => 'PhabricatorEditField', 'PhabricatorSubscribedToObjectEdgeType' => 'PhabricatorEdgeType', @@ -9901,6 +9914,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsEditController' => 'PhabricatorController', 'PhabricatorSubscriptionsEditEngineExtension' => 'PhabricatorEditEngineExtension', 'PhabricatorSubscriptionsEditor' => 'PhabricatorEditor', + 'PhabricatorSubscriptionsExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorSubscriptionsFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', 'PhabricatorSubscriptionsHeraldAction' => 'HeraldAction', 'PhabricatorSubscriptionsListController' => 'PhabricatorController', diff --git a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php index caf6eb30f5..ad668db376 100644 --- a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php +++ b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php @@ -464,7 +464,7 @@ final class ManiphestTaskSearchEngine ->setLabel(pht('Priority Name')), id(new PhabricatorStringExportField()) ->setKey('subtype') - ->setLabel('string'), + ->setLabel('Subtype'), id(new PhabricatorURIExportField()) ->setKey('uri') ->setLabel(pht('URI')), diff --git a/src/applications/people/query/PhabricatorPeopleSearchEngine.php b/src/applications/people/query/PhabricatorPeopleSearchEngine.php index e0e1b5070e..57ed133df4 100644 --- a/src/applications/people/query/PhabricatorPeopleSearchEngine.php +++ b/src/applications/people/query/PhabricatorPeopleSearchEngine.php @@ -328,9 +328,6 @@ final class PhabricatorPeopleSearchEngine id(new PhabricatorStringExportField()) ->setKey('realName') ->setLabel(pht('Real Name')), - id(new PhabricatorEpochExportField()) - ->setKey('created') - ->setLabel(pht('Date Created')), ); } @@ -342,7 +339,6 @@ final class PhabricatorPeopleSearchEngine $export[] = array( 'username' => $user->getUsername(), 'realName' => $user->getRealName(), - 'created' => $user->getDateCreated(), ); } diff --git a/src/infrastructure/export/engine/PhabricatorLiskExportEngineExtension.php b/src/infrastructure/export/engine/PhabricatorLiskExportEngineExtension.php new file mode 100644 index 0000000000..5162986057 --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorLiskExportEngineExtension.php @@ -0,0 +1,42 @@ +getConfigOption(LiskDAO::CONFIG_TIMESTAMPS)) { + return false; + } + + return true; + } + + public function newExportFields() { + return array( + id(new PhabricatorEpochExportField()) + ->setKey('dateCreated') + ->setLabel(pht('Created')), + id(new PhabricatorEpochExportField()) + ->setKey('dateModified') + ->setLabel(pht('Modified')), + ); + } + + public function newExportData(array $objects) { + $map = array(); + foreach ($objects as $object) { + $map[] = array( + 'dateCreated' => $object->getDateCreated(), + 'dateModified' => $object->getDateModified(), + ); + } + return $map; + } + +} diff --git a/src/infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php b/src/infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php new file mode 100644 index 0000000000..eb3bca3a49 --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php @@ -0,0 +1,60 @@ +setKey('tagPHIDs') + ->setLabel(pht('Tag PHIDs')), + id(new PhabricatorStringListExportField()) + ->setKey('tags') + ->setLabel(pht('Tags')), + ); + } + + public function newExportData(array $objects) { + $viewer = $this->getViewer(); + + $object_phids = mpull($objects, 'getPHID'); + + $projects_query = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs($object_phids) + ->withEdgeTypes( + array( + PhabricatorProjectObjectHasProjectEdgeType::EDGECONST, + )); + $projects_query->execute(); + + $handles = $viewer->loadHandles($projects_query->getDestinationPHIDs()); + + $map = array(); + foreach ($objects as $object) { + $object_phid = $object->getPHID(); + + $project_phids = $projects_query->getDestinationPHIDs( + array($object_phid), + array(PhabricatorProjectObjectHasProjectEdgeType::EDGECONST)); + + $handle_list = $handles->newSublist($project_phids); + $handle_list = iterator_to_array($handle_list); + $handle_names = mpull($handle_list, 'getName'); + $handle_names = array_values($handle_names); + + $map[] = array( + 'tagPHIDs' => $project_phids, + 'tags' => $handle_names, + ); + } + + return $map; + } + +} diff --git a/src/infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php b/src/infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php new file mode 100644 index 0000000000..3e187bc8cd --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php @@ -0,0 +1,53 @@ +getViewer(); + + if (!PhabricatorSpacesNamespaceQuery::getViewerSpacesExist($viewer)) { + return false; + } + + return ($object instanceof PhabricatorSpacesInterface); + } + + public function newExportFields() { + return array( + id(new PhabricatorPHIDExportField()) + ->setKey('spacePHID') + ->setLabel(pht('Space PHID')), + id(new PhabricatorStringExportField()) + ->setKey('space') + ->setLabel(pht('Space')), + ); + } + + public function newExportData(array $objects) { + $viewer = $this->getViewer(); + + $space_phids = array(); + foreach ($objects as $object) { + $space_phids[] = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( + $object); + } + $handles = $viewer->loadHandles($space_phids); + + $map = array(); + foreach ($objects as $object) { + $space_phid = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( + $object); + + $map[] = array( + 'spacePHID' => $space_phid, + 'space' => $handles[$space_phid]->getName(), + ); + } + + return $map; + } + +} diff --git a/src/infrastructure/export/engine/PhabricatorSubscriptionsExportEngineExtension.php b/src/infrastructure/export/engine/PhabricatorSubscriptionsExportEngineExtension.php new file mode 100644 index 0000000000..8aedb38fa8 --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorSubscriptionsExportEngineExtension.php @@ -0,0 +1,60 @@ +setKey('subscriberPHIDs') + ->setLabel(pht('Subscriber PHIDs')), + id(new PhabricatorStringListExportField()) + ->setKey('subscribers') + ->setLabel(pht('Subscribers')), + ); + } + + public function newExportData(array $objects) { + $viewer = $this->getViewer(); + + $object_phids = mpull($objects, 'getPHID'); + + $projects_query = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs($object_phids) + ->withEdgeTypes( + array( + PhabricatorObjectHasSubscriberEdgeType::EDGECONST, + )); + $projects_query->execute(); + + $handles = $viewer->loadHandles($projects_query->getDestinationPHIDs()); + + $map = array(); + foreach ($objects as $object) { + $object_phid = $object->getPHID(); + + $project_phids = $projects_query->getDestinationPHIDs( + array($object_phid), + array(PhabricatorObjectHasSubscriberEdgeType::EDGECONST)); + + $handle_list = $handles->newSublist($project_phids); + $handle_list = iterator_to_array($handle_list); + $handle_names = mpull($handle_list, 'getName'); + $handle_names = array_values($handle_names); + + $map[] = array( + 'subscriberPHIDs' => $project_phids, + 'subscribers' => $handle_names, + ); + } + + return $map; + } + +} diff --git a/src/infrastructure/export/field/PhabricatorExportField.php b/src/infrastructure/export/field/PhabricatorExportField.php index 85e21b3e37..7ee0918595 100644 --- a/src/infrastructure/export/field/PhabricatorExportField.php +++ b/src/infrastructure/export/field/PhabricatorExportField.php @@ -25,7 +25,13 @@ abstract class PhabricatorExportField } public function getTextValue($value) { - return (string)$this->getNaturalValue($value); + $natural_value = $this->getNaturalValue($value); + + if ($natural_value === null) { + return null; + } + + return (string)$natural_value; } public function getNaturalValue($value) { diff --git a/src/infrastructure/export/field/PhabricatorListExportField.php b/src/infrastructure/export/field/PhabricatorListExportField.php new file mode 100644 index 0000000000..67c3e06ff5 --- /dev/null +++ b/src/infrastructure/export/field/PhabricatorListExportField.php @@ -0,0 +1,10 @@ + Date: Mon, 29 Jan 2018 08:56:49 -0800 Subject: [PATCH 18/89] Remove the old, non-modular Excel export workflow from Maniphest Summary: Depends on D18960. Ref T13049. Now that Maniphest fully supports "Export Data", remove the old hard-coded version. This is a backward compatibility break with the handful of installs that might have defined a custom export by subclassing `ManiphestExcelFormat`. I suspect this is almost zero installs, and that the additional data in the new format may serve most of the needs of this tiny number of installs. They can upgrade to `ExportEngineExtensions` fairly easily if this isn't true. Test Plan: - Viewed Maniphest, no longer saw the old export workflow. - Grepped for `export` and similar strings to try to hunt everything down. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18961 --- src/__phutil_library_map__.php | 8 - .../PhabricatorManiphestApplication.php | 1 - .../controller/ManiphestExportController.php | 135 ----------------- .../export/ManiphestExcelDefaultFormat.php | 140 ------------------ .../maniphest/export/ManiphestExcelFormat.php | 35 ----- .../ManiphestExcelFormatTestCase.php | 10 -- .../view/ManiphestTaskResultListView.php | 13 +- ...PhabricatorApplicationSearchController.php | 2 +- 8 files changed, 2 insertions(+), 342 deletions(-) delete mode 100644 src/applications/maniphest/controller/ManiphestExportController.php delete mode 100644 src/applications/maniphest/export/ManiphestExcelDefaultFormat.php delete mode 100644 src/applications/maniphest/export/ManiphestExcelFormat.php delete mode 100644 src/applications/maniphest/export/__tests__/ManiphestExcelFormatTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 132c112879..1f71d97460 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1525,10 +1525,6 @@ phutil_register_library_map(array( 'ManiphestEditProjectsCapability' => 'applications/maniphest/capability/ManiphestEditProjectsCapability.php', 'ManiphestEditStatusCapability' => 'applications/maniphest/capability/ManiphestEditStatusCapability.php', 'ManiphestEmailCommand' => 'applications/maniphest/command/ManiphestEmailCommand.php', - 'ManiphestExcelDefaultFormat' => 'applications/maniphest/export/ManiphestExcelDefaultFormat.php', - 'ManiphestExcelFormat' => 'applications/maniphest/export/ManiphestExcelFormat.php', - 'ManiphestExcelFormatTestCase' => 'applications/maniphest/export/__tests__/ManiphestExcelFormatTestCase.php', - 'ManiphestExportController' => 'applications/maniphest/controller/ManiphestExportController.php', 'ManiphestGetTaskTransactionsConduitAPIMethod' => 'applications/maniphest/conduit/ManiphestGetTaskTransactionsConduitAPIMethod.php', 'ManiphestHovercardEngineExtension' => 'applications/maniphest/engineextension/ManiphestHovercardEngineExtension.php', 'ManiphestInfoConduitAPIMethod' => 'applications/maniphest/conduit/ManiphestInfoConduitAPIMethod.php', @@ -6780,10 +6776,6 @@ phutil_register_library_map(array( 'ManiphestEditProjectsCapability' => 'PhabricatorPolicyCapability', 'ManiphestEditStatusCapability' => 'PhabricatorPolicyCapability', 'ManiphestEmailCommand' => 'MetaMTAEmailTransactionCommand', - 'ManiphestExcelDefaultFormat' => 'ManiphestExcelFormat', - 'ManiphestExcelFormat' => 'Phobject', - 'ManiphestExcelFormatTestCase' => 'PhabricatorTestCase', - 'ManiphestExportController' => 'ManiphestController', 'ManiphestGetTaskTransactionsConduitAPIMethod' => 'ManiphestConduitAPIMethod', 'ManiphestHovercardEngineExtension' => 'PhabricatorHovercardEngineExtension', 'ManiphestInfoConduitAPIMethod' => 'ManiphestConduitAPIMethod', diff --git a/src/applications/maniphest/application/PhabricatorManiphestApplication.php b/src/applications/maniphest/application/PhabricatorManiphestApplication.php index 4376eb3f32..0075770863 100644 --- a/src/applications/maniphest/application/PhabricatorManiphestApplication.php +++ b/src/applications/maniphest/application/PhabricatorManiphestApplication.php @@ -57,7 +57,6 @@ final class PhabricatorManiphestApplication extends PhabricatorApplication { $this->getEditRoutePattern('edit/') => 'ManiphestTaskEditController', ), - 'export/(?P[^/]+)/' => 'ManiphestExportController', 'subpriority/' => 'ManiphestSubpriorityController', ), ); diff --git a/src/applications/maniphest/controller/ManiphestExportController.php b/src/applications/maniphest/controller/ManiphestExportController.php deleted file mode 100644 index 1201c50478..0000000000 --- a/src/applications/maniphest/controller/ManiphestExportController.php +++ /dev/null @@ -1,135 +0,0 @@ -getViewer(); - $key = $request->getURIData('key'); - - $ok = @include_once 'PHPExcel.php'; - if (!$ok) { - $dialog = $this->newDialog(); - - $inst1 = pht( - 'This system does not have PHPExcel installed. This software '. - 'component is required to export tasks to Excel. Have your system '. - 'administrator install it from:'); - - $inst2 = pht( - 'Your PHP "%s" needs to be updated to include the '. - 'PHPExcel Classes directory.', - 'include_path'); - - $dialog->setTitle(pht('Excel Export Not Configured')); - $dialog->appendChild(hsprintf( - '

%s

'. - '
'. - '

'. - ''. - 'https://github.com/PHPOffice/PHPExcel'. - ''. - '

'. - '
'. - '

%s

', - $inst1, - $inst2)); - - $dialog->addCancelButton('/maniphest/'); - return id(new AphrontDialogResponse())->setDialog($dialog); - } - - // TODO: PHPExcel has a dependency on the PHP zip extension. We should test - // for that here, since it fatals if we don't have the ZipArchive class. - - $saved = id(new PhabricatorSavedQueryQuery()) - ->setViewer($viewer) - ->withQueryKeys(array($key)) - ->executeOne(); - if (!$saved) { - $engine = id(new ManiphestTaskSearchEngine()) - ->setViewer($viewer); - if ($engine->isBuiltinQuery($key)) { - $saved = $engine->buildSavedQueryFromBuiltin($key); - } - if (!$saved) { - return new Aphront404Response(); - } - } - - $formats = ManiphestExcelFormat::loadAllFormats(); - $export_formats = array(); - foreach ($formats as $format_class => $format_object) { - $export_formats[$format_class] = $format_object->getName(); - } - - if (!$request->isDialogFormPost()) { - $dialog = new AphrontDialogView(); - $dialog->setUser($viewer); - - $dialog->setTitle(pht('Export Tasks to Excel')); - $dialog->appendChild( - phutil_tag( - 'p', - array(), - pht('Do you want to export the query results to Excel?'))); - - $form = id(new PHUIFormLayoutView()) - ->appendChild( - id(new AphrontFormSelectControl()) - ->setLabel(pht('Format:')) - ->setName('excel-format') - ->setOptions($export_formats)); - - $dialog->appendChild($form); - - $dialog->addCancelButton('/maniphest/'); - $dialog->addSubmitButton(pht('Export to Excel')); - return id(new AphrontDialogResponse())->setDialog($dialog); - } - - $format = idx($formats, $request->getStr('excel-format')); - if ($format === null) { - throw new Exception(pht('Excel format object not found.')); - } - - $saved->makeEphemeral(); - $saved->setParameter('limit', PHP_INT_MAX); - - $engine = id(new ManiphestTaskSearchEngine()) - ->setViewer($viewer); - - $query = $engine->buildQueryFromSavedQuery($saved); - $query->setViewer($viewer); - $tasks = $query->execute(); - - $all_projects = array_mergev(mpull($tasks, 'getProjectPHIDs')); - $all_assigned = mpull($tasks, 'getOwnerPHID'); - - $handles = id(new PhabricatorHandleQuery()) - ->setViewer($viewer) - ->withPHIDs(array_merge($all_projects, $all_assigned)) - ->execute(); - - $workbook = new PHPExcel(); - $format->buildWorkbook($workbook, $tasks, $handles, $viewer); - $writer = PHPExcel_IOFactory::createWriter($workbook, 'Excel2007'); - - ob_start(); - $writer->save('php://output'); - $data = ob_get_clean(); - - $mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; - - return id(new AphrontFileResponse()) - ->setMimeType($mime) - ->setDownload($format->getFileName().'.xlsx') - ->setContent($data); - } - -} diff --git a/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php b/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php deleted file mode 100644 index 1451ae504e..0000000000 --- a/src/applications/maniphest/export/ManiphestExcelDefaultFormat.php +++ /dev/null @@ -1,140 +0,0 @@ -setActiveSheetIndex(0); - $sheet->setTitle(pht('Tasks')); - - $widths = array( - null, - 15, - null, - 10, - 15, - 15, - 60, - 30, - 20, - 100, - ); - - foreach ($widths as $col => $width) { - if ($width !== null) { - $sheet->getColumnDimension($this->col($col))->setWidth($width); - } - } - - $status_map = ManiphestTaskStatus::getTaskStatusMap(); - $pri_map = ManiphestTaskPriority::getTaskPriorityMap(); - - $date_format = null; - - $rows = array(); - $rows[] = array( - pht('ID'), - pht('Owner'), - pht('Status'), - pht('Priority'), - pht('Date Created'), - pht('Date Updated'), - pht('Title'), - pht('Tags'), - pht('URI'), - pht('Description'), - ); - - $is_date = array( - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - ); - - $header_format = array( - 'font' => array( - 'bold' => true, - ), - ); - - foreach ($tasks as $task) { - $task_owner = null; - if ($task->getOwnerPHID()) { - $task_owner = $handles[$task->getOwnerPHID()]->getName(); - } - - $projects = array(); - foreach ($task->getProjectPHIDs() as $phid) { - $projects[] = $handles[$phid]->getName(); - } - $projects = implode(', ', $projects); - - $rows[] = array( - 'T'.$task->getID(), - $task_owner, - idx($status_map, $task->getStatus(), '?'), - idx($pri_map, $task->getPriority(), '?'), - $this->computeExcelDate($task->getDateCreated()), - $this->computeExcelDate($task->getDateModified()), - $task->getTitle(), - $projects, - PhabricatorEnv::getProductionURI('/T'.$task->getID()), - id(new PhutilUTF8StringTruncator()) - ->setMaximumBytes(512) - ->truncateString($task->getDescription()), - ); - } - - foreach ($rows as $row => $cols) { - foreach ($cols as $col => $spec) { - $cell_name = $this->col($col).($row + 1); - $cell = $sheet - ->setCellValue($cell_name, $spec, $return_cell = true); - - if ($row == 0) { - $sheet->getStyle($cell_name)->applyFromArray($header_format); - } - - if ($is_date[$col]) { - $code = PHPExcel_Style_NumberFormat::FORMAT_DATE_YYYYMMDD2; - $sheet - ->getStyle($cell_name) - ->getNumberFormat() - ->setFormatCode($code); - } else { - $cell->setDataType(PHPExcel_Cell_DataType::TYPE_STRING); - } - } - } - } - - private function col($n) { - return chr(ord('A') + $n); - } - -} diff --git a/src/applications/maniphest/export/ManiphestExcelFormat.php b/src/applications/maniphest/export/ManiphestExcelFormat.php deleted file mode 100644 index e455a63656..0000000000 --- a/src/applications/maniphest/export/ManiphestExcelFormat.php +++ /dev/null @@ -1,35 +0,0 @@ -setAncestorClass(__CLASS__) - ->setSortMethod('getOrder') - ->execute(); - } - - abstract public function getName(); - abstract public function getFileName(); - - public function getOrder() { - return 0; - } - - protected function computeExcelDate($epoch) { - $seconds_per_day = (60 * 60 * 24); - $offset = ($seconds_per_day * 25569); - - return ($epoch + $offset) / $seconds_per_day; - } - - /** - * @phutil-external-symbol class PHPExcel - */ - abstract public function buildWorkbook( - PHPExcel $workbook, - array $tasks, - array $handles, - PhabricatorUser $user); - -} diff --git a/src/applications/maniphest/export/__tests__/ManiphestExcelFormatTestCase.php b/src/applications/maniphest/export/__tests__/ManiphestExcelFormatTestCase.php deleted file mode 100644 index a3c312fcd2..0000000000 --- a/src/applications/maniphest/export/__tests__/ManiphestExcelFormatTestCase.php +++ /dev/null @@ -1,10 +0,0 @@ -assertTrue(true); - } - -} diff --git a/src/applications/maniphest/view/ManiphestTaskResultListView.php b/src/applications/maniphest/view/ManiphestTaskResultListView.php index b4cf9a544c..6aafcbdccb 100644 --- a/src/applications/maniphest/view/ManiphestTaskResultListView.php +++ b/src/applications/maniphest/view/ManiphestTaskResultListView.php @@ -175,8 +175,7 @@ final class ManiphestTaskResultListView extends ManiphestView { } if (!$user->isLoggedIn()) { - // Don't show the batch editor or excel export for logged-out users. - // Technically we //could// let them export, but ehh. + // Don't show the batch editor for logged-out users. return null; } @@ -220,14 +219,6 @@ final class ManiphestTaskResultListView extends ManiphestView { ), pht("Bulk Edit Selected \xC2\xBB")); - $export = javelin_tag( - 'a', - array( - 'href' => '/maniphest/export/'.$saved_query->getQueryKey().'/', - 'class' => 'button button-grey', - ), - pht('Export to Excel')); - $hidden = phutil_tag( 'div', array( @@ -239,14 +230,12 @@ final class ManiphestTaskResultListView extends ManiphestView { ''. ''. ''. - ''. ''. ''. ''. '
%s%s%s%s%s%s
', $select_all, $select_none, - $export, '', $submit, $hidden); diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index f0bf9bb365..ea46886a15 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -154,7 +154,7 @@ final class PhabricatorApplicationSearchController $saved_query = $engine->buildSavedQueryFromRequest($request); // Save the query to generate a query key, so "Save Custom Query..." and - // other features like Maniphest's "Export..." work correctly. + // other features like "Bulk Edit" and "Export Data" work correctly. $engine->saveQuery($saved_query); } From 84df1220858f61f13ee33506edfc8ec61af920c7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 29 Jan 2018 10:29:49 -0800 Subject: [PATCH 19/89] When exporting more than 1,000 records, export in the background Summary: Depends on D18961. Ref T13049. Currently, longer exports don't give the user any feedback, and exports that take longer than 30 seconds are likely to timeout. For small exports (up to 1,000 rows) continue doing the export in the web process. For large exports, queue a bulk job and do them in the workers instead. This sends the user through the bulk operation UI and is similar to bulk edits. It's a little clunky for now, but you get your data at the end, which is far better than hanging for 30 seconds and then fataling. Test Plan: Exported small result sets, got the same workflow as before. Exported very large result sets, went through the bulk flow, got reasonable results out. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18962 --- src/__phutil_library_map__.php | 6 + ...PhabricatorDaemonBulkJobViewController.php | 12 +- ...PhabricatorApplicationSearchController.php | 79 ++++---- .../bulk/PhabricatorWorkerBulkJobType.php | 20 +++ .../bulk/PhabricatorWorkerBulkJobWorker.php | 4 + .../PhabricatorWorkerSingleBulkJobType.php | 27 +++ .../storage/PhabricatorWorkerBulkJob.php | 4 + .../export/engine/PhabricatorExportEngine.php | 168 ++++++++++++++++++ .../PhabricatorExportEngineBulkJobType.php | 118 ++++++++++++ 9 files changed, 380 insertions(+), 58 deletions(-) create mode 100644 src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php create mode 100644 src/infrastructure/export/engine/PhabricatorExportEngine.php create mode 100644 src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 1f71d97460..06f619d74e 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2845,6 +2845,8 @@ phutil_register_library_map(array( 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 'PhabricatorExcelExportFormat' => 'infrastructure/export/format/PhabricatorExcelExportFormat.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', + 'PhabricatorExportEngine' => 'infrastructure/export/engine/PhabricatorExportEngine.php', + 'PhabricatorExportEngineBulkJobType' => 'infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php', 'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php', 'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php', 'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php', @@ -4419,6 +4421,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php', 'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php', 'PhabricatorWorkerSchemaSpec' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php', + 'PhabricatorWorkerSingleBulkJobType' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php', 'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php', 'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php', 'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php', @@ -8286,6 +8289,8 @@ phutil_register_library_map(array( 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 'PhabricatorExcelExportFormat' => 'PhabricatorExportFormat', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', + 'PhabricatorExportEngine' => 'Phobject', + 'PhabricatorExportEngineBulkJobType' => 'PhabricatorWorkerSingleBulkJobType', 'PhabricatorExportEngineExtension' => 'Phobject', 'PhabricatorExportField' => 'Phobject', 'PhabricatorExportFormat' => 'Phobject', @@ -10154,6 +10159,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorWorkerPermanentFailureException' => 'Exception', 'PhabricatorWorkerSchemaSpec' => 'PhabricatorConfigSchemaSpec', + 'PhabricatorWorkerSingleBulkJobType' => 'PhabricatorWorkerBulkJobType', 'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController', diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php index f7aa396e94..00fb297fe0 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php @@ -71,18 +71,10 @@ final class PhabricatorDaemonBulkJobViewController $viewer = $this->getViewer(); $curtain = $this->newCurtainView($job); - if ($job->isConfirming()) { - $continue_uri = $job->getMonitorURI(); - } else { - $continue_uri = $job->getDoneURI(); + foreach ($job->getCurtainActions($viewer) as $action) { + $curtain->addAction($action); } - $curtain->addAction( - id(new PhabricatorActionView()) - ->setHref($continue_uri) - ->setIcon('fa-arrow-circle-o-right') - ->setName(pht('Continue'))); - return $curtain; } diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index ea46886a15..6c7d80eafe 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -444,6 +444,17 @@ final class PhabricatorApplicationSearchController $format_key = head_key($format_options); } + // Check if this is a large result set or not. If we're exporting a + // large amount of data, we'll build the actual export file in the daemons. + + $threshold = 1000; + $query = $engine->buildQueryFromSavedQuery($saved_query); + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize($threshold + 1); + $objects = $engine->executeQuery($query, $pager); + $object_count = count($objects); + $is_large_export = ($object_count > $threshold); + $errors = array(); $e_format = null; @@ -475,59 +486,31 @@ final class PhabricatorApplicationSearchController if (!$errors) { $this->writeExportFormatPreference($format_key); - $query = $engine->buildQueryFromSavedQuery($saved_query); - - // NOTE: We aren't reading the pager from the request. Exports always - // affect the entire result set. - $pager = $engine->newPagerForSavedQuery($saved_query); - $pager->setPageSize(0x7FFFFFFF); - - $objects = $engine->executeQuery($query, $pager); - - $extension = $format->getFileExtension(); - $mime_type = $format->getMIMEContentType(); - $filename = $filename.'.'.$extension; - - $format = id(clone $format) + $export_engine = id(new PhabricatorExportEngine()) ->setViewer($viewer) - ->setTitle($sheet_title); + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setTitle($sheet_title) + ->setFilename($filename) + ->setExportFormat($format); - $export_data = $engine->newExport($objects); - $objects = array_values($objects); + if ($is_large_export) { + $job = $export_engine->newBulkJob($request); - $field_list = $engine->newExportFieldList(); - $field_list = mpull($field_list, null, 'getKey'); + return id(new AphrontRedirectResponse()) + ->setURI($job->getMonitorURI()); + } else { + $file = $export_engine->exportFile(); - $format->addHeaders($field_list); - for ($ii = 0; $ii < count($objects); $ii++) { - $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + return $this->newDialog() + ->setTitle(pht('Download Results')) + ->appendParagraph( + pht('Click the download button to download the exported data.')) + ->addCancelButton($cancel_uri, pht('Done')) + ->setSubmitURI($file->getDownloadURI()) + ->setDisableWorkflowOnSubmit(true) + ->addSubmitButton(pht('Download Data')); } - - $export_result = $format->newFileData(); - - // We have all the data in one big string and aren't actually - // streaming it, but pretending that we are allows us to actviate - // the chunk engine and store large files. - $iterator = new ArrayIterator(array($export_result)); - - $source = id(new PhabricatorIteratorFileUploadSource()) - ->setName($filename) - ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) - ->setMIMEType($mime_type) - ->setRelativeTTL(phutil_units('60 minutes in seconds')) - ->setAuthorPHID($viewer->getPHID()) - ->setIterator($iterator); - - $file = $source->uploadFile(); - - return $this->newDialog() - ->setTitle(pht('Download Results')) - ->appendParagraph( - pht('Click the download button to download the exported data.')) - ->addCancelButton($cancel_uri, pht('Done')) - ->setSubmitURI($file->getDownloadURI()) - ->setDisableWorkflowOnSubmit(true) - ->addSubmitButton(pht('Download Data')); } } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php index a5e29bc101..7854730bea 100644 --- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php @@ -25,4 +25,24 @@ abstract class PhabricatorWorkerBulkJobType extends Phobject { ->execute(); } + public function getCurtainActions( + PhabricatorUser $viewer, + PhabricatorWorkerBulkJob $job) { + + if ($job->isConfirming()) { + $continue_uri = $job->getMonitorURI(); + } else { + $continue_uri = $job->getDoneURI(); + } + + $continue = id(new PhabricatorActionView()) + ->setHref($continue_uri) + ->setIcon('fa-arrow-circle-o-right') + ->setName(pht('Continue')); + + return array( + $continue, + ); + } + } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php index 9117060da6..d11f7e81af 100644 --- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php @@ -77,6 +77,10 @@ abstract class PhabricatorWorkerBulkJobWorker pht('Job actor does not have permission to edit job.')); } + // Allow the worker to fill user caches inline; bulk jobs occasionally + // need to access user preferences. + $actor->setAllowInlineCacheGeneration(true); + return $actor; } diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php new file mode 100644 index 0000000000..f80411348c --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php @@ -0,0 +1,27 @@ +getPHID()); + + return $tasks; + } + +} diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php index 40e43c2ec7..312281617d 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php @@ -180,6 +180,10 @@ final class PhabricatorWorkerBulkJob return $this->getJobImplementation()->getJobName($this); } + public function getCurtainActions(PhabricatorUser $viewer) { + return $this->getJobImplementation()->getCurtainActions($viewer, $this); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/infrastructure/export/engine/PhabricatorExportEngine.php b/src/infrastructure/export/engine/PhabricatorExportEngine.php new file mode 100644 index 0000000000..f2c6a1270b --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportEngine.php @@ -0,0 +1,168 @@ +viewer = $viewer; + return $this; + } + + public function getViewer() { + return $this->viewer; + } + + public function setSearchEngine( + PhabricatorApplicationSearchEngine $search_engine) { + $this->searchEngine = $search_engine; + return $this; + } + + public function getSearchEngine() { + return $this->searchEngine; + } + + public function setSavedQuery(PhabricatorSavedQuery $saved_query) { + $this->savedQuery = $saved_query; + return $this; + } + + public function getSavedQuery() { + return $this->savedQuery; + } + + public function setExportFormat( + PhabricatorExportFormat $export_format) { + $this->exportFormat = $export_format; + return $this; + } + + public function getExportFormat() { + return $this->exportFormat; + } + + public function setFilename($filename) { + $this->filename = $filename; + return $this; + } + + public function getFilename() { + return $this->filename; + } + + public function setTitle($title) { + $this->title = $title; + return $this; + } + + public function getTitle() { + return $this->title; + } + + public function newBulkJob(AphrontRequest $request) { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $saved_query = $this->getSavedQuery(); + $format = $this->getExportFormat(); + + $params = array( + 'engineClass' => get_class($engine), + 'queryKey' => $saved_query->getQueryKey(), + 'formatKey' => $format->getExportFormatKey(), + 'title' => $this->getTitle(), + 'filename' => $this->getFilename(), + ); + + $job = PhabricatorWorkerBulkJob::initializeNewJob( + $viewer, + new PhabricatorExportEngineBulkJobType(), + $params); + + // We queue these jobs directly into STATUS_WAITING without requiring + // a confirmation from the user. + + $xactions = array(); + + $xactions[] = id(new PhabricatorWorkerBulkJobTransaction()) + ->setTransactionType(PhabricatorWorkerBulkJobTransaction::TYPE_STATUS) + ->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING); + + $editor = id(new PhabricatorWorkerBulkJobEditor()) + ->setActor($viewer) + ->setContentSourceFromRequest($request) + ->setContinueOnMissingFields(true) + ->applyTransactions($job, $xactions); + + return $job; + } + + public function exportFile() { + $viewer = $this->getViewer(); + $engine = $this->getSearchEngine(); + $saved_query = $this->getSavedQuery(); + $format = $this->getExportFormat(); + $title = $this->getTitle(); + $filename = $this->getFilename(); + + $query = $engine->buildQueryFromSavedQuery($saved_query); + + $extension = $format->getFileExtension(); + $mime_type = $format->getMIMEContentType(); + $filename = $filename.'.'.$extension; + + $format = id(clone $format) + ->setViewer($viewer) + ->setTitle($title); + + $field_list = $engine->newExportFieldList(); + $field_list = mpull($field_list, null, 'getKey'); + $format->addHeaders($field_list); + + // Iterate over the query results in large page so we don't have to hold + // too much stuff in memory. + $page_size = 1000; + $page_cursor = null; + do { + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize($page_size); + + if ($page_cursor !== null) { + $pager->setAfterID($page_cursor); + } + + $objects = $engine->executeQuery($query, $pager); + $objects = array_values($objects); + $page_cursor = $pager->getNextPageID(); + + $export_data = $engine->newExport($objects); + for ($ii = 0; $ii < count($objects); $ii++) { + $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + } + } while ($pager->getHasMoreResults()); + + $export_result = $format->newFileData(); + + // We have all the data in one big string and aren't actually + // streaming it, but pretending that we are allows us to actviate + // the chunk engine and store large files. + $iterator = new ArrayIterator(array($export_result)); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName($filename) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType($mime_type) + ->setRelativeTTL(phutil_units('60 minutes in seconds')) + ->setAuthorPHID($viewer->getPHID()) + ->setIterator($iterator); + + return $source->uploadFile(); + } + +} diff --git a/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php new file mode 100644 index 0000000000..712127f479 --- /dev/null +++ b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php @@ -0,0 +1,118 @@ +getParameter('filePHID'); + if (!$file_phid) { + $actions[] = id(new PhabricatorActionView()) + ->setHref('#') + ->setIcon('fa-download') + ->setDisabled(true) + ->setName(pht('Exporting Data...')); + } else { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if (!$file) { + $actions[] = id(new PhabricatorActionView()) + ->setHref('#') + ->setIcon('fa-download') + ->setDisabled(true) + ->setName(pht('Temporary File Expired')); + } else { + $actions[] = id(new PhabricatorActionView()) + ->setRenderAsForm(true) + ->setHref($file->getDownloadURI()) + ->setIcon('fa-download') + ->setName(pht('Download Data Export')); + } + } + + return $actions; + } + + + public function runTask( + PhabricatorUser $actor, + PhabricatorWorkerBulkJob $job, + PhabricatorWorkerBulkTask $task) { + + $engine_class = $job->getParameter('engineClass'); + if (!is_subclass_of($engine_class, 'PhabricatorApplicationSearchEngine')) { + throw new Exception( + pht( + 'Unknown search engine class "%s".', + $engine_class)); + } + + $engine = newv($engine_class, array()) + ->setViewer($actor); + + $query_key = $job->getParameter('queryKey'); + if ($engine->isBuiltinQuery($query_key)) { + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); + } else if ($query_key) { + $saved_query = id(new PhabricatorSavedQueryQuery()) + ->setViewer($actor) + ->withQueryKeys(array($query_key)) + ->executeOne(); + } else { + $saved_query = null; + } + + if (!$saved_query) { + throw new Exception( + pht( + 'Failed to load saved query ("%s").', + $query_key)); + } + + $format_key = $job->getParameter('formatKey'); + + $all_formats = PhabricatorExportFormat::getAllExportFormats(); + $format = idx($all_formats, $format_key); + if (!$format) { + throw new Exception( + pht( + 'Unknown export format ("%s").', + $format_key)); + } + + if (!$format->isExportFormatEnabled()) { + throw new Exception( + pht( + 'Export format ("%s") is not enabled.', + $format_key)); + } + + $export_engine = id(new PhabricatorExportEngine()) + ->setViewer($actor) + ->setTitle($job->getParameter('title')) + ->setFilename($job->getParameter('filename')) + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setExportFormat($format); + + $file = $export_engine->exportFile(); + + $job + ->setParameter('filePHID', $file->getPHID()) + ->save(); + } + +} From b27fd05eef0a20176574c7ff433ab59911738a77 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 05:39:09 -0800 Subject: [PATCH 20/89] Add a `bin/bulk export` CLI tool to make debugging and profiling large exports easier Summary: Ref T13049. When stuff executes asynchronously on the bulk workflow it can be hard to inspect directly, and/or a pain to test because you have to go through a bunch of steps to run it again. Make future work here easier by making export triggerable from the CLI. This makes it easy to repeat, inspect with `--trace`, profile with `--xprofile`, etc. Test Plan: - Ran several invalid commands, got sensible error messages. - Ran some valid commands, got exported data. - Used `--xprofile` to look at the profile for a 300MB dump of 100K tasks which took about 40 seconds to export. Nothing jumped out as sketchy to me -- CustomField wrangling is a little slow but most of the time looked like it was being spent legitimately. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18965 --- src/__phutil_library_map__.php | 2 + ...habricatorBulkManagementExportWorkflow.php | 168 ++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 06f619d74e..32985c76c0 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2226,6 +2226,7 @@ phutil_register_library_map(array( 'PhabricatorBulkContentSource' => 'infrastructure/daemon/contentsource/PhabricatorBulkContentSource.php', 'PhabricatorBulkEditGroup' => 'applications/transactions/bulk/PhabricatorBulkEditGroup.php', 'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php', + 'PhabricatorBulkManagementExportWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php', 'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php', 'PhabricatorCSVExportFormat' => 'infrastructure/export/format/PhabricatorCSVExportFormat.php', @@ -7579,6 +7580,7 @@ phutil_register_library_map(array( 'PhabricatorBulkContentSource' => 'PhabricatorContentSource', 'PhabricatorBulkEditGroup' => 'Phobject', 'PhabricatorBulkEngine' => 'Phobject', + 'PhabricatorBulkManagementExportWorkflow' => 'PhabricatorBulkManagementWorkflow', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'PhabricatorBulkManagementWorkflow', 'PhabricatorBulkManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorCSVExportFormat' => 'PhabricatorExportFormat', diff --git a/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php b/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php new file mode 100644 index 0000000000..cd0ed346fc --- /dev/null +++ b/src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php @@ -0,0 +1,168 @@ +setName('export') + ->setExamples('**export** [options]') + ->setSynopsis( + pht('Export data to a flat file (JSON, CSV, Excel, etc).')) + ->setArguments( + array( + array( + 'name' => 'class', + 'param' => 'class', + 'help' => pht( + 'SearchEngine class to export data from.'), + ), + array( + 'name' => 'format', + 'param' => 'format', + 'help' => pht('Export format.'), + ), + array( + 'name' => 'query', + 'param' => 'key', + 'help' => pht( + 'Export the data selected by this query.'), + ), + array( + 'name' => 'output', + 'param' => 'path', + 'help' => pht( + 'Write output to a file. If omitted, output will be sent to '. + 'stdout.'), + ), + array( + 'name' => 'overwrite', + 'help' => pht( + 'If the output file already exists, overwrite it instead of '. + 'raising an error.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $class = $args->getArg('class'); + + if (!strlen($class)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a search engine class to export data from with '. + '"--class".')); + } + + if (!is_subclass_of($class, 'PhabricatorApplicationSearchEngine')) { + throw new PhutilArgumentUsageException( + pht( + 'SearchEngine class ("%s") is unknown.', + $class)); + } + + $engine = newv($class, array()) + ->setViewer($viewer); + + if (!$engine->canExport()) { + throw new PhutilArgumentUsageException( + pht( + 'SearchEngine class ("%s") does not support data export.', + $class)); + } + + $query_key = $args->getArg('query'); + if (!strlen($query_key)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a query to export with "--query".')); + } + + if ($engine->isBuiltinQuery($query_key)) { + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); + } else if ($query_key) { + $saved_query = id(new PhabricatorSavedQueryQuery()) + ->setViewer($viewer) + ->withQueryKeys(array($query_key)) + ->executeOne(); + } else { + $saved_query = null; + } + + if (!$saved_query) { + throw new PhutilArgumentUsageException( + pht( + 'Failed to load saved query ("%s").', + $query_key)); + } + + $format_key = $args->getArg('format'); + if (!strlen($format_key)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify an export format with "--format".')); + } + + $all_formats = PhabricatorExportFormat::getAllExportFormats(); + $format = idx($all_formats, $format_key); + if (!$format) { + throw new PhutilArgumentUsageException( + pht( + 'Unknown export format ("%s"). Known formats are: %s.', + $format_key, + implode(', ', array_keys($all_formats)))); + } + + if (!$format->isExportFormatEnabled()) { + throw new PhutilArgumentUsageException( + pht( + 'Export format ("%s") is not enabled.', + $format_key)); + } + + $is_overwrite = $args->getArg('overwrite'); + $output_path = $args->getArg('output'); + + if (!strlen($output_path) && $is_overwrite) { + throw new PhutilArgumentUsageException( + pht( + 'Flag "--overwrite" has no effect without "--output".')); + } + + if (!$is_overwrite) { + if (Filesystem::pathExists($output_path)) { + throw new PhutilArgumentUsageException( + pht( + 'Output path already exists. Use "--overwrite" to overwrite '. + 'it.')); + } + } + + $export_engine = id(new PhabricatorExportEngine()) + ->setViewer($viewer) + ->setTitle(pht('Export')) + ->setFilename(pht('export')) + ->setSearchEngine($engine) + ->setSavedQuery($saved_query) + ->setExportFormat($format); + + $file = $export_engine->exportFile(); + + $iterator = $file->getFileDataIterator(); + + if (strlen($output_path)) { + foreach ($iterator as $chunk) { + Filesystem::appendFile($output_path, $chunk); + } + } else { + foreach ($iterator as $chunk) { + echo $chunk; + } + } + + return 0; + } + +} From 91108cf83826c4587adcad73008b309669b49dee Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 06:09:35 -0800 Subject: [PATCH 21/89] Upgrade user account activity logs to modern construction Summary: Depends on D18965. Ref T13049. Move this Query and SearchEngine to be a little more modern, to prepare for Export support. Test Plan: - Used all the query fields, viewed activity logs via People and Settings. - I'm not sure the "Session" query is used/useful and may remove it before I'm done here, but I just left it in place for now. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18966 --- .../query/PhabricatorPeopleLogQuery.php | 41 ++--- .../PhabricatorPeopleLogSearchEngine.php | 157 ++++++------------ .../people/view/PhabricatorUserLogView.php | 15 +- .../PhabricatorActivitySettingsPanel.php | 3 +- 4 files changed, 71 insertions(+), 145 deletions(-) diff --git a/src/applications/people/query/PhabricatorPeopleLogQuery.php b/src/applications/people/query/PhabricatorPeopleLogQuery.php index 9bcdc53f49..e6e2506bee 100644 --- a/src/applications/people/query/PhabricatorPeopleLogQuery.php +++ b/src/applications/people/query/PhabricatorPeopleLogQuery.php @@ -40,70 +40,61 @@ final class PhabricatorPeopleLogQuery return $this; } - protected function loadPage() { - $table = new PhabricatorUserLog(); - $conn_r = $table->establishConnection('r'); - - $data = queryfx_all( - $conn_r, - 'SELECT * FROM %T %Q %Q %Q', - $table->getTableName(), - $this->buildWhereClause($conn_r), - $this->buildOrderClause($conn_r), - $this->buildLimitClause($conn_r)); - - return $table->loadAllFromArray($data); + public function newResultObject() { + return new PhabricatorUserLog(); } - protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { - $where = array(); + protected function loadPage() { + return $this->loadStandardPage($this->newResultObject()); + } + + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); if ($this->actorPHIDs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'actorPHID IN (%Ls)', $this->actorPHIDs); } if ($this->userPHIDs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'userPHID IN (%Ls)', $this->userPHIDs); } if ($this->relatedPHIDs !== null) { $where[] = qsprintf( - $conn_r, - 'actorPHID IN (%Ls) OR userPHID IN (%Ls)', + $conn, + '(actorPHID IN (%Ls) OR userPHID IN (%Ls))', $this->relatedPHIDs, $this->relatedPHIDs); } if ($this->sessionKeys !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'session IN (%Ls)', $this->sessionKeys); } if ($this->actions !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'action IN (%Ls)', $this->actions); } if ($this->remoteAddressPrefix !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'remoteAddr LIKE %>', $this->remoteAddressPrefix); } - $where[] = $this->buildPagingClause($conn_r); - - return $this->formatWhereClause($where); + return $where; } public function getQueryApplicationClass() { diff --git a/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php b/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php index 851ef113d0..aea576693f 100644 --- a/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php +++ b/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php @@ -15,34 +15,8 @@ final class PhabricatorPeopleLogSearchEngine return 500; } - public function buildSavedQueryFromRequest(AphrontRequest $request) { - $saved = new PhabricatorSavedQuery(); - - $saved->setParameter( - 'userPHIDs', - $this->readUsersFromRequest($request, 'users')); - - $saved->setParameter( - 'actorPHIDs', - $this->readUsersFromRequest($request, 'actors')); - - $saved->setParameter( - 'actions', - $this->readListFromRequest($request, 'actions')); - - $saved->setParameter( - 'ip', - $request->getStr('ip')); - - $saved->setParameter( - 'sessions', - $this->readListFromRequest($request, 'sessions')); - - return $saved; - } - - public function buildQueryFromSavedQuery(PhabricatorSavedQuery $saved) { - $query = id(new PhabricatorPeopleLogQuery()); + public function newQuery() { + $query = new PhabricatorPeopleLogQuery(); // NOTE: If the viewer isn't an administrator, always restrict the query to // related records. This echoes the policy logic of these logs. This is @@ -54,82 +28,61 @@ final class PhabricatorPeopleLogSearchEngine $query->withRelatedPHIDs(array($viewer->getPHID())); } - $actor_phids = $saved->getParameter('actorPHIDs', array()); - if ($actor_phids) { - $query->withActorPHIDs($actor_phids); + return $query; + } + + protected function buildQueryFromParameters(array $map) { + $query = $this->newQuery(); + + if ($map['userPHIDs']) { + $query->withUserPHIDs($map['userPHIDs']); } - $user_phids = $saved->getParameter('userPHIDs', array()); - if ($user_phids) { - $query->withUserPHIDs($user_phids); + if ($map['actorPHIDs']) { + $query->withActorPHIDs($map['actorPHIDs']); } - $actions = $saved->getParameter('actions', array()); - if ($actions) { - $query->withActions($actions); + if ($map['actions']) { + $query->withActions($map['actions']); } - $remote_prefix = $saved->getParameter('ip'); - if (strlen($remote_prefix)) { - $query->withRemoteAddressprefix($remote_prefix); + if (strlen($map['ip'])) { + $query->withRemoteAddressPrefix($map['ip']); } - $sessions = $saved->getParameter('sessions', array()); - if ($sessions) { - $query->withSessionKeys($sessions); + if ($map['sessions']) { + $query->withSessionKeys($map['sessions']); } return $query; } - public function buildSearchForm( - AphrontFormView $form, - PhabricatorSavedQuery $saved) { - - $actor_phids = $saved->getParameter('actorPHIDs', array()); - $user_phids = $saved->getParameter('userPHIDs', array()); - - $actions = $saved->getParameter('actions', array()); - $remote_prefix = $saved->getParameter('ip'); - $sessions = $saved->getParameter('sessions', array()); - - $actions = array_fuse($actions); - $action_control = id(new AphrontFormCheckboxControl()) - ->setLabel(pht('Actions')); - $action_types = PhabricatorUserLog::getActionTypeMap(); - foreach ($action_types as $type => $label) { - $action_control->addCheckbox( - 'actions[]', - $type, - $label, - isset($actions[$label])); - } - - $form - ->appendControl( - id(new AphrontFormTokenizerControl()) - ->setDatasource(new PhabricatorPeopleDatasource()) - ->setName('actors') - ->setLabel(pht('Actors')) - ->setValue($actor_phids)) - ->appendControl( - id(new AphrontFormTokenizerControl()) - ->setDatasource(new PhabricatorPeopleDatasource()) - ->setName('users') - ->setLabel(pht('Users')) - ->setValue($user_phids)) - ->appendChild($action_control) - ->appendChild( - id(new AphrontFormTextControl()) - ->setLabel(pht('Filter IP')) - ->setName('ip') - ->setValue($remote_prefix)) - ->appendChild( - id(new AphrontFormTextControl()) - ->setLabel(pht('Sessions')) - ->setName('sessions') - ->setValue(implode(', ', $sessions))); - + protected function buildCustomSearchFields() { + return array( + id(new PhabricatorUsersSearchField()) + ->setKey('userPHIDs') + ->setAliases(array('users', 'user', 'userPHID')) + ->setLabel(pht('Users')) + ->setDescription(pht('Search for activity affecting specific users.')), + id(new PhabricatorUsersSearchField()) + ->setKey('actorPHIDs') + ->setAliases(array('actors', 'actor', 'actorPHID')) + ->setLabel(pht('Actors')) + ->setDescription(pht('Search for activity by specific users.')), + id(new PhabricatorSearchCheckboxesField()) + ->setKey('actions') + ->setLabel(pht('Actions')) + ->setDescription(pht('Search for particular types of activity.')) + ->setOptions(PhabricatorUserLog::getActionTypeMap()), + id(new PhabricatorSearchTextField()) + ->setKey('ip') + ->setLabel(pht('Filter IP')) + ->setDescription(pht('Search for actions by remote address.')), + id(new PhabricatorSearchStringListField()) + ->setKey('sessions') + ->setLabel(pht('Sessions')) + ->setDescription(pht('Search for activity in particular sessions.')), + ); } protected function getURI($path) { @@ -156,19 +109,6 @@ final class PhabricatorPeopleLogSearchEngine return parent::buildSavedQueryFromBuiltin($query_key); } - protected function getRequiredHandlePHIDsForResultList( - array $logs, - PhabricatorSavedQuery $query) { - - $phids = array(); - foreach ($logs as $log) { - $phids[$log->getActorPHID()] = true; - $phids[$log->getUserPHID()] = true; - } - - return array_keys($phids); - } - protected function renderResultList( array $logs, PhabricatorSavedQuery $query, @@ -179,16 +119,13 @@ final class PhabricatorPeopleLogSearchEngine $table = id(new PhabricatorUserLogView()) ->setUser($viewer) - ->setLogs($logs) - ->setHandles($handles); + ->setLogs($logs); if ($viewer->getIsAdmin()) { $table->setSearchBaseURI($this->getApplicationURI('logs/')); } - $result = new PhabricatorApplicationSearchResultView(); - $result->setTable($table); - - return $result; + return id(new PhabricatorApplicationSearchResultView()) + ->setTable($table); } } diff --git a/src/applications/people/view/PhabricatorUserLogView.php b/src/applications/people/view/PhabricatorUserLogView.php index d648b248f7..309540c611 100644 --- a/src/applications/people/view/PhabricatorUserLogView.php +++ b/src/applications/people/view/PhabricatorUserLogView.php @@ -3,7 +3,6 @@ final class PhabricatorUserLogView extends AphrontView { private $logs; - private $handles; private $searchBaseURI; public function setSearchBaseURI($search_base_uri) { @@ -17,17 +16,17 @@ final class PhabricatorUserLogView extends AphrontView { return $this; } - public function setHandles(array $handles) { - assert_instances_of($handles, 'PhabricatorObjectHandle'); - $this->handles = $handles; - return $this; - } - public function render() { $logs = $this->logs; - $handles = $this->handles; $viewer = $this->getUser(); + $phids = array(); + foreach ($logs as $log) { + $phids[] = $log->getActorPHID(); + $phids[] = $log->getUserPHID(); + } + $handles = $viewer->loadHandles($phids); + $action_map = PhabricatorUserLog::getActionTypeMap(); $base_uri = $this->searchBaseURI; diff --git a/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php b/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php index 50f951d661..eeeca27663 100644 --- a/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php +++ b/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php @@ -43,8 +43,7 @@ final class PhabricatorActivitySettingsPanel extends PhabricatorSettingsPanel { $table = id(new PhabricatorUserLogView()) ->setUser($viewer) - ->setLogs($logs) - ->setHandles($handles); + ->setLogs($logs); $panel = $this->newBox(pht('Account Activity Logs'), $table); From a5b8be0316ce617208072a3fa402cf61e90b0402 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 06:27:36 -0800 Subject: [PATCH 22/89] Support export of user activity logs Summary: Depends on D18966. Ref T13049. Adds export support to user activity logs. These don't have PHIDs. We could add them, but just make the "phid" column test if the objects have PHIDs or not for now. Test Plan: - Exported user activity logs, got sensible output (with no PHIDs). - Exported some users to make sure I didn't break PHIDs, got an export with PHIDs. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18967 --- .../PhabricatorPeopleApplication.php | 5 +- .../PhabricatorPeopleLogSearchEngine.php | 98 +++++++++++++++++++ .../PhabricatorApplicationSearchEngine.php | 23 ++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/src/applications/people/application/PhabricatorPeopleApplication.php b/src/applications/people/application/PhabricatorPeopleApplication.php index 28405ca92c..6322b29b24 100644 --- a/src/applications/people/application/PhabricatorPeopleApplication.php +++ b/src/applications/people/application/PhabricatorPeopleApplication.php @@ -42,8 +42,9 @@ final class PhabricatorPeopleApplication extends PhabricatorApplication { return array( '/people/' => array( $this->getQueryRoutePattern() => 'PhabricatorPeopleListController', - 'logs/(?:query/(?P[^/]+)/)?' - => 'PhabricatorPeopleLogsController', + 'logs/' => array( + $this->getQueryRoutePattern() => 'PhabricatorPeopleLogsController', + ), 'invite/' => array( '(?:query/(?P[^/]+)/)?' => 'PhabricatorPeopleInviteListController', diff --git a/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php b/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php index aea576693f..1a3ae4dea1 100644 --- a/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php +++ b/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php @@ -128,4 +128,102 @@ final class PhabricatorPeopleLogSearchEngine return id(new PhabricatorApplicationSearchResultView()) ->setTable($table); } + + protected function newExportFields() { + $viewer = $this->requireViewer(); + + $fields = array( + $fields[] = id(new PhabricatorPHIDExportField()) + ->setKey('actorPHID') + ->setLabel(pht('Actor PHID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('actor') + ->setLabel(pht('Actor')), + $fields[] = id(new PhabricatorPHIDExportField()) + ->setKey('userPHID') + ->setLabel(pht('User PHID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('user') + ->setLabel(pht('User')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('action') + ->setLabel(pht('Action')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('actionName') + ->setLabel(pht('Action Name')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('session') + ->setLabel(pht('Session')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('old') + ->setLabel(pht('Old Value')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('new') + ->setLabel(pht('New Value')), + ); + + if ($viewer->getIsAdmin()) { + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('remoteAddress') + ->setLabel(pht('Remote Address')); + } + + return $fields; + } + + protected function newExportData(array $logs) { + $viewer = $this->requireViewer(); + + + $phids = array(); + foreach ($logs as $log) { + $phids[] = $log->getUserPHID(); + $phids[] = $log->getActorPHID(); + } + $handles = $viewer->loadHandles($phids); + + $action_map = PhabricatorUserLog::getActionTypeMap(); + + $export = array(); + foreach ($logs as $log) { + + $user_phid = $log->getUserPHID(); + if ($user_phid) { + $user_name = $handles[$user_phid]->getName(); + } else { + $user_name = null; + } + + $actor_phid = $log->getActorPHID(); + if ($actor_phid) { + $actor_name = $handles[$actor_phid]->getName(); + } else { + $actor_name = null; + } + + $action = $log->getAction(); + $action_name = idx($action_map, $action, pht('Unknown ("%s")', $action)); + + $map = array( + 'actorPHID' => $actor_phid, + 'actor' => $actor_name, + 'userPHID' => $user_phid, + 'user' => $user_name, + 'action' => $action, + 'actionName' => $action_name, + 'session' => substr($log->getSession(), 0, 6), + 'old' => $log->getOldValue(), + 'new' => $log->getNewValue(), + ); + + if ($viewer->getIsAdmin()) { + $map['remoteAddress'] = $log->getRemoteAddr(); + } + + $export[] = $map; + } + + return $export; + } + } diff --git a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php index 3de7b9c4b9..b808291a52 100644 --- a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php +++ b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php @@ -1455,15 +1455,20 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { } final public function newExportFieldList() { + $object = $this->newResultObject(); + $builtin_fields = array( id(new PhabricatorIDExportField()) ->setKey('id') ->setLabel(pht('ID')), - id(new PhabricatorPHIDExportField()) - ->setKey('phid') - ->setLabel(pht('PHID')), ); + if ($object->getConfigOption(LiskDAO::CONFIG_AUX_PHID)) { + $builtin_fields[] = id(new PhabricatorPHIDExportField()) + ->setKey('phid') + ->setLabel(pht('PHID')); + } + $fields = mpull($builtin_fields, null, 'getKey'); $export_fields = $this->newExportFields(); @@ -1507,15 +1512,23 @@ abstract class PhabricatorApplicationSearchEngine extends Phobject { } final public function newExport(array $objects) { + $object = $this->newResultObject(); + $has_phid = $object->getConfigOption(LiskDAO::CONFIG_AUX_PHID); + $objects = array_values($objects); $n = count($objects); $maps = array(); foreach ($objects as $object) { - $maps[] = array( + $map = array( 'id' => $object->getID(), - 'phid' => $object->getPHID(), ); + + if ($has_phid) { + $map['phid'] = $object->getPHID(); + } + + $maps[] = $map; } $export_data = $this->newExportData($objects); From 5b22412f246e74abf0d95d497d416466be488a8f Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 07:47:34 -0800 Subject: [PATCH 23/89] Support data export on push logs Summary: Depends on D18967. Ref T13049. Nothing too fancy going on here. Test Plan: Exported push logs, looked at the export, seemed sensible. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18968 --- .../PhabricatorDiffusionApplication.php | 2 +- .../PhabricatorRepositoryPushLogQuery.php | 53 +++---- ...abricatorRepositoryPushLogSearchEngine.php | 142 ++++++++++++++++++ .../storage/PhabricatorRepositoryPushLog.php | 22 +++ 4 files changed, 187 insertions(+), 32 deletions(-) diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php index d932149a75..e619ecb1ad 100644 --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -121,7 +121,7 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication { $this->getEditRoutePattern('edit/') => 'DiffusionRepositoryEditController', 'pushlog/' => array( - '(?:query/(?P[^/]+)/)?' => 'DiffusionPushLogListController', + $this->getQueryRoutePattern() => 'DiffusionPushLogListController', 'view/(?P\d+)/' => 'DiffusionPushEventViewController', ), 'pulllog/' => array( diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php b/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php index 94a0b6922e..2c5f7b0d0d 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php @@ -46,19 +46,12 @@ final class PhabricatorRepositoryPushLogQuery return $this; } + public function newResultObject() { + return new PhabricatorRepositoryPushLog(); + } + protected function loadPage() { - $table = new PhabricatorRepositoryPushLog(); - $conn_r = $table->establishConnection('r'); - - $data = queryfx_all( - $conn_r, - 'SELECT * FROM %T %Q %Q %Q', - $table->getTableName(), - $this->buildWhereClause($conn_r), - $this->buildOrderClause($conn_r), - $this->buildLimitClause($conn_r)); - - return $table->loadAllFromArray($data); + return $this->loadStandardPage($this->newResultObject()); } protected function willFilterPage(array $logs) { @@ -82,61 +75,59 @@ final class PhabricatorRepositoryPushLogQuery return $logs; } - protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { - $where = array(); + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); - if ($this->ids) { + if ($this->ids !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'id IN (%Ld)', $this->ids); } - if ($this->phids) { + if ($this->phids !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'phid IN (%Ls)', $this->phids); } - if ($this->repositoryPHIDs) { + if ($this->repositoryPHIDs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'repositoryPHID IN (%Ls)', $this->repositoryPHIDs); } - if ($this->pusherPHIDs) { + if ($this->pusherPHIDs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'pusherPHID in (%Ls)', $this->pusherPHIDs); } - if ($this->pushEventPHIDs) { + if ($this->pushEventPHIDs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'pushEventPHID in (%Ls)', $this->pushEventPHIDs); } - if ($this->refTypes) { + if ($this->refTypes !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'refType IN (%Ls)', $this->refTypes); } - if ($this->newRefs) { + if ($this->newRefs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'refNew IN (%Ls)', $this->newRefs); } - $where[] = $this->buildPagingClause($conn_r); - - return $this->formatWhereClause($where); + return $where; } public function getQueryApplicationClass() { diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php index d171b80999..b2234ab6c7 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php @@ -82,4 +82,146 @@ final class PhabricatorRepositoryPushLogSearchEngine ->setTable($table); } + protected function newExportFields() { + $viewer = $this->requireViewer(); + + $fields = array( + $fields[] = id(new PhabricatorIDExportField()) + ->setKey('pushID') + ->setLabel(pht('Push ID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('protocol') + ->setLabel(pht('Protocol')), + $fields[] = id(new PhabricatorPHIDExportField()) + ->setKey('repositoryPHID') + ->setLabel(pht('Repository PHID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('repository') + ->setLabel(pht('Repository')), + $fields[] = id(new PhabricatorPHIDExportField()) + ->setKey('pusherPHID') + ->setLabel(pht('Pusher PHID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('pusher') + ->setLabel(pht('Pusher')), + $fields[] = id(new PhabricatorPHIDExportField()) + ->setKey('devicePHID') + ->setLabel(pht('Device PHID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('device') + ->setLabel(pht('Device')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('type') + ->setLabel(pht('Ref Type')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('name') + ->setLabel(pht('Ref Name')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('old') + ->setLabel(pht('Ref Old')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('new') + ->setLabel(pht('Ref New')), + $fields[] = id(new PhabricatorIntExportField()) + ->setKey('flags') + ->setLabel(pht('Flags')), + $fields[] = id(new PhabricatorStringListExportField()) + ->setKey('flagNames') + ->setLabel(pht('Flag Names')), + $fields[] = id(new PhabricatorIntExportField()) + ->setKey('result') + ->setLabel(pht('Result')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('resultName') + ->setLabel(pht('Result Name')), + ); + + if ($viewer->getIsAdmin()) { + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('remoteAddress') + ->setLabel(pht('Remote Address')); + } + + return $fields; + } + + protected function newExportData(array $logs) { + $viewer = $this->requireViewer(); + + $phids = array(); + foreach ($logs as $log) { + $phids[] = $log->getPusherPHID(); + $phids[] = $log->getDevicePHID(); + $phids[] = $log->getPushEvent()->getRepositoryPHID(); + } + $handles = $viewer->loadHandles($phids); + + $flag_map = PhabricatorRepositoryPushLog::getFlagDisplayNames(); + $reject_map = PhabricatorRepositoryPushLog::getRejectCodeDisplayNames(); + + $export = array(); + foreach ($logs as $log) { + $event = $log->getPushEvent(); + + $repository_phid = $event->getRepositoryPHID(); + if ($repository_phid) { + $repository_name = $handles[$repository_phid]->getName(); + } else { + $repository_name = null; + } + + $pusher_phid = $log->getPusherPHID(); + if ($pusher_phid) { + $pusher_name = $handles[$pusher_phid]->getName(); + } else { + $pusher_name = null; + } + + $device_phid = $log->getDevicePHID(); + if ($device_phid) { + $device_name = $handles[$device_phid]->getName(); + } else { + $device_name = null; + } + + $flags = $log->getChangeFlags(); + $flag_names = array(); + foreach ($flag_map as $flag_key => $flag_name) { + if (($flags & $flag_key) === $flag_key) { + $flag_names[] = $flag_name; + } + } + + $result = $event->getRejectCode(); + $result_name = idx($reject_map, $result, pht('Unknown ("%s")', $result)); + + $map = array( + 'pushID' => $event->getID(), + 'protocol' => $event->getRemoteProtocol(), + 'repositoryPHID' => $repository_phid, + 'repository' => $repository_name, + 'pusherPHID' => $pusher_phid, + 'pusher' => $pusher_name, + 'devicePHID' => $device_phid, + 'device' => $device_name, + 'type' => $log->getRefType(), + 'name' => $log->getRefName(), + 'old' => $log->getRefOld(), + 'new' => $log->getRefNew(), + 'flags' => $flags, + 'flagNames' => $flag_names, + 'result' => $result, + 'resultName' => $result_name, + ); + + if ($viewer->getIsAdmin()) { + $map['remoteAddress'] = $event->getRemoteAddress(); + } + + $export[] = $map; + } + + return $export; + } + } diff --git a/src/applications/repository/storage/PhabricatorRepositoryPushLog.php b/src/applications/repository/storage/PhabricatorRepositoryPushLog.php index 4e099209c6..81a564a91f 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryPushLog.php +++ b/src/applications/repository/storage/PhabricatorRepositoryPushLog.php @@ -55,6 +55,28 @@ final class PhabricatorRepositoryPushLog ->setPusherPHID($viewer->getPHID()); } + public static function getFlagDisplayNames() { + return array( + self::CHANGEFLAG_ADD => pht('Create'), + self::CHANGEFLAG_DELETE => pht('Delete'), + self::CHANGEFLAG_APPEND => pht('Append'), + self::CHANGEFLAG_REWRITE => pht('Rewrite'), + self::CHANGEFLAG_DANGEROUS => pht('Dangerous'), + self::CHANGEFLAG_ENORMOUS => pht('Enormous'), + ); + } + + public static function getRejectCodeDisplayNames() { + return array( + self::REJECT_ACCEPT => pht('Accepted'), + self::REJECT_DANGEROUS => pht('Rejected: Dangerous'), + self::REJECT_HERALD => pht('Rejected: Herald'), + self::REJECT_EXTERNAL => pht('Rejected: External Hook'), + self::REJECT_BROKEN => pht('Rejected: Broken'), + self::REJECT_ENORMOUS => pht('Rejected: Enormous'), + ); + } + public static function getHeraldChangeFlagConditionOptions() { return array( self::CHANGEFLAG_ADD => From 0d5379ee1778fd049216c171382b1dbb0173e900 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 08:03:29 -0800 Subject: [PATCH 24/89] Fix an export bug where queries specified in the URI ("?param=value") were ignored when filtering the result set Summary: Depends on D18968. Ref T13049. Currently, if you visit `/query/?param=value`, there is no `queryKey` for the page but we build a query later on. Right now, we incorrectly link to `/query/all/export/` in this case (and export too many results), but we should actually link to `/query//export/` to export only the desired/previewed results. Swap the logic around a little bit so we look at the query we're actually executing, not the original URI, to figure out the query key we should use when building the link. Test Plan: Visited a `/?param=value` page, exported data, got a subset of the full data instead of everything. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18969 --- .../PhabricatorApplicationSearchController.php | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index 6c7d80eafe..cc8bcefff1 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -7,6 +7,7 @@ final class PhabricatorApplicationSearchController private $navigation; private $queryKey; private $preface; + private $activeQuery; public function setPreface($preface) { $this->preface = $preface; @@ -45,6 +46,14 @@ final class PhabricatorApplicationSearchController return $this->searchEngine; } + protected function getActiveQuery() { + if (!$this->activeQuery) { + throw new Exception(pht('There is no active query yet.')); + } + + return $this->activeQuery; + } + protected function validateDelegatingController() { $parent = $this->getDelegatingController(); @@ -158,6 +167,8 @@ final class PhabricatorApplicationSearchController $engine->saveQuery($saved_query); } + $this->activeQuery = $saved_query; + $nav->selectFilter( 'query/'.$saved_query->getQueryKey(), 'query/advanced'); @@ -867,10 +878,8 @@ final class PhabricatorApplicationSearchController $engine = $this->getSearchEngine(); $engine_class = get_class($engine); - $query_key = $this->getQueryKey(); - if (!$query_key) { - $query_key = $engine->getDefaultQueryKey(); - } + + $query_key = $this->getActiveQuery()->getQueryKey(); $can_use = $engine->canUseInPanelContext(); $is_installed = PhabricatorApplication::isClassInstalledForViewer( From 75bc86589f10dd520bba41dc52bd3458e22b1f6a Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 11:47:55 -0800 Subject: [PATCH 25/89] Add date range filtering for activity, push, and pull logs Summary: Ref T13049. This is just a general nice-to-have so you don't have to export a 300MB file if you want to check the last month of data or whatever. Test Plan: Applied filters to all three logs, got appropriate date-range result sets. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18970 --- .../query/DiffusionPullLogSearchEngine.php | 12 ++++++++++ .../query/PhabricatorPeopleLogQuery.php | 22 +++++++++++++++++++ .../PhabricatorPeopleLogSearchEngine.php | 12 ++++++++++ .../PhabricatorRepositoryPullEventQuery.php | 22 +++++++++++++++++++ .../PhabricatorRepositoryPushLogQuery.php | 22 +++++++++++++++++++ ...abricatorRepositoryPushLogSearchEngine.php | 12 ++++++++++ .../storage/PhabricatorRepositoryPushLog.php | 3 +++ 7 files changed, 105 insertions(+) diff --git a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php index fc7cee52ce..db5d4b9fcb 100644 --- a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php +++ b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php @@ -26,6 +26,12 @@ final class DiffusionPullLogSearchEngine $query->withPullerPHIDs($map['pullerPHIDs']); } + if ($map['createdStart'] || $map['createdEnd']) { + $query->withEpochBetween( + $map['createdStart'], + $map['createdEnd']); + } + return $query; } @@ -44,6 +50,12 @@ final class DiffusionPullLogSearchEngine ->setLabel(pht('Pullers')) ->setDescription( pht('Search for pull logs by specific users.')), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Created After')) + ->setKey('createdStart'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Created Before')) + ->setKey('createdEnd'), ); } diff --git a/src/applications/people/query/PhabricatorPeopleLogQuery.php b/src/applications/people/query/PhabricatorPeopleLogQuery.php index e6e2506bee..fc6a87b335 100644 --- a/src/applications/people/query/PhabricatorPeopleLogQuery.php +++ b/src/applications/people/query/PhabricatorPeopleLogQuery.php @@ -9,6 +9,8 @@ final class PhabricatorPeopleLogQuery private $sessionKeys; private $actions; private $remoteAddressPrefix; + private $dateCreatedMin; + private $dateCreatedMax; public function withActorPHIDs(array $actor_phids) { $this->actorPHIDs = $actor_phids; @@ -40,6 +42,12 @@ final class PhabricatorPeopleLogQuery return $this; } + public function withDateCreatedBetween($min, $max) { + $this->dateCreatedMin = $min; + $this->dateCreatedMax = $max; + return $this; + } + public function newResultObject() { return new PhabricatorUserLog(); } @@ -94,6 +102,20 @@ final class PhabricatorPeopleLogQuery $this->remoteAddressPrefix); } + if ($this->dateCreatedMin !== null) { + $where[] = qsprintf( + $conn, + 'dateCreated >= %d', + $this->dateCreatedMin); + } + + if ($this->dateCreatedMax !== null) { + $where[] = qsprintf( + $conn, + 'dateCreated <= %d', + $this->dateCreatedMax); + } + return $where; } diff --git a/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php b/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php index 1a3ae4dea1..b052456cd3 100644 --- a/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php +++ b/src/applications/people/query/PhabricatorPeopleLogSearchEngine.php @@ -54,6 +54,12 @@ final class PhabricatorPeopleLogSearchEngine $query->withSessionKeys($map['sessions']); } + if ($map['createdStart'] || $map['createdEnd']) { + $query->withDateCreatedBetween( + $map['createdStart'], + $map['createdEnd']); + } + return $query; } @@ -82,6 +88,12 @@ final class PhabricatorPeopleLogSearchEngine ->setKey('sessions') ->setLabel(pht('Sessions')) ->setDescription(pht('Search for activity in particular sessions.')), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Created After')) + ->setKey('createdStart'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Created Before')) + ->setKey('createdEnd'), ); } diff --git a/src/applications/repository/query/PhabricatorRepositoryPullEventQuery.php b/src/applications/repository/query/PhabricatorRepositoryPullEventQuery.php index af60ee0383..ce14a6f831 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPullEventQuery.php +++ b/src/applications/repository/query/PhabricatorRepositoryPullEventQuery.php @@ -7,6 +7,8 @@ final class PhabricatorRepositoryPullEventQuery private $phids; private $repositoryPHIDs; private $pullerPHIDs; + private $epochMin; + private $epochMax; public function withIDs(array $ids) { $this->ids = $ids; @@ -28,6 +30,12 @@ final class PhabricatorRepositoryPullEventQuery return $this; } + public function withEpochBetween($min, $max) { + $this->epochMin = $min; + $this->epochMax = $max; + return $this; + } + public function newResultObject() { return new PhabricatorRepositoryPullEvent(); } @@ -103,6 +111,20 @@ final class PhabricatorRepositoryPullEventQuery $this->pullerPHIDs); } + if ($this->epochMin !== null) { + $where[] = qsprintf( + $conn, + 'epoch >= %d', + $this->epochMin); + } + + if ($this->epochMax !== null) { + $where[] = qsprintf( + $conn, + 'epoch <= %d', + $this->epochMax); + } + return $where; } diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php b/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php index 2c5f7b0d0d..cb097fae2f 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php @@ -10,6 +10,8 @@ final class PhabricatorRepositoryPushLogQuery private $refTypes; private $newRefs; private $pushEventPHIDs; + private $epochMin; + private $epochMax; public function withIDs(array $ids) { $this->ids = $ids; @@ -46,6 +48,12 @@ final class PhabricatorRepositoryPushLogQuery return $this; } + public function withEpochBetween($min, $max) { + $this->epochMin = $min; + $this->epochMax = $max; + return $this; + } + public function newResultObject() { return new PhabricatorRepositoryPushLog(); } @@ -127,6 +135,20 @@ final class PhabricatorRepositoryPushLogQuery $this->newRefs); } + if ($this->epochMin !== null) { + $where[] = qsprintf( + $conn, + 'epoch >= %d', + $this->epochMin); + } + + if ($this->epochMax !== null) { + $where[] = qsprintf( + $conn, + 'epoch <= %d', + $this->epochMax); + } + return $where; } diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php index b2234ab6c7..8ad76987f9 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php @@ -26,6 +26,12 @@ final class PhabricatorRepositoryPushLogSearchEngine $query->withPusherPHIDs($map['pusherPHIDs']); } + if ($map['createdStart'] || $map['createdEnd']) { + $query->withEpochBetween( + $map['createdStart'], + $map['createdEnd']); + } + return $query; } @@ -44,6 +50,12 @@ final class PhabricatorRepositoryPushLogSearchEngine ->setLabel(pht('Pushers')) ->setDescription( pht('Search for pull logs by specific users.')), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Created After')) + ->setKey('createdStart'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Created Before')) + ->setKey('createdEnd'), ); } diff --git a/src/applications/repository/storage/PhabricatorRepositoryPushLog.php b/src/applications/repository/storage/PhabricatorRepositoryPushLog.php index 81a564a91f..c2d3456da6 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryPushLog.php +++ b/src/applications/repository/storage/PhabricatorRepositoryPushLog.php @@ -124,6 +124,9 @@ final class PhabricatorRepositoryPushLog 'key_pusher' => array( 'columns' => array('pusherPHID'), ), + 'key_epoch' => array( + 'columns' => array('epoch'), + ), ), ) + parent::getConfiguration(); } From 8a2863e3f7f8de79fbdd25eb3dc05605b0be2784 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 12:01:26 -0800 Subject: [PATCH 26/89] Change the "can see remote address?" policy to "is administrator?" everywhere Summary: Depends on D18970. Ref T13049. Currently, the policy for viewing remote addresses is: - In activity logs: administrators. - In push and pull logs: users who can edit the corresponding repository. This sort of makes sense, but is also sort of weird. Particularly, I think it's kind of hard to understand and predict, and hard to guess that this is the behavior we implement. The actual implementation is complex, too. Instead, just use the rule "administrators can see remote addresses" consistently across all applications. This should generally be more strict than the old rule, because administrators could usually have seen everyone's address in the activity logs anyway. It's also simpler and more expected, and I don't really know of any legit use cases for the "repository editor" rule. Test Plan: Viewed pull/push/activity logs as non-admin. Saw remote addresses as an admin, and none as a non-admin. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18971 --- .../query/DiffusionPullLogSearchEngine.php | 20 +++++++++-- .../view/DiffusionPullLogListView.php | 36 +++++++------------ .../view/DiffusionPushLogListView.php | 25 ++++--------- 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php index db5d4b9fcb..dfdfceb519 100644 --- a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php +++ b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php @@ -60,7 +60,9 @@ final class DiffusionPullLogSearchEngine } protected function newExportFields() { - return array( + $viewer = $this->requireViewer(); + + $fields = array( id(new PhabricatorPHIDExportField()) ->setKey('repositoryPHID') ->setLabel(pht('Repository PHID')), @@ -86,6 +88,14 @@ final class DiffusionPullLogSearchEngine ->setKey('date') ->setLabel(pht('Date')), ); + + if ($viewer->getIsAdmin()) { + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('remoteAddress') + ->setLabel(pht('Remote Address')); + } + + return $fields; } protected function newExportData(array $events) { @@ -117,7 +127,7 @@ final class DiffusionPullLogSearchEngine $puller_name = null; } - $export[] = array( + $map = array( 'repositoryPHID' => $repository_phid, 'repository' => $repository_name, 'pullerPHID' => $puller_phid, @@ -127,6 +137,12 @@ final class DiffusionPullLogSearchEngine 'code' => $event->getResultCode(), 'date' => $event->getEpoch(), ); + + if ($viewer->getIsAdmin()) { + $map['remoteAddress'] = $event->getRemoteAddress(); + } + + $export[] = $map; } return $export; diff --git a/src/applications/diffusion/view/DiffusionPullLogListView.php b/src/applications/diffusion/view/DiffusionPullLogListView.php index f2e3280eba..8df35e2922 100644 --- a/src/applications/diffusion/view/DiffusionPullLogListView.php +++ b/src/applications/diffusion/view/DiffusionPullLogListView.php @@ -22,24 +22,10 @@ final class DiffusionPullLogListView extends AphrontView { } $handles = $viewer->loadHandles($handle_phids); - // Figure out which repositories are editable. We only let you see remote - // IPs if you have edit capability on a repository. - $editable_repos = array(); - if ($events) { - $editable_repos = id(new PhabricatorRepositoryQuery()) - ->setViewer($viewer) - ->requireCapabilities( - array( - PhabricatorPolicyCapability::CAN_VIEW, - PhabricatorPolicyCapability::CAN_EDIT, - )) - ->withPHIDs(mpull($events, 'getRepositoryPHID')) - ->execute(); - $editable_repos = mpull($editable_repos, null, 'getPHID'); - } + // Only administrators can view remote addresses. + $remotes_visible = $viewer->getIsAdmin(); $rows = array(); - $any_host = false; foreach ($events as $event) { if ($event->getRepositoryPHID()) { $repository = $event->getRepository(); @@ -47,13 +33,10 @@ final class DiffusionPullLogListView extends AphrontView { $repository = null; } - // Reveal this if it's valid and the user can edit the repository. For - // invalid requests you currently have to go fishing in the database. - $remote_address = '-'; - if ($repository) { - if (isset($editable_repos[$event->getRepositoryPHID()])) { - $remote_address = $event->getRemoteAddress(); - } + if ($remotes_visible) { + $remote_address = $event->getRemoteAddress(); + } else { + $remote_address = null; } $event_id = $event->getID(); @@ -107,6 +90,13 @@ final class DiffusionPullLogListView extends AphrontView { '', 'n', 'right', + )) + ->setColumnVisibility( + array( + true, + true, + true, + $remotes_visible, )); return $table; diff --git a/src/applications/diffusion/view/DiffusionPushLogListView.php b/src/applications/diffusion/view/DiffusionPushLogListView.php index 303f0519f6..77f28671c6 100644 --- a/src/applications/diffusion/view/DiffusionPushLogListView.php +++ b/src/applications/diffusion/view/DiffusionPushLogListView.php @@ -25,31 +25,18 @@ final class DiffusionPushLogListView extends AphrontView { $handles = $viewer->loadHandles($handle_phids); - // Figure out which repositories are editable. We only let you see remote - // IPs if you have edit capability on a repository. - $editable_repos = array(); - if ($logs) { - $editable_repos = id(new PhabricatorRepositoryQuery()) - ->setViewer($viewer) - ->requireCapabilities( - array( - PhabricatorPolicyCapability::CAN_VIEW, - PhabricatorPolicyCapability::CAN_EDIT, - )) - ->withPHIDs(mpull($logs, 'getRepositoryPHID')) - ->execute(); - $editable_repos = mpull($editable_repos, null, 'getPHID'); - } + // Only administrators can view remote addresses. + $remotes_visible = $viewer->getIsAdmin(); $rows = array(); $any_host = false; foreach ($logs as $log) { $repository = $log->getRepository(); - // Reveal this if it's valid and the user can edit the repository. - $remote_address = '-'; - if (isset($editable_repos[$log->getRepositoryPHID()])) { + if ($remotes_visible) { $remote_address = $log->getPushEvent()->getRemoteAddress(); + } else { + $remote_address = null; } $event_id = $log->getPushEvent()->getID(); @@ -142,7 +129,7 @@ final class DiffusionPushLogListView extends AphrontView { true, true, true, - true, + $remotes_visible, true, $any_host, )); From ff98f6f522b171bcdabf45933c775e1a4089792e Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 12:13:28 -0800 Subject: [PATCH 27/89] Make the remote address rules for Settings > Activity Logs more consistent Summary: Depends on D18971. Ref T13049. The rule is currently "you can see IP addresses for actions which affect your account". There's some legitimate motivation for this, since it's good if you can see that someone you don't recognize has been trying to log into your account. However, this includes cases where an administrator disables/enables your account, or promotes/demotes you to administrator. In these cases, //their// IP is shown! Make the rule: - Administrators can see it (consistent with everything else). - You can see your own actions. - You can see actions which affected you that have no actor (these are things like login attempts). - You can't see other stuff: usually, administrators changing your account settings. Test Plan: Viewed activity log as a non-admin, no longer saw administrator's IP address disclosed in "Demote from Admin" log. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18972 --- .../people/view/PhabricatorUserLogView.php | 58 +++++++++++++++---- .../PhabricatorActivitySettingsPanel.php | 15 ----- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/applications/people/view/PhabricatorUserLogView.php b/src/applications/people/view/PhabricatorUserLogView.php index 309540c611..72a9378a1a 100644 --- a/src/applications/people/view/PhabricatorUserLogView.php +++ b/src/applications/people/view/PhabricatorUserLogView.php @@ -30,31 +30,65 @@ final class PhabricatorUserLogView extends AphrontView { $action_map = PhabricatorUserLog::getActionTypeMap(); $base_uri = $this->searchBaseURI; + $viewer_phid = $viewer->getPHID(); + $rows = array(); foreach ($logs as $log) { - $ip = $log->getRemoteAddr(); $session = substr($log->getSession(), 0, 6); - if ($base_uri) { - $ip = phutil_tag( - 'a', - array( - 'href' => $base_uri.'?ip='.$ip.'#R', - ), - $ip); + $actor_phid = $log->getActorPHID(); + $user_phid = $log->getUserPHID(); + + if ($viewer->getIsAdmin()) { + $can_see_ip = true; + } else if ($viewer_phid == $actor_phid) { + // You can see the address if you took the action. + $can_see_ip = true; + } else if (!$actor_phid && ($viewer_phid == $user_phid)) { + // You can see the address if it wasn't authenticated and applied + // to you (partial login). + $can_see_ip = true; + } else { + // You can't see the address when an administrator disables your + // account, since it's their address. + $can_see_ip = false; + } + + if ($can_see_ip) { + $ip = $log->getRemoteAddr(); + if ($base_uri) { + $ip = phutil_tag( + 'a', + array( + 'href' => $base_uri.'?ip='.$ip.'#R', + ), + $ip); + } + } else { + $ip = null; } $action = $log->getAction(); $action_name = idx($action_map, $action, $action); + if ($actor_phid) { + $actor_name = $handles[$actor_phid]->renderLink(); + } else { + $actor_name = null; + } + + if ($user_phid) { + $user_name = $handles[$user_phid]->renderLink(); + } else { + $user_name = null; + } + $rows[] = array( phabricator_date($log->getDateCreated(), $viewer), phabricator_time($log->getDateCreated(), $viewer), $action_name, - $log->getActorPHID() - ? $handles[$log->getActorPHID()]->getName() - : null, - $username = $handles[$log->getUserPHID()]->renderLink(), + $actor_name, + $user_name, $ip, $session, ); diff --git a/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php b/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php index eeeca27663..2759f3a26c 100644 --- a/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php +++ b/src/applications/settings/panel/PhabricatorActivitySettingsPanel.php @@ -26,21 +26,6 @@ final class PhabricatorActivitySettingsPanel extends PhabricatorSettingsPanel { ->withRelatedPHIDs(array($user->getPHID())) ->executeWithCursorPager($pager); - $phids = array(); - foreach ($logs as $log) { - $phids[] = $log->getUserPHID(); - $phids[] = $log->getActorPHID(); - } - - if ($phids) { - $handles = id(new PhabricatorHandleQuery()) - ->setViewer($viewer) - ->withPHIDs($phids) - ->execute(); - } else { - $handles = array(); - } - $table = id(new PhabricatorUserLogView()) ->setUser($viewer) ->setLogs($logs); From 1e3d1271ada02ee80693901286152422d6e9d731 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 13:01:09 -0800 Subject: [PATCH 28/89] Make push log "flags", "reject code" human readable; add crumbs to pull/push logs Summary: Depends on D18972. Ref T13049. Currently, the "flags" columns renders an inscrutible bitmask which you have to go hunt down in the code. Show a list of flags in human-readable text instead. The "code" column renders a meaningless integer code. Show a text description instead. The pull logs and push logs pages don't have a crumb to go back up out of the current query. Add one. Test Plan: Viewed push logs, no more arcane numbers. Saw and clicked crumbs on each log page. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13049 Differential Revision: https://secure.phabricator.com/D18973 --- .../DiffusionPullLogListController.php | 5 ++++ .../DiffusionPushLogListController.php | 5 ++++ .../view/DiffusionPushLogListView.php | 30 +++++++++++++++---- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/applications/diffusion/controller/DiffusionPullLogListController.php b/src/applications/diffusion/controller/DiffusionPullLogListController.php index 29712be2a4..f3b570c815 100644 --- a/src/applications/diffusion/controller/DiffusionPullLogListController.php +++ b/src/applications/diffusion/controller/DiffusionPullLogListController.php @@ -9,4 +9,9 @@ final class DiffusionPullLogListController ->buildResponse(); } + protected function buildApplicationCrumbs() { + return parent::buildApplicationCrumbs() + ->addTextCrumb(pht('Pull Logs'), $this->getApplicationURI('pulllog/')); + } + } diff --git a/src/applications/diffusion/controller/DiffusionPushLogListController.php b/src/applications/diffusion/controller/DiffusionPushLogListController.php index 658e21674d..a6c612eebe 100644 --- a/src/applications/diffusion/controller/DiffusionPushLogListController.php +++ b/src/applications/diffusion/controller/DiffusionPushLogListController.php @@ -9,4 +9,9 @@ final class DiffusionPushLogListController ->buildResponse(); } + protected function buildApplicationCrumbs() { + return parent::buildApplicationCrumbs() + ->addTextCrumb(pht('Push Logs'), $this->getApplicationURI('pushlog/')); + } + } diff --git a/src/applications/diffusion/view/DiffusionPushLogListView.php b/src/applications/diffusion/view/DiffusionPushLogListView.php index 77f28671c6..e101625752 100644 --- a/src/applications/diffusion/view/DiffusionPushLogListView.php +++ b/src/applications/diffusion/view/DiffusionPushLogListView.php @@ -28,6 +28,9 @@ final class DiffusionPushLogListView extends AphrontView { // Only administrators can view remote addresses. $remotes_visible = $viewer->getIsAdmin(); + $flag_map = PhabricatorRepositoryPushLog::getFlagDisplayNames(); + $reject_map = PhabricatorRepositoryPushLog::getRejectCodeDisplayNames(); + $rows = array(); $any_host = false; foreach ($logs as $log) { @@ -59,6 +62,23 @@ final class DiffusionPushLogListView extends AphrontView { $device = null; } + $flags = $log->getChangeFlags(); + $flag_names = array(); + foreach ($flag_map as $flag_key => $flag_name) { + if (($flags & $flag_key) === $flag_key) { + $flag_names[] = $flag_name; + } + } + $flag_names = phutil_implode_html( + phutil_tag('br'), + $flag_names); + + $reject_code = $log->getPushEvent()->getRejectCode(); + $reject_label = idx( + $reject_map, + $reject_code, + pht('Unknown ("%s")', $reject_code)); + $rows[] = array( phutil_tag( 'a', @@ -85,10 +105,8 @@ final class DiffusionPushLogListView extends AphrontView { 'href' => $repository->getCommitURI($log->getRefNew()), ), $log->getRefNewShort()), - - // TODO: Make these human-readable. - $log->getChangeFlags(), - $log->getPushEvent()->getRejectCode(), + $flag_names, + $reject_label, $viewer->formatShortDateTime($log->getEpoch()), ); } @@ -107,7 +125,7 @@ final class DiffusionPushLogListView extends AphrontView { pht('Old'), pht('New'), pht('Flags'), - pht('Code'), + pht('Result'), pht('Date'), )) ->setColumnClasses( @@ -122,6 +140,8 @@ final class DiffusionPushLogListView extends AphrontView { 'wide', 'n', 'n', + '', + '', 'right', )) ->setColumnVisibility( From c9df8f77c8b5cabb5fe2a3c8874c82d12d1cf431 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 31 Jan 2018 10:37:38 -0800 Subject: [PATCH 29/89] Fix transcription of single-value bulk edit fields ("Assign to") Summary: See PHI333. Some of the cleanup at the tail end of the bulk edit changes made "Assign To" stop working properly, since we don't strip the `array(...)` off the `array(PHID)` value we receive. Test Plan: - Used bulk editor to assign and unassign tasks (single value datasource). - Used bulk editor to change projects (multi-value datasource). Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18975 --- resources/celerity/map.php | 12 ++++++------ .../edittype/PhabricatorPHIDListEditType.php | 12 ++++++++++++ webroot/rsrc/js/phuix/PHUIXFormControl.js | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 661ac58c15..d9aebf32bc 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -532,7 +532,7 @@ return array( 'rsrc/js/phuix/PHUIXButtonView.js' => '8a91e1ac', 'rsrc/js/phuix/PHUIXDropdownMenu.js' => '04b2ae03', 'rsrc/js/phuix/PHUIXExample.js' => '68af71ca', - 'rsrc/js/phuix/PHUIXFormControl.js' => '1dd0870c', + 'rsrc/js/phuix/PHUIXFormControl.js' => '16ad6224', 'rsrc/js/phuix/PHUIXIconView.js' => 'bff6884b', ), 'symbols' => array( @@ -884,7 +884,7 @@ return array( 'phuix-autocomplete' => 'e0731603', 'phuix-button-view' => '8a91e1ac', 'phuix-dropdown-menu' => '04b2ae03', - 'phuix-form-control-view' => '1dd0870c', + 'phuix-form-control-view' => '16ad6224', 'phuix-icon-view' => 'bff6884b', 'policy-css' => '957ea14c', 'policy-edit-css' => '815c66f7', @@ -995,6 +995,10 @@ return array( 'aphront-typeahead-control-css', 'phui-tag-view-css', ), + '16ad6224' => array( + 'javelin-install', + 'javelin-dom', + ), '17bb8539' => array( 'javelin-behavior', 'javelin-stratcom', @@ -1029,10 +1033,6 @@ return array( 'javelin-request', 'javelin-uri', ), - '1dd0870c' => array( - 'javelin-install', - 'javelin-dom', - ), '1e911d0f' => array( 'javelin-stratcom', 'javelin-request', diff --git a/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php b/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php index eb728a6808..07489521b6 100644 --- a/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php +++ b/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php @@ -55,4 +55,16 @@ abstract class PhabricatorPHIDListEditType } } + public function getTransactionValueFromBulkEdit($value) { + if (!$this->getIsSingleValue()) { + return $value; + } + + if ($value) { + return head($value); + } + + return null; + } + } diff --git a/webroot/rsrc/js/phuix/PHUIXFormControl.js b/webroot/rsrc/js/phuix/PHUIXFormControl.js index cd67e89691..eb5f32963d 100644 --- a/webroot/rsrc/js/phuix/PHUIXFormControl.js +++ b/webroot/rsrc/js/phuix/PHUIXFormControl.js @@ -293,7 +293,7 @@ JX.install('PHUIXFormControl', { }, _newPoints: function(spec) { - return this._newText(); + return this._newText(spec); }, _newText: function(spec) { From f9336e56940fb9ae00ac7fd967eaf0ef974463c9 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 30 Jan 2018 15:58:05 -0800 Subject: [PATCH 30/89] Mangle cells that look a little bit like formulas in CSV files Summary: Fixes T12800. See that task for discussion. When a cell in a CSV begins with "=", "+", "-", or "@", mangle the content to discourage Excel from executing it. This is clumsy, but we support other formats (e.g., JSON) which preserve the data faithfully and you should probably be using JSON if you're going to do anything programmatic with it. We could add two formats or a checkbox or a warning or something but cells with these symbols are fairly rare anyway. Some possible exceptions I can think of are "user monograms" (but we don't export those right now) and "negative numbers" (but also no direct export today). We can add exceptions for those as they arise. Test Plan: Exported a task named `=cmd|'/C evil.exe'!A0`, saw the title get mangled with "(!)" in front. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T12800 Differential Revision: https://secure.phabricator.com/D18974 --- .../export/format/PhabricatorCSVExportFormat.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/infrastructure/export/format/PhabricatorCSVExportFormat.php b/src/infrastructure/export/format/PhabricatorCSVExportFormat.php index 67f0a4963a..8f3879d5fb 100644 --- a/src/infrastructure/export/format/PhabricatorCSVExportFormat.php +++ b/src/infrastructure/export/format/PhabricatorCSVExportFormat.php @@ -42,6 +42,16 @@ final class PhabricatorCSVExportFormat private function addRow(array $values) { $row = array(); foreach ($values as $value) { + + // Excel is extremely interested in executing arbitrary code it finds in + // untrusted CSV files downloaded from the internet. When a cell looks + // like it might be too tempting for Excel to ignore, mangle the value + // to dissuade remote code execution. See T12800. + + if (preg_match('/^\s*[+=@-]/', $value)) { + $value = '(!) '.$value; + } + if (preg_match('/\s|,|\"/', $value)) { $value = str_replace('"', '""', $value); $value = '"'.$value.'"'; From 6d5f265a57957d9b7bbba54a0355bd004b8619aa Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 31 Jan 2018 11:11:29 -0800 Subject: [PATCH 31/89] Accept `null` via `conduit.edit` to unassign a task Summary: See . This unusual field doesn't actually accept `null`, although the documentation says it does and that was the intent. Accept `null`, and show `phid|null` in the docs. Test Plan: Viewed docs, saw `phid|null`. Unassigned with `null`. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18976 --- .../ConduitPHIDParameterType.php | 31 +++++++++++++++++-- .../maniphest/editor/ManiphestEditEngine.php | 1 + .../PhabricatorPHIDListEditField.php | 16 ++++++++-- .../edittype/PhabricatorPHIDListEditType.php | 21 +++++++------ 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/applications/conduit/parametertype/ConduitPHIDParameterType.php b/src/applications/conduit/parametertype/ConduitPHIDParameterType.php index 3bb45697dc..eafee4d453 100644 --- a/src/applications/conduit/parametertype/ConduitPHIDParameterType.php +++ b/src/applications/conduit/parametertype/ConduitPHIDParameterType.php @@ -3,9 +3,26 @@ final class ConduitPHIDParameterType extends ConduitParameterType { + private $isNullable; + + public function setIsNullable($is_nullable) { + $this->isNullable = $is_nullable; + return $this; + } + + public function getIsNullable() { + return $this->isNullable; + } + protected function getParameterValue(array $request, $key, $strict) { $value = parent::getParameterValue($request, $key, $strict); + if ($this->getIsNullable()) { + if ($value === null) { + return $value; + } + } + if (!is_string($value)) { $this->raiseValidationException( $request, @@ -17,7 +34,11 @@ final class ConduitPHIDParameterType } protected function getParameterTypeName() { - return 'phid'; + if ($this->getIsNullable()) { + return 'phid|null'; + } else { + return 'phid'; + } } protected function getParameterFormatDescriptions() { @@ -27,9 +48,15 @@ final class ConduitPHIDParameterType } protected function getParameterExamples() { - return array( + $examples = array( '"PHID-WXYZ-1111222233334444"', ); + + if ($this->getIsNullable()) { + $examples[] = 'null'; + } + + return $examples; } } diff --git a/src/applications/maniphest/editor/ManiphestEditEngine.php b/src/applications/maniphest/editor/ManiphestEditEngine.php index 359ba493d4..c270104034 100644 --- a/src/applications/maniphest/editor/ManiphestEditEngine.php +++ b/src/applications/maniphest/editor/ManiphestEditEngine.php @@ -196,6 +196,7 @@ EODOCS pht('New task owner, or `null` to unassign.')) ->setTransactionType(ManiphestTaskOwnerTransaction::TRANSACTIONTYPE) ->setIsCopyable(true) + ->setIsNullable(true) ->setSingleValue($object->getOwnerPHID()) ->setCommentActionLabel(pht('Assign / Claim')) ->setCommentActionValue($owner_value), diff --git a/src/applications/transactions/editfield/PhabricatorPHIDListEditField.php b/src/applications/transactions/editfield/PhabricatorPHIDListEditField.php index b084c142e5..d2b647b9aa 100644 --- a/src/applications/transactions/editfield/PhabricatorPHIDListEditField.php +++ b/src/applications/transactions/editfield/PhabricatorPHIDListEditField.php @@ -5,6 +5,7 @@ abstract class PhabricatorPHIDListEditField private $useEdgeTransactions; private $isSingleValue; + private $isNullable; public function setUseEdgeTransactions($use_edge_transactions) { $this->useEdgeTransactions = $use_edge_transactions; @@ -30,13 +31,23 @@ abstract class PhabricatorPHIDListEditField return $this->isSingleValue; } + public function setIsNullable($is_nullable) { + $this->isNullable = $is_nullable; + return $this; + } + + public function getIsNullable() { + return $this->isNullable; + } + protected function newHTTPParameterType() { return new AphrontPHIDListHTTPParameterType(); } protected function newConduitParameterType() { if ($this->getIsSingleValue()) { - return new ConduitPHIDParameterType(); + return id(new ConduitPHIDParameterType()) + ->setIsNullable($this->getIsNullable()); } else { return new ConduitPHIDListParameterType(); } @@ -99,7 +110,8 @@ abstract class PhabricatorPHIDListEditField } return id(new PhabricatorDatasourceEditType()) - ->setIsSingleValue($this->getIsSingleValue()); + ->setIsSingleValue($this->getIsSingleValue()) + ->setIsNullable($this->getIsNullable()); } protected function newBulkEditTypes() { diff --git a/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php b/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php index 07489521b6..763f6ff001 100644 --- a/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php +++ b/src/applications/transactions/edittype/PhabricatorPHIDListEditType.php @@ -6,6 +6,7 @@ abstract class PhabricatorPHIDListEditType private $datasource; private $isSingleValue; private $defaultValue; + private $isNullable; public function setDatasource(PhabricatorTypeaheadDatasource $datasource) { $this->datasource = $datasource; @@ -30,16 +31,17 @@ abstract class PhabricatorPHIDListEditType return $this; } - public function getDefaultValue() { - return $this->defaultValue; + public function setIsNullable($is_nullable) { + $this->isNullable = $is_nullable; + return $this; } - public function getValueType() { - if ($this->getIsSingleValue()) { - return 'phid'; - } else { - return 'list'; - } + public function getIsNullable() { + return $this->isNullable; + } + + public function getDefaultValue() { + return $this->defaultValue; } protected function newConduitParameterType() { @@ -49,7 +51,8 @@ abstract class PhabricatorPHIDListEditType } if ($this->getIsSingleValue()) { - return new ConduitPHIDParameterType(); + return id(new ConduitPHIDParameterType()) + ->setIsNullable($this->getIsNullable()); } else { return new ConduitPHIDListParameterType(); } From f535981c0d61c4ddfa956c0d4baae8573dcc9e94 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 31 Jan 2018 11:21:24 -0800 Subject: [PATCH 32/89] Fix a missing getSSHUser() callsite Summary: See . I renamed this method in D18912 but missed this callsite since the workflow doesn't live alongside the other ones. Test Plan: Ran `git push` in an LFS repository over SSH. Before: fatal; after: clean push. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18977 --- .../diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php b/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php index 41c009455d..56297f4d8b 100644 --- a/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php +++ b/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php @@ -84,7 +84,7 @@ final class DiffusionGitLFSAuthenticateWorkflow // This works even if normal HTTP repository operations are not available // on this host, and does not require the user to have a VCS password. - $user = $this->getUser(); + $user = $this->getSSHUser(); $authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization( $repository, From 032f5b22941f85340b70da6d164d9dc17cb1b0f9 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 31 Jan 2018 12:19:19 -0800 Subject: [PATCH 33/89] Allow revisions to revert commits and one another, and commits to revert revisions Summary: Ref T13057. This makes "reverts" syntax more visible and useful. In particular, you can now `Reverts Dxx` in a revision or commit, and `Reverts ` from a revision. When you do, the corresponding object will get a more-visible cross-reference marker in its timeline: {F5405517} From here, we can look at surfacing revert information more heavily, since we can now query it on revision/commit pages via edges. Test Plan: Used "reverts " and "reverts " in Differential and Diffusion, got sensible results in the timeline. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13057 Differential Revision: https://secure.phabricator.com/D18978 --- .../audit/editor/PhabricatorAuditEditor.php | 56 ++++++++++++++-- .../editor/DifferentialTransactionEditor.php | 39 ++++++++++- ...iffusionCommitRevertedByCommitEdgeType.php | 12 ++-- .../DiffusionCommitRevertsCommitEdgeType.php | 12 ++-- ...habricatorRepositoryCommitHeraldWorker.php | 26 +------- .../PhabricatorApplicationTransaction.php | 14 ++++ .../PhabricatorUSEnglishTranslation.php | 64 +++++++++---------- 7 files changed, 149 insertions(+), 74 deletions(-) diff --git a/src/applications/audit/editor/PhabricatorAuditEditor.php b/src/applications/audit/editor/PhabricatorAuditEditor.php index 0cf6339239..049733f777 100644 --- a/src/applications/audit/editor/PhabricatorAuditEditor.php +++ b/src/applications/audit/editor/PhabricatorAuditEditor.php @@ -358,9 +358,17 @@ final class PhabricatorAuditEditor array $changes, PhutilMarkupEngine $engine) { - // we are only really trying to find unmentionable phids here... - // don't bother with this outside initial commit (i.e. create) - // transaction + $actor = $this->getActor(); + $result = array(); + + // Some interactions (like "Fixes Txxx" interacting with Maniphest) have + // already been processed, so we're only re-parsing them here to avoid + // generating an extra redundant mention. Other interactions are being + // processed for the first time. + + // We're only recognizing magic in the commit message itself, not in + // audit comments. + $is_commit = false; foreach ($xactions as $xaction) { switch ($xaction->getTransactionType()) { @@ -370,8 +378,6 @@ final class PhabricatorAuditEditor } } - // "result" is always an array.... - $result = array(); if (!$is_commit) { return $result; } @@ -403,6 +409,46 @@ final class PhabricatorAuditEditor ->withNames($monograms) ->execute(); $phid_map[] = mpull($objects, 'getPHID', 'getPHID'); + + + $reverts_refs = id(new DifferentialCustomFieldRevertsParser()) + ->parseCorpus($huge_block); + $reverts = array_mergev(ipull($reverts_refs, 'monograms')); + if ($reverts) { + // Only allow commits to revert other commits in the same repository. + $reverted_commits = id(new DiffusionCommitQuery()) + ->setViewer($actor) + ->withRepository($object->getRepository()) + ->withIdentifiers($reverts) + ->execute(); + + $reverted_revisions = id(new PhabricatorObjectQuery()) + ->setViewer($actor) + ->withNames($reverts) + ->withTypes( + array( + DifferentialRevisionPHIDType::TYPECONST, + )) + ->execute(); + + $reverted_phids = + mpull($reverted_commits, 'getPHID', 'getPHID') + + mpull($reverted_revisions, 'getPHID', 'getPHID'); + + // NOTE: Skip any write attempts if a user cleverly implies a commit + // reverts itself, although this would be exceptionally clever in Git + // or Mercurial. + unset($reverted_phids[$object->getPHID()]); + + $reverts_edge = DiffusionCommitRevertsCommitEdgeType::EDGECONST; + $result[] = id(new PhabricatorAuditTransaction()) + ->setTransactionType(PhabricatorTransactions::TYPE_EDGE) + ->setMetadataValue('edge:type', $reverts_edge) + ->setNewValue(array('+' => $reverted_phids)); + + $phid_map[] = $reverted_phids; + } + $phid_map = array_mergev($phid_map); $this->setUnmentionablePHIDMap($phid_map); diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index ecd1e6c95c..3a1537b01d 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -919,7 +919,44 @@ final class DifferentialTransactionEditor } } - $this->setUnmentionablePHIDMap(array_merge($task_phids, $rev_phids)); + $revert_refs = id(new DifferentialCustomFieldRevertsParser()) + ->parseCorpus($content_block); + + $revert_monograms = array(); + foreach ($revert_refs as $match) { + foreach ($match['monograms'] as $monogram) { + $revert_monograms[] = $monogram; + } + } + + if ($revert_monograms) { + $revert_objects = id(new PhabricatorObjectQuery()) + ->setViewer($this->getActor()) + ->withNames($revert_monograms) + ->withTypes( + array( + DifferentialRevisionPHIDType::TYPECONST, + PhabricatorRepositoryCommitPHIDType::TYPECONST, + )) + ->execute(); + + $revert_phids = mpull($revert_objects, 'getPHID', 'getPHID'); + + // Don't let an object revert itself, although other silly stuff like + // cycles of objects reverting each other is not prevented. + unset($revert_phids[$object->getPHID()]); + + $revert_type = DiffusionCommitRevertsCommitEdgeType::EDGECONST; + $edges[$revert_type] = $revert_phids; + } else { + $revert_phids = array(); + } + + $this->setUnmentionablePHIDMap( + array_merge( + $task_phids, + $rev_phids, + $revert_phids)); $result = array(); foreach ($edges as $type => $specs) { diff --git a/src/applications/diffusion/edge/DiffusionCommitRevertedByCommitEdgeType.php b/src/applications/diffusion/edge/DiffusionCommitRevertedByCommitEdgeType.php index 53c39e54c9..ae59a3b1e6 100644 --- a/src/applications/diffusion/edge/DiffusionCommitRevertedByCommitEdgeType.php +++ b/src/applications/diffusion/edge/DiffusionCommitRevertedByCommitEdgeType.php @@ -19,7 +19,7 @@ final class DiffusionCommitRevertedByCommitEdgeType $add_edges) { return pht( - '%s added %s reverting commit(s): %s.', + '%s added %s reverting change(s): %s.', $actor, $add_count, $add_edges); @@ -31,7 +31,7 @@ final class DiffusionCommitRevertedByCommitEdgeType $rem_edges) { return pht( - '%s removed %s reverting commit(s): %s.', + '%s removed %s reverting change(s): %s.', $actor, $rem_count, $rem_edges); @@ -46,7 +46,7 @@ final class DiffusionCommitRevertedByCommitEdgeType $rem_edges) { return pht( - '%s edited reverting commit(s), added %s: %s; removed %s: %s.', + '%s edited reverting change(s), added %s: %s; removed %s: %s.', $actor, $add_count, $add_edges, @@ -61,7 +61,7 @@ final class DiffusionCommitRevertedByCommitEdgeType $add_edges) { return pht( - '%s added %s reverting commit(s) for %s: %s.', + '%s added %s reverting change(s) for %s: %s.', $actor, $add_count, $object, @@ -75,7 +75,7 @@ final class DiffusionCommitRevertedByCommitEdgeType $rem_edges) { return pht( - '%s removed %s reverting commit(s) for %s: %s.', + '%s removed %s reverting change(s) for %s: %s.', $actor, $rem_count, $object, @@ -92,7 +92,7 @@ final class DiffusionCommitRevertedByCommitEdgeType $rem_edges) { return pht( - '%s edited reverting commit(s) for %s, added %s: %s; removed %s: %s.', + '%s edited reverting change(s) for %s, added %s: %s; removed %s: %s.', $actor, $object, $add_count, diff --git a/src/applications/diffusion/edge/DiffusionCommitRevertsCommitEdgeType.php b/src/applications/diffusion/edge/DiffusionCommitRevertsCommitEdgeType.php index 8f32797173..ee0223c966 100644 --- a/src/applications/diffusion/edge/DiffusionCommitRevertsCommitEdgeType.php +++ b/src/applications/diffusion/edge/DiffusionCommitRevertsCommitEdgeType.php @@ -22,7 +22,7 @@ final class DiffusionCommitRevertsCommitEdgeType extends PhabricatorEdgeType { $add_edges) { return pht( - '%s added %s reverted commit(s): %s.', + '%s added %s reverted change(s): %s.', $actor, $add_count, $add_edges); @@ -34,7 +34,7 @@ final class DiffusionCommitRevertsCommitEdgeType extends PhabricatorEdgeType { $rem_edges) { return pht( - '%s removed %s reverted commit(s): %s.', + '%s removed %s reverted change(s): %s.', $actor, $rem_count, $rem_edges); @@ -49,7 +49,7 @@ final class DiffusionCommitRevertsCommitEdgeType extends PhabricatorEdgeType { $rem_edges) { return pht( - '%s edited reverted commit(s), added %s: %s; removed %s: %s.', + '%s edited reverted change(s), added %s: %s; removed %s: %s.', $actor, $add_count, $add_edges, @@ -64,7 +64,7 @@ final class DiffusionCommitRevertsCommitEdgeType extends PhabricatorEdgeType { $add_edges) { return pht( - '%s added %s reverted commit(s) for %s: %s.', + '%s added %s reverted change(s) for %s: %s.', $actor, $add_count, $object, @@ -78,7 +78,7 @@ final class DiffusionCommitRevertsCommitEdgeType extends PhabricatorEdgeType { $rem_edges) { return pht( - '%s removed %s reverted commit(s) for %s: %s.', + '%s removed %s reverted change(s) for %s: %s.', $actor, $rem_count, $object, @@ -95,7 +95,7 @@ final class DiffusionCommitRevertsCommitEdgeType extends PhabricatorEdgeType { $rem_edges) { return pht( - '%s edited reverted commit(s) for %s, added %s: %s; removed %s: %s.', + '%s edited reverted change(s) for %s, added %s: %s; removed %s: %s.', $actor, $object, $add_count, diff --git a/src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php b/src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php index 9fb6667924..818d1e8781 100644 --- a/src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php +++ b/src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php @@ -15,6 +15,7 @@ final class PhabricatorRepositoryCommitHeraldWorker protected function parseCommit( PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit) { + $viewer = PhabricatorUser::getOmnipotentUser(); if ($this->shouldSkipImportStep()) { // This worker has no followup tasks, so we can just bail out @@ -50,7 +51,7 @@ final class PhabricatorRepositoryCommitHeraldWorker id(new PhabricatorDiffusionApplication())->getPHID()); $editor = id(new PhabricatorAuditEditor()) - ->setActor(PhabricatorUser::getOmnipotentUser()) + ->setActor($viewer) ->setActingAsPHID($acting_as_phid) ->setContinueOnMissingFields(true) ->setContinueOnNoEffect(true) @@ -69,29 +70,6 @@ final class PhabricatorRepositoryCommitHeraldWorker 'committerPHID' => $data->getCommitDetail('committerPHID'), )); - $reverts_refs = id(new DifferentialCustomFieldRevertsParser()) - ->parseCorpus($data->getCommitMessage()); - $reverts = array_mergev(ipull($reverts_refs, 'monograms')); - - if ($reverts) { - $reverted_commits = id(new DiffusionCommitQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withRepository($repository) - ->withIdentifiers($reverts) - ->execute(); - $reverted_commit_phids = mpull($reverted_commits, 'getPHID', 'getPHID'); - - // NOTE: Skip any write attempts if a user cleverly implies a commit - // reverts itself. - unset($reverted_commit_phids[$commit->getPHID()]); - - $reverts_edge = DiffusionCommitRevertsCommitEdgeType::EDGECONST; - $xactions[] = id(new PhabricatorAuditTransaction()) - ->setTransactionType(PhabricatorTransactions::TYPE_EDGE) - ->setMetadataValue('edge:type', $reverts_edge) - ->setNewValue(array('+' => array_fuse($reverted_commit_phids))); - } - try { $raw_patch = $this->loadRawPatchText($repository, $commit); } catch (Exception $ex) { diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php index 97b57009ec..d5c28b83bf 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php @@ -458,6 +458,12 @@ abstract class PhabricatorApplicationTransaction case PhabricatorTransactions::TYPE_JOIN_POLICY: return 'fa-lock'; case PhabricatorTransactions::TYPE_EDGE: + switch ($this->getMetadataValue('edge:type')) { + case DiffusionCommitRevertedByCommitEdgeType::EDGECONST: + return 'fa-undo'; + case DiffusionCommitRevertsCommitEdgeType::EDGECONST: + return 'fa-ambulance'; + } return 'fa-link'; case PhabricatorTransactions::TYPE_BUILDABLE: return 'fa-wrench'; @@ -496,6 +502,14 @@ abstract class PhabricatorApplicationTransaction return 'black'; } break; + case PhabricatorTransactions::TYPE_EDGE: + switch ($this->getMetadataValue('edge:type')) { + case DiffusionCommitRevertedByCommitEdgeType::EDGECONST: + return 'pink'; + case DiffusionCommitRevertsCommitEdgeType::EDGECONST: + return 'sky'; + } + break; case PhabricatorTransactions::TYPE_BUILDABLE: switch ($this->getNewValue()) { case HarbormasterBuildable::STATUS_PASSED: diff --git a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php index 40cedacf06..eb74f8debf 100644 --- a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php +++ b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php @@ -676,73 +676,73 @@ final class PhabricatorUSEnglishTranslation '%s edited commit(s), added %s: %s; removed %s: %s.' => '%s edited commits, added %3$s; removed %5$s.', - '%s added %s reverted commit(s): %s.' => array( + '%s added %s reverted change(s): %s.' => array( array( - '%s added a reverted commit: %3$s.', - '%s added reverted commits: %3$s.', + '%s added a reverted change: %3$s.', + '%s added reverted changes: %3$s.', ), ), - '%s removed %s reverted commit(s): %s.' => array( + '%s removed %s reverted change(s): %s.' => array( array( - '%s removed a reverted commit: %3$s.', - '%s removed reverted commits: %3$s.', + '%s removed a reverted change: %3$s.', + '%s removed reverted changes: %3$s.', ), ), - '%s edited reverted commit(s), added %s: %s; removed %s: %s.' => - '%s edited reverted commits, added %3$s; removed %5$s.', + '%s edited reverted change(s), added %s: %s; removed %s: %s.' => + '%s edited reverted changes, added %3$s; removed %5$s.', - '%s added %s reverted commit(s) for %s: %s.' => array( + '%s added %s reverted change(s) for %s: %s.' => array( array( - '%s added a reverted commit for %3$s: %4$s.', - '%s added reverted commits for %3$s: %4$s.', + '%s added a reverted change for %3$s: %4$s.', + '%s added reverted changes for %3$s: %4$s.', ), ), - '%s removed %s reverted commit(s) for %s: %s.' => array( + '%s removed %s reverted change(s) for %s: %s.' => array( array( - '%s removed a reverted commit for %3$s: %4$s.', - '%s removed reverted commits for %3$s: %4$s.', + '%s removed a reverted change for %3$s: %4$s.', + '%s removed reverted changes for %3$s: %4$s.', ), ), - '%s edited reverted commit(s) for %s, added %s: %s; removed %s: %s.' => - '%s edited reverted commits for %2$s, added %4$s; removed %6$s.', + '%s edited reverted change(s) for %s, added %s: %s; removed %s: %s.' => + '%s edited reverted changes for %2$s, added %4$s; removed %6$s.', - '%s added %s reverting commit(s): %s.' => array( + '%s added %s reverting change(s): %s.' => array( array( - '%s added a reverting commit: %3$s.', - '%s added reverting commits: %3$s.', + '%s added a reverting change: %3$s.', + '%s added reverting changes: %3$s.', ), ), - '%s removed %s reverting commit(s): %s.' => array( + '%s removed %s reverting change(s): %s.' => array( array( - '%s removed a reverting commit: %3$s.', - '%s removed reverting commits: %3$s.', + '%s removed a reverting change: %3$s.', + '%s removed reverting changes: %3$s.', ), ), - '%s edited reverting commit(s), added %s: %s; removed %s: %s.' => - '%s edited reverting commits, added %3$s; removed %5$s.', + '%s edited reverting change(s), added %s: %s; removed %s: %s.' => + '%s edited reverting changes, added %3$s; removed %5$s.', - '%s added %s reverting commit(s) for %s: %s.' => array( + '%s added %s reverting change(s) for %s: %s.' => array( array( - '%s added a reverting commit for %3$s: %4$s.', - '%s added reverting commits for %3$s: %4$s.', + '%s added a reverting change for %3$s: %4$s.', + '%s added reverting changes for %3$s: %4$s.', ), ), - '%s removed %s reverting commit(s) for %s: %s.' => array( + '%s removed %s reverting change(s) for %s: %s.' => array( array( - '%s removed a reverting commit for %3$s: %4$s.', - '%s removed reverting commits for %3$s: %4$s.', + '%s removed a reverting change for %3$s: %4$s.', + '%s removed reverting changes for %3$s: %4$s.', ), ), - '%s edited reverting commit(s) for %s, added %s: %s; removed %s: %s.' => - '%s edited reverting commits for %s, added %4$s; removed %6$s.', + '%s edited reverting change(s) for %s, added %s: %s; removed %s: %s.' => + '%s edited reverting changes for %s, added %4$s; removed %6$s.', '%s changed project member(s), added %d: %s; removed %d: %s.' => '%s changed project members, added %3$s; removed %5$s.', From 7b2b5cd91e018f3479a20056da79a82dd1008d74 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 06:16:32 -0800 Subject: [PATCH 34/89] Add basic support for a "Must Encrypt" mail flag which prevents unsecured content transmission Summary: Ref T13053. See PHI291. For particularly sensitive objects (like security issues), installs may reasonably wish to prevent details from being sent in plaintext over email. This adds a "Must Encrypt" mail behavior, which discards mail content and all identifying details, replacing it with a link to the `/mail/` application. Users can follow the link to view the message over HTTPS. The flag discards body content, attachments, and headers which imply things about the content of the object. It retains threading headers and headers which may uniquely identify the object as long as they don't disclose anyting about the content. The `bin/mail list-outbound` command now flags these messages with a `#` mark. The `bin/mail show-outbound` command now shows sent/suppressed headers and the body content as delivered (if it differs from the original body content). The `/mail/` web UI now shows a tag for messages marked with this flag. For now, there is no way to actually set this flag on mail. Test Plan: - Forced this flag on, made comments and took actions to send mail. - Reviewed mail with `bin/mail` and `/mail/` in the web UI, saw all content information omitted. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18983 --- .../PhabricatorMetaMTAMailViewController.php | 17 ++ ...atorMailManagementListOutboundWorkflow.php | 2 + ...atorMailManagementShowOutboundWorkflow.php | 57 +++++-- .../storage/PhabricatorMetaMTAMail.php | 146 ++++++++++++++++-- 4 files changed, 197 insertions(+), 25 deletions(-) diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 80da535a9e..20bbc425b5 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -32,6 +32,23 @@ final class PhabricatorMetaMTAMailViewController $color = PhabricatorMailOutboundStatus::getStatusColor($status); $header->setStatus($icon, $color, $name); + if ($mail->getMustEncrypt()) { + Javelin::initBehavior('phabricator-tooltips'); + $header->addTag( + id(new PHUITagView()) + ->setType(PHUITagView::TYPE_SHADE) + ->setColor('blue') + ->setName(pht('Must Encrypt')) + ->setIcon('fa-shield blue') + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => pht( + 'Message content can only be transmitted over secure '. + 'channels.'), + ))); + } + $crumbs = $this->buildApplicationCrumbs() ->addTextCrumb(pht('Mail %d', $mail->getID())) ->setBorder(true); diff --git a/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php index b4aa76379e..a83dafb0a8 100644 --- a/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php @@ -37,6 +37,7 @@ final class PhabricatorMailManagementListOutboundWorkflow $table = id(new PhutilConsoleTable()) ->setShowHeader(false) ->addColumn('id', array('title' => pht('ID'))) + ->addColumn('encrypt', array('title' => pht('#'))) ->addColumn('status', array('title' => pht('Status'))) ->addColumn('subject', array('title' => pht('Subject'))); @@ -45,6 +46,7 @@ final class PhabricatorMailManagementListOutboundWorkflow $table->addRow(array( 'id' => $mail->getID(), + 'encrypt' => ($mail->getMustEncrypt() ? '#' : ' '), 'status' => PhabricatorMailOutboundStatus::getStatusName($status), 'subject' => $mail->getSubject(), )); diff --git a/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php index 54a91861ae..0fc7dd14b9 100644 --- a/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php @@ -79,7 +79,7 @@ final class PhabricatorMailManagementShowOutboundWorkflow $info = array(); - $info[] = pht('PROPERTIES'); + $info[] = $this->newSectionHeader(pht('PROPERTIES')); $info[] = pht('ID: %d', $message->getID()); $info[] = pht('Status: %s', $message->getStatus()); $info[] = pht('Related PHID: %s', $message->getRelatedPHID()); @@ -87,15 +87,17 @@ final class PhabricatorMailManagementShowOutboundWorkflow $ignore = array( 'body' => true, + 'body.sent' => true, 'html-body' => true, 'headers' => true, 'attachments' => true, 'headers.sent' => true, + 'headers.unfiltered' => true, 'authors.sent' => true, ); $info[] = null; - $info[] = pht('PARAMETERS'); + $info[] = $this->newSectionHeader(pht('PARAMETERS')); $parameters = $message->getParameters(); foreach ($parameters as $key => $value) { if (isset($ignore[$key])) { @@ -110,22 +112,40 @@ final class PhabricatorMailManagementShowOutboundWorkflow } $info[] = null; - $info[] = pht('HEADERS'); + $info[] = $this->newSectionHeader(pht('HEADERS')); $headers = $message->getDeliveredHeaders(); - if (!$headers) { + $unfiltered = $message->getUnfilteredHeaders(); + if (!$unfiltered) { $headers = $message->generateHeaders(); + $unfiltered = $headers; } + $header_map = array(); foreach ($headers as $header) { list($name, $value) = $header; - $info[] = "{$name}: {$value}"; + $header_map[$name.':'.$value] = true; + } + + foreach ($unfiltered as $header) { + list($name, $value) = $header; + $was_sent = isset($header_map[$name.':'.$value]); + + if ($was_sent) { + $marker = ' '; + } else { + $marker = '#'; + } + + $info[] = "{$marker} {$name}: {$value}"; } $attachments = idx($parameters, 'attachments'); if ($attachments) { $info[] = null; - $info[] = pht('ATTACHMENTS'); + + $info[] = $this->newSectionHeader(pht('ATTACHMENTS')); + foreach ($attachments as $attachment) { $info[] = idx($attachment, 'filename', pht('Unnamed File')); } @@ -136,7 +156,9 @@ final class PhabricatorMailManagementShowOutboundWorkflow $actors = $message->getDeliveredActors(); if ($actors) { $info[] = null; - $info[] = pht('RECIPIENTS'); + + $info[] = $this->newSectionHeader(pht('RECIPIENTS')); + foreach ($actors as $actor_phid => $actor_info) { $actor = idx($all_actors, $actor_phid); if ($actor) { @@ -162,15 +184,22 @@ final class PhabricatorMailManagementShowOutboundWorkflow } $info[] = null; - $info[] = pht('TEXT BODY'); + $info[] = $this->newSectionHeader(pht('TEXT BODY')); if (strlen($message->getBody())) { - $info[] = $message->getBody(); + $info[] = tsprintf('%B', $message->getBody()); } else { $info[] = pht('(This message has no text body.)'); } + $delivered_body = $message->getDeliveredBody(); + if ($delivered_body !== null) { + $info[] = null; + $info[] = $this->newSectionHeader(pht('BODY AS DELIVERED'), true); + $info[] = tsprintf('%B', $delivered_body); + } + $info[] = null; - $info[] = pht('HTML BODY'); + $info[] = $this->newSectionHeader(pht('HTML BODY')); if (strlen($message->getHTMLBody())) { $info[] = $message->getHTMLBody(); $info[] = null; @@ -186,4 +215,12 @@ final class PhabricatorMailManagementShowOutboundWorkflow } } + private function newSectionHeader($label, $emphasize = false) { + if ($emphasize) { + return tsprintf('** %s **', $label); + } else { + return tsprintf('** %s **', $label); + } + } + } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 20b1482036..c203e86530 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -21,7 +21,10 @@ final class PhabricatorMetaMTAMail public function __construct() { $this->status = PhabricatorMailOutboundStatus::STATUS_QUEUE; - $this->parameters = array('sensitive' => true); + $this->parameters = array( + 'sensitive' => true, + 'mustEncrypt' => false, + ); parent::__construct(); } @@ -247,6 +250,15 @@ final class PhabricatorMetaMTAMail return $this->getParam('sensitive', true); } + public function setMustEncrypt($bool) { + $this->setParam('mustEncrypt', $bool); + return $this; + } + + public function getMustEncrypt() { + return $this->getParam('mustEncrypt', false); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -431,6 +443,7 @@ final class PhabricatorMetaMTAMail unset($params['is-first-message']); $is_threaded = (bool)idx($params, 'thread-id'); + $must_encrypt = $this->getMustEncrypt(); $reply_to_name = idx($params, 'reply-to-name', ''); unset($params['reply-to-name']); @@ -502,6 +515,11 @@ final class PhabricatorMetaMTAMail mpull($cc_actors, 'getEmailAddress')); break; case 'attachments': + // If the mail content must be encrypted, don't add attachments. + if ($must_encrypt) { + break; + } + $value = $this->getAttachments(); foreach ($value as $attachment) { $mailer->addAttachment( @@ -521,14 +539,20 @@ final class PhabricatorMetaMTAMail $subject[] = trim(idx($params, 'subject-prefix')); - $vary_prefix = idx($params, 'vary-subject-prefix'); - if ($vary_prefix != '') { - if ($this->shouldVarySubject($preferences)) { - $subject[] = $vary_prefix; + // If mail content must be encrypted, we replace the subject with + // a generic one. + if ($must_encrypt) { + $subject[] = pht('Object Updated'); + } else { + $vary_prefix = idx($params, 'vary-subject-prefix'); + if ($vary_prefix != '') { + if ($this->shouldVarySubject($preferences)) { + $subject[] = $vary_prefix; + } } - } - $subject[] = $value; + $subject[] = $value; + } $mailer->setSubject(implode(' ', array_filter($subject))); break; @@ -567,7 +591,22 @@ final class PhabricatorMetaMTAMail } } - $body = idx($params, 'body', ''); + $raw_body = idx($params, 'body', ''); + $body = $raw_body; + if ($must_encrypt) { + $parts = array(); + $parts[] = pht( + 'The content for this message can only be transmitted over a '. + 'secure channel. To view the message content, follow this '. + 'link:'); + + $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); + + $body = implode("\n\n", $parts); + } else { + $body = $raw_body; + } + $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); if (strlen($body) > $max) { $body = id(new PhutilUTF8StringTruncator()) @@ -578,18 +617,32 @@ final class PhabricatorMetaMTAMail } $mailer->setBody($body); - $html_emails = $this->shouldSendHTML($preferences); - if ($html_emails && isset($params['html-body'])) { + // If we sent a different message body than we were asked to, record + // what we actually sent to make debugging and diagnostics easier. + if ($body !== $raw_body) { + $this->setParam('body.sent', $body); + } + + if ($must_encrypt) { + $send_html = false; + } else { + $send_html = $this->shouldSendHTML($preferences); + } + + if ($send_html && isset($params['html-body'])) { $mailer->setHTMLBody($params['html-body']); } // Pass the headers to the mailer, then save the state so we can show - // them in the web UI. - foreach ($headers as $header) { + // them in the web UI. If the mail must be encrypted, we remove headers + // which are not on a strict whitelist to avoid disclosing information. + $filtered_headers = $this->filterHeaders($headers, $must_encrypt); + foreach ($filtered_headers as $header) { list($header_key, $header_value) = $header; $mailer->addHeader($header_key, $header_value); } - $this->setParam('headers.sent', $headers); + $this->setParam('headers.unfiltered', $headers); + $this->setParam('headers.sent', $filtered_headers); // Save the final deliverability outcomes and reasoning so we can // explain why things happened the way they did. @@ -1002,8 +1055,6 @@ final class PhabricatorMetaMTAMail // Some clients respect this to suppress OOF and other auto-responses. $headers[] = array('X-Auto-Response-Suppress', 'All'); - // If the message has mailtags, filter out any recipients who don't want - // to receive this type of mail. $mailtags = $this->getParam('mailtags'); if ($mailtags) { $tag_header = array(); @@ -1028,6 +1079,10 @@ final class PhabricatorMetaMTAMail $headers[] = array('Precedence', 'bulk'); } + if ($this->getMustEncrypt()) { + $headers[] = array('X-Phabricator-Must-Encrypt', 'Yes'); + } + return $headers; } @@ -1035,6 +1090,19 @@ final class PhabricatorMetaMTAMail return $this->getParam('headers.sent'); } + public function getUnfilteredHeaders() { + $unfiltered = $this->getParam('headers.unfiltered'); + + if ($unfiltered === null) { + // Older versions of Phabricator did not filter headers, and thus did + // not record unfiltered headers. If we don't have unfiltered header + // data just return the delivered headers for compatibility. + return $this->getDeliveredHeaders(); + } + + return $unfiltered; + } + public function getDeliveredActors() { return $this->getParam('actors.sent'); } @@ -1047,6 +1115,54 @@ final class PhabricatorMetaMTAMail return $this->getParam('routingmap.sent'); } + public function getDeliveredBody() { + return $this->getParam('body.sent'); + } + + private function filterHeaders(array $headers, $must_encrypt) { + if (!$must_encrypt) { + return $headers; + } + + $whitelist = array( + 'In-Reply-To', + 'Message-ID', + 'Precedence', + 'References', + 'Thread-Index', + + 'X-Mail-Transport-Agent', + 'X-Auto-Response-Suppress', + + 'X-Phabricator-Sent-This-Message', + 'X-Phabricator-Must-Encrypt', + ); + + // NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags". + // This header contains a significant amount of meaningful information + // about the object. + + $whitelist_map = array(); + foreach ($whitelist as $term) { + $whitelist_map[phutil_utf8_strtolower($term)] = true; + } + + foreach ($headers as $key => $header) { + list($name, $value) = $header; + $name = phutil_utf8_strtolower($name); + + if (!isset($whitelist_map[$name])) { + unset($headers[$key]); + } + } + + return $headers; + } + + public function getURI() { + return '/mail/detail/'.$this->getID().'/'; + } + /* -( Routing )------------------------------------------------------------ */ From cbe4e68c072239608a5b59d50dec399cb36699b0 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 09:19:14 -0800 Subject: [PATCH 35/89] Add a Herald action to trigger "Must Encrypt" for mail Summary: Depends on D18983. Ref T13053. Adds a new Herald action to activate the "must encrypt" flag and drop mail content. Test Plan: - Created a new Herald rule: {F5407075} - Created a "dog task" (woof woof, unsecure) and a "duck task" (quack quack, secure). - Viewed mail for both in `bin/mail` and web UI, saw appropriate security/encryption behavior. - Viewed "Must Encrypt" in "Headers" tab for the duck mail, saw why the mail was encrypted (link to Herald rule). Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18984 --- src/__phutil_library_map__.php | 2 + .../herald/adapter/HeraldAdapter.php | 14 +++++ .../PhabricatorMetaMTAMailViewController.php | 9 +++ ...PhabricatorMailMustEncryptHeraldAction.php | 62 +++++++++++++++++++ .../PhabricatorMetaMTAEmailHeraldAction.php | 4 ++ .../storage/PhabricatorMetaMTAMail.php | 9 +++ ...habricatorApplicationTransactionEditor.php | 11 ++++ 7 files changed, 111 insertions(+) create mode 100644 src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 32985c76c0..f0e2d29cfc 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3189,6 +3189,7 @@ phutil_register_library_map(array( 'PhabricatorMailManagementUnverifyWorkflow' => 'applications/metamta/management/PhabricatorMailManagementUnverifyWorkflow.php', 'PhabricatorMailManagementVolumeWorkflow' => 'applications/metamta/management/PhabricatorMailManagementVolumeWorkflow.php', 'PhabricatorMailManagementWorkflow' => 'applications/metamta/management/PhabricatorMailManagementWorkflow.php', + 'PhabricatorMailMustEncryptHeraldAction' => 'applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php', 'PhabricatorMailOutboundMailHeraldAdapter' => 'applications/metamta/herald/PhabricatorMailOutboundMailHeraldAdapter.php', 'PhabricatorMailOutboundRoutingHeraldAction' => 'applications/metamta/herald/PhabricatorMailOutboundRoutingHeraldAction.php', 'PhabricatorMailOutboundRoutingSelfEmailHeraldAction' => 'applications/metamta/herald/PhabricatorMailOutboundRoutingSelfEmailHeraldAction.php', @@ -8674,6 +8675,7 @@ phutil_register_library_map(array( 'PhabricatorMailManagementUnverifyWorkflow' => 'PhabricatorMailManagementWorkflow', 'PhabricatorMailManagementVolumeWorkflow' => 'PhabricatorMailManagementWorkflow', 'PhabricatorMailManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'PhabricatorMailMustEncryptHeraldAction' => 'HeraldAction', 'PhabricatorMailOutboundMailHeraldAdapter' => 'HeraldAdapter', 'PhabricatorMailOutboundRoutingHeraldAction' => 'HeraldAction', 'PhabricatorMailOutboundRoutingSelfEmailHeraldAction' => 'PhabricatorMailOutboundRoutingHeraldAction', diff --git a/src/applications/herald/adapter/HeraldAdapter.php b/src/applications/herald/adapter/HeraldAdapter.php index 9d56f474ff..cc0fdbd3b5 100644 --- a/src/applications/herald/adapter/HeraldAdapter.php +++ b/src/applications/herald/adapter/HeraldAdapter.php @@ -39,6 +39,7 @@ abstract class HeraldAdapter extends Phobject { private $edgeCache = array(); private $forbiddenActions = array(); private $viewer; + private $mustEncryptReasons = array(); public function getEmailPHIDs() { return array_values($this->emailPHIDs); @@ -1182,4 +1183,17 @@ abstract class HeraldAdapter extends Phobject { return $this->forbiddenActions[$action]; } + +/* -( Must Encrypt )------------------------------------------------------- */ + + + final public function addMustEncryptReason($reason) { + $this->mustEncryptReasons[] = $reason; + return $this; + } + + final public function getMustEncryptReasons() { + return $this->mustEncryptReasons; + } + } diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 20bbc425b5..1aca34c2ea 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -175,6 +175,15 @@ final class PhabricatorMetaMTAMailViewController $properties->addProperty($key, $value); } + $encrypt_phids = $mail->getMustEncryptReasons(); + if ($encrypt_phids) { + $properties->addProperty( + pht('Must Encrypt'), + $viewer->loadHandles($encrypt_phids) + ->renderList()); + } + + return $properties; } diff --git a/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php new file mode 100644 index 0000000000..f8cf7ee204 --- /dev/null +++ b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php @@ -0,0 +1,62 @@ +getRule()->getPHID(); + + $adapter = $this->getAdapter(); + $adapter->addMustEncryptReason($rule_phid); + + $this->logEffect(self::DO_MUST_ENCRYPT, array($rule_phid)); + } + + protected function getActionEffectMap() { + return array( + self::DO_MUST_ENCRYPT => array( + 'icon' => 'fa-shield', + 'color' => 'blue', + 'name' => pht('Must Encrypt'), + ), + ); + } + + protected function renderActionEffectDescription($type, $data) { + switch ($type) { + case self::DO_MUST_ENCRYPT: + return pht( + 'Made it a requirement that mail content be transmitted only '. + 'over secure channels.'); + } + } + +} diff --git a/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php b/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php index 74fb879fe7..383b8ebd36 100644 --- a/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php +++ b/src/applications/metamta/herald/PhabricatorMetaMTAEmailHeraldAction.php @@ -13,6 +13,10 @@ abstract class PhabricatorMetaMTAEmailHeraldAction } public function supportsObject($object) { + return self::isMailGeneratingObject($object); + } + + public static function isMailGeneratingObject($object) { // NOTE: This implementation lacks generality, but there's no great way to // figure out if something generates email right now. diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index c203e86530..a9736c1766 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -259,6 +259,15 @@ final class PhabricatorMetaMTAMail return $this->getParam('mustEncrypt', false); } + public function setMustEncryptReasons(array $reasons) { + $this->setParam('mustEncryptReasons', $reasons); + return $this; + } + + public function getMustEncryptReasons() { + return $this->getParam('mustEncryptReasons', array()); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 155592fc4e..0dc6a06fbf 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -71,6 +71,7 @@ abstract class PhabricatorApplicationTransactionEditor private $mailShouldSend = false; private $modularTypes; private $silent; + private $mustEncrypt; private $transactionQueue = array(); @@ -2549,6 +2550,13 @@ abstract class PhabricatorApplicationTransactionEditor $this->loadHandles($xactions); $mail = $this->buildMailForTarget($object, $xactions, $target); + + if ($this->mustEncrypt) { + $mail + ->setMustEncrypt(true) + ->setMustEncryptReasons($this->mustEncrypt); + } + } catch (Exception $ex) { $caught = $ex; } @@ -3214,6 +3222,8 @@ abstract class PhabricatorApplicationTransactionEditor $adapter->getQueuedHarbormasterBuildRequests()); } + $this->mustEncrypt = $adapter->getMustEncryptReasons(); + return array_merge( $this->didApplyHeraldRules($object, $adapter, $xscript), $adapter->getQueuedTransactions()); @@ -3558,6 +3568,7 @@ abstract class PhabricatorApplicationTransactionEditor 'feedRelatedPHIDs', 'feedShouldPublish', 'mailShouldSend', + 'mustEncrypt', ); } From eb06aca951cee6b01ceecf57286a892c77348d81 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 11:28:06 -0800 Subject: [PATCH 36/89] Support DestructionEngine in MetaMTAMail Summary: Depends on D18984. Ref T13053. See D13408 for the original change and why this doesn't use DestructionEngine right now. The quick version is: - It causes us to write a destruction log, which is slightly silly (we're deleting one thing and creating another). - It's a little bit slower than not using DestructionEngine. However, it gets us some stuff for free that's likely relevant now (e.g., Herald Transcript cleanup) and I'm planning to move attachments to Files, but want to be able to delete them when mail is destroyed. The destruction log is a touch silly, but those records are very small and that log gets GC'd later without generating new logs. We could silence the log from the GC if it's ever an issue. Test Plan: Used `bin/remove destroy` and `bin/garbage collect --collector mail.sent` to destroy mail and collect garbage. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18985 --- src/__phutil_library_map__.php | 1 + .../MetaMTAMailSentGarbageCollector.php | 3 ++- .../storage/PhabricatorMetaMTAMail.php | 26 ++++++++----------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index f0e2d29cfc..aff0d5e36f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -8739,6 +8739,7 @@ phutil_register_library_map(array( 'PhabricatorMetaMTAMail' => array( 'PhabricatorMetaMTADAO', 'PhabricatorPolicyInterface', + 'PhabricatorDestructibleInterface', ), 'PhabricatorMetaMTAMailBody' => 'Phobject', 'PhabricatorMetaMTAMailBodyTestCase' => 'PhabricatorTestCase', diff --git a/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php b/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php index c9ca274436..dacd46d187 100644 --- a/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php +++ b/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php @@ -18,8 +18,9 @@ final class MetaMTAMailSentGarbageCollector 'dateCreated < %d LIMIT 100', $this->getGarbageEpoch()); + $engine = new PhabricatorDestructionEngine(); foreach ($mails as $mail) { - $mail->delete(); + $engine->destroyObject($mail); } return (count($mails) == 100); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index a9736c1766..d5111529f3 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -5,7 +5,9 @@ */ final class PhabricatorMetaMTAMail extends PhabricatorMetaMTADAO - implements PhabricatorPolicyInterface { + implements + PhabricatorPolicyInterface, + PhabricatorDestructibleInterface { const RETRY_DELAY = 5; @@ -1041,20 +1043,6 @@ final class PhabricatorMetaMTAMail } } - public function delete() { - $this->openTransaction(); - queryfx( - $this->establishConnection('w'), - 'DELETE FROM %T WHERE src = %s AND type = %d', - PhabricatorEdgeConfig::TABLE_NAME_EDGE, - $this->getPHID(), - PhabricatorMetaMTAMailHasRecipientEdgeType::EDGECONST); - $ret = parent::delete(); - $this->saveTransaction(); - - return $ret; - } - public function generateHeaders() { $headers = array(); @@ -1306,4 +1294,12 @@ final class PhabricatorMetaMTAMail } +/* -( PhabricatorDestructibleInterface )----------------------------------- */ + + + public function destroyObjectPermanently( + PhabricatorDestructionEngine $engine) { + $this->delete(); + } + } From 6d90c7ad92b35e95ceda1b94772b7120cea84acd Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 1 Feb 2018 11:17:47 -0800 Subject: [PATCH 37/89] Save mail attachments in Files, not on the actual objects Summary: Depends on D18985. Ref T13053. See PHI125. Currently, mail attachments are just encoded onto the actual objects in the `MetaMTAMail` table. This fails if attachments can't be encoded in JSON -- e.g., they aren't UTF8. This happens most often when revisions or commits attach patches to mail and those patches contain source code changes for files that are not encoded in UTF8. Instead, save attachments in (and load attachments from) Files. Test Plan: Enabled patches for mail, created a revision, saw it attach a patch. Viewed mail in web UI, saw link to download patch. Followed link, saw sensible file. Checked database, saw a `filePHID`. Destroyed mail with `bin/remove destroy`, saw attached files also destroyed. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18986 --- .../PhabricatorMetaMTAMailViewController.php | 6 +++ .../storage/PhabricatorMetaMTAAttachment.php | 44 ++++++++++++++++--- .../storage/PhabricatorMetaMTAMail.php | 41 +++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 1aca34c2ea..03d340bac9 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -151,6 +151,12 @@ final class PhabricatorMetaMTAMailViewController $properties->addTextContent($body); + $file_phids = $mail->getAttachmentFilePHIDs(); + if ($file_phids) { + $properties->addProperty( + pht('Attached Files'), + $viewer->loadHandles($file_phids)->renderList()); + } return $properties; } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php b/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php index b26bb0b8a7..256bee46e8 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAAttachment.php @@ -1,9 +1,12 @@ setData($data); @@ -39,18 +42,49 @@ final class PhabricatorMetaMTAAttachment extends Phobject { } public function toDictionary() { + if (!$this->file) { + $iterator = new ArrayIterator(array($this->getData())); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName($this->getFilename()) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType($this->getMimeType()) + ->setIterator($iterator); + + $this->file = $source->uploadFile(); + } + return array( 'filename' => $this->getFilename(), 'mimetype' => $this->getMimeType(), - 'data' => $this->getData(), + 'filePHID' => $this->file->getPHID(), ); } public static function newFromDictionary(array $dict) { - return new PhabricatorMetaMTAAttachment( + $file = null; + + $file_phid = idx($dict, 'filePHID'); + if ($file_phid) { + $file = id(new PhabricatorFileQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if ($file) { + $dict['data'] = $file->loadFileData(); + } + } + + $attachment = new self( idx($dict, 'data'), idx($dict, 'filename'), idx($dict, 'mimetype')); + + if ($file) { + $attachment->file = $file; + } + + return $attachment; } } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index d5111529f3..a61951650c 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -197,6 +197,35 @@ final class PhabricatorMetaMTAMail return $result; } + public function getAttachmentFilePHIDs() { + $file_phids = array(); + + $dictionaries = $this->getParam('attachments'); + if ($dictionaries) { + foreach ($dictionaries as $dictionary) { + $file_phid = idx($dictionary, 'filePHID'); + if ($file_phid) { + $file_phids[] = $file_phid; + } + } + } + + return $file_phids; + } + + public function loadAttachedFiles(PhabricatorUser $viewer) { + $file_phids = $this->getAttachmentFilePHIDs(); + + if (!$file_phids) { + return array(); + } + + return id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs($file_phids) + ->execute(); + } + public function setAttachments(array $attachments) { assert_instances_of($attachments, 'PhabricatorMetaMTAAttachment'); $this->setParam('attachments', mpull($attachments, 'toDictionary')); @@ -526,6 +555,12 @@ final class PhabricatorMetaMTAMail mpull($cc_actors, 'getEmailAddress')); break; case 'attachments': + $attached_viewer = PhabricatorUser::getOmnipotentUser(); + $files = $this->loadAttachedFiles($attached_viewer); + foreach ($files as $file) { + $file->attachToObject($this->getPHID()); + } + // If the mail content must be encrypted, don't add attachments. if ($must_encrypt) { break; @@ -1299,6 +1334,12 @@ final class PhabricatorMetaMTAMail public function destroyObjectPermanently( PhabricatorDestructionEngine $engine) { + + $files = $this->loadAttachedFiles($engine->getViewer()); + foreach ($files as $file) { + $engine->destroyObject($file); + } + $this->delete(); } From 55f7cdb99b3f40b94aa07b1d29c8eaf344db34da Mon Sep 17 00:00:00 2001 From: epriestley Date: Fri, 2 Feb 2018 14:57:25 -0800 Subject: [PATCH 38/89] Fix a bad classname reference in the "Must Encrypt" action --- .../metamta/herald/PhabricatorMailMustEncryptHeraldAction.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php index f8cf7ee204..027e1bb733 100644 --- a/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php +++ b/src/applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php @@ -16,7 +16,7 @@ final class PhabricatorMailMustEncryptHeraldAction 'Require mail content be transmitted only over secure channels.'); } public function supportsObject($object) { - return self::isMailGeneratingObject($object); + return PhabricatorMetaMTAEmailHeraldAction::isMailGeneratingObject($object); } public function getActionGroupKey() { From 956c4058e64cf3ec338e08e5218980e7f93f72fa Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 4 Feb 2018 05:55:01 -0800 Subject: [PATCH 39/89] Add a `bin/conduit call` support binary Summary: Ref T13060. See PHI343. Triaging this bug required figuring out where in the pipeline UTF8 was being dropped, and bisecting the pipeline required making calls to Conduit. Currently, there's no easy way to debug/inspect arbitrary Conduit calls, especially when they are `diffusion.*` calls which route to a different host (even if you have a real session and use the web console for these, you just see an HTTP service call to the target host in DarkConsole). Add a `bin/conduit` utility to make this kind of debugging easier, with an eye toward the Phacility production cluster (or other similar clusters) specifically. Test Plan: - Ran `echo '{}' | bin/conduit call --method conduit.ping --input -` and similar. - Used a similar approach to successfully diagnose the UTF8 issue in T13060. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13060 Differential Revision: https://secure.phabricator.com/D18987 --- bin/conduit | 1 + scripts/setup/manage_conduit.php | 21 ++++++ src/__phutil_library_map__.php | 4 ++ ...abricatorConduitCallManagementWorkflow.php | 66 +++++++++++++++++++ .../PhabricatorConduitManagementWorkflow.php | 4 ++ 5 files changed, 96 insertions(+) create mode 120000 bin/conduit create mode 100755 scripts/setup/manage_conduit.php create mode 100644 src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php create mode 100644 src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php diff --git a/bin/conduit b/bin/conduit new file mode 120000 index 0000000000..9221340a93 --- /dev/null +++ b/bin/conduit @@ -0,0 +1 @@ +../scripts/setup/manage_conduit.php \ No newline at end of file diff --git a/scripts/setup/manage_conduit.php b/scripts/setup/manage_conduit.php new file mode 100755 index 0000000000..07384e7ed8 --- /dev/null +++ b/scripts/setup/manage_conduit.php @@ -0,0 +1,21 @@ +#!/usr/bin/env php +setTagline(pht('manage Conduit')); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorConduitManagementWorkflow') + ->execute(); +$workflows[] = new PhutilHelpArgumentWorkflow(); +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index aff0d5e36f..7076cde010 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2425,6 +2425,7 @@ phutil_register_library_map(array( 'PhabricatorCommonPasswords' => 'applications/auth/constants/PhabricatorCommonPasswords.php', 'PhabricatorConduitAPIController' => 'applications/conduit/controller/PhabricatorConduitAPIController.php', 'PhabricatorConduitApplication' => 'applications/conduit/application/PhabricatorConduitApplication.php', + 'PhabricatorConduitCallManagementWorkflow' => 'applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php', 'PhabricatorConduitCertificateToken' => 'applications/conduit/storage/PhabricatorConduitCertificateToken.php', 'PhabricatorConduitConsoleController' => 'applications/conduit/controller/PhabricatorConduitConsoleController.php', 'PhabricatorConduitContentSource' => 'infrastructure/contentsource/PhabricatorConduitContentSource.php', @@ -2435,6 +2436,7 @@ phutil_register_library_map(array( 'PhabricatorConduitLogController' => 'applications/conduit/controller/PhabricatorConduitLogController.php', 'PhabricatorConduitLogQuery' => 'applications/conduit/query/PhabricatorConduitLogQuery.php', 'PhabricatorConduitLogSearchEngine' => 'applications/conduit/query/PhabricatorConduitLogSearchEngine.php', + 'PhabricatorConduitManagementWorkflow' => 'applications/conduit/management/PhabricatorConduitManagementWorkflow.php', 'PhabricatorConduitMethodCallLog' => 'applications/conduit/storage/PhabricatorConduitMethodCallLog.php', 'PhabricatorConduitMethodQuery' => 'applications/conduit/query/PhabricatorConduitMethodQuery.php', 'PhabricatorConduitRequestExceptionHandler' => 'aphront/handler/PhabricatorConduitRequestExceptionHandler.php', @@ -7822,6 +7824,7 @@ phutil_register_library_map(array( 'PhabricatorCommonPasswords' => 'Phobject', 'PhabricatorConduitAPIController' => 'PhabricatorConduitController', 'PhabricatorConduitApplication' => 'PhabricatorApplication', + 'PhabricatorConduitCallManagementWorkflow' => 'PhabricatorConduitManagementWorkflow', 'PhabricatorConduitCertificateToken' => 'PhabricatorConduitDAO', 'PhabricatorConduitConsoleController' => 'PhabricatorConduitController', 'PhabricatorConduitContentSource' => 'PhabricatorContentSource', @@ -7832,6 +7835,7 @@ phutil_register_library_map(array( 'PhabricatorConduitLogController' => 'PhabricatorConduitController', 'PhabricatorConduitLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorConduitLogSearchEngine' => 'PhabricatorApplicationSearchEngine', + 'PhabricatorConduitManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorConduitMethodCallLog' => array( 'PhabricatorConduitDAO', 'PhabricatorPolicyInterface', diff --git a/src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php b/src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php new file mode 100644 index 0000000000..6cb3bd2409 --- /dev/null +++ b/src/applications/conduit/management/PhabricatorConduitCallManagementWorkflow.php @@ -0,0 +1,66 @@ +setName('call') + ->setSynopsis(pht('Call a Conduit method..')) + ->setArguments( + array( + array( + 'name' => 'method', + 'param' => 'method', + 'help' => pht('Method to call.'), + ), + array( + 'name' => 'input', + 'param' => 'input', + 'help' => pht( + 'File to read parameters from, or "-" to read from '. + 'stdin.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $method = $args->getArg('method'); + if (!strlen($method)) { + throw new PhutilArgumentUsageException( + pht('Specify a method to call with "--method".')); + } + + $input = $args->getArg('input'); + if (!strlen($input)) { + throw new PhutilArgumentUsageException( + pht('Specify a file to read parameters from with "--input".')); + } + + if ($input === '-') { + fprintf(STDERR, tsprintf("%s\n", pht('Reading input from stdin...'))); + $input_json = file_get_contents('php://stdin'); + } else { + $input_json = Filesystem::readFile($input); + } + + $params = phutil_json_decode($input_json); + + $result = id(new ConduitCall($method, $params)) + ->setUser($viewer) + ->execute(); + + $output = array( + 'result' => $result, + ); + + echo tsprintf( + "%B\n", + id(new PhutilJSON())->encodeFormatted($output)); + + return 0; + } + +} diff --git a/src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php b/src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php new file mode 100644 index 0000000000..4abb250fc4 --- /dev/null +++ b/src/applications/conduit/management/PhabricatorConduitManagementWorkflow.php @@ -0,0 +1,4 @@ + Date: Sun, 4 Feb 2018 06:01:49 -0800 Subject: [PATCH 40/89] Always setlocale() to en_US.UTF-8 for the main process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Depends on D18987. See PHI343. Fixes T13060. See also T7339. When the main process starts up with `LANG=POSIX` (this is the default on Ubuntu) and we later try to run a subprocess with a UTF8 character in the argument list (like `git cat-file blob 🐑.txt`), the argument is not passed to the subprocess correctly. We already set `LANG=en_US.UTF-8` in the //subprocess// environment, but this only controls behavior for the subprocess itself. It appears that the argument list encoding before the actual subprocess starts depends on the parent process's locale setting, which makes some degree of sense. Setting `putenv('LANG=en_US.UTF-8')` has no effect on this, but my guess is that the parent process's locale setting is read at startup (rather than read anew from `LANG` every time) and not changed by further modifications of `LANG`. Using `setlocale(...)` does appear to fix this. Ideally, installs would probably set some UTF-8-compatible LANG setting as the default. However, this makes setup harder and I couldn't figure out how to do it on our production Ubuntu AMI after spending a reasonable amount of time at it (see T13060). Since it's very rare that this setting matters, try to just do the right thing. This may fail if "en_US.UTF-8" isn't available, but I think warnings/remedies to this are in the scope of T7339, since we want this locale to exist for other legitimate reasons anyway. Test Plan: - Applied this fix in production, processed the failing worker task from PHI343 after kicking Apache hard enough. - Ran locally with `setlocale(LC_ALL, 'duck.quack')` to make sure a bad/invalid/unavailable setting didn't break anything, didn't hit any issues. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13060 Differential Revision: https://secure.phabricator.com/D18988 --- support/startup/PhabricatorStartup.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/support/startup/PhabricatorStartup.php b/support/startup/PhabricatorStartup.php index 1911a46b8a..212b057376 100644 --- a/support/startup/PhabricatorStartup.php +++ b/support/startup/PhabricatorStartup.php @@ -395,6 +395,11 @@ final class PhabricatorStartup { if (function_exists('libxml_disable_entity_loader')) { libxml_disable_entity_loader(true); } + + // See T13060. If the locale for this process (the parent process) is not + // a UTF-8 locale we can encounter problems when launching subprocesses + // which receive UTF-8 parameters in their command line argument list. + @setlocale(LC_ALL, 'en_US.UTF-8'); } From b3880975e5a17e0202055e235556316b34146cab Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 10:24:17 -0800 Subject: [PATCH 41/89] =?UTF-8?q?Add=20aliases=20for=20"party"=20emoji=20(?= =?UTF-8?q?=F0=9F=8E=89)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: This is currently `:tada:`, which I'd never have guessed. (This isn't a super scalable approach, but this emoji is in particularly common use. See also T12644.) Test Plan: Typed `:party`, `:confet`, etc. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D18993 --- resources/emoji/manifest.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/resources/emoji/manifest.json b/resources/emoji/manifest.json index 2d8388d277..47568fb43d 100644 --- a/resources/emoji/manifest.json +++ b/resources/emoji/manifest.json @@ -1622,5 +1622,9 @@ "zipper_mouth": "\ud83e\udd10", "zzz": "\ud83d\udca4", "100": "\ud83d\udcaf", - "1234": "\ud83d\udd22" + "1234": "\ud83d\udd22", + + "party": "\ud83c\udf89", + "celebration": "\ud83c\udf89", + "confetti": "\ud83c\udf89" } From ef121b3e17cfa9649820938e2092a5c98183c8f1 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 05:58:21 -0800 Subject: [PATCH 42/89] Fix a Herald repetition policy selection error for rule types which support only one policy Summary: Ref T13048. See . When a rule supports only one repetition policy (always "every time") like "Commit Hook" rules, we don't render a control for `repetition_policy` and fail to update it when saving. Before the changes to support the new "if the rule did not match the last time" policy, this workflow just defaulted to "every time" if the input was invalid, but this was changed by accident in D18926 when I removed some of the toInt/toString juggling code. (This patch also prevents users from fiddling with the form to create a rule which evaluates with an invalid policy; this wasn't validated before.) Test Plan: - Created new "Commit Hook" (only one policy available) rule. - Saved existing "Commit Hook" rule. - Created new "Task" (multiple policies) rule. - Saved existing Task rule. - Set task rule to each repetition policy, saved, verified the save worked. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13048 Differential Revision: https://secure.phabricator.com/D18992 --- .../controller/HeraldRuleController.php | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/applications/herald/controller/HeraldRuleController.php b/src/applications/herald/controller/HeraldRuleController.php index c61c29e90e..d400f8ae90 100644 --- a/src/applications/herald/controller/HeraldRuleController.php +++ b/src/applications/herald/controller/HeraldRuleController.php @@ -265,7 +265,15 @@ final class HeraldRuleController extends HeraldController { $new_name = $request->getStr('name'); $match_all = ($request->getStr('must_match') == 'all'); - $repetition_policy_param = $request->getStr('repetition_policy'); + $repetition_policy = $request->getStr('repetition_policy'); + + // If the user selected an invalid policy, or there's only one possible + // value so we didn't render a control, adjust the value to the first + // valid policy value. + $repetition_options = $this->getRepetitionOptionMap($adapter); + if (!isset($repetition_options[$repetition_policy])) { + $repetition_policy = head_key($repetition_options); + } $e_name = true; $errors = array(); @@ -348,7 +356,7 @@ final class HeraldRuleController extends HeraldController { $match_all, $conditions, $actions, - $repetition_policy_param); + $repetition_policy); $xactions = array(); $xactions[] = id(new HeraldRuleTransaction()) @@ -373,7 +381,7 @@ final class HeraldRuleController extends HeraldController { // mutate current rule, so it would be sent to the client in the right state $rule->setMustMatchAll((int)$match_all); $rule->setName($new_name); - $rule->setRepetitionPolicyStringConstant($repetition_policy_param); + $rule->setRepetitionPolicyStringConstant($repetition_policy); $rule->attachConditions($conditions); $rule->attachActions($actions); @@ -594,13 +602,9 @@ final class HeraldRuleController extends HeraldController { */ private function renderRepetitionSelector($rule, HeraldAdapter $adapter) { $repetition_policy = $rule->getRepetitionPolicyStringConstant(); - - $repetition_options = $adapter->getRepetitionOptions(); - $repetition_names = HeraldRule::getRepetitionPolicySelectOptionMap(); - $repetition_map = array_select_keys($repetition_names, $repetition_options); - + $repetition_map = $this->getRepetitionOptionMap($adapter); if (count($repetition_map) < 2) { - return head($repetition_names); + return head($repetition_map); } else { return AphrontFormSelectControl::renderSelectTag( $repetition_policy, @@ -611,6 +615,11 @@ final class HeraldRuleController extends HeraldController { } } + private function getRepetitionOptionMap(HeraldAdapter $adapter) { + $repetition_options = $adapter->getRepetitionOptions(); + $repetition_names = HeraldRule::getRepetitionPolicySelectOptionMap(); + return array_select_keys($repetition_names, $repetition_options); + } protected function buildTokenizerTemplates() { $template = new AphrontTokenizerTemplateView(); From c3f95bc410a3173fc1d2eccc392e0d4b6b6c65a4 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 4 Feb 2018 09:04:12 -0800 Subject: [PATCH 43/89] Add basic support for mail "stamps" to improve client mail routing Summary: Ref T10448. Currently, we use "mail tags" (in {nav Settings > Email Preferences}) to give users some ability to route mail. There are a number of major issues with this: - It isn't modular and can't be extended by third-party applications. - The UI is a giant mess of 5,000 individual settings. - Settings don't map clearly to actual edits. - A lot of stuff isn't covered by any setting. This adds a new system, called "mail stamps", which is similar to "mail tags" but tries to fix all these problems. I called these "stamps" because: stamps make sense with mail; we can't throw away the old system just yet and need to keep it around for a bit; we don't use this term for anything else; it avoids confusion with project tags. (Conceptually, imagine these as ink stamps like "RETURN TO SENDER" or "FRAGILE", not actual postage stamps.) The only real "trick" here is that later versions of this will need to enumerate possible stamps for an object and maybe all possible stamps for all objects in the system. This is why stamp generation is separated into a "template" phase and a "value" phase. In future changes, the "template" phase can be used on its own to generate documentation and typeaheads and let users build rules. This may need some more refinement before it really works since I haven't built any of that yet. Also adds a preference for getting stamps in the header only (default) or header and body (better for Gmail, which can't route based on headers). Test Plan: Fiddled with preference, sent some mail and saw a "STAMPS" setting in the body and an "X-Phabricator-Stamps" header. {F5411694} Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T10448 Differential Revision: https://secure.phabricator.com/D18991 --- src/__phutil_library_map__.php | 10 ++ .../engine/PhabricatorMailEngineExtension.php | 47 ++++++ .../replyhandler/PhabricatorMailTarget.php | 33 +++- .../metamta/stamp/PhabricatorMailStamp.php | 88 +++++++++++ .../stamp/PhabricatorStringMailStamp.php | 16 ++ .../storage/PhabricatorMetaMTAMail.php | 29 ++++ .../setting/PhabricatorEmailStampsSetting.php | 47 ++++++ ...habricatorApplicationTransactionEditor.php | 149 ++++++++++++++++++ ...orApplicationObjectMailEngineExtension.php | 92 +++++++++++ 9 files changed, 507 insertions(+), 4 deletions(-) create mode 100644 src/applications/metamta/engine/PhabricatorMailEngineExtension.php create mode 100644 src/applications/metamta/stamp/PhabricatorMailStamp.php create mode 100644 src/applications/metamta/stamp/PhabricatorStringMailStamp.php create mode 100644 src/applications/settings/setting/PhabricatorEmailStampsSetting.php create mode 100644 src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 7076cde010..6fa14fbacb 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1958,6 +1958,7 @@ phutil_register_library_map(array( 'PhabricatorApplicationEditHTTPParameterHelpView' => 'applications/transactions/view/PhabricatorApplicationEditHTTPParameterHelpView.php', 'PhabricatorApplicationEditor' => 'applications/meta/editor/PhabricatorApplicationEditor.php', 'PhabricatorApplicationEmailCommandsController' => 'applications/meta/controller/PhabricatorApplicationEmailCommandsController.php', + 'PhabricatorApplicationObjectMailEngineExtension' => 'applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php', 'PhabricatorApplicationPanelController' => 'applications/meta/controller/PhabricatorApplicationPanelController.php', 'PhabricatorApplicationPolicyChangeTransaction' => 'applications/meta/xactions/PhabricatorApplicationPolicyChangeTransaction.php', 'PhabricatorApplicationProfileMenuItem' => 'applications/search/menuitem/PhabricatorApplicationProfileMenuItem.php', @@ -2828,6 +2829,7 @@ phutil_register_library_map(array( 'PhabricatorEmailPreferencesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailPreferencesSettingsPanel.php', 'PhabricatorEmailRePrefixSetting' => 'applications/settings/setting/PhabricatorEmailRePrefixSetting.php', 'PhabricatorEmailSelfActionsSetting' => 'applications/settings/setting/PhabricatorEmailSelfActionsSetting.php', + 'PhabricatorEmailStampsSetting' => 'applications/settings/setting/PhabricatorEmailStampsSetting.php', 'PhabricatorEmailTagsSetting' => 'applications/settings/setting/PhabricatorEmailTagsSetting.php', 'PhabricatorEmailVarySubjectsSetting' => 'applications/settings/setting/PhabricatorEmailVarySubjectsSetting.php', 'PhabricatorEmailVerificationController' => 'applications/auth/controller/PhabricatorEmailVerificationController.php', @@ -3174,6 +3176,7 @@ phutil_register_library_map(array( 'PhabricatorMailEmailHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailHeraldField.php', 'PhabricatorMailEmailHeraldFieldGroup' => 'applications/metamta/herald/PhabricatorMailEmailHeraldFieldGroup.php', 'PhabricatorMailEmailSubjectHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailSubjectHeraldField.php', + 'PhabricatorMailEngineExtension' => 'applications/metamta/engine/PhabricatorMailEngineExtension.php', 'PhabricatorMailImplementationAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationAdapter.php', 'PhabricatorMailImplementationAmazonSESAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php', 'PhabricatorMailImplementationMailgunAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php', @@ -3202,6 +3205,7 @@ phutil_register_library_map(array( 'PhabricatorMailReplyHandler' => 'applications/metamta/replyhandler/PhabricatorMailReplyHandler.php', 'PhabricatorMailRoutingRule' => 'applications/metamta/constants/PhabricatorMailRoutingRule.php', 'PhabricatorMailSetupCheck' => 'applications/config/check/PhabricatorMailSetupCheck.php', + 'PhabricatorMailStamp' => 'applications/metamta/stamp/PhabricatorMailStamp.php', 'PhabricatorMailTarget' => 'applications/metamta/replyhandler/PhabricatorMailTarget.php', 'PhabricatorMailgunConfigOptions' => 'applications/config/option/PhabricatorMailgunConfigOptions.php', 'PhabricatorMainMenuBarExtension' => 'view/page/menu/PhabricatorMainMenuBarExtension.php', @@ -4204,6 +4208,7 @@ phutil_register_library_map(array( 'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php', 'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php', 'PhabricatorStringListExportField' => 'infrastructure/export/field/PhabricatorStringListExportField.php', + 'PhabricatorStringMailStamp' => 'applications/metamta/stamp/PhabricatorStringMailStamp.php', 'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php', 'PhabricatorSubmitEditField' => 'applications/transactions/editfield/PhabricatorSubmitEditField.php', 'PhabricatorSubscribableInterface' => 'applications/subscriptions/interface/PhabricatorSubscribableInterface.php', @@ -7269,6 +7274,7 @@ phutil_register_library_map(array( 'PhabricatorApplicationEditHTTPParameterHelpView' => 'AphrontView', 'PhabricatorApplicationEditor' => 'PhabricatorApplicationTransactionEditor', 'PhabricatorApplicationEmailCommandsController' => 'PhabricatorApplicationsController', + 'PhabricatorApplicationObjectMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorApplicationPanelController' => 'PhabricatorApplicationsController', 'PhabricatorApplicationPolicyChangeTransaction' => 'PhabricatorApplicationTransactionType', 'PhabricatorApplicationProfileMenuItem' => 'PhabricatorProfileMenuItem', @@ -8276,6 +8282,7 @@ phutil_register_library_map(array( 'PhabricatorEmailPreferencesSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorEmailRePrefixSetting' => 'PhabricatorSelectSetting', 'PhabricatorEmailSelfActionsSetting' => 'PhabricatorSelectSetting', + 'PhabricatorEmailStampsSetting' => 'PhabricatorSelectSetting', 'PhabricatorEmailTagsSetting' => 'PhabricatorInternalSetting', 'PhabricatorEmailVarySubjectsSetting' => 'PhabricatorSelectSetting', 'PhabricatorEmailVerificationController' => 'PhabricatorAuthController', @@ -8662,6 +8669,7 @@ phutil_register_library_map(array( 'PhabricatorMailEmailHeraldField' => 'HeraldField', 'PhabricatorMailEmailHeraldFieldGroup' => 'HeraldFieldGroup', 'PhabricatorMailEmailSubjectHeraldField' => 'PhabricatorMailEmailHeraldField', + 'PhabricatorMailEngineExtension' => 'Phobject', 'PhabricatorMailImplementationAdapter' => 'Phobject', 'PhabricatorMailImplementationAmazonSESAdapter' => 'PhabricatorMailImplementationPHPMailerLiteAdapter', 'PhabricatorMailImplementationMailgunAdapter' => 'PhabricatorMailImplementationAdapter', @@ -8690,6 +8698,7 @@ phutil_register_library_map(array( 'PhabricatorMailReplyHandler' => 'Phobject', 'PhabricatorMailRoutingRule' => 'Phobject', 'PhabricatorMailSetupCheck' => 'PhabricatorSetupCheck', + 'PhabricatorMailStamp' => 'Phobject', 'PhabricatorMailTarget' => 'Phobject', 'PhabricatorMailgunConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMainMenuBarExtension' => 'Phobject', @@ -9907,6 +9916,7 @@ phutil_register_library_map(array( 'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType', 'PhabricatorStringListEditField' => 'PhabricatorEditField', 'PhabricatorStringListExportField' => 'PhabricatorListExportField', + 'PhabricatorStringMailStamp' => 'PhabricatorMailStamp', 'PhabricatorStringSetting' => 'PhabricatorSetting', 'PhabricatorSubmitEditField' => 'PhabricatorEditField', 'PhabricatorSubscribedToObjectEdgeType' => 'PhabricatorEdgeType', diff --git a/src/applications/metamta/engine/PhabricatorMailEngineExtension.php b/src/applications/metamta/engine/PhabricatorMailEngineExtension.php new file mode 100644 index 0000000000..36675dda4a --- /dev/null +++ b/src/applications/metamta/engine/PhabricatorMailEngineExtension.php @@ -0,0 +1,47 @@ +getPhobjectClassConstant('EXTENSIONKEY'); + } + + final public function setViewer($viewer) { + $this->viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + final public function setEditor( + PhabricatorApplicationTransactionEditor $editor) { + $this->editor = $editor; + return $this; + } + + final public function getEditor() { + return $this->editor; + } + + abstract public function supportsObject($object); + abstract public function newMailStampTemplates($object); + abstract public function newMailStamps($object, array $xactions); + + final public static function getAllExtensions() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getExtensionKey') + ->execute(); + } + + final protected function getMailStamp($key) { + return $this->getEditor()->getMailStamp($key); + } + +} diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index c607087b22..0463b6cdd7 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -58,23 +58,48 @@ final class PhabricatorMailTarget extends Phobject { public function willSendMail(PhabricatorMetaMTAMail $mail) { $viewer = $this->getViewer(); + $show_stamps = $mail->shouldRenderMailStampsInBody($viewer); + + $body = $mail->getBody(); + $html_body = $mail->getHTMLBody(); + $has_html = (strlen($html_body) > 0); + + if ($show_stamps) { + $stamps = $mail->getMailStamps(); + + $body .= "\n"; + $body .= pht('STAMPS'); + $body .= "\n"; + $body .= implode(', ', $stamps); + $body .= "\n"; + + if ($has_html) { + $html = array(); + $html[] = phutil_tag('strong', array(), pht('STAMPS')); + $html[] = phutil_tag('br'); + $html[] = phutil_implode_html(', ', $stamps); + $html[] = phutil_tag('br'); + $html = phutil_tag('div', array(), $html); + $html_body .= hsprintf('%s', $html); + } + } + $mail->addPHIDHeaders('X-Phabricator-To', $this->rawToPHIDs); $mail->addPHIDHeaders('X-Phabricator-Cc', $this->rawCCPHIDs); $to_handles = $viewer->loadHandles($this->rawToPHIDs); $cc_handles = $viewer->loadHandles($this->rawCCPHIDs); - $body = $mail->getBody(); $body .= "\n"; $body .= $this->getRecipientsSummary($to_handles, $cc_handles); - $mail->setBody($body); - $html_body = $mail->getHTMLBody(); - if (strlen($html_body)) { + if ($has_html) { $html_body .= hsprintf( '%s', $this->getRecipientsSummaryHTML($to_handles, $cc_handles)); } + + $mail->setBody($body); $mail->setHTMLBody($html_body); $reply_to = $this->getReplyTo(); diff --git a/src/applications/metamta/stamp/PhabricatorMailStamp.php b/src/applications/metamta/stamp/PhabricatorMailStamp.php new file mode 100644 index 0000000000..9b425a4bdf --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorMailStamp.php @@ -0,0 +1,88 @@ +getPhobjectClassConstant('STAMPTYPE'); + } + + final public function setKey($key) { + $this->key = $key; + return $this; + } + + final public function getKey() { + return $this->key; + } + + final protected function setRawValue($value) { + $this->value = $value; + return $this; + } + + final protected function getRawValue() { + return $this->value; + } + + final public function setViewer(PhabricatorUser $viewer) { + $this->viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + final public function setLabel($label) { + $this->label = $label; + return $this; + } + + final public function getLabel() { + return $this->label; + } + + public function setValue($value) { + return $this->setRawValue($value); + } + + final public function toDictionary() { + return array( + 'type' => $this->getStampType(), + 'key' => $this->getKey(), + 'value' => $this->getValueForDictionary(), + ); + } + + final public static function getAllStamps() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getStampType') + ->execute(); + } + + protected function getValueForDictionary() { + return $this->getRawValue(); + } + + public function setValueFromDictionary($value) { + return $this->setRawValue($value); + } + + public function getValueForRendering() { + return $this->getRawValue(); + } + + abstract public function renderStamps($value); + + final protected function renderStamp($key, $value = null) { + return $key.'('.$value.')'; + } + +} diff --git a/src/applications/metamta/stamp/PhabricatorStringMailStamp.php b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php new file mode 100644 index 0000000000..98d472ad48 --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php @@ -0,0 +1,16 @@ +renderStamp($this->getKey(), $value); + } + +} diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index a61951650c..a7734b7ae8 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -299,6 +299,22 @@ final class PhabricatorMetaMTAMail return $this->getParam('mustEncryptReasons', array()); } + public function setMailStamps(array $stamps) { + return $this->setParam('stamps', $stamps); + } + + public function getMailStamps() { + return $this->getParam('stamps', array()); + } + + public function setMailStampMetadata($metadata) { + return $this->setParam('stampMetadata', $metadata); + } + + public function getMailStampMetadata() { + return $this->getParam('stampMetadata', array()); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -637,6 +653,11 @@ final class PhabricatorMetaMTAMail } } + $stamps = $this->getMailStamps(); + if ($stamps) { + $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); + } + $raw_body = idx($params, 'body', ''); $body = $raw_body; if ($must_encrypt) { @@ -1304,6 +1325,14 @@ final class PhabricatorMetaMTAMail return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL); } + public function shouldRenderMailStampsInBody($viewer) { + $preferences = $this->loadPreferences($viewer->getPHID()); + $value = $preferences->getSettingValue( + PhabricatorEmailStampsSetting::SETTINGKEY); + + return ($value == PhabricatorEmailStampsSetting::VALUE_BODY_STAMPS); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/settings/setting/PhabricatorEmailStampsSetting.php b/src/applications/settings/setting/PhabricatorEmailStampsSetting.php new file mode 100644 index 0000000000..39403f40a0 --- /dev/null +++ b/src/applications/settings/setting/PhabricatorEmailStampsSetting.php @@ -0,0 +1,47 @@ + pht('Mail Headers'), + self::VALUE_BODY_STAMPS => pht('Mail Headers and Body'), + ); + } + +} diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 0dc6a06fbf..47b975c093 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -72,6 +72,8 @@ abstract class PhabricatorApplicationTransactionEditor private $modularTypes; private $silent; private $mustEncrypt; + private $stampTemplates = array(); + private $mailStamps = array(); private $transactionQueue = array(); @@ -1181,6 +1183,12 @@ abstract class PhabricatorApplicationTransactionEditor $this->mailShouldSend = true; $this->mailToPHIDs = $this->getMailTo($object); $this->mailCCPHIDs = $this->getMailCC($object); + + $mail_xactions = $this->getTransactionsForMail($object, $xactions); + $stamps = $this->newMailStamps($object, $xactions); + foreach ($stamps as $stamp) { + $this->mailStamps[] = $stamp->toDictionary(); + } } if ($this->shouldPublishFeedStory($object, $xactions)) { @@ -2611,6 +2619,7 @@ abstract class PhabricatorApplicationTransactionEditor $mail_tags = $this->getMailTags($object, $mail_xactions); $action = $this->getMailAction($object, $mail_xactions); + $stamps = $this->generateMailStamps($object, $this->mailStamps); if (PhabricatorEnv::getEnvConfig('metamta.email-preferences')) { $this->addEmailPreferenceSectionToMailBody( @@ -2649,6 +2658,18 @@ abstract class PhabricatorApplicationTransactionEditor $mail->setParentMessageID($this->getParentMessageID()); } + // If we have stamps, attach the raw dictionary version (not the actual + // objects) to the mail so that debugging tools can see what we used to + // render the final list. + if ($this->mailStamps) { + $mail->setMailStampMetadata($this->mailStamps); + } + + // If we have rendered stamps, attach them to the mail. + if ($stamps) { + $mail->setMailStamps($stamps); + } + return $target->willSendMail($mail); } @@ -3569,6 +3590,7 @@ abstract class PhabricatorApplicationTransactionEditor 'feedShouldPublish', 'mailShouldSend', 'mustEncrypt', + 'mailStamps', ); } @@ -3961,4 +3983,131 @@ abstract class PhabricatorApplicationTransactionEditor return $editor; } + +/* -( Stamps )------------------------------------------------------------- */ + + + public function newMailStampTemplates($object) { + $actor = $this->getActor(); + + $templates = array(); + + $extensions = $this->newMailExtensions($object); + foreach ($extensions as $extension) { + $stamps = $extension->newMailStampTemplates($object); + foreach ($stamps as $stamp) { + $key = $stamp->getKey(); + if (isset($templates[$key])) { + throw new Exception( + pht( + 'Mail extension ("%s") defines a stamp template with the '. + 'same key ("%s") as another template. Each stamp template '. + 'must have a unique key.', + get_class($extension), + $key)); + } + + $stamp->setViewer($actor); + + $templates[$key] = $stamp; + } + } + + return $templates; + } + + final public function getMailStamp($key) { + if (!isset($this->stampTemplates)) { + throw new PhutilInvalidStateException('newMailStampTemplates'); + } + + if (!isset($this->stampTemplates[$key])) { + throw new Exception( + pht( + 'Editor ("%s") has no mail stamp template with provided key ("%s").', + get_class($this), + $key)); + } + + return $this->stampTemplates[$key]; + } + + private function newMailStamps($object, array $xactions) { + $actor = $this->getActor(); + + $this->stampTemplates = $this->newMailStampTemplates($object); + + $extensions = $this->newMailExtensions($object); + $stamps = array(); + foreach ($extensions as $extension) { + $extension->newMailStamps($object, $xactions); + } + + return $this->stampTemplates; + } + + private function newMailExtensions($object) { + $actor = $this->getActor(); + + $all_extensions = PhabricatorMailEngineExtension::getAllExtensions(); + + $extensions = array(); + foreach ($all_extensions as $key => $template) { + $extension = id(clone $template) + ->setViewer($actor) + ->setEditor($this); + + if ($extension->supportsObject($object)) { + $extensions[$key] = $extension; + } + } + + return $extensions; + } + + private function generateMailStamps($object, $data) { + if (!$data || !is_array($data)) { + return null; + } + + $templates = $this->newMailStampTemplates($object); + foreach ($data as $spec) { + if (!is_array($spec)) { + continue; + } + + $key = idx($spec, 'key'); + if (!isset($templates[$key])) { + continue; + } + + $type = idx($spec, 'type'); + if ($templates[$key]->getStampType() !== $type) { + continue; + } + + $value = idx($spec, 'value'); + $templates[$key]->setValueFromDictionary($value); + } + + $results = array(); + foreach ($templates as $template) { + $value = $template->getValueForRendering(); + + $rendered = $template->renderStamps($value); + if ($rendered === null) { + continue; + } + + $rendered = (array)$rendered; + foreach ($rendered as $stamp) { + $results[] = $stamp; + } + } + + sort($results); + + return $results; + } + } diff --git a/src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php new file mode 100644 index 0000000000..bf441df71c --- /dev/null +++ b/src/applications/transactions/engineextension/PhabricatorApplicationObjectMailEngineExtension.php @@ -0,0 +1,92 @@ +setKey('application') + ->setLabel(pht('Application')), + ); + + if ($this->hasMonogram($object)) { + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('monogram') + ->setLabel(pht('Object Monogram')); + } + + if ($this->hasPHID($object)) { + // This is a PHID, but we always want to render it as a raw string, so + // use a string mail stamp. + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('phid') + ->setLabel(pht('Object PHID')); + + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('object-type') + ->setLabel(pht('Object Type')); + } + + return $templates; + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $application = null; + $class = $editor->getEditorApplicationClass(); + if (PhabricatorApplication::isClassInstalledForViewer($class, $viewer)) { + $application = newv($class, array()); + } + + if ($application) { + $application_name = $application->getName(); + $this->getMailStamp('application') + ->setValue($application_name); + } + + if ($this->hasMonogram($object)) { + $monogram = $object->getMonogram(); + $this->getMailStamp('monogram') + ->setValue($monogram); + } + + if ($this->hasPHID($object)) { + $object_phid = $object->getPHID(); + + $this->getMailStamp('phid') + ->setValue($object_phid); + + $phid_type = phid_get_type($object_phid); + if ($phid_type != PhabricatorPHIDConstants::PHID_TYPE_UNKNOWN) { + $this->getMailStamp('object-type') + ->setValue($phid_type); + } + } + } + + private function hasPHID($object) { + if (!($object instanceof LiskDAO)) { + return false; + } + + if (!$object->getConfigOption(LiskDAO::CONFIG_AUX_PHID)) { + return false; + } + + return true; + } + + private function hasMonogram($object) { + return method_exists($object, 'getMonogram'); + } + +} From 9de54aedb57801fcee9f9da839cf730063f98ca7 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 10:37:39 -0800 Subject: [PATCH 44/89] Remove inconsistent and confusing use of the term "multiplex" in mail Summary: Ref T13053. Because I previously misunderstood what "multiplex" means, I used it in various contradictory and inconsistent ways. We can send mail in two ways: either one mail to everyone with a big "To" and a big "Cc" (not default; better for mailing lists) or one mail to each recipient with just them in "To" (default; better for almost everything else). "Multiplexing" is combining multiple signals over a single channel, so it more accurately describes the big to/cc. However, it is sometimes used to descibe the other approach. Since it's ambiguous and I've tainted it through misuse, get rid of it and use more clear language. (There's still some likely misuse in the SMS stuff, and a couple of legitimate uses in other contexts.) Test Plan: Grepped for `multiplex`, saw less of it. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18994 --- .../config/option/PhabricatorMetaMTAConfigOptions.php | 6 +++--- .../PhabricatorMailImplementationPHPMailerAdapter.php | 4 ++-- .../PhabricatorMailImplementationPHPMailerLiteAdapter.php | 4 ++-- .../metamta/storage/PhabricatorMetaMTAMail.php | 8 ++++---- .../panel/PhabricatorEmailFormatSettingsPanel.php | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 6c846daa57..8c2d2265bc 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -153,9 +153,9 @@ EODOC $adapter_doc_name)); $placeholder_description = $this->deformat(pht(<<mailer->Encoding = $encoding; // By default, PHPMailer sends one mail per recipient. We handle - // multiplexing higher in the stack, so tell it to send mail exactly - // like we ask. + // combining or separating To and Cc higher in the stack, so tell it to + // send mail exactly like we ask. $this->mailer->SingleTo = false; $mailer = PhabricatorEnv::getEnvConfig('phpmailer.mailer'); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index f072e769c3..668f9353ec 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -22,8 +22,8 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter $this->mailer->Encoding = $encoding; // By default, PHPMailerLite sends one mail per recipient. We handle - // multiplexing higher in the stack, so tell it to send mail exactly - // like we ask. + // combining or separating To and Cc higher in the stack, so tell it to + // send mail exactly like we ask. $this->mailer->SingleTo = false; } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index a7734b7ae8..f19e5e951a 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -507,8 +507,8 @@ final class PhabricatorMetaMTAMail $add_cc = array(); $add_to = array(); - // If multiplexing is enabled, some recipients will be in "Cc" - // rather than "To". We'll move them to "To" later (or supply a + // If we're sending one mail to everyone, some recipients will be in + // "Cc" rather than "To". We'll move them to "To" later (or supply a // dummy "To") but need to look for the recipient in either the // "To" or "Cc" fields here. $target_phid = head(idx($params, 'to', array())); @@ -847,7 +847,7 @@ final class PhabricatorMetaMTAMail return base64_encode($base); } - public static function shouldMultiplexAllMail() { + public static function shouldMailEachRecipient() { return PhabricatorEnv::getEnvConfig('metamta.one-mail-per-recipient'); } @@ -1290,7 +1290,7 @@ final class PhabricatorMetaMTAMail private function loadPreferences($target_phid) { $viewer = PhabricatorUser::getOmnipotentUser(); - if (self::shouldMultiplexAllMail()) { + if (self::shouldMailEachRecipient()) { $preferences = id(new PhabricatorUserPreferencesQuery()) ->setViewer($viewer) ->withUserPHIDs(array($target_phid)) diff --git a/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php b/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php index 107816f2eb..bdc3c994b3 100644 --- a/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php +++ b/src/applications/settings/panel/PhabricatorEmailFormatSettingsPanel.php @@ -14,7 +14,7 @@ final class PhabricatorEmailFormatSettingsPanel } public function isUserPanel() { - return PhabricatorMetaMTAMail::shouldMultiplexAllMail(); + return PhabricatorMetaMTAMail::shouldMailEachRecipient(); } public function isManagementPanel() { From 3131e733a85cb8be5f6119ca8d4d7a0b23f29009 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 10:31:56 -0800 Subject: [PATCH 45/89] Add Editor-based mail stamps: actor, via, silent, encrypted, new, mention, self-actor, self-mention Summary: Ref T13053. Adds more mail tags with information available on the Editor object. Test Plan: Banged around in Maniphest, viewed the resulting mail, all the stamps seemed to align with reality. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18995 --- src/__phutil_library_map__.php | 8 ++ .../stamp/PhabricatorBoolMailStamp.php | 16 ++++ .../stamp/PhabricatorPHIDMailStamp.php | 36 +++++++++ .../stamp/PhabricatorViewerMailStamp.php | 35 ++++++++ .../phid/PhabricatorPeopleUserPHIDType.php | 11 ++- .../phid/PhabricatorObjectHandle.php | 10 +++ .../PhabricatorProjectProjectPHIDType.php | 3 +- ...habricatorApplicationTransactionEditor.php | 6 +- .../PhabricatorEditorMailEngineExtension.php | 81 +++++++++++++++++++ 9 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 src/applications/metamta/stamp/PhabricatorBoolMailStamp.php create mode 100644 src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php create mode 100644 src/applications/metamta/stamp/PhabricatorViewerMailStamp.php create mode 100644 src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 6fa14fbacb..2a84849171 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2220,6 +2220,7 @@ phutil_register_library_map(array( 'PhabricatorBoardResponseEngine' => 'applications/project/engine/PhabricatorBoardResponseEngine.php', 'PhabricatorBoolConfigType' => 'applications/config/type/PhabricatorBoolConfigType.php', 'PhabricatorBoolEditField' => 'applications/transactions/editfield/PhabricatorBoolEditField.php', + 'PhabricatorBoolMailStamp' => 'applications/metamta/stamp/PhabricatorBoolMailStamp.php', 'PhabricatorBritishEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorBritishEnglishTranslation.php', 'PhabricatorBuiltinDraftEngine' => 'applications/transactions/draft/PhabricatorBuiltinDraftEngine.php', 'PhabricatorBuiltinFileCachePurger' => 'applications/cache/purger/PhabricatorBuiltinFileCachePurger.php', @@ -2813,6 +2814,7 @@ phutil_register_library_map(array( 'PhabricatorEditPage' => 'applications/transactions/editengine/PhabricatorEditPage.php', 'PhabricatorEditType' => 'applications/transactions/edittype/PhabricatorEditType.php', 'PhabricatorEditor' => 'infrastructure/PhabricatorEditor.php', + 'PhabricatorEditorMailEngineExtension' => 'applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php', 'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php', 'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php', 'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php', @@ -3440,6 +3442,7 @@ phutil_register_library_map(array( 'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php', 'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php', 'PhabricatorPHIDListExportField' => 'infrastructure/export/field/PhabricatorPHIDListExportField.php', + 'PhabricatorPHIDMailStamp' => 'applications/metamta/stamp/PhabricatorPHIDMailStamp.php', 'PhabricatorPHIDResolver' => 'applications/phid/resolver/PhabricatorPHIDResolver.php', 'PhabricatorPHIDType' => 'applications/phid/type/PhabricatorPHIDType.php', 'PhabricatorPHIDTypeTestCase' => 'applications/phid/type/__tests__/PhabricatorPHIDTypeTestCase.php', @@ -4395,6 +4398,7 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'applications/draft/storage/PhabricatorVersionedDraft.php', 'PhabricatorVeryWowEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorVeryWowEnglishTranslation.php', 'PhabricatorViewerDatasource' => 'applications/people/typeahead/PhabricatorViewerDatasource.php', + 'PhabricatorViewerMailStamp' => 'applications/metamta/stamp/PhabricatorViewerMailStamp.php', 'PhabricatorWatcherHasObjectEdgeType' => 'applications/transactions/edges/PhabricatorWatcherHasObjectEdgeType.php', 'PhabricatorWebContentSource' => 'infrastructure/contentsource/PhabricatorWebContentSource.php', 'PhabricatorWebServerSetupCheck' => 'applications/config/check/PhabricatorWebServerSetupCheck.php', @@ -7582,6 +7586,7 @@ phutil_register_library_map(array( 'PhabricatorBoardResponseEngine' => 'Phobject', 'PhabricatorBoolConfigType' => 'PhabricatorTextConfigType', 'PhabricatorBoolEditField' => 'PhabricatorEditField', + 'PhabricatorBoolMailStamp' => 'PhabricatorMailStamp', 'PhabricatorBritishEnglishTranslation' => 'PhutilTranslation', 'PhabricatorBuiltinDraftEngine' => 'PhabricatorDraftEngine', 'PhabricatorBuiltinFileCachePurger' => 'PhabricatorCachePurger', @@ -8267,6 +8272,7 @@ phutil_register_library_map(array( 'PhabricatorEditPage' => 'Phobject', 'PhabricatorEditType' => 'Phobject', 'PhabricatorEditor' => 'Phobject', + 'PhabricatorEditorMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting', 'PhabricatorEditorSetting' => 'PhabricatorStringSetting', 'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', @@ -8973,6 +8979,7 @@ phutil_register_library_map(array( 'PhabricatorPHIDListEditField' => 'PhabricatorEditField', 'PhabricatorPHIDListEditType' => 'PhabricatorEditType', 'PhabricatorPHIDListExportField' => 'PhabricatorListExportField', + 'PhabricatorPHIDMailStamp' => 'PhabricatorMailStamp', 'PhabricatorPHIDResolver' => 'Phobject', 'PhabricatorPHIDType' => 'Phobject', 'PhabricatorPHIDTypeTestCase' => 'PhutilTestCase', @@ -10137,6 +10144,7 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'PhabricatorDraftDAO', 'PhabricatorVeryWowEnglishTranslation' => 'PhutilTranslation', 'PhabricatorViewerDatasource' => 'PhabricatorTypeaheadDatasource', + 'PhabricatorViewerMailStamp' => 'PhabricatorMailStamp', 'PhabricatorWatcherHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorWebContentSource' => 'PhabricatorContentSource', 'PhabricatorWebServerSetupCheck' => 'PhabricatorSetupCheck', diff --git a/src/applications/metamta/stamp/PhabricatorBoolMailStamp.php b/src/applications/metamta/stamp/PhabricatorBoolMailStamp.php new file mode 100644 index 0000000000..d274df67fe --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorBoolMailStamp.php @@ -0,0 +1,16 @@ +renderStamp($this->getKey()); + } + +} diff --git a/src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php b/src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php new file mode 100644 index 0000000000..575ad16f6a --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorPHIDMailStamp.php @@ -0,0 +1,36 @@ +getViewer(); + $handles = $viewer->loadHandles($value); + + $results = array(); + foreach ($value as $phid) { + $handle = $handles[$phid]; + + $mail_name = $handle->getMailStampName(); + if ($mail_name === null) { + $mail_name = $handle->getPHID(); + } + + $results[] = $this->renderStamp($this->getKey(), $mail_name); + } + + return $results; + } + +} diff --git a/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php b/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php new file mode 100644 index 0000000000..4eb2028ed8 --- /dev/null +++ b/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php @@ -0,0 +1,35 @@ +getViewer()->getPHID(); + if (!$viewer_phid) { + return null; + } + + if (!$value) { + return null; + } + + $value = (array)$value; + $value = array_fuse($value); + + if (!isset($value[$viewer_phid])) { + return null; + } + + return $this->renderStamp($this->getKey()); + } + +} diff --git a/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php b/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php index f0512e91f1..7867f098f1 100644 --- a/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php +++ b/src/applications/people/phid/PhabricatorPeopleUserPHIDType.php @@ -39,11 +39,14 @@ final class PhabricatorPeopleUserPHIDType extends PhabricatorPHIDType { foreach ($handles as $phid => $handle) { $user = $objects[$phid]; $realname = $user->getRealName(); + $username = $user->getUsername(); - $handle->setName($user->getUsername()); - $handle->setURI('/p/'.$user->getUsername().'/'); - $handle->setFullName($user->getFullName()); - $handle->setImageURI($user->getProfileImageURI()); + $handle + ->setName($username) + ->setURI('/p/'.$username.'/') + ->setFullName($user->getFullName()) + ->setImageURI($user->getProfileImageURI()) + ->setMailStampName('@'.$username); if ($user->getIsMailingList()) { $handle->setIcon('fa-envelope-o'); diff --git a/src/applications/phid/PhabricatorObjectHandle.php b/src/applications/phid/PhabricatorObjectHandle.php index 1e6812b53b..ba93dbcead 100644 --- a/src/applications/phid/PhabricatorObjectHandle.php +++ b/src/applications/phid/PhabricatorObjectHandle.php @@ -31,6 +31,7 @@ final class PhabricatorObjectHandle private $subtitle; private $tokenIcon; private $commandLineObjectName; + private $mailStampName; private $stateIcon; private $stateColor; @@ -134,6 +135,15 @@ final class PhabricatorObjectHandle return $this->objectName; } + public function setMailStampName($mail_stamp_name) { + $this->mailStampName = $mail_stamp_name; + return $this; + } + + public function getMailStampName() { + return $this->mailStampName; + } + public function setURI($uri) { $this->uri = $uri; return $this; diff --git a/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php b/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php index 3aa6088780..9247966d75 100644 --- a/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php +++ b/src/applications/project/phid/PhabricatorProjectProjectPHIDType.php @@ -45,11 +45,12 @@ final class PhabricatorProjectProjectPHIDType extends PhabricatorPHIDType { if (strlen($slug)) { $handle->setObjectName('#'.$slug); + $handle->setMailStampName('#'.$slug); $handle->setURI("/tag/{$slug}/"); } else { // We set the name to the project's PHID to avoid a parse error when a // project has no hashtag (as is the case with milestones by default). - // See T12659 for more details + // See T12659 for more details. $handle->setCommandLineObjectName($project->getPHID()); $handle->setURI("/project/view/{$id}/"); } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 47b975c093..f82cd91701 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -179,7 +179,7 @@ abstract class PhabricatorApplicationTransactionEditor return $this->isNewObject; } - protected function getMentionedPHIDs() { + public function getMentionedPHIDs() { return $this->mentionedPHIDs; } @@ -201,6 +201,10 @@ abstract class PhabricatorApplicationTransactionEditor return $this->silent; } + public function getMustEncrypt() { + return $this->mustEncrypt; + } + public function setIsInverseEdgeEditor($is_inverse_edge_editor) { $this->isInverseEdgeEditor = $is_inverse_edge_editor; return $this; diff --git a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php new file mode 100644 index 0000000000..acfc7ec833 --- /dev/null +++ b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php @@ -0,0 +1,81 @@ +setKey('actor') + ->setLabel(pht('Acting User')); + + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('via') + ->setLabel(pht('Via Content Source')); + + $templates[] = id(new PhabricatorBoolMailStamp()) + ->setKey('silent') + ->setLabel(pht('Silent Edit')); + + $templates[] = id(new PhabricatorBoolMailStamp()) + ->setKey('encrypted') + ->setLabel(pht('Encryption Required')); + + $templates[] = id(new PhabricatorBoolMailStamp()) + ->setKey('new') + ->setLabel(pht('New Object')); + + $templates[] = id(new PhabricatorPHIDMailStamp()) + ->setKey('mention') + ->setLabel(pht('Mentioned User')); + + $templates[] = id(new PhabricatorViewerMailStamp()) + ->setKey('self-actor') + ->setLabel(pht('You Acted')); + + $templates[] = id(new PhabricatorViewerMailStamp()) + ->setKey('self-mention') + ->setLabel(pht('You Were Mentioned')); + + return $templates; + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $this->getMailStamp('actor') + ->setValue($editor->getActingAsPHID()); + + $content_source = $editor->getContentSource(); + $this->getMailStamp('via') + ->setValue($content_source->getSourceTypeConstant()); + + $this->getMailStamp('silent') + ->setValue($editor->getIsSilent()); + + $this->getMailStamp('encrypted') + ->setValue($editor->getMustEncrypt()); + + $this->getMailStamp('new') + ->setValue($editor->getIsNewObject()); + + $mentioned_phids = $editor->getMentionedPHIDs(); + $this->getMailStamp('mention') + ->setValue($mentioned_phids); + + $this->getMailStamp('self-actor') + ->setValue($editor->getActingAsPHID()); + + $this->getMailStamp('self-mention') + ->setValue($mentioned_phids); + } + +} From 7d475eb09af74b42da52ca258d2fbf130fca6a3e Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 11:09:36 -0800 Subject: [PATCH 46/89] Add more mail stamps: tasks, subscribers, projects, spaces Summary: Ref T13053. Adds task stamps plus the major infrastructure applications. Test Plan: {F5413058} Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18996 --- src/__phutil_library_map__.php | 8 +++ .../ManiphestMailEngineExtension.php | 58 +++++++++++++++++++ .../phid/PhabricatorOwnersPackagePHIDType.php | 1 + ...PhabricatorProjectsMailEngineExtension.php | 32 ++++++++++ .../PhabricatorSpacesMailEngineExtension.php | 35 +++++++++++ .../PhabricatorSpacesNamespacePHIDType.php | 8 ++- ...icatorSubscriptionsMailEngineExtension.php | 32 ++++++++++ 7 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php create mode 100644 src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php create mode 100644 src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php create mode 100644 src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 2a84849171..4742ce410e 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1528,6 +1528,7 @@ phutil_register_library_map(array( 'ManiphestGetTaskTransactionsConduitAPIMethod' => 'applications/maniphest/conduit/ManiphestGetTaskTransactionsConduitAPIMethod.php', 'ManiphestHovercardEngineExtension' => 'applications/maniphest/engineextension/ManiphestHovercardEngineExtension.php', 'ManiphestInfoConduitAPIMethod' => 'applications/maniphest/conduit/ManiphestInfoConduitAPIMethod.php', + 'ManiphestMailEngineExtension' => 'applications/maniphest/engineextension/ManiphestMailEngineExtension.php', 'ManiphestNameIndex' => 'applications/maniphest/storage/ManiphestNameIndex.php', 'ManiphestPointsConfigType' => 'applications/maniphest/config/ManiphestPointsConfigType.php', 'ManiphestPrioritiesConfigType' => 'applications/maniphest/config/ManiphestPrioritiesConfigType.php', @@ -3853,6 +3854,7 @@ phutil_register_library_map(array( 'PhabricatorProjectsEditField' => 'applications/transactions/editfield/PhabricatorProjectsEditField.php', 'PhabricatorProjectsExportEngineExtension' => 'infrastructure/export/engine/PhabricatorProjectsExportEngineExtension.php', 'PhabricatorProjectsFulltextEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsFulltextEngineExtension.php', + 'PhabricatorProjectsMailEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php', 'PhabricatorProjectsMembersSearchEngineAttachment' => 'applications/project/engineextension/PhabricatorProjectsMembersSearchEngineAttachment.php', 'PhabricatorProjectsMembershipIndexEngineExtension' => 'applications/project/engineextension/PhabricatorProjectsMembershipIndexEngineExtension.php', 'PhabricatorProjectsPolicyRule' => 'applications/project/policyrule/PhabricatorProjectsPolicyRule.php', @@ -4146,6 +4148,7 @@ phutil_register_library_map(array( 'PhabricatorSpacesExportEngineExtension' => 'infrastructure/export/engine/PhabricatorSpacesExportEngineExtension.php', 'PhabricatorSpacesInterface' => 'applications/spaces/interface/PhabricatorSpacesInterface.php', 'PhabricatorSpacesListController' => 'applications/spaces/controller/PhabricatorSpacesListController.php', + 'PhabricatorSpacesMailEngineExtension' => 'applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php', 'PhabricatorSpacesNamespace' => 'applications/spaces/storage/PhabricatorSpacesNamespace.php', 'PhabricatorSpacesNamespaceArchiveTransaction' => 'applications/spaces/xaction/PhabricatorSpacesNamespaceArchiveTransaction.php', 'PhabricatorSpacesNamespaceDatasource' => 'applications/spaces/typeahead/PhabricatorSpacesNamespaceDatasource.php', @@ -4230,6 +4233,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsFulltextEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsFulltextEngineExtension.php', 'PhabricatorSubscriptionsHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsHeraldAction.php', 'PhabricatorSubscriptionsListController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsListController.php', + 'PhabricatorSubscriptionsMailEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSelfHeraldAction.php', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSubscribersHeraldAction.php', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php', @@ -6795,6 +6799,7 @@ phutil_register_library_map(array( 'ManiphestGetTaskTransactionsConduitAPIMethod' => 'ManiphestConduitAPIMethod', 'ManiphestHovercardEngineExtension' => 'PhabricatorHovercardEngineExtension', 'ManiphestInfoConduitAPIMethod' => 'ManiphestConduitAPIMethod', + 'ManiphestMailEngineExtension' => 'PhabricatorMailEngineExtension', 'ManiphestNameIndex' => 'ManiphestDAO', 'ManiphestPointsConfigType' => 'PhabricatorJSONConfigType', 'ManiphestPrioritiesConfigType' => 'PhabricatorJSONConfigType', @@ -9482,6 +9487,7 @@ phutil_register_library_map(array( 'PhabricatorProjectsEditField' => 'PhabricatorTokenizerEditField', 'PhabricatorProjectsExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorProjectsFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', + 'PhabricatorProjectsMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorProjectsMembersSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', 'PhabricatorProjectsMembershipIndexEngineExtension' => 'PhabricatorIndexEngineExtension', 'PhabricatorProjectsPolicyRule' => 'PhabricatorPolicyRule', @@ -9851,6 +9857,7 @@ phutil_register_library_map(array( 'PhabricatorSpacesExportEngineExtension' => 'PhabricatorExportEngineExtension', 'PhabricatorSpacesInterface' => 'PhabricatorPHIDInterface', 'PhabricatorSpacesListController' => 'PhabricatorSpacesController', + 'PhabricatorSpacesMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorSpacesNamespace' => array( 'PhabricatorSpacesDAO', 'PhabricatorPolicyInterface', @@ -9941,6 +9948,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsFulltextEngineExtension' => 'PhabricatorFulltextEngineExtension', 'PhabricatorSubscriptionsHeraldAction' => 'HeraldAction', 'PhabricatorSubscriptionsListController' => 'PhabricatorController', + 'PhabricatorSubscriptionsMailEngineExtension' => 'PhabricatorMailEngineExtension', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', diff --git a/src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php b/src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php new file mode 100644 index 0000000000..ee38bdf604 --- /dev/null +++ b/src/applications/maniphest/engineextension/ManiphestMailEngineExtension.php @@ -0,0 +1,58 @@ +setKey('author') + ->setLabel(pht('Author')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('task-owner') + ->setLabel(pht('Task Owner')), + id(new PhabricatorBoolMailStamp()) + ->setKey('task-unassigned') + ->setLabel(pht('Task Unassigned')), + id(new PhabricatorStringMailStamp()) + ->setKey('task-priority') + ->setLabel(pht('Task Priority')), + id(new PhabricatorStringMailStamp()) + ->setKey('task-status') + ->setLabel(pht('Task Status')), + id(new PhabricatorStringMailStamp()) + ->setKey('subtype') + ->setLabel(pht('Subtype')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $this->getMailStamp('author') + ->setValue($object->getAuthorPHID()); + + $this->getMailStamp('task-owner') + ->setValue($object->getOwnerPHID()); + + $this->getMailStamp('task-unassigned') + ->setValue(!$object->getOwnerPHID()); + + $this->getMailStamp('task-priority') + ->setValue($object->getPriority()); + + $this->getMailStamp('task-status') + ->setValue($object->getStatus()); + + $this->getMailStamp('subtype') + ->setValue($object->getSubtype()); + } + +} diff --git a/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php b/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php index cfbaf6eeb2..fbff6a2103 100644 --- a/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php +++ b/src/applications/owners/phid/PhabricatorOwnersPackagePHIDType.php @@ -45,6 +45,7 @@ final class PhabricatorOwnersPackagePHIDType extends PhabricatorPHIDType { ->setName($monogram) ->setFullName("{$monogram}: {$name}") ->setCommandLineObjectName("{$monogram} {$name}") + ->setMailStampName($monogram) ->setURI($uri); if ($package->isArchived()) { diff --git a/src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php b/src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php new file mode 100644 index 0000000000..6f92f87b11 --- /dev/null +++ b/src/applications/project/engineextension/PhabricatorProjectsMailEngineExtension.php @@ -0,0 +1,32 @@ +setKey('tag') + ->setLabel(pht('Tagged with Project')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $project_phids = PhabricatorEdgeQuery::loadDestinationPHIDs( + $object->getPHID(), + PhabricatorProjectObjectHasProjectEdgeType::EDGECONST); + + $this->getMailStamp('tag') + ->setValue($project_phids); + } + +} diff --git a/src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php b/src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php new file mode 100644 index 0000000000..7ddbda05fe --- /dev/null +++ b/src/applications/spaces/engineextension/PhabricatorSpacesMailEngineExtension.php @@ -0,0 +1,35 @@ +setKey('space') + ->setLabel(pht('Space')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + if (!PhabricatorSpacesNamespaceQuery::getSpacesExist()) { + return; + } + + $space_phid = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( + $object); + + $this->getMailStamp('space') + ->setValue($space_phid); + } + +} diff --git a/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php b/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php index 86371d6420..1399e71c8e 100644 --- a/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php +++ b/src/applications/spaces/phid/PhabricatorSpacesNamespacePHIDType.php @@ -36,9 +36,11 @@ final class PhabricatorSpacesNamespacePHIDType $monogram = $namespace->getMonogram(); $name = $namespace->getNamespaceName(); - $handle->setName($name); - $handle->setFullName(pht('%s %s', $monogram, $name)); - $handle->setURI('/'.$monogram); + $handle + ->setName($name) + ->setFullName(pht('%s %s', $monogram, $name)) + ->setURI('/'.$monogram) + ->setMailStampName($monogram); if ($namespace->getIsArchived()) { $handle->setStatus(PhabricatorObjectHandle::STATUS_CLOSED); diff --git a/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php new file mode 100644 index 0000000000..122fad4b0d --- /dev/null +++ b/src/applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php @@ -0,0 +1,32 @@ +setKey('subscriber') + ->setLabel(pht('Subscriber')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $subscriber_phids = PhabricatorEdgeQuery::loadDestinationPHIDs( + $object->getPHID(), + PhabricatorObjectHasSubscriberEdgeType::EDGECONST); + + $this->getMailStamp('subscriber') + ->setValue($subscriber_phids); + } + +} From 1bf64e5cbcf48adf05ef60cb9eb948d4056092b2 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 11:44:31 -0800 Subject: [PATCH 47/89] Add Differential and Herald mail stamps and some refinements Summary: Ref T13053. Adds revision stamps (status, reviewers, etc). Adds Herald rule stamps, like the existing X-Herald-Rules header. Removes the "self" stamps, since you can just write a rule against `whatever(@epriestley)` equivalently. If there's routing logic around this, it can live in the routing layer. This avoids tons of self-actor, self-mention, self-reviewer, self-blocking-reviewer, self-resigned-reviewer, etc., stamps. Use `natcasesort()` instead of `sort()` so that numeric values (like monograms) sort `9, 80, 700` instead of `700, 80, 9`. Remove the commas from rendering since they don't really add anything. Test Plan: Edited tasks and revisions, looked at mail stamps, saw stamps that looked pretty reasonable (with no more self stuff, no more commas, sorting numbers, and Herald stamps). Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18997 --- src/__phutil_library_map__.php | 4 +- .../DifferentialMailEngineExtension.php | 80 +++++++++++++++++++ .../storage/DifferentialReviewer.php | 5 ++ .../replyhandler/PhabricatorMailTarget.php | 4 +- .../stamp/PhabricatorStringMailStamp.php | 14 +++- .../stamp/PhabricatorViewerMailStamp.php | 35 -------- ...habricatorRepositoryRepositoryPHIDType.php | 8 +- ...habricatorApplicationTransactionEditor.php | 21 ++++- .../PhabricatorEditorMailEngineExtension.php | 17 ++-- 9 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 src/applications/differential/engineextension/DifferentialMailEngineExtension.php delete mode 100644 src/applications/metamta/stamp/PhabricatorViewerMailStamp.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 4742ce410e..c642f13f90 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -487,6 +487,7 @@ phutil_register_library_map(array( 'DifferentialLintField' => 'applications/differential/customfield/DifferentialLintField.php', 'DifferentialLintStatus' => 'applications/differential/constants/DifferentialLintStatus.php', 'DifferentialLocalCommitsView' => 'applications/differential/view/DifferentialLocalCommitsView.php', + 'DifferentialMailEngineExtension' => 'applications/differential/engineextension/DifferentialMailEngineExtension.php', 'DifferentialMailView' => 'applications/differential/mail/DifferentialMailView.php', 'DifferentialManiphestTasksField' => 'applications/differential/customfield/DifferentialManiphestTasksField.php', 'DifferentialModernHunk' => 'applications/differential/storage/DifferentialModernHunk.php', @@ -4402,7 +4403,6 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'applications/draft/storage/PhabricatorVersionedDraft.php', 'PhabricatorVeryWowEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorVeryWowEnglishTranslation.php', 'PhabricatorViewerDatasource' => 'applications/people/typeahead/PhabricatorViewerDatasource.php', - 'PhabricatorViewerMailStamp' => 'applications/metamta/stamp/PhabricatorViewerMailStamp.php', 'PhabricatorWatcherHasObjectEdgeType' => 'applications/transactions/edges/PhabricatorWatcherHasObjectEdgeType.php', 'PhabricatorWebContentSource' => 'infrastructure/contentsource/PhabricatorWebContentSource.php', 'PhabricatorWebServerSetupCheck' => 'applications/config/check/PhabricatorWebServerSetupCheck.php', @@ -5612,6 +5612,7 @@ phutil_register_library_map(array( 'DifferentialLintField' => 'DifferentialHarbormasterField', 'DifferentialLintStatus' => 'Phobject', 'DifferentialLocalCommitsView' => 'AphrontView', + 'DifferentialMailEngineExtension' => 'PhabricatorMailEngineExtension', 'DifferentialMailView' => 'Phobject', 'DifferentialManiphestTasksField' => 'DifferentialCoreCustomField', 'DifferentialModernHunk' => 'DifferentialHunk', @@ -10152,7 +10153,6 @@ phutil_register_library_map(array( 'PhabricatorVersionedDraft' => 'PhabricatorDraftDAO', 'PhabricatorVeryWowEnglishTranslation' => 'PhutilTranslation', 'PhabricatorViewerDatasource' => 'PhabricatorTypeaheadDatasource', - 'PhabricatorViewerMailStamp' => 'PhabricatorMailStamp', 'PhabricatorWatcherHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorWebContentSource' => 'PhabricatorContentSource', 'PhabricatorWebServerSetupCheck' => 'PhabricatorSetupCheck', diff --git a/src/applications/differential/engineextension/DifferentialMailEngineExtension.php b/src/applications/differential/engineextension/DifferentialMailEngineExtension.php new file mode 100644 index 0000000000..24aa9fb329 --- /dev/null +++ b/src/applications/differential/engineextension/DifferentialMailEngineExtension.php @@ -0,0 +1,80 @@ +setKey('author') + ->setLabel(pht('Author')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('reviewer') + ->setLabel(pht('Reviewer')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('blocking-reviewer') + ->setLabel(pht('Reviewer')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('resigned-reviewer') + ->setLabel(pht('Reviewer')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('revision-repository') + ->setLabel(pht('Revision Repository')), + id(new PhabricatorPHIDMailStamp()) + ->setKey('revision-status') + ->setLabel(pht('Revision Status')), + ); + } + + public function newMailStamps($object, array $xactions) { + $editor = $this->getEditor(); + $viewer = $this->getViewer(); + + $revision = id(new DifferentialRevisionQuery()) + ->setViewer($viewer) + ->needReviewers(true) + ->withPHIDs(array($object->getPHID())) + ->executeOne(); + + $reviewers = array(); + $blocking = array(); + $resigned = array(); + foreach ($revision->getReviewers() as $reviewer) { + $reviewer_phid = $reviewer->getReviewerPHID(); + + if ($reviewer->isResigned()) { + $resigned[] = $reviewer_phid; + } else { + $reviewers[] = $reviewer_phid; + if ($reviewer->isBlocking()) { + $reviewers[] = $blocking; + } + } + } + + $this->getMailStamp('author') + ->setValue($revision->getAuthorPHID()); + + $this->getMailStamp('reviewer') + ->setValue($reviewers); + + $this->getMailStamp('blocking-reviewer') + ->setValue($blocking); + + $this->getMailStamp('resigned-reviewer') + ->setValue($resigned); + + $this->getMailStamp('revision-repository') + ->setValue($revision->getRepositoryPHID()); + + $this->getMailStamp('revision-status') + ->setValue($revision->getModernRevisionStatus()); + } + +} diff --git a/src/applications/differential/storage/DifferentialReviewer.php b/src/applications/differential/storage/DifferentialReviewer.php index 9df149e788..e3f9bdaf8d 100644 --- a/src/applications/differential/storage/DifferentialReviewer.php +++ b/src/applications/differential/storage/DifferentialReviewer.php @@ -69,6 +69,11 @@ final class DifferentialReviewer return ($this->getReviewerStatus() == $status_resigned); } + public function isBlocking() { + $status_blocking = DifferentialReviewerStatus::STATUS_BLOCKING; + return ($this->getReviewerStatus() == $status_blocking); + } + public function isRejected($diff_phid) { $status_rejected = DifferentialReviewerStatus::STATUS_REJECTED; diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index 0463b6cdd7..bbf17be3fd 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -70,14 +70,14 @@ final class PhabricatorMailTarget extends Phobject { $body .= "\n"; $body .= pht('STAMPS'); $body .= "\n"; - $body .= implode(', ', $stamps); + $body .= implode(' ', $stamps); $body .= "\n"; if ($has_html) { $html = array(); $html[] = phutil_tag('strong', array(), pht('STAMPS')); $html[] = phutil_tag('br'); - $html[] = phutil_implode_html(', ', $stamps); + $html[] = phutil_implode_html(' ', $stamps); $html[] = phutil_tag('br'); $html = phutil_tag('div', array(), $html); $html_body .= hsprintf('%s', $html); diff --git a/src/applications/metamta/stamp/PhabricatorStringMailStamp.php b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php index 98d472ad48..b6210afb4e 100644 --- a/src/applications/metamta/stamp/PhabricatorStringMailStamp.php +++ b/src/applications/metamta/stamp/PhabricatorStringMailStamp.php @@ -6,11 +6,21 @@ final class PhabricatorStringMailStamp const STAMPTYPE = 'string'; public function renderStamps($value) { - if (!strlen($value)) { + if ($value === null || $value === '') { return null; } - return $this->renderStamp($this->getKey(), $value); + $value = (array)$value; + if (!$value) { + return null; + } + + $results = array(); + foreach ($value as $v) { + $results[] = $this->renderStamp($this->getKey(), $v); + } + + return $results; } } diff --git a/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php b/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php deleted file mode 100644 index 4eb2028ed8..0000000000 --- a/src/applications/metamta/stamp/PhabricatorViewerMailStamp.php +++ /dev/null @@ -1,35 +0,0 @@ -getViewer()->getPHID(); - if (!$viewer_phid) { - return null; - } - - if (!$value) { - return null; - } - - $value = (array)$value; - $value = array_fuse($value); - - if (!isset($value[$viewer_phid])) { - return null; - } - - return $this->renderStamp($this->getKey()); - } - -} diff --git a/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php b/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php index c938b31a65..ba78b0fe7a 100644 --- a/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php +++ b/src/applications/repository/phid/PhabricatorRepositoryRepositoryPHIDType.php @@ -41,9 +41,11 @@ final class PhabricatorRepositoryRepositoryPHIDType $name = $repository->getName(); $uri = $repository->getURI(); - $handle->setName($monogram); - $handle->setFullName("{$monogram} {$name}"); - $handle->setURI($uri); + $handle + ->setName($monogram) + ->setFullName("{$monogram} {$name}") + ->setURI($uri) + ->setMailStampName($monogram); } } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index f82cd91701..ac4d56ba16 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -205,6 +205,25 @@ abstract class PhabricatorApplicationTransactionEditor return $this->mustEncrypt; } + public function getHeraldRuleMonograms() { + // Convert the stored "<123>, <456>" string into a list: "H123", "H456". + $list = $this->heraldHeader; + $list = preg_split('/[, ]+/', $list); + + foreach ($list as $key => $item) { + $item = trim($item, '<>'); + + if (!is_numeric($item)) { + unset($list[$key]); + continue; + } + + $list[$key] = 'H'.$item; + } + + return $list; + } + public function setIsInverseEdgeEditor($is_inverse_edge_editor) { $this->isInverseEdgeEditor = $is_inverse_edge_editor; return $this; @@ -4109,7 +4128,7 @@ abstract class PhabricatorApplicationTransactionEditor } } - sort($results); + natcasesort($results); return $results; } diff --git a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php index acfc7ec833..29d10d641f 100644 --- a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php +++ b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php @@ -36,13 +36,9 @@ final class PhabricatorEditorMailEngineExtension ->setKey('mention') ->setLabel(pht('Mentioned User')); - $templates[] = id(new PhabricatorViewerMailStamp()) - ->setKey('self-actor') - ->setLabel(pht('You Acted')); - - $templates[] = id(new PhabricatorViewerMailStamp()) - ->setKey('self-mention') - ->setLabel(pht('You Were Mentioned')); + $templates[] = id(new PhabricatorStringMailStamp()) + ->setKey('herald') + ->setLabel(pht('Herald Rule')); return $templates; } @@ -71,11 +67,8 @@ final class PhabricatorEditorMailEngineExtension $this->getMailStamp('mention') ->setValue($mentioned_phids); - $this->getMailStamp('self-actor') - ->setValue($editor->getActingAsPHID()); - - $this->getMailStamp('self-mention') - ->setValue($mentioned_phids); + $this->getMailStamp('herald') + ->setValue($editor->getHeraldRuleMonograms()); } } From 56bf0690804fe4c77085b1320df9a5e994d3163c Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 04:07:22 -0800 Subject: [PATCH 48/89] Try running Herald when performing inverse edge edits Summary: Ref T13053. When you mention one object on another (or link two objects together with an action like "Edit Parent Revisions"), we write a transaction on each side to add the "alice added subtask X" and "alice added parent task Y" items to the timeline. This behavior now causes problems in T13053 with the "Must Encrypt" flag because it prevents the flag from being applied to the corresponding "inverse edge" mail. This was added in rP5050389f as a quick workaround for a fatal related to Editors not having enough data to apply Herald on mentions. However, that was in 2014, and since then: - Herald got a significant rewrite to modularize all the rules and adapters. - Editing got a significant upgrade in EditEngine and most edit workflows now operate through EditEngine. - We generally do more editing on more pathways, everything is more modular, and we have standardized how data is loaded to a greater degree. I suspect there's no longer a problem with just running Herald here, and can't reproduce one. If anything does crop up, it's probably easy (and desirable) to fix it. This makes Herald fire a little more often: if someone writes a rule, mentioning or creating a relationship to old tasks will now make the rule act. Offhand, that seems fine. If it turns out to be weird, we can probably tailor Herald's behavior. Test Plan: I wasn't able to break anything: - Mentioned a task on another task (original issue). - Linked tasks with commits, mocks, revisions. - Linked revisions with commits, tasks. - Mentioned users, revisions, and commits. - Verified that mail generated by creating links (e.g., Revision > Edit Tasks) now gets the "Must Encrypt" flag properly. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D18999 --- .../editor/PhabricatorApplicationTransactionEditor.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index ac4d56ba16..c5390de362 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -1095,13 +1095,6 @@ abstract class PhabricatorApplicationTransactionEditor // We are the Herald editor, so stop work here and return the updated // transactions. return $xactions; - } else if ($this->getIsInverseEdgeEditor()) { - // If we're applying inverse edge transactions, don't trigger Herald. - // From a product perspective, the current set of inverse edges (most - // often, mentions) aren't things users would expect to trigger Herald. - // From a technical perspective, objects loaded by the inverse editor may - // not have enough data to execute rules. At least for now, just stop - // Herald from executing when applying inverse edges. } else if ($this->shouldApplyHeraldRules($object, $xactions)) { // We are not the Herald editor, so try to apply Herald rules. $herald_xactions = $this->applyHeraldRules($object, $xactions); From a5bbadbaba0259350ccc6f6a86786373e935aee5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 16:00:45 -0800 Subject: [PATCH 49/89] Fix another Git 2.16.0 CLI compatibility issue Summary: This command also needs a "." instead of an empty string now. (This powers the file browser typeahead in Diffusion.) Test Plan: Will test in production since there's still no easy 2.16 installer for macOS. Reviewers: amckinley Reviewed By: amckinley Differential Revision: https://secure.phabricator.com/D19010 --- .../conduit/DiffusionQueryPathsConduitAPIMethod.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php index be2f07f2c6..09c07ec28f 100644 --- a/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionQueryPathsConduitAPIMethod.php @@ -37,7 +37,11 @@ final class DiffusionQueryPathsConduitAPIMethod $commit = $request->getValue('commit'); $repository = $drequest->getRepository(); - // http://comments.gmane.org/gmane.comp.version-control.git/197735 + // Recent versions of Git don't work if you pass the empty string, and + // require "." to list everything. + if (!strlen($path)) { + $path = '.'; + } $future = $repository->getLocalCommandFuture( 'ls-tree --name-only -r -z %s -- %s', From 150a04791c06c91ab587fe11ea2a50ba8b40cd45 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 05:43:37 -0800 Subject: [PATCH 50/89] Fix bad NUX link in Legalpad search view Summary: See . This URI isn't correct. Test Plan: Visited {nav Use Results > New User State} in developer mode, clicked green button. Before: 404. After: taken to the edit screen. Differential Revision: https://secure.phabricator.com/D19024 --- .../legalpad/query/LegalpadDocumentSearchEngine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php b/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php index 1b608b02e3..a245417483 100644 --- a/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php +++ b/src/applications/legalpad/query/LegalpadDocumentSearchEngine.php @@ -176,7 +176,7 @@ final class LegalpadDocumentSearchEngine $create_button = id(new PHUIButtonView()) ->setTag('a') ->setText(pht('Create a Document')) - ->setHref('/legalpad/create/') + ->setHref('/legalpad/edit/') ->setColor(PHUIButtonView::GREEN); $icon = $this->getApplication()->getIcon(); From 1485debcbda2bae25884dbd54055ccb3fecd9580 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 14:22:54 -0800 Subject: [PATCH 51/89] Prepare mail transmission to support failover across multiple mailers Summary: Ref T13053. Ref T12677. This restructures the calls and error handling logic so that we can pass in a list of multiple mailers and get retry logic. This doesn't actually ever use multiple mailers yet, and shouldn't change any behavior. I'll add multiple-mailer coverage a little further in, since there's currently no way to effectively test which of several mailers ended up transmitting a message. Test Plan: - This has test coverage; tests still pass. - Poked around locally doing things that send mail, saw mail appear to send. I'm not attached to a real mailer though so my confidence in local testing is only so-so. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D18998 --- .../storage/PhabricatorMetaMTAMail.php | 693 +++++++++--------- .../PhabricatorMetaMTAMailTestCase.php | 8 +- 2 files changed, 367 insertions(+), 334 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index f19e5e951a..179eb18088 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -461,359 +461,392 @@ final class PhabricatorMetaMTAMail /** * Attempt to deliver an email immediately, in this process. * - * @param bool Try to deliver this email even if it has already been - * delivered or is in backoff after a failed delivery attempt. - * @param PhabricatorMailImplementationAdapter Use a specific mail adapter, - * instead of the default. - * * @return void */ - public function sendNow( - $force_send = false, - PhabricatorMailImplementationAdapter $mailer = null) { - - if ($mailer === null) { - $mailer = $this->buildDefaultMailer(); + public function sendNow() { + if ($this->getStatus() != PhabricatorMailOutboundStatus::STATUS_QUEUE) { + throw new Exception(pht('Trying to send an already-sent mail!')); } - if (!$force_send) { - if ($this->getStatus() != PhabricatorMailOutboundStatus::STATUS_QUEUE) { - throw new Exception(pht('Trying to send an already-sent mail!')); - } - } + $mailers = array( + $this->buildDefaultMailer(), + ); - try { - $headers = $this->generateHeaders(); + return $this->sendWithMailers($mailers); + } - $params = $this->parameters; - $actors = $this->loadAllActors(); - $deliverable_actors = $this->filterDeliverableActors($actors); + public function sendWithMailers(array $mailers) { + $exceptions = array(); + foreach ($mailers as $template_mailer) { + $mailer = null; - $default_from = PhabricatorEnv::getEnvConfig('metamta.default-address'); - if (empty($params['from'])) { - $mailer->setFrom($default_from); + try { + $mailer = $this->buildMailer($template_mailer); + } catch (Exception $ex) { + $exceptions[] = $ex; + continue; } - $is_first = idx($params, 'is-first-message'); - unset($params['is-first-message']); - - $is_threaded = (bool)idx($params, 'thread-id'); - $must_encrypt = $this->getMustEncrypt(); - - $reply_to_name = idx($params, 'reply-to-name', ''); - unset($params['reply-to-name']); - - $add_cc = array(); - $add_to = array(); - - // If we're sending one mail to everyone, some recipients will be in - // "Cc" rather than "To". We'll move them to "To" later (or supply a - // dummy "To") but need to look for the recipient in either the - // "To" or "Cc" fields here. - $target_phid = head(idx($params, 'to', array())); - if (!$target_phid) { - $target_phid = head(idx($params, 'cc', array())); + if (!$mailer) { + // If we don't get a mailer back, that means the mail doesn't + // actually need to be sent (for example, because recipients have + // declined to receive the mail). Void it and return. + return $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID) + ->save(); } - $preferences = $this->loadPreferences($target_phid); - - foreach ($params as $key => $value) { - switch ($key) { - case 'raw-from': - list($from_email, $from_name) = $value; - $mailer->setFrom($from_email, $from_name); - break; - case 'from': - $from = $value; - $actor_email = null; - $actor_name = null; - $actor = idx($actors, $from); - if ($actor) { - $actor_email = $actor->getEmailAddress(); - $actor_name = $actor->getName(); - } - $can_send_as_user = $actor_email && - PhabricatorEnv::getEnvConfig('metamta.can-send-as-user'); - - if ($can_send_as_user) { - $mailer->setFrom($actor_email, $actor_name); - } else { - $from_email = coalesce($actor_email, $default_from); - $from_name = coalesce($actor_name, pht('Phabricator')); - - if (empty($params['reply-to'])) { - $params['reply-to'] = $from_email; - $params['reply-to-name'] = $from_name; - } - - $mailer->setFrom($default_from, $from_name); - } - break; - case 'reply-to': - $mailer->addReplyTo($value, $reply_to_name); - break; - case 'to': - $to_phids = $this->expandRecipients($value); - $to_actors = array_select_keys($deliverable_actors, $to_phids); - $add_to = array_merge( - $add_to, - mpull($to_actors, 'getEmailAddress')); - break; - case 'raw-to': - $add_to = array_merge($add_to, $value); - break; - case 'cc': - $cc_phids = $this->expandRecipients($value); - $cc_actors = array_select_keys($deliverable_actors, $cc_phids); - $add_cc = array_merge( - $add_cc, - mpull($cc_actors, 'getEmailAddress')); - break; - case 'attachments': - $attached_viewer = PhabricatorUser::getOmnipotentUser(); - $files = $this->loadAttachedFiles($attached_viewer); - foreach ($files as $file) { - $file->attachToObject($this->getPHID()); - } - - // If the mail content must be encrypted, don't add attachments. - if ($must_encrypt) { - break; - } - - $value = $this->getAttachments(); - foreach ($value as $attachment) { - $mailer->addAttachment( - $attachment->getData(), - $attachment->getFilename(), - $attachment->getMimeType()); - } - break; - case 'subject': - $subject = array(); - - if ($is_threaded) { - if ($this->shouldAddRePrefix($preferences)) { - $subject[] = 'Re:'; - } - } - - $subject[] = trim(idx($params, 'subject-prefix')); - - // If mail content must be encrypted, we replace the subject with - // a generic one. - if ($must_encrypt) { - $subject[] = pht('Object Updated'); - } else { - $vary_prefix = idx($params, 'vary-subject-prefix'); - if ($vary_prefix != '') { - if ($this->shouldVarySubject($preferences)) { - $subject[] = $vary_prefix; - } - } - - $subject[] = $value; - } - - $mailer->setSubject(implode(' ', array_filter($subject))); - break; - case 'thread-id': - - // NOTE: Gmail freaks out about In-Reply-To and References which - // aren't in the form ""; this is also required - // by RFC 2822, although some clients are more liberal in what they - // accept. - $domain = PhabricatorEnv::getEnvConfig('metamta.domain'); - $value = '<'.$value.'@'.$domain.'>'; - - if ($is_first && $mailer->supportsMessageIDHeader()) { - $headers[] = array('Message-ID', $value); - } else { - $in_reply_to = $value; - $references = array($value); - $parent_id = $this->getParentMessageID(); - if ($parent_id) { - $in_reply_to = $parent_id; - // By RFC 2822, the most immediate parent should appear last - // in the "References" header, so this order is intentional. - $references[] = $parent_id; - } - $references = implode(' ', $references); - $headers[] = array('In-Reply-To', $in_reply_to); - $headers[] = array('References', $references); - } - $thread_index = $this->generateThreadIndex($value, $is_first); - $headers[] = array('Thread-Index', $thread_index); - break; - default: - // Other parameters are handled elsewhere or are not relevant to - // constructing the message. - break; - } - } - - $stamps = $this->getMailStamps(); - if ($stamps) { - $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); - } - - $raw_body = idx($params, 'body', ''); - $body = $raw_body; - if ($must_encrypt) { - $parts = array(); - $parts[] = pht( - 'The content for this message can only be transmitted over a '. - 'secure channel. To view the message content, follow this '. - 'link:'); - - $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); - - $body = implode("\n\n", $parts); - } else { - $body = $raw_body; - } - - $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); - if (strlen($body) > $max) { - $body = id(new PhutilUTF8StringTruncator()) - ->setMaximumBytes($max) - ->truncateString($body); - $body .= "\n"; - $body .= pht('(This email was truncated at %d bytes.)', $max); - } - $mailer->setBody($body); - - // If we sent a different message body than we were asked to, record - // what we actually sent to make debugging and diagnostics easier. - if ($body !== $raw_body) { - $this->setParam('body.sent', $body); - } - - if ($must_encrypt) { - $send_html = false; - } else { - $send_html = $this->shouldSendHTML($preferences); - } - - if ($send_html && isset($params['html-body'])) { - $mailer->setHTMLBody($params['html-body']); - } - - // Pass the headers to the mailer, then save the state so we can show - // them in the web UI. If the mail must be encrypted, we remove headers - // which are not on a strict whitelist to avoid disclosing information. - $filtered_headers = $this->filterHeaders($headers, $must_encrypt); - foreach ($filtered_headers as $header) { - list($header_key, $header_value) = $header; - $mailer->addHeader($header_key, $header_value); - } - $this->setParam('headers.unfiltered', $headers); - $this->setParam('headers.sent', $filtered_headers); - - // Save the final deliverability outcomes and reasoning so we can - // explain why things happened the way they did. - $actor_list = array(); - foreach ($actors as $actor) { - $actor_list[$actor->getPHID()] = array( - 'deliverable' => $actor->isDeliverable(), - 'reasons' => $actor->getDeliverabilityReasons(), - ); - } - $this->setParam('actors.sent', $actor_list); - - $this->setParam('routing.sent', $this->getParam('routing')); - $this->setParam('routingmap.sent', $this->getRoutingRuleMap()); - - if (!$add_to && !$add_cc) { - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID); - $this->setMessage( - pht( - 'Message has no valid recipients: all To/Cc are disabled, '. - 'invalid, or configured not to receive this mail.')); - return $this->save(); - } - - if ($this->getIsErrorEmail()) { - $all_recipients = array_merge($add_to, $add_cc); - if ($this->shouldRateLimitMail($all_recipients)) { - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID); - $this->setMessage( + try { + $ok = $mailer->send(); + if (!$ok) { + // TODO: At some point, we should clean this up and make all mailers + // throw. + throw new Exception( pht( - 'This is an error email, but one or more recipients have '. - 'exceeded the error email rate limit. Declining to deliver '. - 'message.')); - return $this->save(); + 'Mail adapter encountered an unexpected, unspecified '. + 'failure.')); } + } catch (PhabricatorMetaMTAPermanentFailureException $ex) { + // If any mailer raises a permanent failure, stop trying to send the + // mail with other mailers. + $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_FAIL) + ->setMessage($ex->getMessage()) + ->save(); + + throw $ex; + } catch (Exception $ex) { + $exceptions[] = $ex; + continue; } - if (PhabricatorEnv::getEnvConfig('phabricator.silent')) { - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID); + return $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT) + ->save(); + } + + // If we make it here, no mailer could send the mail but no mailer failed + // permanently either. We update the error message for the mail, but leave + // it in the current status (usually, STATUS_QUEUE) and try again later. + + $messages = array(); + foreach ($exceptions as $ex) { + $messages[] = $ex->getMessage(); + } + $messages = implode("\n\n", $messages); + + $this + ->setMessage($messages) + ->save(); + + if (count($exceptions) === 1) { + throw head($exceptions); + } + + throw new PhutilAggregateException( + pht('Encountered multiple exceptions while transmitting mail.'), + $exceptions); + } + + private function buildMailer(PhabricatorMailImplementationAdapter $mailer) { + $headers = $this->generateHeaders(); + + $params = $this->parameters; + + $actors = $this->loadAllActors(); + $deliverable_actors = $this->filterDeliverableActors($actors); + + $default_from = PhabricatorEnv::getEnvConfig('metamta.default-address'); + if (empty($params['from'])) { + $mailer->setFrom($default_from); + } + + $is_first = idx($params, 'is-first-message'); + unset($params['is-first-message']); + + $is_threaded = (bool)idx($params, 'thread-id'); + $must_encrypt = $this->getMustEncrypt(); + + $reply_to_name = idx($params, 'reply-to-name', ''); + unset($params['reply-to-name']); + + $add_cc = array(); + $add_to = array(); + + // If we're sending one mail to everyone, some recipients will be in + // "Cc" rather than "To". We'll move them to "To" later (or supply a + // dummy "To") but need to look for the recipient in either the + // "To" or "Cc" fields here. + $target_phid = head(idx($params, 'to', array())); + if (!$target_phid) { + $target_phid = head(idx($params, 'cc', array())); + } + + $preferences = $this->loadPreferences($target_phid); + + foreach ($params as $key => $value) { + switch ($key) { + case 'raw-from': + list($from_email, $from_name) = $value; + $mailer->setFrom($from_email, $from_name); + break; + case 'from': + $from = $value; + $actor_email = null; + $actor_name = null; + $actor = idx($actors, $from); + if ($actor) { + $actor_email = $actor->getEmailAddress(); + $actor_name = $actor->getName(); + } + $can_send_as_user = $actor_email && + PhabricatorEnv::getEnvConfig('metamta.can-send-as-user'); + + if ($can_send_as_user) { + $mailer->setFrom($actor_email, $actor_name); + } else { + $from_email = coalesce($actor_email, $default_from); + $from_name = coalesce($actor_name, pht('Phabricator')); + + if (empty($params['reply-to'])) { + $params['reply-to'] = $from_email; + $params['reply-to-name'] = $from_name; + } + + $mailer->setFrom($default_from, $from_name); + } + break; + case 'reply-to': + $mailer->addReplyTo($value, $reply_to_name); + break; + case 'to': + $to_phids = $this->expandRecipients($value); + $to_actors = array_select_keys($deliverable_actors, $to_phids); + $add_to = array_merge( + $add_to, + mpull($to_actors, 'getEmailAddress')); + break; + case 'raw-to': + $add_to = array_merge($add_to, $value); + break; + case 'cc': + $cc_phids = $this->expandRecipients($value); + $cc_actors = array_select_keys($deliverable_actors, $cc_phids); + $add_cc = array_merge( + $add_cc, + mpull($cc_actors, 'getEmailAddress')); + break; + case 'attachments': + $attached_viewer = PhabricatorUser::getOmnipotentUser(); + $files = $this->loadAttachedFiles($attached_viewer); + foreach ($files as $file) { + $file->attachToObject($this->getPHID()); + } + + // If the mail content must be encrypted, don't add attachments. + if ($must_encrypt) { + break; + } + + $value = $this->getAttachments(); + foreach ($value as $attachment) { + $mailer->addAttachment( + $attachment->getData(), + $attachment->getFilename(), + $attachment->getMimeType()); + } + break; + case 'subject': + $subject = array(); + + if ($is_threaded) { + if ($this->shouldAddRePrefix($preferences)) { + $subject[] = 'Re:'; + } + } + + $subject[] = trim(idx($params, 'subject-prefix')); + + // If mail content must be encrypted, we replace the subject with + // a generic one. + if ($must_encrypt) { + $subject[] = pht('Object Updated'); + } else { + $vary_prefix = idx($params, 'vary-subject-prefix'); + if ($vary_prefix != '') { + if ($this->shouldVarySubject($preferences)) { + $subject[] = $vary_prefix; + } + } + + $subject[] = $value; + } + + $mailer->setSubject(implode(' ', array_filter($subject))); + break; + case 'thread-id': + + // NOTE: Gmail freaks out about In-Reply-To and References which + // aren't in the form ""; this is also required + // by RFC 2822, although some clients are more liberal in what they + // accept. + $domain = PhabricatorEnv::getEnvConfig('metamta.domain'); + $value = '<'.$value.'@'.$domain.'>'; + + if ($is_first && $mailer->supportsMessageIDHeader()) { + $headers[] = array('Message-ID', $value); + } else { + $in_reply_to = $value; + $references = array($value); + $parent_id = $this->getParentMessageID(); + if ($parent_id) { + $in_reply_to = $parent_id; + // By RFC 2822, the most immediate parent should appear last + // in the "References" header, so this order is intentional. + $references[] = $parent_id; + } + $references = implode(' ', $references); + $headers[] = array('In-Reply-To', $in_reply_to); + $headers[] = array('References', $references); + } + $thread_index = $this->generateThreadIndex($value, $is_first); + $headers[] = array('Thread-Index', $thread_index); + break; + default: + // Other parameters are handled elsewhere or are not relevant to + // constructing the message. + break; + } + } + + $stamps = $this->getMailStamps(); + if ($stamps) { + $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); + } + + $raw_body = idx($params, 'body', ''); + $body = $raw_body; + if ($must_encrypt) { + $parts = array(); + $parts[] = pht( + 'The content for this message can only be transmitted over a '. + 'secure channel. To view the message content, follow this '. + 'link:'); + + $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); + + $body = implode("\n\n", $parts); + } else { + $body = $raw_body; + } + + $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); + if (strlen($body) > $max) { + $body = id(new PhutilUTF8StringTruncator()) + ->setMaximumBytes($max) + ->truncateString($body); + $body .= "\n"; + $body .= pht('(This email was truncated at %d bytes.)', $max); + } + $mailer->setBody($body); + + // If we sent a different message body than we were asked to, record + // what we actually sent to make debugging and diagnostics easier. + if ($body !== $raw_body) { + $this->setParam('body.sent', $body); + } + + if ($must_encrypt) { + $send_html = false; + } else { + $send_html = $this->shouldSendHTML($preferences); + } + + if ($send_html && isset($params['html-body'])) { + $mailer->setHTMLBody($params['html-body']); + } + + // Pass the headers to the mailer, then save the state so we can show + // them in the web UI. If the mail must be encrypted, we remove headers + // which are not on a strict whitelist to avoid disclosing information. + $filtered_headers = $this->filterHeaders($headers, $must_encrypt); + foreach ($filtered_headers as $header) { + list($header_key, $header_value) = $header; + $mailer->addHeader($header_key, $header_value); + } + $this->setParam('headers.unfiltered', $headers); + $this->setParam('headers.sent', $filtered_headers); + + // Save the final deliverability outcomes and reasoning so we can + // explain why things happened the way they did. + $actor_list = array(); + foreach ($actors as $actor) { + $actor_list[$actor->getPHID()] = array( + 'deliverable' => $actor->isDeliverable(), + 'reasons' => $actor->getDeliverabilityReasons(), + ); + } + $this->setParam('actors.sent', $actor_list); + + $this->setParam('routing.sent', $this->getParam('routing')); + $this->setParam('routingmap.sent', $this->getRoutingRuleMap()); + + if (!$add_to && !$add_cc) { + $this->setMessage( + pht( + 'Message has no valid recipients: all To/Cc are disabled, '. + 'invalid, or configured not to receive this mail.')); + + return null; + } + + if ($this->getIsErrorEmail()) { + $all_recipients = array_merge($add_to, $add_cc); + if ($this->shouldRateLimitMail($all_recipients)) { $this->setMessage( pht( - 'Phabricator is running in silent mode. See `%s` '. - 'in the configuration to change this setting.', - 'phabricator.silent')); - return $this->save(); + 'This is an error email, but one or more recipients have '. + 'exceeded the error email rate limit. Declining to deliver '. + 'message.')); + + return null; } - - // Some mailers require a valid "To:" in order to deliver mail. If we - // don't have any "To:", try to fill it in with a placeholder "To:". - // If that also fails, move the "Cc:" line to "To:". - if (!$add_to) { - $placeholder_key = 'metamta.placeholder-to-recipient'; - $placeholder = PhabricatorEnv::getEnvConfig($placeholder_key); - if ($placeholder !== null) { - $add_to = array($placeholder); - } else { - $add_to = $add_cc; - $add_cc = array(); - } - } - - $add_to = array_unique($add_to); - $add_cc = array_diff(array_unique($add_cc), $add_to); - - $mailer->addTos($add_to); - if ($add_cc) { - $mailer->addCCs($add_cc); - } - } catch (Exception $ex) { - $this - ->setStatus(PhabricatorMailOutboundStatus::STATUS_FAIL) - ->setMessage($ex->getMessage()) - ->save(); - - throw $ex; } - try { - $ok = $mailer->send(); - if (!$ok) { - // TODO: At some point, we should clean this up and make all mailers - // throw. - throw new Exception( - pht('Mail adapter encountered an unexpected, unspecified failure.')); - } + if (PhabricatorEnv::getEnvConfig('phabricator.silent')) { + $this->setMessage( + pht( + 'Phabricator is running in silent mode. See `%s` '. + 'in the configuration to change this setting.', + 'phabricator.silent')); - $this->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT); - $this->save(); - - return $this; - } catch (PhabricatorMetaMTAPermanentFailureException $ex) { - $this - ->setStatus(PhabricatorMailOutboundStatus::STATUS_FAIL) - ->setMessage($ex->getMessage()) - ->save(); - - throw $ex; - } catch (Exception $ex) { - $this - ->setMessage($ex->getMessage()."\n".$ex->getTraceAsString()) - ->save(); - - throw $ex; + return null; } + + // Some mailers require a valid "To:" in order to deliver mail. If we + // don't have any "To:", try to fill it in with a placeholder "To:". + // If that also fails, move the "Cc:" line to "To:". + if (!$add_to) { + $placeholder_key = 'metamta.placeholder-to-recipient'; + $placeholder = PhabricatorEnv::getEnvConfig($placeholder_key); + if ($placeholder !== null) { + $add_to = array($placeholder); + } else { + $add_to = $add_cc; + $add_cc = array(); + } + } + + $add_to = array_unique($add_to); + $add_cc = array_diff(array_unique($add_cc), $add_to); + + $mailer->addTos($add_to); + if ($add_cc) { + $mailer->addCCs($add_cc); + } + + return $mailer; } private function generateThreadIndex($seed, $is_first_mail) { diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php index 635913439d..9f14e0c4e1 100644 --- a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php +++ b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php @@ -18,7 +18,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mail->addTos(array($phid)); $mailer = new PhabricatorMailImplementationTestAdapter(); - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); $this->assertEqual( PhabricatorMailOutboundStatus::STATUS_SENT, $mail->getStatus()); @@ -31,7 +31,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mailer = new PhabricatorMailImplementationTestAdapter(); $mailer->setFailTemporarily(true); try { - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); } catch (Exception $ex) { // Ignore. } @@ -47,7 +47,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mailer = new PhabricatorMailImplementationTestAdapter(); $mailer->setFailPermanently(true); try { - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); } catch (Exception $ex) { // Ignore. } @@ -191,7 +191,7 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $mail = new PhabricatorMetaMTAMail(); $mail->setThreadID($thread_id, $is_first_mail); - $mail->sendNow($force = true, $mailer); + $mail->sendWithMailers(array($mailer)); $guts = $mailer->getGuts(); $dict = ipull($guts['headers'], 1, 0); From 7765299f8399f58ed96b91ee1ef0b2cabf562617 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 04:28:03 -0800 Subject: [PATCH 52/89] Mask the sender for "Must Encrypt" mail Summary: Depends on D18998. Ref T13053. When we send "Must Encrypt" mail, we currently send it with a normal "From" address. This discloses a little information about the object (for example, if the Director of Silly Walks is interacting with a "must encrypt" object, the vulnerability is probably related to Silly Walks), so anonymize who is interacting with the object. Test Plan: Processed some mail. (The actual final "From" is ephemeral and a little tricky to examine and I didn't actually transmit mail over the network, but it should be obvious if this works or not on `secure`.) Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19000 --- src/applications/metamta/storage/PhabricatorMetaMTAMail.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 179eb18088..317f9be8df 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -592,6 +592,12 @@ final class PhabricatorMetaMTAMail $mailer->setFrom($from_email, $from_name); break; case 'from': + // If the mail content must be encrypted, disguise the sender. + if ($must_encrypt) { + $mailer->setFrom($default_from, pht('Phabricator')); + break; + } + $from = $value; $actor_email = null; $actor_name = null; From 7f2c90fbd12b10e10abc3098c080baae96d4040f Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 5 Feb 2018 15:58:07 -0800 Subject: [PATCH 53/89] Prepare for multiple mailers of the same type Summary: Depends on D19000. Ref T13053. Ref T12677. Currently, most mailers are configured with a bunch of `.setting-name` global config options. This means that you can't configure two different SMTP servers, which is a reasonable thing to want to do in the brave new world of mail failover. It also means you can't configure two Mailgun accounts or two SES accounts. Although this might seem a little silly, we've had more service disruptions because of policy issues / administrative error (where a particular account was disabled) than actual downtime, so maybe it's not completely ridiculous. Realign mailers so they can take configuration directly in an explicit way. A later change will add new configuration to take advantage of this and let us move away from having ~10 global options for this stuff eventually. (This also makes writing third-party mailers easier.) Test Plan: Processed some mail, ran existing unit tests. But I wasn't especially thorough. I expect later changes to provide some tools to make this more testable, so I'll vet each provider more thoroughly and add coverage for multiple mailers after that stuff is ready. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19002 --- .../PhabricatorMailImplementationAdapter.php | 40 ++++++++++++++ ...atorMailImplementationAmazonSESAdapter.php | 36 ++++++++++-- ...icatorMailImplementationMailgunAdapter.php | 27 ++++++++- ...atorMailImplementationPHPMailerAdapter.php | 55 ++++++++++++++++--- ...MailImplementationPHPMailerLiteAdapter.php | 24 +++++++- ...catorMailImplementationSendGridAdapter.php | 27 ++++++++- ...abricatorMailImplementationTestAdapter.php | 18 +++++- .../storage/PhabricatorMetaMTAMail.php | 29 +++++++--- .../PhabricatorMetaMTAMailTestCase.php | 4 +- 9 files changed, 229 insertions(+), 31 deletions(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 3363301909..514306758d 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -2,6 +2,9 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { + private $key; + private $options = array(); + abstract public function setFrom($email, $name = ''); abstract public function addReplyTo($email, $name = ''); abstract public function addTos(array $emails); @@ -12,6 +15,7 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { abstract public function setHTMLBody($html_body); abstract public function setSubject($subject); + /** * Some mailers, notably Amazon SES, do not support us setting a specific * Message-ID header. @@ -32,4 +36,40 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { */ abstract public function send(); + final public function setKey($key) { + $this->key = $key; + return $this; + } + + final public function getKey() { + return $this->key; + } + + final public function getOption($key) { + if (!array_key_exists($key, $this->options)) { + throw new Exception( + pht( + 'Mailer ("%s") is attempting to access unknown option ("%s").', + get_class($this), + $key)); + } + + return $this->options[$key]; + } + + final public function setOptions(array $options) { + $this->validateOptions($options); + $this->options = $options; + return $this; + } + + abstract protected function validateOptions(array $options); + + abstract public function newDefaultOptions(); + abstract public function newLegacyOptions(); + + public function prepareForSend() { + return; + } + } diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php index 5b03cd86ac..850b83f1dd 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php @@ -6,8 +6,8 @@ final class PhabricatorMailImplementationAmazonSESAdapter private $message; private $isHTML; - public function __construct() { - parent::__construct(); + public function prepareForSend() { + parent::prepareForSend(); $this->mailer->Mailer = 'amazon-ses'; $this->mailer->customMailer = $this; } @@ -17,13 +17,39 @@ final class PhabricatorMailImplementationAmazonSESAdapter return false; } + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'access-key' => 'string', + 'secret-key' => 'string', + 'endpoint' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'access-key' => null, + 'secret-key' => null, + 'endpoint' => null, + ); + } + + public function newLegacyOptions() { + return array( + 'access-key' => PhabricatorEnv::getEnvConfig('amazon-ses.access-key'), + 'secret-key' => PhabricatorEnv::getEnvConfig('amazon-ses.secret-key'), + 'endpoint' => PhabricatorEnv::getEnvConfig('amazon-ses.endpoint'), + ); + } + /** * @phutil-external-symbol class SimpleEmailService */ public function executeSend($body) { - $key = PhabricatorEnv::getEnvConfig('amazon-ses.access-key'); - $secret = PhabricatorEnv::getEnvConfig('amazon-ses.secret-key'); - $endpoint = PhabricatorEnv::getEnvConfig('amazon-ses.endpoint'); + $key = $this->getOption('access-key'); + $secret = $this->getOption('secret-key'); + $endpoint = $this->getOption('endpoint'); $root = phutil_get_library_root('phabricator'); $root = dirname($root); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index cfe6491fe0..a7be6731eb 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -71,9 +71,32 @@ final class PhabricatorMailImplementationMailgunAdapter return true; } + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'api-key' => 'string', + 'domain' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'api-key' => null, + 'domain' => null, + ); + } + + public function newLegacyOptions() { + return array( + 'api-key' => PhabricatorEnv::getEnvConfig('mailgun.api-key'), + 'domain' => PhabricatorEnv::getEnvConfig('mailgun.domain'), + ); + } + public function send() { - $key = PhabricatorEnv::getEnvConfig('mailgun.api-key'); - $domain = PhabricatorEnv::getEnvConfig('mailgun.domain'); + $key = $this->getOption('api-key'); + $domain = $this->getOption('domain'); $params = array(); $params['to'] = implode(', ', idx($this->params, 'tos', array())); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php index f4d7e8e156..0eb59629a6 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php @@ -5,17 +5,55 @@ final class PhabricatorMailImplementationPHPMailerAdapter private $mailer; + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'host' => 'string|null', + 'port' => 'int', + 'user' => 'string|null', + 'password' => 'string|null', + 'protocol' => 'string|null', + 'encoding' => 'string', + 'mailer' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'host' => null, + 'port' => 25, + 'user' => null, + 'password' => null, + 'protocol' => null, + 'encoding' => 'base64', + 'mailer' => 'smtp', + ); + } + + public function newLegacyOptions() { + return array( + 'host' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-host'), + 'port' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-port'), + 'user' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-user'), + 'password' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-passsword'), + 'protocol' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-protocol'), + 'encoding' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'), + 'mailer' => PhabricatorEnv::getEnvConfig('phpmailer.mailer'), + ); + } + /** * @phutil-external-symbol class PHPMailer */ - public function __construct() { + public function prepareForSend() { $root = phutil_get_library_root('phabricator'); $root = dirname($root); require_once $root.'/externals/phpmailer/class.phpmailer.php'; $this->mailer = new PHPMailer($use_exceptions = true); $this->mailer->CharSet = 'utf-8'; - $encoding = PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'); + $encoding = $this->getOption('encoding'); $this->mailer->Encoding = $encoding; // By default, PHPMailer sends one mail per recipient. We handle @@ -23,20 +61,19 @@ final class PhabricatorMailImplementationPHPMailerAdapter // send mail exactly like we ask. $this->mailer->SingleTo = false; - $mailer = PhabricatorEnv::getEnvConfig('phpmailer.mailer'); + $mailer = $this->getOption('mailer'); if ($mailer == 'smtp') { $this->mailer->IsSMTP(); - $this->mailer->Host = PhabricatorEnv::getEnvConfig('phpmailer.smtp-host'); - $this->mailer->Port = PhabricatorEnv::getEnvConfig('phpmailer.smtp-port'); - $user = PhabricatorEnv::getEnvConfig('phpmailer.smtp-user'); + $this->mailer->Host = $this->getOption('host'); + $this->mailer->Port = $this->getOption('port'); + $user = $this->getOption('user'); if ($user) { $this->mailer->SMTPAuth = true; $this->mailer->Username = $user; - $this->mailer->Password = - PhabricatorEnv::getEnvConfig('phpmailer.smtp-password'); + $this->mailer->Password = $this->getOption('password'); } - $protocol = PhabricatorEnv::getEnvConfig('phpmailer.smtp-protocol'); + $protocol = $this->getOption('protocol'); if ($protocol) { $protocol = phutil_utf8_strtolower($protocol); $this->mailer->SMTPSecure = $protocol; diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index 668f9353ec..4fd8387252 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -8,17 +8,37 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter protected $mailer; + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'encoding' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'encoding' => 'base64', + ); + } + + public function newLegacyOptions() { + return array( + 'encoding' => PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'), + ); + } + /** * @phutil-external-symbol class PHPMailerLite */ - public function __construct() { + public function prepareForSend() { $root = phutil_get_library_root('phabricator'); $root = dirname($root); require_once $root.'/externals/phpmailer/class.phpmailer-lite.php'; $this->mailer = new PHPMailerLite($use_exceptions = true); $this->mailer->CharSet = 'utf-8'; - $encoding = PhabricatorEnv::getEnvConfig('phpmailer.smtp-encoding'); + $encoding = $this->getOption('encoding'); $this->mailer->Encoding = $encoding; // By default, PHPMailerLite sends one mail per recipient. We handle diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php index 566d33fd14..9cd8dd19b4 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php @@ -8,6 +8,29 @@ final class PhabricatorMailImplementationSendGridAdapter private $params = array(); + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'api-user' => 'string', + 'api-key' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'api-user' => null, + 'api-key' => null, + ); + } + + public function newLegacyOptions() { + return array( + 'api-user' => PhabricatorEnv::getEnvConfig('sendgrid.api-user'), + 'api-key' => PhabricatorEnv::getEnvConfig('sendgrid.api-key'), + ); + } + public function setFrom($email, $name = '') { $this->params['from'] = $email; $this->params['from-name'] = $name; @@ -73,8 +96,8 @@ final class PhabricatorMailImplementationSendGridAdapter public function send() { - $user = PhabricatorEnv::getEnvConfig('sendgrid.api-user'); - $key = PhabricatorEnv::getEnvConfig('sendgrid.api-key'); + $user = $this->getOption('api-user'); + $key = $this->getOption('api-key'); if (!$user || !$key) { throw new Exception( diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php index 0ea2af916f..bd64076a59 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php @@ -8,9 +8,23 @@ final class PhabricatorMailImplementationTestAdapter extends PhabricatorMailImplementationAdapter { private $guts = array(); - private $config; + private $config = array(); - public function __construct(array $config = array()) { + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array()); + } + + public function newDefaultOptions() { + return array(); + } + + public function newLegacyOptions() { + return array(); + } + + public function prepareForSend(array $config = array()) { $this->config = $config; } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 317f9be8df..eb1f1fbea2 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -453,11 +453,6 @@ final class PhabricatorMetaMTAMail return $result; } - public function buildDefaultMailer() { - return PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); - } - - /** * Attempt to deliver an email immediately, in this process. * @@ -468,13 +463,31 @@ final class PhabricatorMetaMTAMail throw new Exception(pht('Trying to send an already-sent mail!')); } - $mailers = array( - $this->buildDefaultMailer(), - ); + $mailers = $this->newMailers(); return $this->sendWithMailers($mailers); } + private function newMailers() { + $mailers = array(); + + $mailer = PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); + + $defaults = $mailer->newDefaultOptions(); + $options = $mailer->newLegacyOptions(); + + $options = $options + $defaults; + + $mailer + ->setKey('default') + ->setOptions($options); + + $mailer->prepareForSend(); + + $mailers[] = $mailer; + + return $mailers; + } public function sendWithMailers(array $mailers) { $exceptions = array(); diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php index 9f14e0c4e1..6e72b129b1 100644 --- a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php +++ b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php @@ -182,7 +182,9 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { $supports_message_id, $is_first_mail) { - $mailer = new PhabricatorMailImplementationTestAdapter( + $mailer = new PhabricatorMailImplementationTestAdapter(); + + $mailer->prepareForSend( array( 'supportsMessageIDHeader' => $supports_message_id, )); From c868ee9c07d0e8b7a6622f1de263027efa0773ea Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 05:23:47 -0800 Subject: [PATCH 54/89] Introduce and document a new `cluster.mailers` option for configuring multiple mailers Summary: Depends on D19002. Ref T13053. Ref T12677. Adds a new option to allow configuration of multiple mailers. Nothing actually uses this yet. Test Plan: Tried to set it to various bad values, got reasonable error messages. Read documentation. Reviewers: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19003 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMetaMTAConfigOptions.php | 20 +- .../PhabricatorMailImplementationAdapter.php | 12 + ...atorMailImplementationAmazonSESAdapter.php | 2 + ...icatorMailImplementationMailgunAdapter.php | 2 + ...atorMailImplementationPHPMailerAdapter.php | 2 + ...MailImplementationPHPMailerLiteAdapter.php | 2 + ...catorMailImplementationSendGridAdapter.php | 2 + ...abricatorMailImplementationTestAdapter.php | 2 + .../configuring_outbound_email.diviner | 303 +++++++++++------- .../PhabricatorClusterMailersConfigType.php | 100 ++++++ 11 files changed, 323 insertions(+), 126 deletions(-) create mode 100644 src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c642f13f90..fb2b0ccf0c 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2411,6 +2411,7 @@ phutil_register_library_map(array( 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php', 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php', 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterMailersConfigType' => 'infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php', 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php', 'PhabricatorClusterSearchConfigType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigType.php', 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php', @@ -7824,6 +7825,7 @@ phutil_register_library_map(array( 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterMailersConfigType' => 'PhabricatorJSONConfigType', 'PhabricatorClusterNoHostForRoleException' => 'Exception', 'PhabricatorClusterSearchConfigType' => 'PhabricatorJSONConfigType', 'PhabricatorClusterServiceHealthRecord' => 'Phobject', diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 8c2d2265bc..43734abea0 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -138,19 +138,14 @@ EODOC , 'metamta.public-replies')); - $adapter_doc_href = PhabricatorEnv::getDoclink( - 'Configuring Outbound Email'); - $adapter_doc_name = pht('Configuring Outbound Email'); $adapter_description = $this->deformat(pht(<<deformat(pht(<<deformat(pht(<<newOption('cluster.mailers', 'cluster.mailers', null) + ->setLocked(true) + ->setDescription($mailers_description), $this->newOption( 'metamta.default-address', 'string', diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 514306758d..923574d16d 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -5,6 +5,18 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { private $key; private $options = array(); + final public function getAdapterType() { + return $this->getPhobjectClassConstant('ADAPTERTYPE'); + } + + final public static function getAllAdapters() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getAdapterType') + ->execute(); + } + + abstract public function setFrom($email, $name = ''); abstract public function addReplyTo($email, $name = ''); abstract public function addTos(array $emails); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php index 850b83f1dd..22cc102262 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAmazonSESAdapter.php @@ -3,6 +3,8 @@ final class PhabricatorMailImplementationAmazonSESAdapter extends PhabricatorMailImplementationPHPMailerLiteAdapter { + const ADAPTERTYPE = 'ses'; + private $message; private $isHTML; diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index a7be6731eb..bed0dada63 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -6,6 +6,8 @@ final class PhabricatorMailImplementationMailgunAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'mailgun'; + private $params = array(); private $attachments = array(); diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php index 0eb59629a6..3ca6366730 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php @@ -3,6 +3,8 @@ final class PhabricatorMailImplementationPHPMailerAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'smtp'; + private $mailer; protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php index 4fd8387252..1f21a993c9 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php @@ -6,6 +6,8 @@ class PhabricatorMailImplementationPHPMailerLiteAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'sendmail'; + protected $mailer; protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php index 9cd8dd19b4..be2a837053 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php @@ -6,6 +6,8 @@ final class PhabricatorMailImplementationSendGridAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'sendgrid'; + private $params = array(); protected function validateOptions(array $options) { diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php index bd64076a59..8a8d0de0c2 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php @@ -7,6 +7,8 @@ final class PhabricatorMailImplementationTestAdapter extends PhabricatorMailImplementationAdapter { + const ADAPTERTYPE = 'test'; + private $guts = array(); private $config = array(); diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 2a95f49bc3..0c7ef529fa 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -3,43 +3,40 @@ Instructions for configuring Phabricator to send mail. -= Overview = +Overview +======== -Phabricator can send outbound email via several different providers, called -"Adapters". +Phabricator can send outbound email through several different mail services, +including a local mailer or various third-party services. Options include: | Send Mail With | Setup | Cost | Inbound | Notes | |---------|-------|------|---------|-------| | Mailgun | Easy | Cheap | Yes | Recommended | | Amazon SES | Easy | Cheap | No | Recommended | -| SendGrid | Medium | Cheap | Yes | Discouraged (See Note) | +| SendGrid | Medium | Cheap | Yes | Discouraged | | External SMTP | Medium | Varies | No | Gmail, etc. | -| Local SMTP | Hard | Free | No | (Default) sendmail, postfix, etc | -| Custom | Hard | Free | No | Write an adapter for some other service. | +| Local SMTP | Hard | Free | No | sendmail, postfix, etc | +| Custom | Hard | Free | No | Write a custom mailer for some other service. | | Drop in a Hole | Easy | Free | No | Drops mail in a deep, dark hole. | -Of these options, sending mail via local SMTP is the default, but usually -requires some configuration to get working. See below for details on how to -select and configure a delivery method. +See below for details on how to select and configure mail delivery for each +mailer. Overall, Mailgun and SES are much easier to set up, and using one of them is recommended. In particular, Mailgun will also let you set up inbound email easily. If you have some internal mail service you'd like to use you can also -write a custom adapter, but this requires digging into the code. +write a custom mailer, but this requires digging into the code. Phabricator sends mail in the background, so the daemons need to be running for it to be able to deliver mail. You should receive setup warnings if they are not. For more information on using daemons, see @{article:Managing Daemons with phd}. -**Note on SendGrid**: Users have experienced a number of odd issues with -SendGrid, compared to fewer issues with other mailers. We discourage SendGrid -unless you're already using it. If you send to SendGrid via SMTP, you may need -to adjust `phpmailer.smtp-encoding`. -= Basics = +Basics +====== Regardless of how outbound email is delivered, you should configure these keys in your configuration: @@ -51,33 +48,113 @@ in your configuration: - **metamta.can-send-as-user** should be left as `false` in most cases, but see the documentation for details. -= Configuring Mail Adapters = -To choose how mail will be sent, change the `metamta.mail-adapter` key in -your configuration. Possible values are listed in the UI: +Configuring Mailers +=================== - - `PhabricatorMailImplementationAmazonMailgunAdapter`: use Mailgun, see - "Adapter: Mailgun". - - `PhabricatorMailImplementationAmazonSESAdapter`: use Amazon SES, see - "Adapter: Amazon SES". - - `PhabricatorMailImplementationPHPMailerLiteAdapter`: default, uses - "sendmail", see "Adapter: Sendmail". - - `PhabricatorMailImplementationPHPMailerAdapter`: uses SMTP, see - "Adapter: SMTP". - - `PhabricatorMailImplementationSendGridAdapter`: use SendGrid, see - "Adapter: SendGrid". - - `Some Custom Class You Write`: use a custom adapter you write, see - "Adapter: Custom". - - `PhabricatorMailImplementationTestAdapter`: this will - **completely disable** outbound mail. You can use this if you don't want to - send outbound mail, or want to skip this step for now and configure it - later. +Configure one or more mailers by listing them in the the `cluster.mailers` +configuration option. Most installs only need to configure one mailer, but you +can configure multiple mailers to provide greater availability in the event of +a service disruption. -= Adapter: Sendmail = +A valid `cluster.mailers` configuration looks something like this: -This is the default, and selected by choosing -`PhabricatorMailImplementationPHPMailerLiteAdapter` as the value for -**metamta.mail-adapter**. This requires a `sendmail` binary to be installed on +```lang=json +[ + { + "key": "mycompany-mailgun", + "type": "mailgun", + "options": { + "domain": "mycompany.com", + "api-key": "..." + } + }, + ... +] +``` + +The supported keys for each mailer are: + + - `key`: Required string. A unique name for this mailer. + - `type`: Required string. Identifies the type of mailer. See below for + options. + - `priority`: Optional string. Advanced option which controls load balancing + and failover behavior. See below for details. + - `options`: Optional map. Additional options for the mailer type. + +The `type` field can be used to select these third-party mailers: + + - `mailgun`: Use Mailgun. + - `ses`: Use Amazon SES. + - `sendgrid`: Use Sendgrid. + +It also supports these local mailers: + + - `sendmail`: Use the local `sendmail` binary. + - `smtp`: Connect directly to an SMTP server. + - `test`: Internal mailer for testing. Does not send mail. + +You can also write your own mailer by extending +`PhabricatorMailImplementationAdapter`. + +Once you've selected a mailer, find the corresponding section below for +instructions on configuring it. + + +Mailer: Mailgun +=============== + +Mailgun is a third-party email delivery service. You can learn more at +. Mailgun is easy to configure and works well. + +To use this mailer, set `type` to `mailgun`, then configure these `options`: + + - `api-key`: Required string. Your Mailgun API key. + - `domain`: Required string. Your Mailgun domain. + + +Mailer: Amazon SES +================== + +Amazon SES is Amazon's cloud email service. You can learn more at +. + +To use this mailer, set `type` to `ses`, then configure these `options`: + + - `access-key`: Required string. Your Amazon SES access key. + - `secret-key`: Required string. Your Amazon SES secret key. + - `endpoint`: Required string. Your Amazon SES endpoint. + +NOTE: Amazon SES **requires you to verify your "From" address**. Configure +which "From" address to use by setting "`metamta.default-address`" in your +config, then follow the Amazon SES verification process to verify it. You +won't be able to send email until you do this! + + +Mailer: SendGrid +================ + +SendGrid is a third-party email delivery service. You can learn more at +. + +You can configure SendGrid in two ways: you can send via SMTP or via the REST +API. To use SMTP, configure Phabricator to use an `smtp` mailer. + +To use the REST API mailer, set `type` to `sendgrid`, then configure +these `options`: + + - `api-user`: Required string. Your SendGrid login name. + - `api-key`: Required string. Your SendGrid API key. + +NOTE: Users have experienced a number of odd issues with SendGrid, compared to +fewer issues with other mailers. We discourage SendGrid unless you're already +using it. + + +Mailer: Sendmail +================ + +This requires a `sendmail` binary to be installed on the system. Most MTAs (e.g., sendmail, qmail, postfix) should do this, but your machine may not have one installed by default. For install instructions, consult the documentation for your favorite MTA. @@ -88,96 +165,32 @@ document. If you can already send outbound email from the command line or know how to configure it, this option is straightforward. If you have no idea how to do any of this, strongly consider using Mailgun or Amazon SES instead. -If you experience issues with mail getting mangled (for example, arriving with -too many or too few newlines) you may try adjusting `phpmailer.smtp-encoding`. +To use this mailer, set `type` to `sendmail`. There are no `options` to +configure. -= Adapter: SMTP = + +Mailer: STMP +============ You can use this adapter to send mail via an external SMTP server, like Gmail. -To do this, set these configuration keys: - - **metamta.mail-adapter**: set to - `PhabricatorMailImplementationPHPMailerAdapter`. - - **phpmailer.mailer**: set to `smtp`. - - **phpmailer.smtp-host**: set to hostname of your SMTP server. - - **phpmailer.smtp-port**: set to port of your SMTP server. - - **phpmailer.smtp-user**: set to your username used for authentication. - - **phpmailer.smtp-password**: set to your password used for authentication. - - **phpmailer.smtp-protocol**: set to `tls` or `ssl` if necessary. Use +To use this mailer, set `type` to `smtp`, then configure these `options`: + + - `host`: Required string. The hostname of your SMTP server. + - `user`: Optional string. Username used for authentication. + - `password`: Optional string. Password for authentication. + - `protocol`: Optional string. Set to `tls` or `ssl` if necessary. Use `ssl` for Gmail. - - **phpmailer.smtp-encoding**: Normally safe to leave as the default, but - adjusting it may help resolve mail mangling issues (for example, mail - arriving with too many or too few newlines). -= Adapter: Mailgun = -Mailgun is an email delivery service. You can learn more at -. Mailgun isn't free, but is very easy to configure -and works well. +Disable Mail +============ -To use Mailgun, sign up for an account, then set these configuration keys: +To disable mail, just don't configure any mailers. - - **metamta.mail-adapter**: set to - `PhabricatorMailImplementationMailgunAdapter`. - - **mailgun.api-key**: set to your Mailgun API key. - - **mailgun.domain**: set to your Mailgun domain. -= Adapter: Amazon SES = - -Amazon SES is Amazon's cloud email service. It is not free, but is easier to -configure than sendmail and can simplify outbound email configuration. To use -Amazon SES, you need to sign up for an account with Amazon at -. - -To configure Phabricator to use Amazon SES, set these configuration keys: - - - **metamta.mail-adapter**: set to - "PhabricatorMailImplementationAmazonSESAdapter". - - **amazon-ses.access-key**: set to your Amazon SES access key. - - **amazon-ses.secret-key**: set to your Amazon SES secret key. - - **amazon-ses.endpoint**: Set to your Amazon SES endpoint. - -NOTE: Amazon SES **requires you to verify your "From" address**. Configure which -"From" address to use by setting "`metamta.default-address`" in your config, -then follow the Amazon SES verification process to verify it. You won't be able -to send email until you do this! - -= Adapter: SendGrid = - -SendGrid is an email delivery service like Amazon SES. You can learn more at -. It is easy to configure, but not free. - -You can configure SendGrid in two ways: you can send via SMTP or via the REST -API. To use SMTP, just configure `sendmail` and leave Phabricator's setup -with defaults. To use the REST API, follow the instructions in this section. - -To configure Phabricator to use SendGrid, set these configuration keys: - - - **metamta.mail-adapter**: set to - "PhabricatorMailImplementationSendGridAdapter". - - **sendgrid.api-user**: set to your SendGrid login name. - - **sendgrid.api-key**: set to your SendGrid password. - -If you're logged into your SendGrid account, you may be able to find this -information easily by visiting . - -= Adapter: Custom = - -You can provide a custom adapter by writing a concrete subclass of -@{class:PhabricatorMailImplementationAdapter} and setting it as the -`metamta.mail-adapter`. - -TODO: This should be better documented once extending Phabricator is better -documented. - -= Adapter: Disable Outbound Mail = - -You can use the @{class:PhabricatorMailImplementationTestAdapter} to completely -disable outbound mail, if you don't want to send mail or don't want to configure -it yet. Just set **metamta.mail-adapter** to -`PhabricatorMailImplementationTestAdapter`. - -= Testing and Debugging Outbound Email = +Testing and Debugging Outbound Email +==================================== You can use the `bin/mail` utility to test, debug, and examine outbound mail. In particular: @@ -191,7 +204,59 @@ Run `bin/mail help ` for more help on using these commands. You can monitor daemons using the Daemon Console (`/daemon/`, or click **Daemon Console** from the homepage). -= Next Steps = + +Priorities +========== + +By default, Phabricator will try each mailer in order: it will try the first +mailer first. If that fails (for example, because the service is not available +at the moment) it will try the second mailer, and so on. + +If you want to load balance between multiple mailers instead of using one as +a primary, you can set `priority`. Phabricator will start with mailers in the +highest priority group and go through them randomly, then fall back to the +next group. + +For example, if you have two SMTP servers and you want to balance requests +between them and then fall back to Mailgun if both fail, configure priorities +like this: + +```lang=json +[ + { + "key": "smtp-uswest", + "type": "smtp", + "priority": 300, + "options": "..." + }, + { + "key": "smtp-useast", + "type": "smtp", + "priority": 300, + "options": "..." + }, + { + "key": "mailgun-fallback", + "type": "mailgun", + "options": "..." + } +} +``` + +Phabricator will start with servers in the highest priority group (the group +with the **largest** `priority` number). In this example, the highest group is +`300`, which has the two SMTP servers. They'll be tried in random order first. + +If both fail, Phabricator will move on to the next priority group. In this +example, there are no other priority groups. + +If it still hasn't sent the mail, Phabricator will try servers which are not +in any priority group, in the configured order. In this example there is +only one such server, so it will try to send via Mailgun. + + +Next Steps +========== Continue by: diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php new file mode 100644 index 0000000000..2a7550c419 --- /dev/null +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -0,0 +1,100 @@ +newException( + pht( + 'Mailer cluster configuration is not valid: it should be a list '. + 'of mailer configurations.')); + } + + foreach ($value as $index => $spec) { + if (!is_array($spec)) { + throw $this->newException( + pht( + 'Mailer cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a mailer, but the value '. + 'with index "%s" is not a dictionary.', + $index)); + } + } + + $adapters = PhabricatorMailImplementationAdapter::getAllAdapters(); + + $map = array(); + foreach ($value as $index => $spec) { + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'key' => 'string', + 'type' => 'string', + 'priority' => 'optional int', + 'options' => 'optional wild', + )); + } catch (Exception $ex) { + throw $this->newException( + pht( + 'Mailer configuration has an invalid mailer specification '. + '(at index "%s"): %s.', + $index, + $ex->getMessage())); + } + + $key = $spec['key']; + if (isset($map[$key])) { + throw $this->newException( + pht( + 'Mailer configuration is invalid: multiple mailers have the same '. + 'key ("%s"). Each mailer must have a unique key.', + $key)); + } + $map[$key] = true; + + $priority = idx($spec, 'priority', 0); + if ($priority <= 0) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") is invalid: priority must be '. + 'greater than 0.', + $key)); + } + + $type = $spec['type']; + if (!isset($adapters[$type])) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") is invalid: mailer type ("%s") is '. + 'unknown. Supported mailer types are: %s.', + $key, + $type, + implode(', ', array_keys($adapters)))); + } + + $options = idx($spec, 'options', array()); + try { + id(clone $adapters[$type])->validateOptions($options); + } catch (Exception $ex) { + throw $this->newException( + pht( + 'Mailer configuration ("%s") specifies invalid options for '. + 'mailer: %s', + $key, + $ex->getMessage())); + } + } + } + +} From 4236952cdbc0e1f8af11b126dc28172997827627 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 06:20:24 -0800 Subject: [PATCH 55/89] Add a `bin/config set --stdin < value.json` flag to make CLI configuration of complex values easier Summary: Depends on D19003. Ref T12677. Ref T13053. For the first time, we're requiring CLI configuration of a complex value (not just a string, integer, bool, etc) to do something fairly standard (send mail). Users sometimes have very reasonable difficulty figuring out how to `./bin/config set key `. Provide an easy way to handle this and make sure it gets appropriate callouts in the documentation. (Also, hide the `cluster.mailers` value rather than just locking it, since it may have API keys or SMTP passwords.) Test Plan: Read documentation, used old and new flags to set configuration. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19004 --- ...PhabricatorConfigManagementSetWorkflow.php | 46 +++++++++++++------ .../PhabricatorMetaMTAConfigOptions.php | 2 +- .../configuration_locked.diviner | 20 ++++++++ .../configuring_outbound_email.diviner | 34 ++++++++++++++ 4 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php index 9f50d2eeed..22b760872e 100644 --- a/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php +++ b/src/applications/config/management/PhabricatorConfigManagementSetWorkflow.php @@ -6,7 +6,9 @@ final class PhabricatorConfigManagementSetWorkflow protected function didConstruct() { $this ->setName('set') - ->setExamples('**set** __key__ __value__') + ->setExamples( + "**set** __key__ __value__\n". + "**set** __key__ --stdin < value.json") ->setSynopsis(pht('Set a local configuration value.')) ->setArguments( array( @@ -16,6 +18,10 @@ final class PhabricatorConfigManagementSetWorkflow 'Update configuration in the database instead of '. 'in local configuration.'), ), + array( + 'name' => 'stdin', + 'help' => pht('Read option value from stdin.'), + ), array( 'name' => 'args', 'wildcard' => true, @@ -31,22 +37,36 @@ final class PhabricatorConfigManagementSetWorkflow pht('Specify a configuration key and a value to set it to.')); } + $is_stdin = $args->getArg('stdin'); + $key = $argv[0]; - if (count($argv) == 1) { - throw new PhutilArgumentUsageException( - pht( - "Specify a value to set the key '%s' to.", - $key)); + if ($is_stdin) { + if (count($argv) > 1) { + throw new PhutilArgumentUsageException( + pht( + 'Too many arguments: expected only a key when using "--stdin".')); + } + + fprintf(STDERR, tsprintf("%s\n", pht('Reading value from stdin...'))); + $value = file_get_contents('php://stdin'); + } else { + if (count($argv) == 1) { + throw new PhutilArgumentUsageException( + pht( + "Specify a value to set the key '%s' to.", + $key)); + } + + if (count($argv) > 2) { + throw new PhutilArgumentUsageException( + pht( + 'Too many arguments: expected one key and one value.')); + } + + $value = $argv[1]; } - $value = $argv[1]; - - if (count($argv) > 2) { - throw new PhutilArgumentUsageException( - pht( - 'Too many arguments: expected one key and one value.')); - } $options = PhabricatorApplicationConfigOptions::loadAllOptions(); if (empty($options[$key])) { diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 43734abea0..0b916150bc 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -202,7 +202,7 @@ EODOC return array( $this->newOption('cluster.mailers', 'cluster.mailers', null) - ->setLocked(true) + ->setHidden(true) ->setDescription($mailers_description), $this->newOption( 'metamta.default-address', diff --git a/src/docs/user/configuration/configuration_locked.diviner b/src/docs/user/configuration/configuration_locked.diviner index 040b838177..fff0da9bdc 100644 --- a/src/docs/user/configuration/configuration_locked.diviner +++ b/src/docs/user/configuration/configuration_locked.diviner @@ -27,6 +27,24 @@ can edit it from the CLI instead, with `bin/config`: phabricator/ $ ./bin/config set ``` +Some configuration options take complicated values which can be difficult +to escape properly for the shell. The easiest way to set these options is +to use the `--stdin` flag. First, put your desired value in a `config.json` +file: + +```name=config.json, lang=json +{ + "duck": "quack", + "cow": "moo" +} +``` + +Then, set it with `--stdin` like this: + +``` +phabricator/ $ ./bin/config set --stdin < config.json +``` + A few settings have alternate CLI tools. Refer to the setting page for details. @@ -98,4 +116,6 @@ Next Steps Continue by: + - learning more about advanced options with + @{Configuration User Guide: Advanced Configuration}; or - returning to the @{article: Configuration Guide}. diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 0c7ef529fa..21abf92736 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -101,6 +101,40 @@ Once you've selected a mailer, find the corresponding section below for instructions on configuring it. +Setting Complex Configuration +============================= + +Mailers can not be edited from the web UI. If mailers could be edited from +the web UI, it would give an attacker who compromised an administrator account +a lot of power: they could redirect mail to a server they control and then +intercept mail for any other account, including password reset mail. + +For more information about locked configuration options, see +@{article:Configuration Guide: Locked and Hidden Configuration}. + +Setting `cluster.mailers` from the command line using `bin/config set` can be +tricky because of shell escaping. The easiest way to do it is to use the +`--stdin` flag. First, put your desired configuration in a file like this: + +```lang=json, name=mailers.json +[ + { + "key": "test-mailer", + "type": "test" + } +] +``` + +Then set the value like this: + +``` +phabricator/ $ ./bin/config set --stdin < mailers.json +``` + +For alternatives and more information on configuration, see +@{article:Configuration User Guide: Advanced Configuration} + + Mailer: Mailgun =============== From 994d2e8e156323e0684dac897e65bc9ba6db42e8 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 06:31:37 -0800 Subject: [PATCH 56/89] Use "cluster.mailers" if it is configured Summary: Depends on D19004. Ref T13053. Ref T12677. If the new `cluster.mailers` is configured, make use of it. Also use it in the Sengrid/Mailgun inbound stuff. Also fix a bug where "Must Encrypt" mail to no recipients could fatal because no `$mail` was returned. Test Plan: Processed some mail locally. The testing on this is still pretty flimsy, but I plan to solidify it in an upcoming change. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19005 --- .../PhabricatorMailImplementationAdapter.php | 10 +++ ...ricatorMetaMTAMailgunReceiveController.php | 23 +++++- ...icatorMetaMTASendGridReceiveController.php | 20 +++++ .../storage/PhabricatorMetaMTAMail.php | 82 ++++++++++++++++--- ...habricatorApplicationTransactionEditor.php | 11 +-- 5 files changed, 126 insertions(+), 20 deletions(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 923574d16d..14c7a63663 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -3,6 +3,7 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { private $key; + private $priority; private $options = array(); final public function getAdapterType() { @@ -57,6 +58,15 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { return $this->key; } + final public function setPriority($priority) { + $this->priority = $priority; + return $this; + } + + final public function getPriority() { + return $this->priority; + } + final public function getOption($key) { if (!array_key_exists($key, $this->options)) { throw new Exception( diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php index 467995a186..4eb53b7120 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php @@ -8,14 +8,31 @@ final class PhabricatorMetaMTAMailgunReceiveController } private function verifyMessage() { - $api_key = PhabricatorEnv::getEnvConfig('mailgun.api-key'); $request = $this->getRequest(); $timestamp = $request->getStr('timestamp'); $token = $request->getStr('token'); $sig = $request->getStr('signature'); - $hash = hash_hmac('sha256', $timestamp.$token, $api_key); - return phutil_hashes_are_identical($sig, $hash); + // An install may configure multiple Mailgun mailers, and we might receive + // inbound mail from any of them. Test the signature to see if it matches + // any configured Mailgun mailer. + + $mailers = PhabricatorMetaMTAMail::newMailers(); + $mailgun_type = PhabricatorMailImplementationMailgunAdapter::ADAPTERTYPE; + foreach ($mailers as $mailer) { + if ($mailer->getAdapterType() != $mailgun_type) { + continue; + } + + $api_key = $mailer->getOption('api-key'); + + $hash = hash_hmac('sha256', $timestamp.$token, $api_key); + if (phutil_hashes_are_identical($sig, $hash)) { + return true; + } + } + + return false; } public function handleRequest(AphrontRequest $request) { diff --git a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php index 0a5e28fcee..99e60caa05 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php @@ -8,6 +8,26 @@ final class PhabricatorMetaMTASendGridReceiveController } public function handleRequest(AphrontRequest $request) { + $mailers = PhabricatorMetaMTAMail::newMailers(); + $sendgrid_type = PhabricatorMailImplementationSendGridAdapter::ADAPTERTYPE; + + // SendGrid doesn't sign payloads so we can't be sure that SendGrid + // actually sent this request, but require a configured SendGrid mailer + // before we activate this endpoint. + + $has_sendgrid = false; + foreach ($mailers as $mailer) { + if ($mailer->getAdapterType() != $sendgrid_type) { + continue; + } + + $has_sendgrid = true; + break; + } + + if (!$has_sendgrid) { + return new Aphront404Response(); + } // No CSRF for SendGrid. $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index eb1f1fbea2..2ab3299c34 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -463,33 +463,85 @@ final class PhabricatorMetaMTAMail throw new Exception(pht('Trying to send an already-sent mail!')); } - $mailers = $this->newMailers(); + $mailers = self::newMailers(); return $this->sendWithMailers($mailers); } - private function newMailers() { + public static function newMailers() { $mailers = array(); - $mailer = PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); + $config = PhabricatorEnv::getEnvConfig('cluster.mailers'); + if ($config === null) { + $mailer = PhabricatorEnv::newObjectFromConfig('metamta.mail-adapter'); - $defaults = $mailer->newDefaultOptions(); - $options = $mailer->newLegacyOptions(); + $defaults = $mailer->newDefaultOptions(); + $options = $mailer->newLegacyOptions(); - $options = $options + $defaults; + $options = $options + $defaults; - $mailer - ->setKey('default') - ->setOptions($options); + $mailer + ->setKey('default') + ->setPriority(-1) + ->setOptions($options); - $mailer->prepareForSend(); + $mailers[] = $mailer; + } else { + $adapters = PhabricatorMailImplementationAdapter::getAllAdapters(); + $next_priority = -1; - $mailers[] = $mailer; + foreach ($config as $spec) { + $type = $spec['type']; + if (!isset($adapters[$type])) { + throw new Exception( + pht( + 'Unknown mailer ("%s")!', + $type)); + } - return $mailers; + $key = $spec['key']; + $mailer = id(clone $adapters[$type]) + ->setKey($key); + + $priority = idx($spec, 'priority'); + if (!$priority) { + $priority = $next_priority; + $next_priority--; + } + $mailer->setPriority($priority); + + $defaults = $mailer->newDefaultOptions(); + $options = idx($spec, 'options', array()) + $defaults; + $mailer->setOptions($options); + } + } + + $sorted = array(); + $groups = mgroup($mailers, 'getPriority'); + ksort($groups); + foreach ($groups as $group) { + // Reorder services within the same priority group randomly. + shuffle($group); + foreach ($group as $mailer) { + $sorted[] = $mailer; + } + } + + foreach ($sorted as $mailer) { + $mailer->prepareForSend(); + } + + return $sorted; } public function sendWithMailers(array $mailers) { + if (!$mailers) { + return $this + ->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID) + ->setMessage(pht('No mailers are configured.')) + ->save(); + } + $exceptions = array(); foreach ($mailers as $template_mailer) { $mailer = null; @@ -865,6 +917,12 @@ final class PhabricatorMetaMTAMail $mailer->addCCs($add_cc); } + // Keep track of which mailer actually ended up accepting the message. + $mailer_key = $mailer->getKey(); + if ($mailer_key !== null) { + $this->setParam('mailer.key', $mailer_key); + } + return $mailer; } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index c5390de362..e4f9607801 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -2575,12 +2575,13 @@ abstract class PhabricatorApplicationTransactionEditor $mail = $this->buildMailForTarget($object, $xactions, $target); - if ($this->mustEncrypt) { - $mail - ->setMustEncrypt(true) - ->setMustEncryptReasons($this->mustEncrypt); + if ($mail) { + if ($this->mustEncrypt) { + $mail + ->setMustEncrypt(true) + ->setMustEncryptReasons($this->mustEncrypt); + } } - } catch (Exception $ex) { $caught = $ex; } From 9947eee182aa9fe04e926098eb00dfd1a758e185 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 06:56:44 -0800 Subject: [PATCH 57/89] Add some test coverage for mailers configuration Summary: Depends on D19005. Ref T12677. Ref T13053. Tests that turning `cluster.mailers` into an actual list of mailers more or less works as expected. Test Plan: Ran unit tests. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19006 --- src/__phutil_library_map__.php | 2 + .../storage/PhabricatorMetaMTAMail.php | 4 +- .../PhabricatorMailConfigTestCase.php | 131 ++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index fb2b0ccf0c..29a97c43d2 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3178,6 +3178,7 @@ phutil_register_library_map(array( 'PhabricatorMacroTransactionQuery' => 'applications/macro/query/PhabricatorMacroTransactionQuery.php', 'PhabricatorMacroTransactionType' => 'applications/macro/xaction/PhabricatorMacroTransactionType.php', 'PhabricatorMacroViewController' => 'applications/macro/controller/PhabricatorMacroViewController.php', + 'PhabricatorMailConfigTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php', 'PhabricatorMailEmailHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailHeraldField.php', 'PhabricatorMailEmailHeraldFieldGroup' => 'applications/metamta/herald/PhabricatorMailEmailHeraldFieldGroup.php', 'PhabricatorMailEmailSubjectHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailSubjectHeraldField.php', @@ -8680,6 +8681,7 @@ phutil_register_library_map(array( 'PhabricatorMacroTransactionQuery' => 'PhabricatorApplicationTransactionQuery', 'PhabricatorMacroTransactionType' => 'PhabricatorModularTransactionType', 'PhabricatorMacroViewController' => 'PhabricatorMacroController', + 'PhabricatorMailConfigTestCase' => 'PhabricatorTestCase', 'PhabricatorMailEmailHeraldField' => 'HeraldField', 'PhabricatorMailEmailHeraldFieldGroup' => 'HeraldFieldGroup', 'PhabricatorMailEmailSubjectHeraldField' => 'PhabricatorMailEmailHeraldField', diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 2ab3299c34..141d3b5e1c 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -513,12 +513,14 @@ final class PhabricatorMetaMTAMail $defaults = $mailer->newDefaultOptions(); $options = idx($spec, 'options', array()) + $defaults; $mailer->setOptions($options); + + $mailers[] = $mailer; } } $sorted = array(); $groups = mgroup($mailers, 'getPriority'); - ksort($groups); + krsort($groups); foreach ($groups as $group) { // Reorder services within the same priority group randomly. shuffle($group); diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php new file mode 100644 index 0000000000..25984a2c1d --- /dev/null +++ b/src/applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php @@ -0,0 +1,131 @@ +newMailersWithConfig( + array( + array( + 'key' => 'A', + 'type' => 'test', + ), + array( + 'key' => 'B', + 'type' => 'test', + ), + )); + + $this->assertEqual( + array('A', 'B'), + mpull($mailers, 'getKey')); + + $mailers = $this->newMailersWithConfig( + array( + array( + 'key' => 'A', + 'priority' => 1, + 'type' => 'test', + ), + array( + 'key' => 'B', + 'priority' => 2, + 'type' => 'test', + ), + )); + + $this->assertEqual( + array('B', 'A'), + mpull($mailers, 'getKey')); + + $mailers = $this->newMailersWithConfig( + array( + array( + 'key' => 'A1', + 'priority' => 300, + 'type' => 'test', + ), + array( + 'key' => 'A2', + 'priority' => 300, + 'type' => 'test', + ), + array( + 'key' => 'B', + 'type' => 'test', + ), + array( + 'key' => 'C', + 'priority' => 400, + 'type' => 'test', + ), + array( + 'key' => 'D', + 'type' => 'test', + ), + )); + + // The "A" servers should be shuffled randomly, so either outcome is + // acceptable. + $option_1 = array('C', 'A1', 'A2', 'B', 'D'); + $option_2 = array('C', 'A2', 'A1', 'B', 'D'); + $actual = mpull($mailers, 'getKey'); + + $this->assertTrue(($actual === $option_1) || ($actual === $option_2)); + + // Make sure that when we're load balancing we actually send traffic to + // both servers reasonably often. + $saw_a1 = false; + $saw_a2 = false; + $attempts = 0; + while (true) { + $mailers = $this->newMailersWithConfig( + array( + array( + 'key' => 'A1', + 'priority' => 300, + 'type' => 'test', + ), + array( + 'key' => 'A2', + 'priority' => 300, + 'type' => 'test', + ), + )); + + $first_key = head($mailers)->getKey(); + + if ($first_key == 'A1') { + $saw_a1 = true; + } + + if ($first_key == 'A2') { + $saw_a2 = true; + } + + if ($saw_a1 && $saw_a2) { + break; + } + + if ($attempts++ > 1024) { + throw new Exception( + pht( + 'Load balancing between two mail servers did not select both '. + 'servers after an absurd number of attempts.')); + } + } + + $this->assertTrue($saw_a1 && $saw_a2); + } + + private function newMailersWithConfig(array $config) { + $env = PhabricatorEnv::beginScopedEnv(); + $env->overrideEnvConfig('cluster.mailers', $config); + + $mailers = PhabricatorMetaMTAMail::newMailers(); + + unset($env); + return $mailers; + } + +} From 1f53aa27e4596754cd347c3a74fab3876c66f4c8 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 08:24:00 -0800 Subject: [PATCH 58/89] Add unit tests for mail failover behaviors when multiple mailers are configured Summary: Depends on D19006. Ref T13053. Ref T12677. When multiple mailers are configured but one or more fail, test that we recover (or don't) appropriately. Test Plan: Ran unit tests. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13053, T12677 Differential Revision: https://secure.phabricator.com/D19007 --- .../storage/PhabricatorMetaMTAMail.php | 16 ++-- .../PhabricatorMetaMTAMailTestCase.php | 78 +++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 141d3b5e1c..5f859fd1d9 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -315,6 +315,10 @@ final class PhabricatorMetaMTAMail return $this->getParam('stampMetadata', array()); } + public function getMailerKey() { + return $this->getParam('mailer.key'); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -588,6 +592,12 @@ final class PhabricatorMetaMTAMail continue; } + // Keep track of which mailer actually ended up accepting the message. + $mailer_key = $mailer->getKey(); + if ($mailer_key !== null) { + $this->setParam('mailer.key', $mailer_key); + } + return $this ->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT) ->save(); @@ -919,12 +929,6 @@ final class PhabricatorMetaMTAMail $mailer->addCCs($add_cc); } - // Keep track of which mailer actually ended up accepting the message. - $mailer_key = $mailer->getKey(); - if ($mailer_key !== null) { - $this->setParam('mailer.key', $mailer_key); - } - return $mailer; } diff --git a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php index 6e72b129b1..c0045301fd 100644 --- a/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php +++ b/src/applications/metamta/storage/__tests__/PhabricatorMetaMTAMailTestCase.php @@ -253,4 +253,82 @@ final class PhabricatorMetaMTAMailTestCase extends PhabricatorTestCase { ->executeOne(); } + public function testMailerFailover() { + $user = $this->generateNewTestUser(); + $phid = $user->getPHID(); + + $status_sent = PhabricatorMailOutboundStatus::STATUS_SENT; + $status_queue = PhabricatorMailOutboundStatus::STATUS_QUEUE; + $status_fail = PhabricatorMailOutboundStatus::STATUS_FAIL; + + $mailer1 = id(new PhabricatorMailImplementationTestAdapter()) + ->setKey('mailer1'); + + $mailer2 = id(new PhabricatorMailImplementationTestAdapter()) + ->setKey('mailer2'); + + $mailers = array( + $mailer1, + $mailer2, + ); + + // Send mail with both mailers active. The first mailer should be used. + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)) + ->sendWithMailers($mailers); + $this->assertEqual($status_sent, $mail->getStatus()); + $this->assertEqual('mailer1', $mail->getMailerKey()); + + + // If the first mailer fails, the mail should be sent with the second + // mailer. Since we transmitted the mail, this doesn't raise an exception. + $mailer1->setFailTemporarily(true); + + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)) + ->sendWithMailers($mailers); + $this->assertEqual($status_sent, $mail->getStatus()); + $this->assertEqual('mailer2', $mail->getMailerKey()); + + + // If both mailers fail, the mail should remain in queue. + $mailer2->setFailTemporarily(true); + + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)); + + $caught = null; + try { + $mail->sendWithMailers($mailers); + } catch (Exception $ex) { + $caught = $ex; + } + + $this->assertTrue($caught instanceof Exception); + $this->assertEqual($status_queue, $mail->getStatus()); + $this->assertEqual(null, $mail->getMailerKey()); + + $mailer1->setFailTemporarily(false); + $mailer2->setFailTemporarily(false); + + + // If the first mailer fails permanently, the mail should fail even though + // the second mailer isn't configured to fail. + $mailer1->setFailPermanently(true); + + $mail = id(new PhabricatorMetaMTAMail()) + ->addTos(array($phid)); + + $caught = null; + try { + $mail->sendWithMailers($mailers); + } catch (Exception $ex) { + $caught = $ex; + } + + $this->assertTrue($caught instanceof Exception); + $this->assertEqual($status_fail, $mail->getStatus()); + $this->assertEqual(null, $mail->getMailerKey()); + } + } From 19b3fb8863d6b72dbb08d904069dc9a914f95c69 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 6 Feb 2018 09:29:40 -0800 Subject: [PATCH 59/89] Add a Postmark mail adapter so it can be configured as an outbound mailer Summary: Depends on D19007. Ref T12677. Test Plan: Used `bin/mail send-test ... --mailer postmark` to deliver some mail via Postmark. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19009 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMailImplementationAdapter.php | 9 ++ ...catorMailImplementationPostmarkAdapter.php | 112 ++++++++++++++++++ ...bricatorMailManagementSendTestWorkflow.php | 20 ++++ .../storage/PhabricatorMetaMTAMail.php | 10 ++ .../configuring_outbound_email.diviner | 12 ++ 6 files changed, 165 insertions(+) create mode 100644 src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 29a97c43d2..16b3e1257a 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3188,6 +3188,7 @@ phutil_register_library_map(array( 'PhabricatorMailImplementationMailgunAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php', 'PhabricatorMailImplementationPHPMailerAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationPHPMailerAdapter.php', 'PhabricatorMailImplementationPHPMailerLiteAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationPHPMailerLiteAdapter.php', + 'PhabricatorMailImplementationPostmarkAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php', 'PhabricatorMailImplementationSendGridAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationSendGridAdapter.php', 'PhabricatorMailImplementationTestAdapter' => 'applications/metamta/adapter/PhabricatorMailImplementationTestAdapter.php', 'PhabricatorMailManagementListInboundWorkflow' => 'applications/metamta/management/PhabricatorMailManagementListInboundWorkflow.php', @@ -8691,6 +8692,7 @@ phutil_register_library_map(array( 'PhabricatorMailImplementationMailgunAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationPHPMailerAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationPHPMailerLiteAdapter' => 'PhabricatorMailImplementationAdapter', + 'PhabricatorMailImplementationPostmarkAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationSendGridAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailImplementationTestAdapter' => 'PhabricatorMailImplementationAdapter', 'PhabricatorMailManagementListInboundWorkflow' => 'PhabricatorMailManagementWorkflow', diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index 14c7a63663..ce56345194 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -94,4 +94,13 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { return; } + protected function renderAddress($email, $name = null) { + if (strlen($name)) { + // TODO: This needs to be escaped correctly. + return "{$name} <{$email}>"; + } else { + return $email; + } + } + } diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php new file mode 100644 index 0000000000..bd5ee820af --- /dev/null +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php @@ -0,0 +1,112 @@ +parameters['From'] = $this->renderAddress($email, $name); + return $this; + } + + public function addReplyTo($email, $name = '') { + $this->parameters['ReplyTo'] = $this->renderAddress($email, $name); + return $this; + } + + public function addTos(array $emails) { + foreach ($emails as $email) { + $this->parameters['To'][] = $email; + } + return $this; + } + + public function addCCs(array $emails) { + foreach ($emails as $email) { + $this->parameters['Cc'][] = $email; + } + return $this; + } + + public function addAttachment($data, $filename, $mimetype) { + $this->parameters['Attachments'][] = array( + 'Name' => $filename, + 'ContentType' => $mimetype, + 'Content' => base64_encode($data), + ); + + return $this; + } + + public function addHeader($header_name, $header_value) { + $this->parameters['Headers'][] = array( + 'Name' => $header_name, + 'Value' => $header_value, + ); + return $this; + } + + public function setBody($body) { + $this->parameters['TextBody'] = $body; + return $this; + } + + public function setHTMLBody($html_body) { + $this->parameters['HtmlBody'] = $html_body; + return $this; + } + + public function setSubject($subject) { + $this->parameters['Subject'] = $subject; + return $this; + } + + public function supportsMessageIDHeader() { + return true; + } + + protected function validateOptions(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'access-token' => 'string', + )); + } + + public function newDefaultOptions() { + return array( + 'access-token' => null, + ); + } + + public function newLegacyOptions() { + return array(); + } + + public function send() { + $access_token = $this->getOption('access-token'); + + $parameters = $this->parameters; + $flatten = array( + 'To', + 'Cc', + ); + + foreach ($flatten as $key) { + if (isset($parameters[$key])) { + $parameters[$key] = implode(', ', $parameters[$key]); + } + } + + id(new PhutilPostmarkFuture()) + ->setAccessToken($access_token) + ->setMethod('email', $parameters) + ->resolve(); + + return true; + } + +} diff --git a/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php index 9f4e91ca22..152b62fd3f 100644 --- a/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementSendTestWorkflow.php @@ -47,6 +47,11 @@ final class PhabricatorMailManagementSendTestWorkflow 'help' => pht('Attach a file.'), 'repeat' => true, ), + array( + 'name' => 'mailer', + 'param' => 'key', + 'help' => pht('Send with a specific configured mailer.'), + ), array( 'name' => 'html', 'help' => pht('Send as HTML mail.'), @@ -161,6 +166,21 @@ final class PhabricatorMailManagementSendTestWorkflow $mail->setFrom($from->getPHID()); } + $mailer_key = $args->getArg('mailer'); + if ($mailer_key !== null) { + $mailers = PhabricatorMetaMTAMail::newMailers(); + $mailers = mpull($mailers, null, 'getKey'); + if (!isset($mailers[$mailer_key])) { + throw new PhutilArgumentUsageException( + pht( + 'Mailer key ("%s") is not configured. Available keys are: %s.', + $mailer_key, + implode(', ', array_keys($mailers)))); + } + + $mail->setTryMailers(array($mailer_key)); + } + foreach ($attach as $attachment) { $data = Filesystem::readFile($attachment); $name = basename($attachment); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 5f859fd1d9..83cdc7c40f 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -319,6 +319,10 @@ final class PhabricatorMetaMTAMail return $this->getParam('mailer.key'); } + public function setTryMailers(array $mailers) { + return $this->setParam('mailers.try', $mailers); + } + public function setHTMLBody($html) { $this->setParam('html-body', $html); return $this; @@ -469,6 +473,12 @@ final class PhabricatorMetaMTAMail $mailers = self::newMailers(); + $try_mailers = $this->getParam('mailers.try'); + if ($try_mailers) { + $mailers = mpull($mailers, null, 'getKey'); + $mailers = array_select_keys($mailers, $try_mailers); + } + return $this->sendWithMailers($mailers); } diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index 21abf92736..d2daf7a40a 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -12,6 +12,7 @@ including a local mailer or various third-party services. Options include: | Send Mail With | Setup | Cost | Inbound | Notes | |---------|-------|------|---------|-------| | Mailgun | Easy | Cheap | Yes | Recommended | +| Postmark | Easy | Cheap | Yes | Recommended | | Amazon SES | Easy | Cheap | No | Recommended | | SendGrid | Medium | Cheap | Yes | Discouraged | | External SMTP | Medium | Varies | No | Gmail, etc. | @@ -147,6 +148,17 @@ To use this mailer, set `type` to `mailgun`, then configure these `options`: - `domain`: Required string. Your Mailgun domain. +Mailer: Postmark +================ + +Postmark is a third-party email delivery serivice. You can learn more at +. + +To use this mailer, set `type` to `postmark`, then configure these `options`: + + - `access-token`: Required string. Your Postmark access token. + + Mailer: Amazon SES ================== From f090fa7426e42bc085e6be25c2eead0af73eb1b4 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 02:57:28 -0800 Subject: [PATCH 60/89] Use object PHIDs for "Thread-Topic" headers in mail Summary: Depends on D19009. Ref T13053. For "Must Encrypt" mail, we must currently strip the "Thread-Topic" header because it sometimes contains sensitive information about the object. I don't actually know if this header is useful or anyting uses it. My understanding is that it's an Outlook/Exchange thing, but we also implement "Thread-Index" which I think is what Outlook/Exchange actually look at. This header may have done something before we implemented "Thread-Index", or maybe never done anything. Or maybe older versions of Excel/Outlook did something with it and newer versions don't, or do less. So it's possible that an even better fix here would be to simply remove this, but I wasn't able to convince myself of that after Googling for 10 minutes and I don't think it's worth hours of installing Exchange/Outlook to figure out. Instead, I'm just trying to simplify our handling of this header for now, and maybe some day we'll learn more about Exchange/Outlook and can remove it. In a number of cases we already use the object monogram or PHID as a "Thread-Topic" without users ever complaining, so I think that if this header is useful it probably isn't shown to users, or isn't shown very often (e.g., only in a specific "conversation" sub-view?). Just use the object PHID (which should be unique and stable) as a thread-topic, everywhere, automatically. Then allow this header through for "Must Encrypt" mail. Test Plan: Processed some local mail, saw object PHIDs for "Thread-Topic" headers. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19012 --- src/applications/audit/editor/PhabricatorAuditEditor.php | 5 +---- .../auth/editor/PhabricatorAuthSSHKeyEditor.php | 4 +--- src/applications/badges/editor/PhabricatorBadgesEditor.php | 4 +--- .../calendar/editor/PhabricatorCalendarEventEditor.php | 4 +--- src/applications/conpherence/editor/ConpherenceEditor.php | 4 +--- .../countdown/editor/PhabricatorCountdownEditor.php | 3 +-- .../differential/editor/DifferentialTransactionEditor.php | 7 +------ src/applications/files/editor/PhabricatorFileEditor.php | 3 +-- src/applications/fund/editor/FundInitiativeEditor.php | 3 +-- .../legalpad/editor/LegalpadDocumentEditor.php | 4 +--- src/applications/macro/editor/PhabricatorMacroEditor.php | 3 +-- .../maniphest/editor/ManiphestTransactionEditor.php | 3 +-- .../metamta/storage/PhabricatorMetaMTAMail.php | 6 ++++++ .../editor/PhabricatorOwnersPackageTransactionEditor.php | 4 +--- src/applications/paste/editor/PhabricatorPasteEditor.php | 3 +-- src/applications/phame/editor/PhameBlogEditor.php | 4 +--- src/applications/phame/editor/PhamePostEditor.php | 4 +--- src/applications/pholio/editor/PholioMockEditor.php | 4 +--- src/applications/phortune/editor/PhortuneCartEditor.php | 3 +-- .../phriction/editor/PhrictionTransactionEditor.php | 4 +--- .../phurl/editor/PhabricatorPhurlURLEditor.php | 3 +-- src/applications/ponder/editor/PonderAnswerEditor.php | 3 +-- src/applications/ponder/editor/PonderQuestionEditor.php | 4 +--- .../project/editor/PhabricatorProjectTransactionEditor.php | 4 +--- .../releeph/editor/ReleephRequestTransactionalEditor.php | 4 +--- .../worker/PhabricatorRepositoryPushMailWorker.php | 1 - .../slowvote/editor/PhabricatorSlowvoteEditor.php | 3 +-- 27 files changed, 31 insertions(+), 70 deletions(-) diff --git a/src/applications/audit/editor/PhabricatorAuditEditor.php b/src/applications/audit/editor/PhabricatorAuditEditor.php index 049733f777..c39be75366 100644 --- a/src/applications/audit/editor/PhabricatorAuditEditor.php +++ b/src/applications/audit/editor/PhabricatorAuditEditor.php @@ -473,17 +473,14 @@ final class PhabricatorAuditEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $identifier = $object->getCommitIdentifier(); $repository = $object->getRepository(); - $monogram = $repository->getMonogram(); $summary = $object->getSummary(); $name = $repository->formatCommitName($identifier); $subject = "{$name}: {$summary}"; - $thread_topic = "Commit {$monogram}{$identifier}"; $template = id(new PhabricatorMetaMTAMail()) - ->setSubject($subject) - ->addHeader('Thread-Topic', $thread_topic); + ->setSubject($subject); $this->attachPatch( $template, diff --git a/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php b/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php index 569c37403b..3f178c9855 100644 --- a/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php +++ b/src/applications/auth/editor/PhabricatorAuthSSHKeyEditor.php @@ -255,11 +255,9 @@ final class PhabricatorAuthSSHKeyEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $name = $object->getName(); - $phid = $object->getPHID(); $mail = id(new PhabricatorMetaMTAMail()) - ->setSubject(pht('SSH Key %d: %s', $id, $name)) - ->addHeader('Thread-Topic', $phid); + ->setSubject(pht('SSH Key %d: %s', $id, $name)); // The primary value of this mail is alerting users to account compromises, // so force delivery. In particular, this mail should still be delivered diff --git a/src/applications/badges/editor/PhabricatorBadgesEditor.php b/src/applications/badges/editor/PhabricatorBadgesEditor.php index fddc55747c..785d8c989b 100644 --- a/src/applications/badges/editor/PhabricatorBadgesEditor.php +++ b/src/applications/badges/editor/PhabricatorBadgesEditor.php @@ -87,12 +87,10 @@ final class PhabricatorBadgesEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $name = $object->getName(); $id = $object->getID(); - $topic = pht('Badge %d', $id); $subject = pht('Badge %d: %s', $id, $name); return id(new PhabricatorMetaMTAMail()) - ->setSubject($subject) - ->addHeader('Thread-Topic', $topic); + ->setSubject($subject); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php b/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php index 4ab13fd360..f1b72dc0ea 100644 --- a/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php +++ b/src/applications/calendar/editor/PhabricatorCalendarEventEditor.php @@ -309,13 +309,11 @@ final class PhabricatorCalendarEventEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $name = $object->getName(); $monogram = $object->getMonogram(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/conpherence/editor/ConpherenceEditor.php b/src/applications/conpherence/editor/ConpherenceEditor.php index 29ffc22251..7896055f64 100644 --- a/src/applications/conpherence/editor/ConpherenceEditor.php +++ b/src/applications/conpherence/editor/ConpherenceEditor.php @@ -227,11 +227,9 @@ final class ConpherenceEditor extends PhabricatorApplicationTransactionEditor { '%s sent you a message.', $this->getActor()->getUserName()); } - $phid = $object->getPHID(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("Z{$id}: {$title}") - ->addHeader('Thread-Topic', "Z{$id}: {$phid}"); + ->setSubject("Z{$id}: {$title}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/countdown/editor/PhabricatorCountdownEditor.php b/src/applications/countdown/editor/PhabricatorCountdownEditor.php index 322b2ee2c3..2102b3785b 100644 --- a/src/applications/countdown/editor/PhabricatorCountdownEditor.php +++ b/src/applications/countdown/editor/PhabricatorCountdownEditor.php @@ -45,8 +45,7 @@ final class PhabricatorCountdownEditor $name = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index 3a1537b01d..f3583438c8 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -689,15 +689,10 @@ final class DifferentialTransactionEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $title = $object->getTitle(); - - $original_title = $object->getOriginalTitle(); - $subject = "D{$id}: {$title}"; - $thread_topic = "D{$id}: {$original_title}"; return id(new PhabricatorMetaMTAMail()) - ->setSubject($subject) - ->addHeader('Thread-Topic', $thread_topic); + ->setSubject($subject); } protected function getTransactionsForMail( diff --git a/src/applications/files/editor/PhabricatorFileEditor.php b/src/applications/files/editor/PhabricatorFileEditor.php index 6a2b797b40..db974cec65 100644 --- a/src/applications/files/editor/PhabricatorFileEditor.php +++ b/src/applications/files/editor/PhabricatorFileEditor.php @@ -47,8 +47,7 @@ final class PhabricatorFileEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("F{$id}: {$name}") - ->addHeader('Thread-Topic', "F{$id}"); + ->setSubject("F{$id}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/fund/editor/FundInitiativeEditor.php b/src/applications/fund/editor/FundInitiativeEditor.php index e5c372fd12..9175156ffd 100644 --- a/src/applications/fund/editor/FundInitiativeEditor.php +++ b/src/applications/fund/editor/FundInitiativeEditor.php @@ -50,8 +50,7 @@ final class FundInitiativeEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/legalpad/editor/LegalpadDocumentEditor.php b/src/applications/legalpad/editor/LegalpadDocumentEditor.php index 35f2487a81..e4b43186ee 100644 --- a/src/applications/legalpad/editor/LegalpadDocumentEditor.php +++ b/src/applications/legalpad/editor/LegalpadDocumentEditor.php @@ -124,12 +124,10 @@ final class LegalpadDocumentEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); - $phid = $object->getPHID(); $title = $object->getDocumentBody()->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("L{$id}: {$title}") - ->addHeader('Thread-Topic', "L{$id}: {$phid}"); + ->setSubject("L{$id}: {$title}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/macro/editor/PhabricatorMacroEditor.php b/src/applications/macro/editor/PhabricatorMacroEditor.php index 5d28b78f5f..f59c29b426 100644 --- a/src/applications/macro/editor/PhabricatorMacroEditor.php +++ b/src/applications/macro/editor/PhabricatorMacroEditor.php @@ -35,8 +35,7 @@ final class PhabricatorMacroEditor $name = 'Image Macro "'.$name.'"'; return id(new PhabricatorMetaMTAMail()) - ->setSubject($name) - ->addHeader('Thread-Topic', $name); + ->setSubject($name); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/maniphest/editor/ManiphestTransactionEditor.php b/src/applications/maniphest/editor/ManiphestTransactionEditor.php index caf70b8f3c..9c8e3869dc 100644 --- a/src/applications/maniphest/editor/ManiphestTransactionEditor.php +++ b/src/applications/maniphest/editor/ManiphestTransactionEditor.php @@ -206,8 +206,7 @@ final class ManiphestTransactionEditor $title = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("T{$id}: {$title}") - ->addHeader('Thread-Topic', "T{$id}: ".$object->getOriginalTitle()); + ->setSubject("T{$id}: {$title}"); } protected function buildMailBody( diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 83cdc7c40f..f2c8939132 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -1262,6 +1262,11 @@ final class PhabricatorMetaMTAMail $headers[] = array('X-Phabricator-Must-Encrypt', 'Yes'); } + $related_phid = $this->getRelatedPHID(); + if ($related_phid) { + $headers[] = array('Thread-Topic', $related_phid); + } + return $headers; } @@ -1309,6 +1314,7 @@ final class PhabricatorMetaMTAMail 'Precedence', 'References', 'Thread-Index', + 'Thread-Topic', 'X-Mail-Transport-Agent', 'X-Auto-Response-Suppress', diff --git a/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php b/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php index 40657abd57..c6aad6e2bd 100644 --- a/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php +++ b/src/applications/owners/editor/PhabricatorOwnersPackageTransactionEditor.php @@ -46,12 +46,10 @@ final class PhabricatorOwnersPackageTransactionEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($name) - ->addHeader('Thread-Topic', $object->getPHID()); + ->setSubject($name); } protected function buildMailBody( diff --git a/src/applications/paste/editor/PhabricatorPasteEditor.php b/src/applications/paste/editor/PhabricatorPasteEditor.php index 063b72cfc0..c312915727 100644 --- a/src/applications/paste/editor/PhabricatorPasteEditor.php +++ b/src/applications/paste/editor/PhabricatorPasteEditor.php @@ -72,8 +72,7 @@ final class PhabricatorPasteEditor $name = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("P{$id}: {$name}") - ->addHeader('Thread-Topic', "P{$id}"); + ->setSubject("P{$id}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/phame/editor/PhameBlogEditor.php b/src/applications/phame/editor/PhameBlogEditor.php index f30a74065e..c122d8fa3b 100644 --- a/src/applications/phame/editor/PhameBlogEditor.php +++ b/src/applications/phame/editor/PhameBlogEditor.php @@ -48,12 +48,10 @@ final class PhameBlogEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $phid = $object->getPHID(); $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($name) - ->addHeader('Thread-Topic', $phid); + ->setSubject($name); } protected function buildReplyHandler(PhabricatorLiskDAO $object) { diff --git a/src/applications/phame/editor/PhamePostEditor.php b/src/applications/phame/editor/PhamePostEditor.php index 488d7a4938..d95389e549 100644 --- a/src/applications/phame/editor/PhamePostEditor.php +++ b/src/applications/phame/editor/PhamePostEditor.php @@ -61,12 +61,10 @@ final class PhamePostEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $phid = $object->getPHID(); $title = $object->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($title) - ->addHeader('Thread-Topic', $phid); + ->setSubject($title); } protected function buildReplyHandler(PhabricatorLiskDAO $object) { diff --git a/src/applications/pholio/editor/PholioMockEditor.php b/src/applications/pholio/editor/PholioMockEditor.php index c0fcf31f83..6bd49d2e7b 100644 --- a/src/applications/pholio/editor/PholioMockEditor.php +++ b/src/applications/pholio/editor/PholioMockEditor.php @@ -112,11 +112,9 @@ final class PholioMockEditor extends PhabricatorApplicationTransactionEditor { protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $name = $object->getName(); - $original_name = $object->getOriginalName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("M{$id}: {$name}") - ->addHeader('Thread-Topic', "M{$id}: {$original_name}"); + ->setSubject("M{$id}: {$name}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/phortune/editor/PhortuneCartEditor.php b/src/applications/phortune/editor/PhortuneCartEditor.php index 5196e12429..dcf1f2d0e0 100644 --- a/src/applications/phortune/editor/PhortuneCartEditor.php +++ b/src/applications/phortune/editor/PhortuneCartEditor.php @@ -123,8 +123,7 @@ final class PhortuneCartEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject(pht('Order %d: %s', $id, $name)) - ->addHeader('Thread-Topic', pht('Order %s', $id)); + ->setSubject(pht('Order %d: %s', $id, $name)); } protected function buildMailBody( diff --git a/src/applications/phriction/editor/PhrictionTransactionEditor.php b/src/applications/phriction/editor/PhrictionTransactionEditor.php index fcc9fe0474..73aee3fd4c 100644 --- a/src/applications/phriction/editor/PhrictionTransactionEditor.php +++ b/src/applications/phriction/editor/PhrictionTransactionEditor.php @@ -299,12 +299,10 @@ final class PhrictionTransactionEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $title = $object->getContent()->getTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject($title) - ->addHeader('Thread-Topic', $object->getPHID()); + ->setSubject($title); } protected function buildMailBody( diff --git a/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php b/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php index 439d62f84a..49f290c343 100644 --- a/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php +++ b/src/applications/phurl/editor/PhabricatorPhurlURLEditor.php @@ -68,8 +68,7 @@ final class PhabricatorPhurlURLEditor $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("U{$id}: {$name}") - ->addHeader('Thread-Topic', "U{$id}: ".$object->getName()); + ->setSubject("U{$id}: {$name}"); } protected function buildMailBody( diff --git a/src/applications/ponder/editor/PonderAnswerEditor.php b/src/applications/ponder/editor/PonderAnswerEditor.php index bab0e1f72d..37b2fe2cd0 100644 --- a/src/applications/ponder/editor/PonderAnswerEditor.php +++ b/src/applications/ponder/editor/PonderAnswerEditor.php @@ -57,8 +57,7 @@ final class PonderAnswerEditor extends PonderEditor { $id = $object->getID(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("ANSR{$id}") - ->addHeader('Thread-Topic', "ANSR{$id}"); + ->setSubject("ANSR{$id}"); } protected function buildMailBody( diff --git a/src/applications/ponder/editor/PonderQuestionEditor.php b/src/applications/ponder/editor/PonderQuestionEditor.php index 0720f436b9..ba9687bd0d 100644 --- a/src/applications/ponder/editor/PonderQuestionEditor.php +++ b/src/applications/ponder/editor/PonderQuestionEditor.php @@ -146,11 +146,9 @@ final class PonderQuestionEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); $title = $object->getTitle(); - $original_title = $object->getOriginalTitle(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("Q{$id}: {$title}") - ->addHeader('Thread-Topic', "Q{$id}: {$original_title}"); + ->setSubject("Q{$id}: {$title}"); } protected function buildMailBody( diff --git a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php index 2764ce6322..de61c2a09b 100644 --- a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php +++ b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php @@ -219,12 +219,10 @@ final class PhabricatorProjectTransactionEditor } protected function buildMailTemplate(PhabricatorLiskDAO $object) { - $id = $object->getID(); $name = $object->getName(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$name}") - ->addHeader('Thread-Topic', "Project {$id}"); + ->setSubject("{$name}"); } protected function buildMailBody( diff --git a/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php b/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php index 4710557043..da488d9c72 100644 --- a/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php +++ b/src/applications/releeph/editor/ReleephRequestTransactionalEditor.php @@ -196,11 +196,9 @@ final class ReleephRequestTransactionalEditor protected function buildMailTemplate(PhabricatorLiskDAO $object) { $id = $object->getID(); - $phid = $object->getPHID(); $title = $object->getSummaryForDisplay(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("RQ{$id}: {$title}") - ->addHeader('Thread-Topic', "RQ{$id}: {$phid}"); + ->setSubject("RQ{$id}: {$title}"); } protected function getMailTo(PhabricatorLiskDAO $object) { diff --git a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php index 17226a1377..5ffaf0a5c2 100644 --- a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php +++ b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php @@ -124,7 +124,6 @@ final class PhabricatorRepositoryPushMailWorker ->setFrom($event->getPusherPHID()) ->setBody($body->render()) ->setThreadID($event->getPHID(), $is_new = true) - ->addHeader('Thread-Topic', $subject) ->setIsBulk(true); return $target->willSendMail($mail); diff --git a/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php b/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php index 38dbfb12d6..cf088f37d4 100644 --- a/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php +++ b/src/applications/slowvote/editor/PhabricatorSlowvoteEditor.php @@ -48,8 +48,7 @@ final class PhabricatorSlowvoteEditor $name = $object->getQuestion(); return id(new PhabricatorMetaMTAMail()) - ->setSubject("{$monogram}: {$name}") - ->addHeader('Thread-Topic', $monogram); + ->setSubject("{$monogram}: {$name}"); } protected function buildMailBody( From aa74af19834ba96ccaa7bcd54a5f7fa14bcc97bc Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 03:29:33 -0800 Subject: [PATCH 61/89] Remove all "originalTitle"/"originalName" fields from objects Summary: Depends on D19012. Ref T13053. In D19012, I've changed "Thread-Topic" to always use PHIDs. This change drops the selective on-object storage we have to track the original, human-readable title for objects. Even if we end up backing out the "Thread-Topic" change, we'd be better off storing this in a table in the Mail app which just has ``, since then we get the right behavior without needing every object to have this separate field. Test Plan: Grepped for `original`, `originalName`, `originalTitle`, etc. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19013 --- resources/sql/autopatches/20180207.mail.01.task.sql | 2 ++ .../sql/autopatches/20180207.mail.02.revision.sql | 2 ++ resources/sql/autopatches/20180207.mail.03.mock.sql | 2 ++ .../differential/storage/DifferentialRevision.php | 10 ---------- .../maniphest/editor/ManiphestTransactionEditor.php | 1 - src/applications/maniphest/storage/ManiphestTask.php | 10 ---------- src/applications/pholio/storage/PholioMock.php | 2 -- .../pholio/xaction/PholioMockNameTransaction.php | 3 --- src/applications/ponder/storage/PonderQuestion.php | 5 ----- 9 files changed, 6 insertions(+), 31 deletions(-) create mode 100644 resources/sql/autopatches/20180207.mail.01.task.sql create mode 100644 resources/sql/autopatches/20180207.mail.02.revision.sql create mode 100644 resources/sql/autopatches/20180207.mail.03.mock.sql diff --git a/resources/sql/autopatches/20180207.mail.01.task.sql b/resources/sql/autopatches/20180207.mail.01.task.sql new file mode 100644 index 0000000000..f04b90c809 --- /dev/null +++ b/resources/sql/autopatches/20180207.mail.01.task.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task + DROP originalTitle; diff --git a/resources/sql/autopatches/20180207.mail.02.revision.sql b/resources/sql/autopatches/20180207.mail.02.revision.sql new file mode 100644 index 0000000000..881efbcc94 --- /dev/null +++ b/resources/sql/autopatches/20180207.mail.02.revision.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_differential.differential_revision + DROP originalTitle; diff --git a/resources/sql/autopatches/20180207.mail.03.mock.sql b/resources/sql/autopatches/20180207.mail.03.mock.sql new file mode 100644 index 0000000000..360d7cf9a7 --- /dev/null +++ b/resources/sql/autopatches/20180207.mail.03.mock.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_pholio.pholio_mock + DROP originalName; diff --git a/src/applications/differential/storage/DifferentialRevision.php b/src/applications/differential/storage/DifferentialRevision.php index 2c82de164a..938c588857 100644 --- a/src/applications/differential/storage/DifferentialRevision.php +++ b/src/applications/differential/storage/DifferentialRevision.php @@ -20,7 +20,6 @@ final class DifferentialRevision extends DifferentialDAO PhabricatorDraftInterface { protected $title = ''; - protected $originalTitle; protected $status; protected $summary = ''; @@ -98,7 +97,6 @@ final class DifferentialRevision extends DifferentialDAO ), self::CONFIG_COLUMN_SCHEMA => array( 'title' => 'text255', - 'originalTitle' => 'text255', 'status' => 'text32', 'summary' => 'text', 'testPlan' => 'text', @@ -155,14 +153,6 @@ final class DifferentialRevision extends DifferentialDAO return '/'.$this->getMonogram(); } - public function setTitle($title) { - $this->title = $title; - if (!$this->getID()) { - $this->originalTitle = $title; - } - return $this; - } - public function loadIDsByCommitPHIDs($phids) { if (!$phids) { return array(); diff --git a/src/applications/maniphest/editor/ManiphestTransactionEditor.php b/src/applications/maniphest/editor/ManiphestTransactionEditor.php index 9c8e3869dc..66247ca6d0 100644 --- a/src/applications/maniphest/editor/ManiphestTransactionEditor.php +++ b/src/applications/maniphest/editor/ManiphestTransactionEditor.php @@ -522,7 +522,6 @@ final class ManiphestTransactionEditor 'status' => '""', 'priority' => 0, 'title' => '""', - 'originalTitle' => '""', 'description' => '""', 'dateCreated' => 0, 'dateModified' => 0, diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index f72977c5b2..e19886d3ff 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -31,7 +31,6 @@ final class ManiphestTask extends ManiphestDAO protected $subpriority = 0; protected $title = ''; - protected $originalTitle = ''; protected $description = ''; protected $originalEmailSource; protected $mailKey; @@ -83,7 +82,6 @@ final class ManiphestTask extends ManiphestDAO 'status' => 'text64', 'priority' => 'uint32', 'title' => 'sort', - 'originalTitle' => 'text', 'description' => 'text', 'mailKey' => 'bytes20', 'ownerOrdering' => 'text64?', @@ -176,14 +174,6 @@ final class ManiphestTask extends ManiphestDAO return $this; } - public function setTitle($title) { - $this->title = $title; - if (!$this->getID()) { - $this->originalTitle = $title; - } - return $this; - } - public function getMonogram() { return 'T'.$this->getID(); } diff --git a/src/applications/pholio/storage/PholioMock.php b/src/applications/pholio/storage/PholioMock.php index 4aa9ef4055..523733b3df 100644 --- a/src/applications/pholio/storage/PholioMock.php +++ b/src/applications/pholio/storage/PholioMock.php @@ -25,7 +25,6 @@ final class PholioMock extends PholioDAO protected $editPolicy; protected $name; - protected $originalName; protected $description; protected $coverPHID; protected $mailKey; @@ -65,7 +64,6 @@ final class PholioMock extends PholioDAO self::CONFIG_COLUMN_SCHEMA => array( 'name' => 'text128', 'description' => 'text', - 'originalName' => 'text128', 'mailKey' => 'bytes20', 'status' => 'text12', ), diff --git a/src/applications/pholio/xaction/PholioMockNameTransaction.php b/src/applications/pholio/xaction/PholioMockNameTransaction.php index d1231636af..82fb92fe40 100644 --- a/src/applications/pholio/xaction/PholioMockNameTransaction.php +++ b/src/applications/pholio/xaction/PholioMockNameTransaction.php @@ -15,9 +15,6 @@ final class PholioMockNameTransaction public function applyInternalEffects($object, $value) { $object->setName($value); - if ($object->getOriginalName() === null) { - $object->setOriginalName($this->getNewValue()); - } } public function getTitle() { diff --git a/src/applications/ponder/storage/PonderQuestion.php b/src/applications/ponder/storage/PonderQuestion.php index eefcdba9be..17f7ee3fdc 100644 --- a/src/applications/ponder/storage/PonderQuestion.php +++ b/src/applications/ponder/storage/PonderQuestion.php @@ -194,11 +194,6 @@ final class PonderQuestion extends PonderDAO return parent::save(); } - public function getOriginalTitle() { - // TODO: Make this actually save/return the original title. - return $this->getTitle(); - } - public function getFullTitle() { $id = $this->getID(); $title = $this->getTitle(); From 6e5df2dd714e8360251026eb112af9d2930ba789 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 03:49:16 -0800 Subject: [PATCH 62/89] Document that disabling "metamta.one-mail-per-recipient" leaks recipients for "Must Encrypt" Summary: Depends on D19013. Ref T13053. When mail is marked "Must Encrypt", we normally do not include recipient information. However, when `metamta.one-mail-per-recipient` is disabled, the recipient list will leak in the "To" and "Cc" headers. This interaction is probably not very surprising, but document it explicitly for completeness. (Also use "Mail messages" instead of "Mails".) Test Plan: Read documentation in the "Config" application. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19014 --- .../config/option/PhabricatorMetaMTAConfigOptions.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php index 0b916150bc..8a236a883b 100644 --- a/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php +++ b/src/applications/config/option/PhabricatorMetaMTAConfigOptions.php @@ -66,7 +66,9 @@ of each approach are: received a similar message, but can not prevent all stray email arising from "Reply All". - Not supported with a private reply-to address. - - Mails are sent in the server default translation. + - Mail messages are sent in the server default translation. + - Mail that must be delivered over secure channels will leak the recipient + list in the "To" and "Cc" headers. - One mail to each user: - Policy controls work correctly and are enforced per-user. - Recipients need to look in the mail body to see To/Cc. @@ -77,7 +79,7 @@ of each approach are: - "Reply All" will never send extra mail to other users involved in the thread. - Required if private reply-to addresses are configured. - - Mails are sent in the language of user preference. + - Mail messages are sent in the language of user preference. EODOC )); From 085221b0d6f6150047cd38ad5174822fa8f279af Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 03:56:29 -0800 Subject: [PATCH 63/89] In HTML mail, make the text for mail stamps in mail bodies smaller and lighter Summary: Depends on D19014. Ref T13053. Test Plan: Used `./bin/mail show-outbound --id --dump-html > out.html && open out.html` to look at HTML mail, saw smaller, lighter stamp text with better spacing. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19015 --- .../metamta/replyhandler/PhabricatorMailTarget.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index bbf17be3fd..5d8378e8af 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -77,7 +77,13 @@ final class PhabricatorMailTarget extends Phobject { $html = array(); $html[] = phutil_tag('strong', array(), pht('STAMPS')); $html[] = phutil_tag('br'); - $html[] = phutil_implode_html(' ', $stamps); + $html[] = phutil_tag( + 'span', + array( + 'style' => 'font-size: smaller; color: #92969D', + ), + phutil_implode_html(' ', $stamps)); + $html[] = phutil_tag('br'); $html[] = phutil_tag('br'); $html = phutil_tag('div', array(), $html); $html_body .= hsprintf('%s', $html); From 0986c7f6732e02d14f8c8f6a192595d90b9416c9 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 04:05:44 -0800 Subject: [PATCH 64/89] Add a "View Object" button on the web mail view page Summary: Depends on D19015. Ref T13053. Currently, we don't link up hyperlinks in the body of mail viewed in the web UI. We should, but this is a little tricky (see T13053#235074). As a general improvement to make working with "Must Encrypt" mail less painful, add a big button to jump to the related object. Test Plan: {F5415990} Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19016 --- .../PhabricatorMetaMTAMailViewController.php | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php index 03d340bac9..9b33397831 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -75,8 +75,26 @@ final class PhabricatorMetaMTAMailViewController ->setKey('metadata') ->appendChild($this->buildMetadataProperties($mail))); + $header_view = id(new PHUIHeaderView()) + ->setHeader(pht('Mail')); + + $object_phid = $mail->getRelatedPHID(); + if ($object_phid) { + $handles = $viewer->loadHandles(array($object_phid)); + $handle = $handles[$object_phid]; + if ($handle->isComplete() && $handle->getURI()) { + $view_button = id(new PHUIButtonView()) + ->setTag('a') + ->setText(pht('View Object')) + ->setIcon('fa-chevron-right') + ->setHref($handle->getURI()); + + $header_view->addActionLink($view_button); + } + } + $object_box = id(new PHUIObjectBoxView()) - ->setHeaderText(pht('Mail')) + ->setHeader($header_view) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->addTabGroup($tab_group); From 5792032dc9c939112542ec32d0021e6cff9e0aea Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 04:51:09 -0800 Subject: [PATCH 65/89] Support Postmark inbound mail via webhook Summary: Depends on D19016. Ref T13053. Adds a listener for the Postmark webhook. Test Plan: Processed some test mail locally, at least: {F5416053} Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19017 --- src/__phutil_library_map__.php | 2 + .../PhabricatorMetaMTAApplication.php | 1 + ...ricatorMetaMTAMailgunReceiveController.php | 11 +-- ...icatorMetaMTAPostmarkReceiveController.php | 87 +++++++++++++++++++ ...icatorMetaMTASendGridReceiveController.php | 20 ++--- .../storage/PhabricatorMetaMTAMail.php | 14 +++ .../configuring_inbound_email.diviner | 12 +++ 7 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 16b3e1257a..301459869f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3272,6 +3272,7 @@ phutil_register_library_map(array( 'PhabricatorMetaMTAMailgunReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php', 'PhabricatorMetaMTAMemberQuery' => 'applications/metamta/query/PhabricatorMetaMTAMemberQuery.php', 'PhabricatorMetaMTAPermanentFailureException' => 'applications/metamta/exception/PhabricatorMetaMTAPermanentFailureException.php', + 'PhabricatorMetaMTAPostmarkReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php', 'PhabricatorMetaMTAReceivedMail' => 'applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php', 'PhabricatorMetaMTAReceivedMailProcessingException' => 'applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php', 'PhabricatorMetaMTAReceivedMailTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php', @@ -8787,6 +8788,7 @@ phutil_register_library_map(array( 'PhabricatorMetaMTAMailgunReceiveController' => 'PhabricatorMetaMTAController', 'PhabricatorMetaMTAMemberQuery' => 'PhabricatorQuery', 'PhabricatorMetaMTAPermanentFailureException' => 'Exception', + 'PhabricatorMetaMTAPostmarkReceiveController' => 'PhabricatorMetaMTAController', 'PhabricatorMetaMTAReceivedMail' => 'PhabricatorMetaMTADAO', 'PhabricatorMetaMTAReceivedMailProcessingException' => 'Exception', 'PhabricatorMetaMTAReceivedMailTestCase' => 'PhabricatorTestCase', diff --git a/src/applications/metamta/application/PhabricatorMetaMTAApplication.php b/src/applications/metamta/application/PhabricatorMetaMTAApplication.php index adb08aaa24..f53af55035 100644 --- a/src/applications/metamta/application/PhabricatorMetaMTAApplication.php +++ b/src/applications/metamta/application/PhabricatorMetaMTAApplication.php @@ -42,6 +42,7 @@ final class PhabricatorMetaMTAApplication extends PhabricatorApplication { 'detail/(?P[1-9]\d*)/' => 'PhabricatorMetaMTAMailViewController', 'sendgrid/' => 'PhabricatorMetaMTASendGridReceiveController', 'mailgun/' => 'PhabricatorMetaMTAMailgunReceiveController', + 'postmark/' => 'PhabricatorMetaMTAPostmarkReceiveController', ), ); } diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php index 4eb53b7120..3ca2711dcf 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailgunReceiveController.php @@ -17,15 +17,12 @@ final class PhabricatorMetaMTAMailgunReceiveController // inbound mail from any of them. Test the signature to see if it matches // any configured Mailgun mailer. - $mailers = PhabricatorMetaMTAMail::newMailers(); - $mailgun_type = PhabricatorMailImplementationMailgunAdapter::ADAPTERTYPE; + $mailers = PhabricatorMetaMTAMail::newMailersWithTypes( + array( + PhabricatorMailImplementationMailgunAdapter::ADAPTERTYPE, + )); foreach ($mailers as $mailer) { - if ($mailer->getAdapterType() != $mailgun_type) { - continue; - } - $api_key = $mailer->getOption('api-key'); - $hash = hash_hmac('sha256', $timestamp.$token, $api_key); if (phutil_hashes_are_identical($sig, $hash)) { return true; diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php new file mode 100644 index 0000000000..a54da6fb40 --- /dev/null +++ b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php @@ -0,0 +1,87 @@ + idx($data, 'To'), + 'from' => idx($data, 'From'), + 'cc' => idx($data, 'Cc'), + 'subject' => idx($data, 'Subject'), + ) + $raw_headers; + + + $received = id(new PhabricatorMetaMTAReceivedMail()) + ->setHeaders($headers) + ->setBodies( + array( + 'text' => idx($data, 'TextBody'), + 'html' => idx($data, 'HtmlBody'), + )); + + $file_phids = array(); + $attachments = idx($data, 'Attachments', array()); + foreach ($attachments as $attachment) { + $file_data = idx($attachment, 'Content'); + $file_data = base64_decode($file_data); + + try { + $file = PhabricatorFile::newFromFileData( + $file_data, + array( + 'name' => idx($attachment, 'Name'), + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + )); + $file_phids[] = $file->getPHID(); + } catch (Exception $ex) { + phlog($ex); + } + } + $received->setAttachments($file_phids); + + try { + $received->save(); + $received->processReceivedMail(); + } catch (Exception $ex) { + phlog($ex); + } + + return id(new AphrontWebpageResponse()) + ->setContent(pht("Got it! Thanks, Postmark!\n")); + } + +} diff --git a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php index 99e60caa05..6651f85d6c 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php @@ -8,24 +8,14 @@ final class PhabricatorMetaMTASendGridReceiveController } public function handleRequest(AphrontRequest $request) { - $mailers = PhabricatorMetaMTAMail::newMailers(); - $sendgrid_type = PhabricatorMailImplementationSendGridAdapter::ADAPTERTYPE; - // SendGrid doesn't sign payloads so we can't be sure that SendGrid // actually sent this request, but require a configured SendGrid mailer // before we activate this endpoint. - - $has_sendgrid = false; - foreach ($mailers as $mailer) { - if ($mailer->getAdapterType() != $sendgrid_type) { - continue; - } - - $has_sendgrid = true; - break; - } - - if (!$has_sendgrid) { + $mailers = PhabricatorMetaMTAMail::newMailersWithTypes( + array( + PhabricatorMailImplementationSendGridAdapter::ADAPTERTYPE, + )); + if (!$mailers) { return new Aphront404Response(); } diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index f2c8939132..4a9bc68322 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -482,6 +482,20 @@ final class PhabricatorMetaMTAMail return $this->sendWithMailers($mailers); } + public static function newMailersWithTypes(array $types) { + $mailers = self::newMailers(); + $types = array_fuse($types); + + foreach ($mailers as $key => $mailer) { + $mailer_type = $mailer->getAdapterType(); + if (!isset($types[$mailer_type])) { + unset($mailers[$key]); + } + } + + return array_values($mailers); + } + public static function newMailers() { $mailers = array(); diff --git a/src/docs/user/configuration/configuring_inbound_email.diviner b/src/docs/user/configuration/configuring_inbound_email.diviner index 5b47a17831..ada4ddb828 100644 --- a/src/docs/user/configuration/configuring_inbound_email.diviner +++ b/src/docs/user/configuration/configuring_inbound_email.diviner @@ -14,6 +14,7 @@ There are a few approaches available: | Receive Mail With | Setup | Cost | Notes | |--------|-------|------|-------| | Mailgun | Easy | Cheap | Recommended | +| Postmark | Easy | Cheap | Recommended | | SendGrid | Easy | Cheap | | | Local MTA | Extremely Difficult | Free | Strongly discouraged! | @@ -130,6 +131,17 @@ like this: example domain with your actual domain. - Set the `mailgun.api-key` config key to your Mailgun API key. +Postmark Setup +============== + +To process inbound mail from Postmark, configure this URI as your inbound +webhook URI in the Postmark control panel: + +``` +https:///mail/postmark/ +``` + + = SendGrid Setup = To use SendGrid, you need a SendGrid account with access to the "Parse API" for From dbe479f0d9dee38aee22808dc6321cd32e766a1f Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 05:09:21 -0800 Subject: [PATCH 66/89] Don't send error/exception mail to unverified addresses Summary: Depends on D19017. Fixes T12491. Ref T13053. After SES threw us in the dungeon for sending mail to a spamtrap we changed outbound mail rules to stop sending to unverified addresses, except a small amount of registration mail which we can't avoid. However, we'll still reply to random inbound messages with a helpful error, even if the sender is unverified. Instead, only send exception mail back if we know who the sender is. Test Plan: Processed inbound mail with `scripts/mail/mail_handler.php`. No more outbound mail for "bad address", etc. Still got outbound mail for "unknown command !quack". Reviewers: amckinley Maniphest Tasks: T13053, T12491 Differential Revision: https://secure.phabricator.com/D19018 --- .../PhabricatorMetaMTAReceivedMail.php | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php index 18fa7dd2ba..fc98d17010 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php @@ -105,6 +105,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { public function processReceivedMail() { + $sender = null; try { $this->dropMailFromPhabricator(); $this->dropMailAlreadyReceived(); @@ -140,7 +141,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { // This error is explicitly ignored. break; default: - $this->sendExceptionMail($ex); + $this->sendExceptionMail($ex, $sender); break; } @@ -150,7 +151,7 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { ->save(); return $this; } catch (Exception $ex) { - $this->sendExceptionMail($ex); + $this->sendExceptionMail($ex, $sender); $this ->setStatus(MetaMTAReceivedMailStatus::STATUS_UNHANDLED_EXCEPTION) @@ -305,9 +306,14 @@ final class PhabricatorMetaMTAReceivedMail extends PhabricatorMetaMTADAO { return head($accept); } - private function sendExceptionMail(Exception $ex) { - $from = $this->getHeader('from'); - if (!strlen($from)) { + private function sendExceptionMail( + Exception $ex, + PhabricatorUser $viewer = null) { + + // If we've failed to identify a legitimate sender, we don't send them + // an error message back. We want to avoid sending mail to unverified + // addresses. See T12491. + if (!$viewer) { return; } @@ -364,9 +370,8 @@ EOBODY $mail = id(new PhabricatorMetaMTAMail()) ->setIsErrorEmail(true) - ->setForceDelivery(true) ->setSubject($title) - ->addRawTos(array($from)) + ->addTos(array($viewer->getPHID())) ->setBody($body) ->saveAndSend(); } From f214abb63f9df8eb17a0438c7256d9f46957f48b Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 05:52:52 -0800 Subject: [PATCH 67/89] When a change removes recipients from an object, send them one last email Summary: Depends on D19018. Fixes T4776. Ref T13053. When you remove someone from an object's recipient list (for example, by removing them a reviewer, auditor, subscriber, owner or author) we currently do not send them mail about it because they're no longer connected to the object. In many of these cases (Commandeer, Reassign) the actual action in the UI adds them back to the object somehow (as a reviewer or subscriber, respectively) so this doesn't actually matter. However, there's no recovery mechanism for reviewer or subscriber removal. This is slightly bad from a policy/threat viewpoint since it means an attacker can remove all the recipients of an object "somewhat" silently. This isn't really silent, but it's less un-silent than it should be. It's also just not very good from a human interaction perspective: if Alice removes Bob as a reviewer, possibly "against his will", he should be notified about that. In the good case, Alice wrote a nice goodbye note that he should get to read. In the bad case, he should get a chance to correct the mistake. Also add a `removed(@user)` mail stamp so you can route these locally if you want. Test Plan: - Created and edited some different objects without catching anything broken. - Removed subscribers from tasks, saw the final email include the removed recipients with a `removed()` stamp. I'm not totally sure this doesn't have any surprising behavior or break any weird objects, but I think anything that crops up should be easy to fix. Reviewers: amckinley Subscribers: sophiebits Maniphest Tasks: T13053, T4776 Differential Revision: https://secure.phabricator.com/D19019 --- ...habricatorApplicationTransactionEditor.php | 70 +++++++++++++++++++ .../PhabricatorEditorMailEngineExtension.php | 7 ++ 2 files changed, 77 insertions(+) diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index e4f9607801..6bb7679fdc 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -74,6 +74,9 @@ abstract class PhabricatorApplicationTransactionEditor private $mustEncrypt; private $stampTemplates = array(); private $mailStamps = array(); + private $oldTo = array(); + private $oldCC = array(); + private $mailRemovedPHIDs = array(); private $transactionQueue = array(); @@ -916,6 +919,8 @@ abstract class PhabricatorApplicationTransactionEditor $this->willApplyTransactions($object, $xactions); if ($object->getID()) { + $this->buildOldRecipientLists($object, $xactions); + foreach ($xactions as $xaction) { // If any of the transactions require a read lock, hold one and @@ -1200,6 +1205,10 @@ abstract class PhabricatorApplicationTransactionEditor $this->mailToPHIDs = $this->getMailTo($object); $this->mailCCPHIDs = $this->getMailCC($object); + // Add any recipients who were previously on the notification list + // but were removed by this change. + $this->applyOldRecipientLists(); + $mail_xactions = $this->getTransactionsForMail($object, $xactions); $stamps = $this->newMailStamps($object, $xactions); foreach ($stamps as $stamp) { @@ -4127,4 +4136,65 @@ abstract class PhabricatorApplicationTransactionEditor return $results; } + public function getRemovedRecipientPHIDs() { + return $this->mailRemovedPHIDs; + } + + private function buildOldRecipientLists($object, $xactions) { + // See T4776. Before we start making any changes, build a list of the old + // recipients. If a change removes a user from the recipient list for an + // object we still want to notify the user about that change. This allows + // them to respond if they didn't want to be removed. + + if (!$this->shouldSendMail($object, $xactions)) { + return; + } + + $this->oldTo = $this->getMailTo($object); + $this->oldCC = $this->getMailCC($object); + + return $this; + } + + private function applyOldRecipientLists() { + $actor_phid = $this->getActingAsPHID(); + + // If you took yourself off the recipient list (for example, by + // unsubscribing or resigning) assume that you know what you did and + // don't need to be notified. + + // If you just moved from "To" to "Cc" (or vice versa), you're still a + // recipient so we don't need to add you back in. + + $map = array_fuse($this->mailToPHIDs) + array_fuse($this->mailCCPHIDs); + + foreach ($this->oldTo as $phid) { + if ($phid === $actor_phid) { + continue; + } + + if (isset($map[$phid])) { + continue; + } + + $this->mailToPHIDs[] = $phid; + $this->mailRemovedPHIDs[] = $phid; + } + + foreach ($this->oldCC as $phid) { + if ($phid === $actor_phid) { + continue; + } + + if (isset($map[$phid])) { + continue; + } + + $this->mailCCPHIDs[] = $phid; + $this->mailRemovedPHIDs[] = $phid; + } + + return $this; + } + } diff --git a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php index 29d10d641f..5365894429 100644 --- a/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php +++ b/src/applications/transactions/engineextension/PhabricatorEditorMailEngineExtension.php @@ -40,6 +40,10 @@ final class PhabricatorEditorMailEngineExtension ->setKey('herald') ->setLabel(pht('Herald Rule')); + $templates[] = id(new PhabricatorPHIDMailStamp()) + ->setKey('removed') + ->setLabel(pht('Recipient Removed')); + return $templates; } @@ -69,6 +73,9 @@ final class PhabricatorEditorMailEngineExtension $this->getMailStamp('herald') ->setValue($editor->getHeraldRuleMonograms()); + + $this->getMailStamp('removed') + ->setValue($editor->getRemovedRecipientPHIDs()); } } From 1cd3a593784a95c9925bce443d790aaa6c90e996 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 07:08:36 -0800 Subject: [PATCH 68/89] When users resign from revisions, stop expanding projects/packages to include them Summary: Depends on D19019. Ref T13053. Fixes T12689. See PHI178. Currently, if `@alice` resigns from a revision but `#alice-fan-club` is still a subscriber or reviewer, she'll continue to get mail. This is undesirable. When users are associated with an object but have explicitly disengaged in an individal role (currently, only resign in audit/differential) mark them "unexpandable", so that they can no longer be included through implicit membership in a group (a project or package). `@alice` can still get mail if she's a explicit recipient: as an author, owner, or if she adds herself back as a subscriber. Test Plan: - Added `@ducker` and `#users-named-ducker` as reviewers. Ducker got mail. - Resigned as ducker, stopped getting future mail. - Subscribed explicitly, got mail again. - (Plus some `var_dump()` sanity checking in the internals.) Reviewers: amckinley Maniphest Tasks: T13053, T12689 Differential Revision: https://secure.phabricator.com/D19021 --- .../audit/editor/PhabricatorAuditEditor.php | 15 ++++++++-- .../editor/DifferentialTransactionEditor.php | 12 ++++++++ .../storage/DifferentialRevision.php | 10 +++++-- .../PhabricatorMailReplyHandler.php | 30 +++++++++++++++++++ .../PhabricatorRepositoryAuditRequest.php | 9 ++++++ .../storage/PhabricatorRepositoryCommit.php | 3 +- ...habricatorApplicationTransactionEditor.php | 14 +++++++++ 7 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/applications/audit/editor/PhabricatorAuditEditor.php b/src/applications/audit/editor/PhabricatorAuditEditor.php index c39be75366..d142bd60cd 100644 --- a/src/applications/audit/editor/PhabricatorAuditEditor.php +++ b/src/applications/audit/editor/PhabricatorAuditEditor.php @@ -496,7 +496,6 @@ final class PhabricatorAuditEditor $phids[] = $object->getAuthorPHID(); } - $status_resigned = PhabricatorAuditStatusConstants::RESIGNED; foreach ($object->getAudits() as $audit) { if (!$audit->isInteresting()) { // Don't send mail to uninteresting auditors, like packages which @@ -504,7 +503,7 @@ final class PhabricatorAuditEditor continue; } - if ($audit->getAuditStatus() != $status_resigned) { + if (!$audit->isResigned()) { $phids[] = $audit->getAuditorPHID(); } } @@ -514,6 +513,18 @@ final class PhabricatorAuditEditor return $phids; } + protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $phids = array(); + + foreach ($object->getAudits() as $auditor) { + if ($auditor->isResigned()) { + $phids[] = $auditor->getAuditorPHID(); + } + } + + return $phids; + } + protected function buildMailBody( PhabricatorLiskDAO $object, array $xactions) { diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index f3583438c8..cc29d69fe0 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -644,6 +644,18 @@ final class DifferentialTransactionEditor return $phids; } + protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $phids = array(); + + foreach ($object->getReviewers() as $reviewer) { + if ($reviewer->isResigned()) { + $phids[] = $reviewer->getReviewerPHID(); + } + } + + return $phids; + } + protected function getMailAction( PhabricatorLiskDAO $object, array $xactions) { diff --git a/src/applications/differential/storage/DifferentialRevision.php b/src/applications/differential/storage/DifferentialRevision.php index 938c588857..4591315207 100644 --- a/src/applications/differential/storage/DifferentialRevision.php +++ b/src/applications/differential/storage/DifferentialRevision.php @@ -820,9 +820,15 @@ final class DifferentialRevision extends DifferentialDAO } foreach ($reviewers as $reviewer) { - if ($reviewer->getReviewerPHID() == $phid) { - return true; + if ($reviewer->getReviewerPHID() !== $phid) { + continue; } + + if ($reviewer->isResigned()) { + continue; + } + + return true; } return false; diff --git a/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php b/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php index b0ae2de494..f8dd784e3b 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailReplyHandler.php @@ -6,6 +6,7 @@ abstract class PhabricatorMailReplyHandler extends Phobject { private $applicationEmail; private $actor; private $excludePHIDs = array(); + private $unexpandablePHIDs = array(); final public function setMailReceiver($mail_receiver) { $this->validateMailReceiver($mail_receiver); @@ -45,6 +46,15 @@ abstract class PhabricatorMailReplyHandler extends Phobject { return $this->excludePHIDs; } + public function setUnexpandablePHIDs(array $phids) { + $this->unexpandablePHIDs = $phids; + return $this; + } + + public function getUnexpandablePHIDs() { + return $this->unexpandablePHIDs; + } + abstract public function validateMailReceiver($mail_receiver); abstract public function getPrivateReplyHandlerEmailAddress( PhabricatorUser $user); @@ -297,6 +307,16 @@ abstract class PhabricatorMailReplyHandler extends Phobject { $to_result = array(); $cc_result = array(); + // "Unexpandable" users have disengaged from an object (for example, + // by resigning from a revision). + + // If such a user is still a direct recipient (for example, they're still + // on the Subscribers list) they're fair game, but group targets (like + // projects) will no longer include them when expanded. + + $unexpandable = $this->getUnexpandablePHIDs(); + $unexpandable = array_fuse($unexpandable); + $all_phids = array_merge($to, $cc); if ($all_phids) { $map = id(new PhabricatorMetaMTAMemberQuery()) @@ -305,11 +325,21 @@ abstract class PhabricatorMailReplyHandler extends Phobject { ->execute(); foreach ($to as $phid) { foreach ($map[$phid] as $expanded) { + if ($expanded !== $phid) { + if (isset($unexpandable[$expanded])) { + continue; + } + } $to_result[$expanded] = $expanded; } } foreach ($cc as $phid) { foreach ($map[$phid] as $expanded) { + if ($expanded !== $phid) { + if (isset($unexpandable[$expanded])) { + continue; + } + } $cc_result[$expanded] = $expanded; } } diff --git a/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php b/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php index ea86594d9e..e05820c825 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php +++ b/src/applications/repository/storage/PhabricatorRepositoryAuditRequest.php @@ -72,6 +72,15 @@ final class PhabricatorRepositoryAuditRequest return true; } + public function isResigned() { + switch ($this->getAuditStatus()) { + case PhabricatorAuditStatusConstants::RESIGNED: + return true; + } + + return false; + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/repository/storage/PhabricatorRepositoryCommit.php b/src/applications/repository/storage/PhabricatorRepositoryCommit.php index 31a06dcbd4..1c5998d583 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryCommit.php +++ b/src/applications/repository/storage/PhabricatorRepositoryCommit.php @@ -657,7 +657,8 @@ final class PhabricatorRepositoryCommit public function isAutomaticallySubscribed($phid) { // TODO: This should also list auditors, but handling that is a bit messy - // right now because we are not guaranteed to have the data. + // right now because we are not guaranteed to have the data. (It should not + // include resigned auditors.) return ($phid == $this->getAuthorPHID()); } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 6bb7679fdc..167aa05fed 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -77,6 +77,7 @@ abstract class PhabricatorApplicationTransactionEditor private $oldTo = array(); private $oldCC = array(); private $mailRemovedPHIDs = array(); + private $mailUnexpandablePHIDs = array(); private $transactionQueue = array(); @@ -1204,6 +1205,7 @@ abstract class PhabricatorApplicationTransactionEditor $this->mailShouldSend = true; $this->mailToPHIDs = $this->getMailTo($object); $this->mailCCPHIDs = $this->getMailCC($object); + $this->mailUnexpandablePHIDs = $this->newMailUnexpandablePHIDs($object); // Add any recipients who were previously on the notification list // but were removed by this change. @@ -2562,7 +2564,13 @@ abstract class PhabricatorApplicationTransactionEditor $email_cc = $this->mailCCPHIDs; $email_cc = array_merge($email_cc, $this->heraldEmailPHIDs); + $unexpandable = $this->mailUnexpandablePHIDs; + if (!is_array($unexpandable)) { + $unexpandable = array(); + } + $targets = $this->buildReplyHandler($object) + ->setUnexpandablePHIDs($unexpandable) ->getMailTargets($email_to, $email_cc); // Set this explicitly before we start swapping out the effective actor. @@ -2817,6 +2825,11 @@ abstract class PhabricatorApplicationTransactionEditor } + protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + return array(); + } + + /** * @task mail */ @@ -3617,6 +3630,7 @@ abstract class PhabricatorApplicationTransactionEditor 'mailShouldSend', 'mustEncrypt', 'mailStamps', + 'mailUnexpandablePHIDs', ); } From d0a2e3c54f9ae254c7158965480c892d1c03d70c Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 09:39:27 -0800 Subject: [PATCH 69/89] Fix an issue where some Differential edit pathways may not have reviewers attached Summary: Depends on D19021. Ref T13053. When you "Subscribe", or make some other types of edits, we don't necessarily have reviewer data, but may now need it to do the new recipient list logic. I don't have a totally clean way to deal with this in the general case in mind, but just load it for now so that things don't fatal. Test Plan: Subscribed to a revision with the "Subscribe" action. Reviewers: amckinley Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19022 --- .../editor/DifferentialTransactionEditor.php | 25 +++++++++++++++++++ .../storage/DifferentialRevision.php | 4 +++ 2 files changed, 29 insertions(+) diff --git a/src/applications/differential/editor/DifferentialTransactionEditor.php b/src/applications/differential/editor/DifferentialTransactionEditor.php index cc29d69fe0..063df6c602 100644 --- a/src/applications/differential/editor/DifferentialTransactionEditor.php +++ b/src/applications/differential/editor/DifferentialTransactionEditor.php @@ -632,6 +632,8 @@ final class DifferentialTransactionEditor } protected function getMailTo(PhabricatorLiskDAO $object) { + $this->requireReviewers($object); + $phids = array(); $phids[] = $object->getAuthorPHID(); foreach ($object->getReviewers() as $reviewer) { @@ -645,6 +647,8 @@ final class DifferentialTransactionEditor } protected function newMailUnexpandablePHIDs(PhabricatorLiskDAO $object) { + $this->requireReviewers($object); + $phids = array(); foreach ($object->getReviewers() as $reviewer) { @@ -1737,4 +1741,25 @@ final class DifferentialTransactionEditor } } + private function requireReviewers(DifferentialRevision $revision) { + if ($revision->hasAttachedReviewers()) { + return; + } + + $with_reviewers = id(new DifferentialRevisionQuery()) + ->setViewer($this->getActor()) + ->needReviewers(true) + ->withPHIDs(array($revision->getPHID())) + ->executeOne(); + if (!$with_reviewers) { + throw new Exception( + pht( + 'Failed to reload revision ("%s").', + $revision->getPHID())); + } + + $revision->attachReviewers($with_reviewers->getReviewers()); + } + + } diff --git a/src/applications/differential/storage/DifferentialRevision.php b/src/applications/differential/storage/DifferentialRevision.php index 4591315207..e8fdf7e514 100644 --- a/src/applications/differential/storage/DifferentialRevision.php +++ b/src/applications/differential/storage/DifferentialRevision.php @@ -583,6 +583,10 @@ final class DifferentialRevision extends DifferentialDAO return $this; } + public function hasAttachedReviewers() { + return ($this->reviewerStatus !== self::ATTACHABLE); + } + public function getReviewerPHIDs() { $reviewers = $this->getReviewers(); return mpull($reviewers, 'getReviewerPHID'); From 2bb4fc9ecea567b8df06a4965b6d1008a615c078 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 7 Feb 2018 07:13:57 -0800 Subject: [PATCH 70/89] Fix a Phortune billing issue where subscription autopay could charge disabled cards Summary: See support email. There's nothing tricky here, we were just missing a check. The different parts of this got built at different times so I think this was simply overlooked. Also add a redundant check just to future-proof this and be on the safe side. Test Plan: Used `bin/phortune invoice` to charge a pact subscription. After deleting the card, the charge failed with an appropriate error. Reviewers: amckinley Differential Revision: https://secure.phabricator.com/D19020 --- .../query/PhortunePaymentMethodQuery.php | 25 ++++++------------- .../phortune/storage/PhortuneCart.php | 7 ++++++ .../storage/PhortunePaymentMethod.php | 4 +++ .../worker/PhortuneSubscriptionWorker.php | 4 +++ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/applications/phortune/query/PhortunePaymentMethodQuery.php b/src/applications/phortune/query/PhortunePaymentMethodQuery.php index 07d49a5909..42d54805e6 100644 --- a/src/applications/phortune/query/PhortunePaymentMethodQuery.php +++ b/src/applications/phortune/query/PhortunePaymentMethodQuery.php @@ -34,19 +34,12 @@ final class PhortunePaymentMethodQuery return $this; } + public function newResultObject() { + return new PhortunePaymentMethod(); + } + protected function loadPage() { - $table = new PhortunePaymentMethod(); - $conn = $table->establishConnection('r'); - - $rows = queryfx_all( - $conn, - 'SELECT * FROM %T %Q %Q %Q', - $table->getTableName(), - $this->buildWhereClause($conn), - $this->buildOrderClause($conn), - $this->buildLimitClause($conn)); - - return $table->loadAllFromArray($rows); + return $this->loadStandardPage($this->newResultObject()); } protected function willFilterPage(array $methods) { @@ -106,8 +99,8 @@ final class PhortunePaymentMethodQuery return $methods; } - protected function buildWhereClause(AphrontDatabaseConnection $conn) { - $where = array(); + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); if ($this->ids !== null) { $where[] = qsprintf( @@ -144,9 +137,7 @@ final class PhortunePaymentMethodQuery $this->statuses); } - $where[] = $this->buildPagingClause($conn); - - return $this->formatWhereClause($where); + return $where; } public function getQueryApplicationClass() { diff --git a/src/applications/phortune/storage/PhortuneCart.php b/src/applications/phortune/storage/PhortuneCart.php index af2b386dfe..07554ccb87 100644 --- a/src/applications/phortune/storage/PhortuneCart.php +++ b/src/applications/phortune/storage/PhortuneCart.php @@ -118,6 +118,13 @@ final class PhortuneCart extends PhortuneDAO ->setAmountAsCurrency($this->getTotalPriceAsCurrency()); if ($method) { + if (!$method->isActive()) { + throw new Exception( + pht( + 'Attempting to apply a charge using an inactive '. + 'payment method ("%s")!', + $method->getPHID())); + } $charge->setPaymentMethodPHID($method->getPHID()); } diff --git a/src/applications/phortune/storage/PhortunePaymentMethod.php b/src/applications/phortune/storage/PhortunePaymentMethod.php index 8044168ba4..1712d3f973 100644 --- a/src/applications/phortune/storage/PhortunePaymentMethod.php +++ b/src/applications/phortune/storage/PhortunePaymentMethod.php @@ -128,6 +128,10 @@ final class PhortunePaymentMethod extends PhortuneDAO return $month.'/'.$year; } + public function isActive() { + return ($this->getStatus() === self::STATUS_ACTIVE); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/phortune/worker/PhortuneSubscriptionWorker.php b/src/applications/phortune/worker/PhortuneSubscriptionWorker.php index 097fb54dd9..d05aacbb7c 100644 --- a/src/applications/phortune/worker/PhortuneSubscriptionWorker.php +++ b/src/applications/phortune/worker/PhortuneSubscriptionWorker.php @@ -141,6 +141,10 @@ final class PhortuneSubscriptionWorker extends PhabricatorWorker { $method = id(new PhortunePaymentMethodQuery()) ->setViewer($viewer) ->withPHIDs(array($subscription->getDefaultPaymentMethodPHID())) + ->withStatuses( + array( + PhortunePaymentMethod::STATUS_ACTIVE, + )) ->executeOne(); if (!$method) { $issues[] = pht( From 948b0ceca466457b97798eb920554aa353aa112b Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 06:47:24 -0800 Subject: [PATCH 71/89] Configure a whitelist of remote addresses for Postmark inbound webhooks Summary: Ref T13053. Postmark support recommends testing requests against a whitelist of known remote addresses to determine request authenticity. Today, the list can be found here: This is potentially less robust than, e.g., HMAC verification, since they may need to add new datacenters or support IPv6 or something. Users might also have weird network topologies where everything is proxied, and this makes testing/simulating more difficult. Allow users to configure the list so that they don't need to hack things apart if Postmark adds a new datacenter or remote addresses are unreliable for some other reason, but ship with safe defaults for today. Test Plan: Tried to make local requests, got kicked out. Added `0.0.0.0/0` to the list, stopped getting kicked out. I don't have a convenient way to route real Postmark traffic to my development laptop with an authentic remote address so I haven't verified that the published remote address is legitimate, but I'll vet that in production when I go through all the other mailers. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19025 --- ...ricatorMailImplementationPostmarkAdapter.php | 12 ++++++++++++ ...bricatorMetaMTAPostmarkReceiveController.php | 15 +++++++++++++++ .../configuring_inbound_email.diviner | 4 ++++ .../configuring_outbound_email.diviner | 17 +++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php index bd5ee820af..5792ba08f8 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationPostmarkAdapter.php @@ -73,12 +73,24 @@ final class PhabricatorMailImplementationPostmarkAdapter $options, array( 'access-token' => 'string', + 'inbound-addresses' => 'list', )); + + // Make sure this is properly formatted. + PhutilCIDRList::newList($options['inbound-addresses']); } public function newDefaultOptions() { return array( 'access-token' => null, + 'inbound-addresses' => array( + // Via Postmark support circa February 2018, see: + // + // https://postmarkapp.com/support/article/800-ips-for-firewalls + // + // "Configuring Outbound Email" should be updated if this changes. + '50.31.156.6/32', + ), ); } diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php index a54da6fb40..345cd93fe1 100644 --- a/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAPostmarkReceiveController.php @@ -20,6 +20,21 @@ final class PhabricatorMetaMTAPostmarkReceiveController return new Aphront404Response(); } + $remote_address = $request->getRemoteAddress(); + $any_remote_match = false; + foreach ($mailers as $mailer) { + $inbound_addresses = $mailer->getOption('inbound-addresses'); + $cidr_list = PhutilCIDRList::newList($inbound_addresses); + if ($cidr_list->containsAddress($remote_address)) { + $any_remote_match = true; + break; + } + } + + if (!$any_remote_match) { + return new Aphront400Response(); + } + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $raw_input = PhabricatorStartup::getRawInput(); diff --git a/src/docs/user/configuration/configuring_inbound_email.diviner b/src/docs/user/configuration/configuring_inbound_email.diviner index ada4ddb828..f4f367d57e 100644 --- a/src/docs/user/configuration/configuring_inbound_email.diviner +++ b/src/docs/user/configuration/configuring_inbound_email.diviner @@ -141,6 +141,10 @@ webhook URI in the Postmark control panel: https:///mail/postmark/ ``` +See also the Postmark section in @{article:Configuring Outbound Email} for +discussion of the remote address whitelist used to verify that requests this +endpoint receives are authentic requests originating from Postmark. + = SendGrid Setup = diff --git a/src/docs/user/configuration/configuring_outbound_email.diviner b/src/docs/user/configuration/configuring_outbound_email.diviner index d2daf7a40a..37a344c275 100644 --- a/src/docs/user/configuration/configuring_outbound_email.diviner +++ b/src/docs/user/configuration/configuring_outbound_email.diviner @@ -157,6 +157,23 @@ Postmark is a third-party email delivery serivice. You can learn more at To use this mailer, set `type` to `postmark`, then configure these `options`: - `access-token`: Required string. Your Postmark access token. + - `inbound-addresses`: Optional list. Address ranges which you + will accept inbound Postmark HTTP webook requests from. + +The default address list is preconfigured with Postmark's address range, so +you generally will not need to set or adjust it. + +The option accepts a list of CIDR ranges, like `1.2.3.4/16` (IPv4) or +`::ffff:0:0/96` (IPv6). The default ranges are: + +```lang=json +[ + "50.31.156.6/32" +] +``` + +The default address ranges were last updated in February 2018, and were +documented at: Mailer: Amazon SES From 942b17a980888b09980217735b12c9dd583a3b8c Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 08:57:07 -0800 Subject: [PATCH 72/89] Improve correctness of email address escaping in Mailgun/Postmark Summary: Ref T13053. Uses the changes in D19026 to escape mail addresses. Those rules may not be right yet, but they're at least all in one place, have test coverage, and aren't obviously incorrect. Test Plan: Will vet this more extensively when re-testing all mailers. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19027 --- .../adapter/PhabricatorMailImplementationAdapter.php | 5 +++-- .../PhabricatorMailImplementationMailgunAdapter.php | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php index ce56345194..dfbe891651 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationAdapter.php @@ -96,8 +96,9 @@ abstract class PhabricatorMailImplementationAdapter extends Phobject { protected function renderAddress($email, $name = null) { if (strlen($name)) { - // TODO: This needs to be escaped correctly. - return "{$name} <{$email}>"; + return (string)id(new PhutilEmailAddress()) + ->setDisplayName($name) + ->setAddress($email); } else { return $email; } diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index bed0dada63..12c54e0d6a 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -21,7 +21,7 @@ final class PhabricatorMailImplementationMailgunAdapter if (empty($this->params['reply-to'])) { $this->params['reply-to'] = array(); } - $this->params['reply-to'][] = "{$name} <{$email}>"; + $this->params['reply-to'][] = $this->renderAddress($name, $email); return $this; } @@ -110,11 +110,8 @@ final class PhabricatorMailImplementationMailgunAdapter } $from = idx($this->params, 'from'); - if (idx($this->params, 'from-name')) { - $params['from'] = "\"{$this->params['from-name']}\" <{$from}>"; - } else { - $params['from'] = $from; - } + $from_name = idx($this->params, 'from-name'); + $params['from'] = $this->renderAddress($from, $from_name); if (idx($this->params, 'reply-to')) { $replyto = $this->params['reply-to']; From a8f937d3138a4b02203964e995945c58be49293c Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:03:26 -0800 Subject: [PATCH 73/89] Only add the Mail "STAMPS" body section if there are stamps Summary: Ref T13053. Some mail (like push notification mail) doesn't currently generate any stamps. Drop this section if there aren't any stamps on the mail. Test Plan: Will check push mail in production. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19028 --- .../replyhandler/PhabricatorMailTarget.php | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php index 5d8378e8af..4bd5105135 100644 --- a/src/applications/metamta/replyhandler/PhabricatorMailTarget.php +++ b/src/applications/metamta/replyhandler/PhabricatorMailTarget.php @@ -66,27 +66,28 @@ final class PhabricatorMailTarget extends Phobject { if ($show_stamps) { $stamps = $mail->getMailStamps(); + if ($stamps) { + $body .= "\n"; + $body .= pht('STAMPS'); + $body .= "\n"; + $body .= implode(' ', $stamps); + $body .= "\n"; - $body .= "\n"; - $body .= pht('STAMPS'); - $body .= "\n"; - $body .= implode(' ', $stamps); - $body .= "\n"; - - if ($has_html) { - $html = array(); - $html[] = phutil_tag('strong', array(), pht('STAMPS')); - $html[] = phutil_tag('br'); - $html[] = phutil_tag( - 'span', - array( - 'style' => 'font-size: smaller; color: #92969D', - ), - phutil_implode_html(' ', $stamps)); - $html[] = phutil_tag('br'); - $html[] = phutil_tag('br'); - $html = phutil_tag('div', array(), $html); - $html_body .= hsprintf('%s', $html); + if ($has_html) { + $html = array(); + $html[] = phutil_tag('strong', array(), pht('STAMPS')); + $html[] = phutil_tag('br'); + $html[] = phutil_tag( + 'span', + array( + 'style' => 'font-size: smaller; color: #92969D', + ), + phutil_implode_html(' ', $stamps)); + $html[] = phutil_tag('br'); + $html[] = phutil_tag('br'); + $html = phutil_tag('div', array(), $html); + $html_body .= hsprintf('%s', $html); + } } } From bae9f459ab7f0e1dd0a71d4f4dac27d94d518907 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:06:40 -0800 Subject: [PATCH 74/89] Pass HTML bodies to push email Summary: Depends on D19028. Ref T13053. Fixes T6576. An HTML body was built here, but not passed to the actual mail message. Test Plan: Will verify production push mail. Maniphest Tasks: T13053, T6576 Differential Revision: https://secure.phabricator.com/D19029 --- .../repository/worker/PhabricatorRepositoryPushMailWorker.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php index 5ffaf0a5c2..554c2cf772 100644 --- a/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php +++ b/src/applications/repository/worker/PhabricatorRepositoryPushMailWorker.php @@ -123,6 +123,7 @@ final class PhabricatorRepositoryPushMailWorker ->setSubject($subject) ->setFrom($event->getPusherPHID()) ->setBody($body->render()) + ->setHTMLBody($body->renderHTML()) ->setThreadID($event->getPHID(), $is_new = true) ->setIsBulk(true); From 6186f0aa91b6c5d8faa6eecc2a0e98571bb34153 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:19:43 -0800 Subject: [PATCH 75/89] Briefly document mail stamps and remove obsolete header documentation Summary: Fixes T10189. Ref T13053. We haven't sent these headers in a very long time. Briefly mention the new stamps header instead, although I expect to integrate stamp documentation into the UI in a more cohesive way in the future. Test Plan: Read documentation. Maniphest Tasks: T13053, T10189 Differential Revision: https://secure.phabricator.com/D19030 --- .../storage/PhabricatorMetaMTAMail.php | 2 +- src/docs/user/userguide/mail_rules.diviner | 71 +++++-------------- 2 files changed, 20 insertions(+), 53 deletions(-) diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index 4a9bc68322..f16b50bf11 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -829,7 +829,7 @@ final class PhabricatorMetaMTAMail $stamps = $this->getMailStamps(); if ($stamps) { - $headers[] = array('X-Phabricator-Stamps', implode(', ', $stamps)); + $headers[] = array('X-Phabricator-Stamps', implode(' ', $stamps)); } $raw_body = idx($params, 'body', ''); diff --git a/src/docs/user/userguide/mail_rules.diviner b/src/docs/user/userguide/mail_rules.diviner index 3640f5e5a5..61bc3210e9 100644 --- a/src/docs/user/userguide/mail_rules.diviner +++ b/src/docs/user/userguide/mail_rules.diviner @@ -3,7 +3,8 @@ How to effectively manage Phabricator email notifications. -= Overview = +Overview +======== Phabricator uses email as a major notification channel, but the amount of email it sends can seem overwhelming if you're working on an active team. This @@ -13,69 +14,35 @@ By far the best approach to managing mail is to **write mail rules** to categorize mail. Essentially all modern mail clients allow you to quickly write sophisticated rules to route, categorize, or delete email. -= Reducing Email = +Reducing Email +============== You can reduce the amount of email you receive by turning off some types of email in {nav Settings > Email Preferences}. For example, you can turn off email produced by your own actions (like when you comment on a revision), and some types of less-important notifications about events. -= Mail Rules = +Mail Rules +========== The best approach to managing mail is to write mail rules. Simply writing rules to move mail from Differential, Maniphest and Herald to separate folders will vastly simplify mail management. -Phabricator also sets a large number of headers (see below) which can allow you -to write more sophisticated mail rules. +Phabricator also adds mail headers (see below) which can allow you to write +more sophisticated mail rules. -= Mail Headers = +Mail Headers +============ -Phabricator sends a variety of mail headers that can be useful in crafting rules -to route and manage mail. +Phabricator sends various information in mail headers that can be useful in +crafting rules to route and manage mail. To see a full list of headers, use +the "View Raw Message" feature in your mail client. -Headers in plural contain lists. A list containing two items, `1` and -`15` will generally be formatted like this: +The most useful header for routing is generally `X-Phabricator-Stamps`. This +is a list of attributes which describe the object the mail is about and the +actions which the mail informs you about. - X-Header: <1>, <15> - -The intent is to allow you to write a rule which matches against "<1>". If you -just match against "1", you'll incorrectly match "15", but matching "<1>" will -correctly match only "<1>". - -Some other headers use a single value but can be presented multiple times. -It is to support e-mail clients which are not able to create rules using regular -expressions or wildcards (namely Outlook). - -The headers Phabricator adds to mail are: - - - `X-Phabricator-Sent-This-Message`: this is attached to all mail - Phabricator sends. You can use it to differentiate between email from - Phabricator and replies/forwards of Phabricator mail from human beings. - - `X-Phabricator-To`: this is attached to all mail Phabricator sends. - It shows the PHIDs of the original "To" line, before any mutation - by the mailer configuration. - - `X-Phabricator-Cc`: this is attached to all mail Phabricator sends. - It shows the PHIDs of the original "Cc" line, before any mutation by the - mailer configuration. - - `X-Differential-Author`: this is attached to Differential mail and shows - the revision's author. You can use it to filter mail about your revisions - (or other users' revisions). - - `X-Differential-Reviewer`: this is attached to Differential mail and - shows the reviewers. You can use it to filter mail about revisions you - are reviewing, versus revisions you are explicitly CC'd on or CC'd as - a result of Herald rules. - - `X-Differential-Reviewers`: list version of the previous. - - `X-Differential-CC`: this is attached to Differential mail and shows - the CCs on the revision. - - `X-Differential-CCs`: list version of the previous. - - `X-Differential-Explicit-CC`: this is attached to Differential mail and - shows the explicit CCs on the revision (those that were added by humans, - not by Herald). - - `X-Differential-Explicit-CCs`: list version of the previous. - - `X-Phabricator-Mail-Tags`: this is attached to some mail and has - a list of descriptors about the mail. (This is fairly new and subject - to some change.) - - `X-Herald-Rules`: this is attached to some mail and shows Herald rule - IDs which have triggered for the object. You can use this to sort or - categorize mail that has triggered specific rules. +If you use a client which can not perform header matching (like Gmail), you can +change the {nav Settings > Email Format > Send Stamps} setting to include the +stamps in the mail body and then match them with body rules. From bca9c08953bd40c0c534f7d94651e37c3513af85 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:36:47 -0800 Subject: [PATCH 76/89] Add an "Acting user" field to Herald Summary: Ref T13053. Fixes T7804. Adds "Acting user" so you can have "always email me" stuff skip things you did or keep an eye on suspicious interns. For the test console, the current user is the acting user. For pushes, the pusher is the acting user. Test Plan: Wrote acting user rules, triggered them via test console and via multiple actors on real objects. Maniphest Tasks: T13053, T7804 Differential Revision: https://secure.phabricator.com/D19031 --- src/__phutil_library_map__.php | 2 ++ .../engine/DiffusionCommitHookEngine.php | 6 +++- .../herald/adapter/HeraldAdapter.php | 10 ++++++ .../HeraldTestConsoleController.php | 1 + .../herald/field/HeraldActingUserField.php | 32 +++++++++++++++++++ ...habricatorApplicationTransactionEditor.php | 1 + 6 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 src/applications/herald/field/HeraldActingUserField.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 301459869f..73632d11b4 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1346,6 +1346,7 @@ phutil_register_library_map(array( 'HarbormasterWaitForPreviousBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterWaitForPreviousBuildStepImplementation.php', 'HarbormasterWorker' => 'applications/harbormaster/worker/HarbormasterWorker.php', 'HarbormasterWorkingCopyArtifact' => 'applications/harbormaster/artifact/HarbormasterWorkingCopyArtifact.php', + 'HeraldActingUserField' => 'applications/herald/field/HeraldActingUserField.php', 'HeraldAction' => 'applications/herald/action/HeraldAction.php', 'HeraldActionGroup' => 'applications/herald/action/HeraldActionGroup.php', 'HeraldActionRecord' => 'applications/herald/storage/HeraldActionRecord.php', @@ -6589,6 +6590,7 @@ phutil_register_library_map(array( 'HarbormasterWaitForPreviousBuildStepImplementation' => 'HarbormasterBuildStepImplementation', 'HarbormasterWorker' => 'PhabricatorWorker', 'HarbormasterWorkingCopyArtifact' => 'HarbormasterDrydockLeaseArtifact', + 'HeraldActingUserField' => 'HeraldField', 'HeraldAction' => 'Phobject', 'HeraldActionGroup' => 'HeraldGroup', 'HeraldActionRecord' => 'HeraldDAO', diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index 99df0e54af..a0769a51f0 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -297,7 +297,11 @@ final class DiffusionCommitHookEngine extends Phobject { return; } - $adapter_template->setHookEngine($this); + $viewer = $this->getViewer(); + + $adapter_template + ->setHookEngine($this) + ->setActingAsPHID($viewer->getPHID()); $engine = new HeraldEngine(); $rules = null; diff --git a/src/applications/herald/adapter/HeraldAdapter.php b/src/applications/herald/adapter/HeraldAdapter.php index cc0fdbd3b5..940d604019 100644 --- a/src/applications/herald/adapter/HeraldAdapter.php +++ b/src/applications/herald/adapter/HeraldAdapter.php @@ -40,6 +40,7 @@ abstract class HeraldAdapter extends Phobject { private $forbiddenActions = array(); private $viewer; private $mustEncryptReasons = array(); + private $actingAsPHID; public function getEmailPHIDs() { return array_values($this->emailPHIDs); @@ -49,6 +50,15 @@ abstract class HeraldAdapter extends Phobject { return array_values($this->forcedEmailPHIDs); } + final public function setActingAsPHID($acting_as_phid) { + $this->actingAsPHID = $acting_as_phid; + return $this; + } + + final public function getActingAsPHID() { + return $this->actingAsPHID; + } + public function addEmailPHID($phid, $force) { $this->emailPHIDs[$phid] = $phid; if ($force) { diff --git a/src/applications/herald/controller/HeraldTestConsoleController.php b/src/applications/herald/controller/HeraldTestConsoleController.php index 8a7a94963d..4ddab2669b 100644 --- a/src/applications/herald/controller/HeraldTestConsoleController.php +++ b/src/applications/herald/controller/HeraldTestConsoleController.php @@ -41,6 +41,7 @@ final class HeraldTestConsoleController extends HeraldController { $adapter ->setIsNewObject(false) + ->setActingAsPHID($viewer->getPHID()) ->setViewer($viewer); $rules = id(new HeraldRuleQuery()) diff --git a/src/applications/herald/field/HeraldActingUserField.php b/src/applications/herald/field/HeraldActingUserField.php new file mode 100644 index 0000000000..2245c7e9f7 --- /dev/null +++ b/src/applications/herald/field/HeraldActingUserField.php @@ -0,0 +1,32 @@ +getAdapter()->getActingAsPHID(); + } + + protected function getHeraldFieldStandardType() { + return self::STANDARD_PHID; + } + + protected function getDatasource() { + return new PhabricatorPeopleDatasource(); + } + + public function supportsObject($object) { + return true; + } + + public function getFieldGroupKey() { + return HeraldEditFieldGroup::FIELDGROUPKEY; + } + +} diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php index 167aa05fed..8c319fc61e 100644 --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -3254,6 +3254,7 @@ abstract class PhabricatorApplicationTransactionEditor $adapter = $this->buildHeraldAdapter($object, $xactions) ->setContentSource($this->getContentSource()) ->setIsNewObject($this->getIsNewObject()) + ->setActingAsPHID($this->getActingAsPHID()) ->setAppliedTransactions($xactions); if ($this->getApplicationEmail()) { From 0402a79e0e5723a54ecffaf5c6bc1c1d89cbe51b Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 09:46:56 -0800 Subject: [PATCH 77/89] Render object remarkup references in a text context as "Dxxx " Summary: Depends on D19031. Fixes T11389. Currently, we render `Dxxx` in a text context (plain text email) as just a URI. Instead, render it like `Dxxx `. This is more faithful to the original intent and preserves `T123/T456` as two separate, usable links. Test Plan: Wrote `T123/T234` in a task, pulled mail for it with `bin/mail show-outbound`, saw separate clickable links. Maniphest Tasks: T11389 Differential Revision: https://secure.phabricator.com/D19032 --- .../markup/rule/PhabricatorObjectRemarkupRule.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php b/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php index 35c0ecfad0..fbbfa36805 100644 --- a/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php +++ b/src/infrastructure/markup/rule/PhabricatorObjectRemarkupRule.php @@ -75,7 +75,7 @@ abstract class PhabricatorObjectRemarkupRule extends PhutilRemarkupRule { } if ($this->getEngine()->isTextMode()) { - return PhabricatorEnv::getProductionURI($href); + return $text.' <'.PhabricatorEnv::getProductionURI($href).'>'; } else if ($this->getEngine()->isHTMLMailMode()) { $href = PhabricatorEnv::getProductionURI($href); return $this->renderObjectTagForMail($text, $href, $handle); From ab04d2179bf1322ec31f03e46c28bebb0334f135 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 10:37:47 -0800 Subject: [PATCH 78/89] Add "Mute/Unmute" for subscribable objects Summary: Ref T13053. See PHI126. Add an explicit "Mute" action to kill mail and notifications for a particular object. Test Plan: Muted and umuted an object while interacting with it. Saw mail route appropriately. Maniphest Tasks: T13053 Differential Revision: https://secure.phabricator.com/D19033 --- resources/celerity/map.php | 6 +- src/__phutil_library_map__.php | 6 ++ .../metamta/query/PhabricatorMetaMTAActor.php | 4 + .../storage/PhabricatorMetaMTAMail.php | 21 +++++ .../PhabricatorSubscriptionsApplication.php | 5 +- ...PhabricatorSubscriptionsMuteController.php | 92 +++++++++++++++++++ ...habricatorSubscriptionsUIEventListener.php | 55 ++++++++--- .../edges/PhabricatorMutedByEdgeType.php | 16 ++++ .../edges/PhabricatorMutedEdgeType.php | 16 ++++ ...habricatorApplicationTransactionEditor.php | 28 ++++++ .../PhabricatorApplicationTransaction.php | 2 + src/view/layout/PhabricatorActionView.php | 10 ++ webroot/rsrc/css/phui/phui-action-list.css | 17 ++-- 13 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php create mode 100644 src/applications/transactions/edges/PhabricatorMutedByEdgeType.php create mode 100644 src/applications/transactions/edges/PhabricatorMutedEdgeType.php diff --git a/resources/celerity/map.php b/resources/celerity/map.php index d9aebf32bc..17138d6604 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -9,7 +9,7 @@ return array( 'names' => array( 'conpherence.pkg.css' => 'e68cf1fa', 'conpherence.pkg.js' => '15191c65', - 'core.pkg.css' => '51debec3', + 'core.pkg.css' => 'ce8c2a58', 'core.pkg.js' => '4c79d74f', 'darkconsole.pkg.js' => '1f9a31bc', 'differential.pkg.css' => '45951e9e', @@ -136,7 +136,7 @@ return array( 'rsrc/css/phui/object-item/phui-oi-flush-ui.css' => '9d9685d6', 'rsrc/css/phui/object-item/phui-oi-list-view.css' => '6ae18df0', 'rsrc/css/phui/object-item/phui-oi-simple-ui.css' => 'a8beebea', - 'rsrc/css/phui/phui-action-list.css' => 'f7f61a34', + 'rsrc/css/phui/phui-action-list.css' => '0bcd9a45', 'rsrc/css/phui/phui-action-panel.css' => 'b4798122', 'rsrc/css/phui/phui-badge.css' => '22c0cf4f', 'rsrc/css/phui/phui-basic-nav-view.css' => '98c11ab3', @@ -766,7 +766,7 @@ return array( 'path-typeahead' => 'f7fc67ec', 'people-picture-menu-item-css' => 'a06f7f34', 'people-profile-css' => '4df76faf', - 'phabricator-action-list-view-css' => 'f7f61a34', + 'phabricator-action-list-view-css' => '0bcd9a45', 'phabricator-busy' => '59a7976a', 'phabricator-chatlog-css' => 'd295b020', 'phabricator-content-source-view-css' => '4b8b05d4', diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 73632d11b4..9342935ca1 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3291,6 +3291,8 @@ phutil_register_library_map(array( 'PhabricatorMultiFactorSettingsPanel' => 'applications/settings/panel/PhabricatorMultiFactorSettingsPanel.php', 'PhabricatorMultimeterApplication' => 'applications/multimeter/application/PhabricatorMultimeterApplication.php', 'PhabricatorMustVerifyEmailController' => 'applications/auth/controller/PhabricatorMustVerifyEmailController.php', + 'PhabricatorMutedByEdgeType' => 'applications/transactions/edges/PhabricatorMutedByEdgeType.php', + 'PhabricatorMutedEdgeType' => 'applications/transactions/edges/PhabricatorMutedEdgeType.php', 'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php', 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', 'PhabricatorMySQLSearchHost' => 'infrastructure/cluster/search/PhabricatorMySQLSearchHost.php', @@ -4240,6 +4242,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsHeraldAction.php', 'PhabricatorSubscriptionsListController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsListController.php', 'PhabricatorSubscriptionsMailEngineExtension' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsMailEngineExtension.php', + 'PhabricatorSubscriptionsMuteController' => 'applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSelfHeraldAction.php', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'applications/subscriptions/herald/PhabricatorSubscriptionsRemoveSubscribersHeraldAction.php', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'applications/subscriptions/engineextension/PhabricatorSubscriptionsSearchEngineAttachment.php', @@ -8808,6 +8811,8 @@ phutil_register_library_map(array( 'PhabricatorMultiFactorSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorMultimeterApplication' => 'PhabricatorApplication', 'PhabricatorMustVerifyEmailController' => 'PhabricatorAuthController', + 'PhabricatorMutedByEdgeType' => 'PhabricatorEdgeType', + 'PhabricatorMutedEdgeType' => 'PhabricatorEdgeType', 'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorMySQLSearchHost' => 'PhabricatorSearchHost', @@ -9960,6 +9965,7 @@ phutil_register_library_map(array( 'PhabricatorSubscriptionsHeraldAction' => 'HeraldAction', 'PhabricatorSubscriptionsListController' => 'PhabricatorController', 'PhabricatorSubscriptionsMailEngineExtension' => 'PhabricatorMailEngineExtension', + 'PhabricatorSubscriptionsMuteController' => 'PhabricatorController', 'PhabricatorSubscriptionsRemoveSelfHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsRemoveSubscribersHeraldAction' => 'PhabricatorSubscriptionsHeraldAction', 'PhabricatorSubscriptionsSearchEngineAttachment' => 'PhabricatorSearchEngineAttachment', diff --git a/src/applications/metamta/query/PhabricatorMetaMTAActor.php b/src/applications/metamta/query/PhabricatorMetaMTAActor.php index 1f4cc7da12..cf2060a8f7 100644 --- a/src/applications/metamta/query/PhabricatorMetaMTAActor.php +++ b/src/applications/metamta/query/PhabricatorMetaMTAActor.php @@ -21,6 +21,7 @@ final class PhabricatorMetaMTAActor extends Phobject { const REASON_ROUTE_AS_NOTIFICATION = 'route-as-notification'; const REASON_ROUTE_AS_MAIL = 'route-as-mail'; const REASON_UNVERIFIED = 'unverified'; + const REASON_MUTED = 'muted'; private $phid; private $emailAddress; @@ -116,6 +117,7 @@ final class PhabricatorMetaMTAActor extends Phobject { self::REASON_ROUTE_AS_NOTIFICATION => pht('Route as Notification'), self::REASON_ROUTE_AS_MAIL => pht('Route as Mail'), self::REASON_UNVERIFIED => pht('Address Not Verified'), + self::REASON_MUTED => pht('Muted'), ); return idx($names, $reason, pht('Unknown ("%s")', $reason)); @@ -172,6 +174,8 @@ final class PhabricatorMetaMTAActor extends Phobject { 'in Herald.'), self::REASON_UNVERIFIED => pht( 'This recipient does not have a verified primary email address.'), + self::REASON_MUTED => pht( + 'This recipient has muted notifications for this object.'), ); return idx($descriptions, $reason, pht('Unknown Reason ("%s")', $reason)); diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php index f16b50bf11..9dfd6a3eb6 100644 --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -160,6 +160,15 @@ final class PhabricatorMetaMTAMail return $this->getParam('exclude', array()); } + public function setMutedPHIDs(array $muted) { + $this->setParam('muted', $muted); + return $this; + } + + private function getMutedPHIDs() { + return $this->getParam('muted', array()); + } + public function setForceHeraldMailRecipientPHIDs(array $force) { $this->setParam('herald-force-recipients', $force); return $this; @@ -1113,6 +1122,18 @@ final class PhabricatorMetaMTAMail } } + // Exclude muted recipients. We're doing this after saving deliverability + // so that Herald "Send me an email" actions can still punch through a + // mute. + + foreach ($this->getMutedPHIDs() as $muted_phid) { + $muted_actor = idx($actors, $muted_phid); + if (!$muted_actor) { + continue; + } + $muted_actor->setUndeliverable(PhabricatorMetaMTAActor::REASON_MUTED); + } + // For the rest of the rules, order matters. We're going to run all the // possible rules in order from weakest to strongest, and let the strongest // matching rule win. The weaker rules leave annotations behind which help diff --git a/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php b/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php index 56759f5dc3..2de2994a92 100644 --- a/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php +++ b/src/applications/subscriptions/application/PhabricatorSubscriptionsApplication.php @@ -24,7 +24,10 @@ final class PhabricatorSubscriptionsApplication extends PhabricatorApplication { return array( '/subscriptions/' => array( '(?Padd|delete)/'. - '(?P[^/]+)/' => 'PhabricatorSubscriptionsEditController', + '(?P[^/]+)/' => 'PhabricatorSubscriptionsEditController', + 'mute/' => array( + '(?P[^/]+)/' => 'PhabricatorSubscriptionsMuteController', + ), 'list/(?P[^/]+)/' => 'PhabricatorSubscriptionsListController', 'transaction/(?Padd|rem)/(?[^/]+)/' => 'PhabricatorSubscriptionsTransactionController', diff --git a/src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php b/src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php new file mode 100644 index 0000000000..1369643ffc --- /dev/null +++ b/src/applications/subscriptions/controller/PhabricatorSubscriptionsMuteController.php @@ -0,0 +1,92 @@ +getViewer(); + $phid = $request->getURIData('phid'); + + $handle = id(new PhabricatorHandleQuery()) + ->setViewer($viewer) + ->withPHIDs(array($phid)) + ->executeOne(); + + $object = id(new PhabricatorObjectQuery()) + ->setViewer($viewer) + ->withPHIDs(array($phid)) + ->executeOne(); + + if (!($object instanceof PhabricatorSubscribableInterface)) { + return new Aphront400Response(); + } + + $muted_type = PhabricatorMutedByEdgeType::EDGECONST; + + $edge_query = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs(array($object->getPHID())) + ->withEdgeTypes(array($muted_type)) + ->withDestinationPHIDs(array($viewer->getPHID())); + + $edge_query->execute(); + + $is_mute = !$edge_query->getDestinationPHIDs(); + $object_uri = $handle->getURI(); + + if ($request->isFormPost()) { + if ($is_mute) { + $xaction_value = array( + '+' => array_fuse(array($viewer->getPHID())), + ); + } else { + $xaction_value = array( + '-' => array_fuse(array($viewer->getPHID())), + ); + } + + $muted_type = PhabricatorMutedByEdgeType::EDGECONST; + + $xaction = id($object->getApplicationTransactionTemplate()) + ->setTransactionType(PhabricatorTransactions::TYPE_EDGE) + ->setMetadataValue('edge:type', $muted_type) + ->setNewValue($xaction_value); + + $editor = id($object->getApplicationTransactionEditor()) + ->setActor($viewer) + ->setContinueOnNoEffect(true) + ->setContinueOnMissingFields(true) + ->setContentSourceFromRequest($request); + + $editor->applyTransactions( + $object->getApplicationTransactionObject(), + array($xaction)); + + return id(new AphrontReloadResponse())->setURI($object_uri); + } + + $dialog = $this->newDialog() + ->addCancelButton($object_uri); + + if ($is_mute) { + $dialog + ->setTitle(pht('Mute Notifications')) + ->appendParagraph( + pht( + 'Mute this object? You will no longer receive notifications or '. + 'email about it.')) + ->addSubmitButton(pht('Mute')); + } else { + $dialog + ->setTitle(pht('Unmute Notifications')) + ->appendParagraph( + pht( + 'Unmute this object? You will receive notifications and email '. + 'again.')) + ->addSubmitButton(pht('Unmute')); + } + + return $dialog; + } + + +} diff --git a/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php b/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php index 5f371d69f3..caf860117e 100644 --- a/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php +++ b/src/applications/subscriptions/events/PhabricatorSubscriptionsUIEventListener.php @@ -42,6 +42,28 @@ final class PhabricatorSubscriptionsUIEventListener return; } + $src_phid = $object->getPHID(); + $subscribed_type = PhabricatorObjectHasSubscriberEdgeType::EDGECONST; + $muted_type = PhabricatorMutedByEdgeType::EDGECONST; + + $edges = id(new PhabricatorEdgeQuery()) + ->withSourcePHIDs(array($src_phid)) + ->withEdgeTypes( + array( + $subscribed_type, + $muted_type, + )) + ->withDestinationPHIDs(array($user_phid)) + ->execute(); + + if ($user_phid) { + $is_subscribed = isset($edges[$src_phid][$subscribed_type][$user_phid]); + $is_muted = isset($edges[$src_phid][$muted_type][$user_phid]); + } else { + $is_subscribed = false; + $is_muted = false; + } + if ($user_phid && $object->isAutomaticallySubscribed($user_phid)) { $sub_action = id(new PhabricatorActionView()) ->setWorkflow(true) @@ -51,22 +73,9 @@ final class PhabricatorSubscriptionsUIEventListener ->setName(pht('Automatically Subscribed')) ->setIcon('fa-check-circle lightgreytext'); } else { - $subscribed = false; - if ($user->isLoggedIn()) { - $src_phid = $object->getPHID(); - $edge_type = PhabricatorObjectHasSubscriberEdgeType::EDGECONST; - - $edges = id(new PhabricatorEdgeQuery()) - ->withSourcePHIDs(array($src_phid)) - ->withEdgeTypes(array($edge_type)) - ->withDestinationPHIDs(array($user_phid)) - ->execute(); - $subscribed = isset($edges[$src_phid][$edge_type][$user_phid]); - } - $can_interact = PhabricatorPolicyFilter::canInteract($user, $object); - if ($subscribed) { + if ($is_subscribed) { $sub_action = id(new PhabricatorActionView()) ->setWorkflow(true) ->setRenderAsForm(true) @@ -89,8 +98,26 @@ final class PhabricatorSubscriptionsUIEventListener } } + $mute_action = id(new PhabricatorActionView()) + ->setWorkflow(true) + ->setHref('/subscriptions/mute/'.$object->getPHID().'/') + ->setDisabled(!$user_phid); + + if (!$is_muted) { + $mute_action + ->setName(pht('Mute Notifications')) + ->setIcon('fa-volume-up'); + } else { + $mute_action + ->setName(pht('Unmute Notifications')) + ->setIcon('fa-volume-off') + ->setColor(PhabricatorActionView::RED); + } + + $actions = $event->getValue('actions'); $actions[] = $sub_action; + $actions[] = $mute_action; $event->setValue('actions', $actions); } diff --git a/src/applications/transactions/edges/PhabricatorMutedByEdgeType.php b/src/applications/transactions/edges/PhabricatorMutedByEdgeType.php new file mode 100644 index 0000000000..1f592239ba --- /dev/null +++ b/src/applications/transactions/edges/PhabricatorMutedByEdgeType.php @@ -0,0 +1,16 @@ +applyOldRecipientLists(); + if ($object instanceof PhabricatorSubscribableInterface) { + $this->mailMutedPHIDs = PhabricatorEdgeQuery::loadDestinationPHIDs( + $object->getPHID(), + PhabricatorMutedByEdgeType::EDGECONST); + } else { + $this->mailMutedPHIDs = array(); + } + $mail_xactions = $this->getTransactionsForMail($object, $xactions); $stamps = $this->newMailStamps($object, $xactions); foreach ($stamps as $stamp) { @@ -2662,6 +2671,11 @@ abstract class PhabricatorApplicationTransactionEditor $mail_xactions); } + $muted_phids = $this->mailMutedPHIDs; + if (!is_array($muted_phids)) { + $muted_phids = array(); + } + $mail ->setSensitiveContent(false) ->setFrom($this->getActingAsPHID()) @@ -2670,6 +2684,7 @@ abstract class PhabricatorApplicationTransactionEditor ->setThreadID($this->getMailThreadID($object), $this->getIsNewObject()) ->setRelatedPHID($object->getPHID()) ->setExcludeMailRecipientPHIDs($this->getExcludeMailRecipientPHIDs()) + ->setMutedPHIDs($muted_phids) ->setForceHeraldMailRecipientPHIDs($this->heraldForcedEmailPHIDs) ->setMailTags($mail_tags) ->setIsBulk(true) @@ -3186,6 +3201,18 @@ abstract class PhabricatorApplicationTransactionEditor $related_phids = $this->feedRelatedPHIDs; $subscribed_phids = $this->feedNotifyPHIDs; + // Remove muted users from the subscription list so they don't get + // notifications, either. + $muted_phids = $this->mailMutedPHIDs; + if (!is_array($muted_phids)) { + $muted_phids = array(); + } + $subscribed_phids = array_fuse($subscribed_phids); + foreach ($muted_phids as $muted_phid) { + unset($subscribed_phids[$muted_phid]); + } + $subscribed_phids = array_values($subscribed_phids); + $story_type = $this->getFeedStoryType(); $story_data = $this->getFeedStoryData($object, $xactions); @@ -3632,6 +3659,7 @@ abstract class PhabricatorApplicationTransactionEditor 'mustEncrypt', 'mailStamps', 'mailUnexpandablePHIDs', + 'mailMutedPHIDs', ); } diff --git a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php index d5c28b83bf..d27370cd44 100644 --- a/src/applications/transactions/storage/PhabricatorApplicationTransaction.php +++ b/src/applications/transactions/storage/PhabricatorApplicationTransaction.php @@ -643,6 +643,8 @@ abstract class PhabricatorApplicationTransaction case PhabricatorObjectMentionsObjectEdgeType::EDGECONST: case ManiphestTaskHasDuplicateTaskEdgeType::EDGECONST: case ManiphestTaskIsDuplicateOfTaskEdgeType::EDGECONST: + case PhabricatorMutedEdgeType::EDGECONST: + case PhabricatorMutedByEdgeType::EDGECONST: return true; break; case PhabricatorObjectMentionedByObjectEdgeType::EDGECONST: diff --git a/src/view/layout/PhabricatorActionView.php b/src/view/layout/PhabricatorActionView.php index f6de8eca5b..d43cc9428b 100644 --- a/src/view/layout/PhabricatorActionView.php +++ b/src/view/layout/PhabricatorActionView.php @@ -21,6 +21,7 @@ final class PhabricatorActionView extends AphrontView { private $order; private $color; private $type; + private $highlight; const TYPE_DIVIDER = 'type-divider'; const TYPE_LABEL = 'label'; @@ -72,6 +73,15 @@ final class PhabricatorActionView extends AphrontView { return $this->href; } + public function setHighlight($highlight) { + $this->highlight = $highlight; + return $this; + } + + public function getHighlight() { + return $this->highlight; + } + public function setIcon($icon) { $this->icon = $icon; return $this; diff --git a/webroot/rsrc/css/phui/phui-action-list.css b/webroot/rsrc/css/phui/phui-action-list.css index 5e32a1ea0a..e7ee38a8bf 100644 --- a/webroot/rsrc/css/phui/phui-action-list.css +++ b/webroot/rsrc/css/phui/phui-action-list.css @@ -95,15 +95,20 @@ color: {$sky}; } -.device-desktop .phabricator-action-view-href.action-item-red:hover - .phabricator-action-view-item { - background-color: {$sh-redbackground}; - color: {$sh-redtext}; +.phabricator-action-view.action-item-red { + background-color: {$sh-redbackground}; } -.device-desktop .phabricator-action-view-href.action-item-red:hover +.phabricator-action-view.action-item-red .phabricator-action-view-item, +.phabricator-action-view.action-item-red .phabricator-action-view-icon { + color: {$sh-redtext}; +} + +.device-desktop .phabricator-action-view.action-item-red:hover + .phabricator-action-view-item, +.device-desktop .phabricator-action-view.action-item-red:hover .phabricator-action-view-icon { - color: {$red}; + color: {$red}; } .phabricator-action-view-label .phabricator-action-view-item, From 705ff8d33de04b09a31753426850d2c7da9a046a Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 11:40:12 -0800 Subject: [PATCH 79/89] Remove `addHighlight()` action view methods Summary: These didn't actually get used by D19033. Test Plan: Grep. Differential Revision: https://secure.phabricator.com/D19034 --- src/view/layout/PhabricatorActionView.php | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/view/layout/PhabricatorActionView.php b/src/view/layout/PhabricatorActionView.php index d43cc9428b..f6de8eca5b 100644 --- a/src/view/layout/PhabricatorActionView.php +++ b/src/view/layout/PhabricatorActionView.php @@ -21,7 +21,6 @@ final class PhabricatorActionView extends AphrontView { private $order; private $color; private $type; - private $highlight; const TYPE_DIVIDER = 'type-divider'; const TYPE_LABEL = 'label'; @@ -73,15 +72,6 @@ final class PhabricatorActionView extends AphrontView { return $this->href; } - public function setHighlight($highlight) { - $this->highlight = $highlight; - return $this; - } - - public function getHighlight() { - return $this->highlight; - } - public function setIcon($icon) { $this->icon = $icon; return $this; From d1e273daf62fc12393f107804b7e19c55bbb25d9 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 12:41:11 -0800 Subject: [PATCH 80/89] Remove completely pointless load of every repository when viewing a repository URI Summary: See D18176. This query has no effect (other than wasting resources) and the result is unused. `$repository` already has the URI loaded because we load them unconditionally during request initialization. Test Plan: Viewed repository URIs. Subscribers: jmeador Differential Revision: https://secure.phabricator.com/D19036 --- .../controller/DiffusionRepositoryURIViewController.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php b/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php index 91ffbb473f..e923ebfc20 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryURIViewController.php @@ -23,14 +23,10 @@ final class DiffusionRepositoryURIViewController return new Aphront404Response(); } - // For display, reload the URI by loading it through the repository. This + // For display, access the URI by loading it through the repository. This // may adjust builtin URIs for repository configuration, so we may end up // with a different view of builtin URIs than we'd see if we loaded them // directly from the database. See T12884. - $repository_with_uris = id(new PhabricatorRepositoryQuery()) - ->setViewer($viewer) - ->needURIs(true) - ->execute(); $repository_uris = $repository->getURIs(); $repository_uris = mpull($repository_uris, null, 'getID'); From f028aa6f60dd6382b59153e681645b478cdd0e62 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 14:20:32 -0800 Subject: [PATCH 81/89] Track closed date and closing user for tasks explicitly Summary: Ref T4434. Although some of the use cases for this data are better fits for Facts, this data is reasonable to track separately. I have an approximate view of it already ("closed, ordered by date modified") that's useful to review things that were fixed recently. This lets us make that view more effective. This just adds (and populates) the storage. Followups will add Conduit, Export, Search, and UI support. This is slightly tricky because merges work oddly (see T13020). Test Plan: - Ran migration, checked database for sensible results. - Created a task in open/closed status, got the right database values. - Modified a task to close/open it, got the right values. - Merged an open task, got updates. Maniphest Tasks: T4434 Differential Revision: https://secure.phabricator.com/D19037 --- .../20180208.maniphest.01.close.sql | 5 ++ .../20180208.maniphest.02.populate.php | 65 +++++++++++++++++++ .../maniphest/storage/ManiphestTask.php | 11 ++++ .../ManiphestTaskMergedIntoTransaction.php | 2 +- .../ManiphestTaskStatusTransaction.php | 2 +- .../xaction/ManiphestTaskTransactionType.php | 23 +++++++ 6 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 resources/sql/autopatches/20180208.maniphest.01.close.sql create mode 100644 resources/sql/autopatches/20180208.maniphest.02.populate.php diff --git a/resources/sql/autopatches/20180208.maniphest.01.close.sql b/resources/sql/autopatches/20180208.maniphest.01.close.sql new file mode 100644 index 0000000000..856300e9ba --- /dev/null +++ b/resources/sql/autopatches/20180208.maniphest.01.close.sql @@ -0,0 +1,5 @@ +ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task + ADD closedEpoch INT UNSIGNED; + +ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task + ADD closerPHID VARBINARY(64); diff --git a/resources/sql/autopatches/20180208.maniphest.02.populate.php b/resources/sql/autopatches/20180208.maniphest.02.populate.php new file mode 100644 index 0000000000..16aa2bf57b --- /dev/null +++ b/resources/sql/autopatches/20180208.maniphest.02.populate.php @@ -0,0 +1,65 @@ +establishConnection('w'); +$viewer = PhabricatorUser::getOmnipotentUser(); + +foreach (new LiskMigrationIterator($table) as $task) { + if ($task->getClosedEpoch()) { + // Task already has a closed date. + continue; + } + + $status = $task->getStatus(); + if (!ManiphestTaskStatus::isClosedStatus($status)) { + // Task isn't closed. + continue; + } + + // Look through the transactions from newest to oldest until we find one + // where the task was closed. A merge also counts as a close, even though + // it doesn't currently produce a separate transaction. + + $type_merge = ManiphestTaskStatusTransaction::TRANSACTIONTYPE; + $type_status = ManiphestTaskMergedIntoTransaction::TRANSACTIONTYPE; + + $xactions = id(new ManiphestTransactionQuery()) + ->setViewer($viewer) + ->withObjectPHIDs(array($task->getPHID())) + ->withTransactionTypes( + array( + $type_merge, + $type_status, + )) + ->execute(); + foreach ($xactions as $xaction) { + $old = $xaction->getOldValue(); + $new = $xaction->getNewValue(); + + $type = $xaction->getTransactionType(); + + // If this is a status change, but is not a close, don't use it. + // (We always use merges, even though it's possible to merge a task which + // was previously closed: we can't tell when this happens very easily.) + if ($type === $type_status) { + if (!ManiphestTaskStatus::isClosedStatus($new)) { + continue; + } + + if ($old && ManiphestTaskStatus::isClosedStatus($old)) { + continue; + } + } + + queryfx( + $conn, + 'UPDATE %T SET closedEpoch = %d, closerPHID = %ns + WHERE id = %d', + $table->getTableName(), + $xaction->getDateCreated(), + $xaction->getAuthorPHID(), + $task->getID()); + + break; + } +} diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index e19886d3ff..7ff70abd80 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -44,6 +44,9 @@ final class ManiphestTask extends ManiphestDAO protected $points; protected $subtype; + protected $closedEpoch; + protected $closerPHID; + private $subscriberPHIDs = self::ATTACHABLE; private $groupByProjectPHID = self::ATTACHABLE; private $customFields = self::ATTACHABLE; @@ -90,6 +93,8 @@ final class ManiphestTask extends ManiphestDAO 'points' => 'double?', 'bridgedObjectPHID' => 'phid?', 'subtype' => 'text64', + 'closedEpoch' => 'epoch?', + 'closerPHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_phid' => null, @@ -131,6 +136,12 @@ final class ManiphestTask extends ManiphestDAO 'key_subtype' => array( 'columns' => array('subtype'), ), + 'key_closed' => array( + 'columns' => array('closedEpoch'), + ), + 'key_closer' => array( + 'columns' => array('closerPHID', 'closedEpoch'), + ), ), ) + parent::getConfiguration(); } diff --git a/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php b/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php index cd0cad6a39..630f5190ce 100644 --- a/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php +++ b/src/applications/maniphest/xaction/ManiphestTaskMergedIntoTransaction.php @@ -10,7 +10,7 @@ final class ManiphestTaskMergedIntoTransaction } public function applyInternalEffects($object, $value) { - $object->setStatus(ManiphestTaskStatus::getDuplicateStatus()); + $this->updateStatus($object, ManiphestTaskStatus::getDuplicateStatus()); } public function getActionName() { diff --git a/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php b/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php index dd51a63799..6f4b558e05 100644 --- a/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php +++ b/src/applications/maniphest/xaction/ManiphestTaskStatusTransaction.php @@ -10,7 +10,7 @@ final class ManiphestTaskStatusTransaction } public function applyInternalEffects($object, $value) { - $object->setStatus($value); + $this->updateStatus($object, $value); } public function shouldHide() { diff --git a/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php b/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php index c59de163c6..836e7765b8 100644 --- a/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php +++ b/src/applications/maniphest/xaction/ManiphestTaskTransactionType.php @@ -3,4 +3,27 @@ abstract class ManiphestTaskTransactionType extends PhabricatorModularTransactionType { + protected function updateStatus($object, $new_value) { + $old_value = $object->getStatus(); + $object->setStatus($new_value); + + // If this status change closes or opens the task, update the closed + // date and actor PHID. + $old_closed = ManiphestTaskStatus::isClosedStatus($old_value); + $new_closed = ManiphestTaskStatus::isClosedStatus($new_value); + + $is_close = ($new_closed && !$old_closed); + $is_open = (!$new_closed && $old_closed); + + if ($is_close) { + $object + ->setClosedEpoch(PhabricatorTime::getNow()) + ->setCloserPHID($this->getActingAsPHID()); + } else if ($is_open) { + $object + ->setClosedEpoch(null) + ->setCloserPHID(null); + } + } + } From 4c4707e467633032fc56797d6b620b626ff21e18 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 14:48:16 -0800 Subject: [PATCH 82/89] Provide task closed date via Conduit API, data export pipeline, and in list UI Summary: Depends on D19037. Ref T4434. Adds closed date to `maniphest.search` and "Export Data". When a task has been closed, show the closed date with a checkmark in the UI instead of the modified date. Test Plan: - Exported data to CSV, saw close information. - Saw close information in `/maniphest/`. - Queried for close information via `maniphest.search`. Maniphest Tasks: T4434 Differential Revision: https://secure.phabricator.com/D19038 --- .../query/ManiphestTaskSearchEngine.php | 20 ++++++++++++++++++ .../maniphest/storage/ManiphestTask.php | 17 +++++++++++++++ .../maniphest/view/ManiphestTaskListView.php | 21 ++++++++++++++++--- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php index ad668db376..a5c98dc202 100644 --- a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php +++ b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php @@ -456,6 +456,15 @@ final class ManiphestTaskSearchEngine id(new PhabricatorStringExportField()) ->setKey('statusName') ->setLabel(pht('Status Name')), + id(new PhabricatorEpochExportField()) + ->setKey('dateClosed') + ->setLabel(pht('Date Closed')), + id(new PhabricatorPHIDExportField()) + ->setKey('closerPHID') + ->setLabel(pht('Closer PHID')), + id(new PhabricatorStringExportField()) + ->setKey('closer') + ->setLabel(pht('Closer')), id(new PhabricatorStringExportField()) ->setKey('priority') ->setLabel(pht('Priority')), @@ -492,6 +501,7 @@ final class ManiphestTaskSearchEngine foreach ($tasks as $task) { $phids[] = $task->getAuthorPHID(); $phids[] = $task->getOwnerPHID(); + $phids[] = $task->getCloserPHID(); } $handles = $viewer->loadHandles($phids); @@ -512,6 +522,13 @@ final class ManiphestTaskSearchEngine $owner_name = null; } + $closer_phid = $task->getCloserPHID(); + if ($closer_phid) { + $closer_name = $handles[$closer_phid]->getName(); + } else { + $closer_name = null; + } + $status_value = $task->getStatus(); $status_name = ManiphestTaskStatus::getTaskStatusName($status_value); @@ -534,6 +551,9 @@ final class ManiphestTaskSearchEngine 'title' => $task->getTitle(), 'uri' => PhabricatorEnv::getProductionURI($task->getURI()), 'description' => $task->getDescription(), + 'dateClosed' => $task->getClosedEpoch(), + 'closerPHID' => $closer_phid, + 'closer' => $closer_name, ); } diff --git a/src/applications/maniphest/storage/ManiphestTask.php b/src/applications/maniphest/storage/ManiphestTask.php index 7ff70abd80..a93fe58c3f 100644 --- a/src/applications/maniphest/storage/ManiphestTask.php +++ b/src/applications/maniphest/storage/ManiphestTask.php @@ -513,6 +513,16 @@ final class ManiphestTask extends ManiphestDAO ->setKey('subtype') ->setType('string') ->setDescription(pht('Subtype of the task.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('closerPHID') + ->setType('phid?') + ->setDescription( + pht('User who closed the task, if the task is closed.')), + id(new PhabricatorConduitSearchFieldSpecification()) + ->setKey('dateClosed') + ->setType('int?') + ->setDescription( + pht('Epoch timestamp when the task was closed.')), ); } @@ -532,6 +542,11 @@ final class ManiphestTask extends ManiphestDAO 'color' => ManiphestTaskPriority::getTaskPriorityColor($priority_value), ); + $closed_epoch = $this->getClosedEpoch(); + if ($closed_epoch !== null) { + $closed_epoch = (int)$closed_epoch; + } + return array( 'name' => $this->getTitle(), 'description' => array( @@ -543,6 +558,8 @@ final class ManiphestTask extends ManiphestDAO 'priority' => $priority_info, 'points' => $this->getPoints(), 'subtype' => $this->getSubtype(), + 'closerPHID' => $this->getCloserPHID(), + 'dateClosed' => $closed_epoch, ); } diff --git a/src/applications/maniphest/view/ManiphestTaskListView.php b/src/applications/maniphest/view/ManiphestTaskListView.php index de6b386ac8..ba17b8e25d 100644 --- a/src/applications/maniphest/view/ManiphestTaskListView.php +++ b/src/applications/maniphest/view/ManiphestTaskListView.php @@ -86,9 +86,24 @@ final class ManiphestTaskListView extends ManiphestView { $item->setStatusIcon($icon.' '.$color, $tooltip); - $item->addIcon( - 'none', - phabricator_datetime($task->getDateModified(), $this->getUser())); + if ($task->isClosed()) { + $closed_epoch = $task->getClosedEpoch(); + + // We don't expect a task to be closed without a closed epoch, but + // recover if we find one. This can happen with older objects or with + // lipsum test data. + if (!$closed_epoch) { + $closed_epoch = $task->getDateModified(); + } + + $item->addIcon( + 'fa-check-square-o grey', + phabricator_datetime($closed_epoch, $this->getUser())); + } else { + $item->addIcon( + 'none', + phabricator_datetime($task->getDateModified(), $this->getUser())); + } if ($this->showSubpriorityControls) { $item->setGrippable(true); From e26a784dcf3e2e90c05229f3d6f71dfd9594348b Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 15:13:03 -0800 Subject: [PATCH 83/89] Allow tasks to be filtered and ordered by closed date Summary: Depends on D19038. Fixes T4434. Updates the SearchEngine and Query to handle these fields. Test Plan: Filtered and ordered by date and closer. Maniphest Tasks: T4434 Differential Revision: https://secure.phabricator.com/D19039 --- .../maniphest/query/ManiphestTaskQuery.php | 49 ++++++++++++++++++- .../query/ManiphestTaskSearchEngine.php | 22 +++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php index cfc69722d8..f7c1551be7 100644 --- a/src/applications/maniphest/query/ManiphestTaskQuery.php +++ b/src/applications/maniphest/query/ManiphestTaskQuery.php @@ -23,6 +23,9 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { private $parentTaskIDs; private $subtaskIDs; private $subtypes; + private $closedEpochMin; + private $closedEpochMax; + private $closerPHIDs; private $status = 'status-any'; const STATUS_ANY = 'status-any'; @@ -179,6 +182,17 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { return $this; } + public function withClosedEpochBetween($min, $max) { + $this->closedEpochMin = $min; + $this->closedEpochMax = $max; + return $this; + } + + public function withCloserPHIDs(array $phids) { + $this->closerPHIDs = $phids; + return $this; + } + public function needSubscriberPHIDs($bool) { $this->needSubscriberPHIDs = $bool; return $this; @@ -379,6 +393,27 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { $this->dateModifiedBefore); } + if ($this->closedEpochMin !== null) { + $where[] = qsprintf( + $conn, + 'task.closedEpoch >= %d', + $this->closedEpochMin); + } + + if ($this->closedEpochMax !== null) { + $where[] = qsprintf( + $conn, + 'task.closedEpoch <= %d', + $this->closedEpochMax); + } + + if ($this->closerPHIDs !== null) { + $where[] = qsprintf( + $conn, + 'task.closerPHID IN (%Ls)', + $this->closerPHIDs); + } + if ($this->priorities !== null) { $where[] = qsprintf( $conn, @@ -722,7 +757,11 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'outdated' => array( 'vector' => array('-updated', '-id'), 'name' => pht('Date Updated (Oldest First)'), - ), + ), + 'closed' => array( + 'vector' => array('closed', 'id'), + 'name' => pht('Date Closed (Latest First)'), + ), 'title' => array( 'vector' => array('title', 'id'), 'name' => pht('Title'), @@ -741,6 +780,7 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'outdated', 'newest', 'oldest', + 'closed', 'title', )) + $orders; @@ -790,6 +830,12 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'column' => 'dateModified', 'type' => 'int', ), + 'closed' => array( + 'table' => 'task', + 'column' => 'closedEpoch', + 'type' => 'int', + 'null' => 'tail', + ), ); } @@ -808,6 +854,7 @@ final class ManiphestTaskQuery extends PhabricatorCursorPagedPolicyAwareQuery { 'status' => $task->getStatus(), 'title' => $task->getTitle(), 'updated' => $task->getDateModified(), + 'closed' => $task->getClosedEpoch(), ); foreach ($keys as $key) { diff --git a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php index a5c98dc202..565bc7a8f4 100644 --- a/src/applications/maniphest/query/ManiphestTaskSearchEngine.php +++ b/src/applications/maniphest/query/ManiphestTaskSearchEngine.php @@ -126,6 +126,17 @@ final class ManiphestTaskSearchEngine id(new PhabricatorSearchDateField()) ->setLabel(pht('Updated Before')) ->setKey('modifiedEnd'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Closed After')) + ->setKey('closedStart'), + id(new PhabricatorSearchDateField()) + ->setLabel(pht('Closed Before')) + ->setKey('closedEnd'), + id(new PhabricatorUsersSearchField()) + ->setLabel(pht('Closed By')) + ->setKey('closerPHIDs') + ->setAliases(array('closer', 'closerPHID', 'closers')) + ->setDescription(pht('Search for tasks closed by certain users.')), id(new PhabricatorSearchTextField()) ->setLabel(pht('Page Size')) ->setKey('limit'), @@ -153,6 +164,9 @@ final class ManiphestTaskSearchEngine 'createdEnd', 'modifiedStart', 'modifiedEnd', + 'closedStart', + 'closedEnd', + 'closerPHIDs', 'limit', ); } @@ -208,6 +222,14 @@ final class ManiphestTaskSearchEngine $query->withDateModifiedBefore($map['modifiedEnd']); } + if ($map['closedStart'] || $map['closedEnd']) { + $query->withClosedEpochBetween($map['closedStart'], $map['closedEnd']); + } + + if ($map['closerPHIDs']) { + $query->withCloserPHIDs($map['closerPHIDs']); + } + if ($map['hasParents'] !== null) { $query->withOpenParents($map['hasParents']); } From 6ea1b8df9bbb0e6f6f365659b6a6ff14e49168e6 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 15:52:59 -0800 Subject: [PATCH 84/89] Colorize filetree for adds, moves, and deletes Summary: See PHI356. Makes it easier to pick out change types in the filetree view in Differential. Test Plan: Created a diff with adds, copies, moves, deletions, and binary files. Viewed in Differential, had an easier time picking stuff out. Differential Revision: https://secure.phabricator.com/D19040 --- resources/celerity/map.php | 6 +-- .../storage/DifferentialChangeset.php | 45 +++++++++++++++++++ ...rentialChangesetFileTreeSideNavBuilder.php | 10 +++-- .../css/layout/phabricator-filetree-view.css | 12 +++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 17138d6604..70896d400d 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -12,7 +12,7 @@ return array( 'core.pkg.css' => 'ce8c2a58', 'core.pkg.js' => '4c79d74f', 'darkconsole.pkg.js' => '1f9a31bc', - 'differential.pkg.css' => '45951e9e', + 'differential.pkg.css' => '1522c3ad', 'differential.pkg.js' => '19ee9979', 'diffusion.pkg.css' => 'a2d17c7d', 'diffusion.pkg.js' => '6134c5a1', @@ -121,7 +121,7 @@ return array( 'rsrc/css/font/font-awesome.css' => 'e838e088', 'rsrc/css/font/font-lato.css' => 'c7ccd872', 'rsrc/css/font/phui-font-icon-base.css' => '870a7360', - 'rsrc/css/layout/phabricator-filetree-view.css' => 'fccf9f82', + 'rsrc/css/layout/phabricator-filetree-view.css' => 'ea5b30a9', 'rsrc/css/layout/phabricator-source-code-view.css' => 'aea41829', 'rsrc/css/phui/button/phui-button-bar.css' => 'f1ff5494', 'rsrc/css/phui/button/phui-button-simple.css' => '8e1baf68', @@ -784,7 +784,7 @@ return array( 'phabricator-favicon' => '1fe2510c', 'phabricator-feed-css' => 'ecd4ec57', 'phabricator-file-upload' => '680ea2c8', - 'phabricator-filetree-view-css' => 'fccf9f82', + 'phabricator-filetree-view-css' => 'ea5b30a9', 'phabricator-flag-css' => 'bba8f811', 'phabricator-keyboard-shortcut' => '1ae869f2', 'phabricator-keyboard-shortcut-manager' => 'c19dd9b9', diff --git a/src/applications/differential/storage/DifferentialChangeset.php b/src/applications/differential/storage/DifferentialChangeset.php index ebdaeacd0a..dbb06fe72b 100644 --- a/src/applications/differential/storage/DifferentialChangeset.php +++ b/src/applications/differential/storage/DifferentialChangeset.php @@ -221,6 +221,51 @@ final class DifferentialChangeset return $this->assertAttached($this->diff); } + public function newFileTreeIcon() { + $file_type = $this->getFileType(); + $change_type = $this->getChangeType(); + + $change_icons = array( + DifferentialChangeType::TYPE_DELETE => 'fa-file-o', + ); + + if (isset($change_icons[$change_type])) { + $icon = $change_icons[$change_type]; + } else { + $icon = DifferentialChangeType::getIconForFileType($file_type); + } + + $change_colors = array( + DifferentialChangeType::TYPE_ADD => 'green', + DifferentialChangeType::TYPE_DELETE => 'red', + DifferentialChangeType::TYPE_MOVE_AWAY => 'orange', + DifferentialChangeType::TYPE_MOVE_HERE => 'orange', + DifferentialChangeType::TYPE_COPY_HERE => 'orange', + DifferentialChangeType::TYPE_MULTICOPY => 'orange', + ); + + $color = idx($change_colors, $change_type, 'bluetext'); + + return id(new PHUIIconView()) + ->setIcon($icon.' '.$color); + } + + public function getFileTreeClass() { + switch ($this->getChangeType()) { + case DifferentialChangeType::TYPE_ADD: + return 'filetree-added'; + case DifferentialChangeType::TYPE_DELETE: + return 'filetree-deleted'; + case DifferentialChangeType::TYPE_MOVE_AWAY: + case DifferentialChangeType::TYPE_MOVE_HERE: + case DifferentialChangeType::TYPE_COPY_HERE: + case DifferentialChangeType::TYPE_MULTICOPY: + return 'filetree-movecopy'; + } + + return null; + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php index 14050e942c..9781fb0d02 100644 --- a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php +++ b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php @@ -83,6 +83,9 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { while (($path = $path->getNextNode())) { $data = $path->getData(); + $classes = array(); + $classes[] = 'phabricator-filetree-item'; + $name = $path->getName(); $style = 'padding-left: '.(2 + (3 * $path->getDepth())).'px'; @@ -90,8 +93,9 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { if ($data) { $href = '#'.$data->getAnchorName(); $title = $name; - $icon = id(new PHUIIconView()) - ->setIcon('fa-file-text-o bluetext'); + + $icon = $data->newFileTreeIcon(); + $classes[] = $data->getFileTreeClass(); } else { $name .= '/'; $title = $path->getFullPath().'/'; @@ -112,7 +116,7 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { 'href' => $href, 'style' => $style, 'title' => $title, - 'class' => 'phabricator-filetree-item', + 'class' => implode(' ', $classes), ), array($icon, $name_element)); } diff --git a/webroot/rsrc/css/layout/phabricator-filetree-view.css b/webroot/rsrc/css/layout/phabricator-filetree-view.css index b247f5e4f9..21bbe9f0af 100644 --- a/webroot/rsrc/css/layout/phabricator-filetree-view.css +++ b/webroot/rsrc/css/layout/phabricator-filetree-view.css @@ -54,3 +54,15 @@ background-color: {$hovergrey}; border-left: 4px solid {$sky}; } + +.phabricator-filetree .filetree-added { + background: {$sh-greenbackground}; +} + +.phabricator-filetree .filetree-deleted { + background: {$sh-redbackground}; +} + +.phabricator-filetree .filetree-movecopy { + background: {$sh-orangebackground}; +} From 261a4a0e51fd1be6f5b1a89bba620f4ef9ad7e6d Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 16:55:54 -0800 Subject: [PATCH 85/89] Add inline comment counts to the filetree view Summary: See PHI356. Adds inline comment and done counts to the filetree. Also makes the filetree wider by default. Test Plan: Fiddled with filetrees in different browsers on different revisions. Added inlines, marked them done/undone. Differential Revision: https://secure.phabricator.com/D19041 --- resources/celerity/map.php | 78 +++++++++---------- .../view/DifferentialChangesetDetailView.php | 1 + ...rentialChangesetFileTreeSideNavBuilder.php | 12 ++- .../rsrc/css/aphront/phabricator-nav-view.css | 6 +- .../css/layout/phabricator-filetree-view.css | 33 ++++++-- .../rsrc/js/application/diff/DiffChangeset.js | 74 +++++++++++++++++- .../js/application/diff/DiffChangesetList.js | 5 ++ .../rsrc/js/core/behavior-phabricator-nav.js | 4 + 8 files changed, 164 insertions(+), 49 deletions(-) diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 70896d400d..f3ead4de53 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -9,11 +9,11 @@ return array( 'names' => array( 'conpherence.pkg.css' => 'e68cf1fa', 'conpherence.pkg.js' => '15191c65', - 'core.pkg.css' => 'ce8c2a58', - 'core.pkg.js' => '4c79d74f', + 'core.pkg.css' => 'e4f098a5', + 'core.pkg.js' => '3ac6e174', 'darkconsole.pkg.js' => '1f9a31bc', - 'differential.pkg.css' => '1522c3ad', - 'differential.pkg.js' => '19ee9979', + 'differential.pkg.css' => '113e692c', + 'differential.pkg.js' => '5d53d5ce', 'diffusion.pkg.css' => 'a2d17c7d', 'diffusion.pkg.js' => '6134c5a1', 'favicon.ico' => '30672e08', @@ -31,7 +31,7 @@ return array( 'rsrc/css/aphront/multi-column.css' => '84cc6640', 'rsrc/css/aphront/notification.css' => '457861ec', 'rsrc/css/aphront/panel-view.css' => '8427b78d', - 'rsrc/css/aphront/phabricator-nav-view.css' => 'faf6a6fc', + 'rsrc/css/aphront/phabricator-nav-view.css' => '028126f6', 'rsrc/css/aphront/table-view.css' => '8c9bbafe', 'rsrc/css/aphront/tokenizer.css' => '15d5ff71', 'rsrc/css/aphront/tooltip.css' => '173b9431', @@ -121,7 +121,7 @@ return array( 'rsrc/css/font/font-awesome.css' => 'e838e088', 'rsrc/css/font/font-lato.css' => 'c7ccd872', 'rsrc/css/font/phui-font-icon-base.css' => '870a7360', - 'rsrc/css/layout/phabricator-filetree-view.css' => 'ea5b30a9', + 'rsrc/css/layout/phabricator-filetree-view.css' => 'b912ad97', 'rsrc/css/layout/phabricator-source-code-view.css' => 'aea41829', 'rsrc/css/phui/button/phui-button-bar.css' => 'f1ff5494', 'rsrc/css/phui/button/phui-button-simple.css' => '8e1baf68', @@ -395,8 +395,8 @@ return array( 'rsrc/js/application/dashboard/behavior-dashboard-move-panels.js' => '408bf173', 'rsrc/js/application/dashboard/behavior-dashboard-query-panel-select.js' => '453c5375', 'rsrc/js/application/dashboard/behavior-dashboard-tab-panel.js' => 'd4eecc63', - 'rsrc/js/application/diff/DiffChangeset.js' => '99abf4cd', - 'rsrc/js/application/diff/DiffChangesetList.js' => '3b77efdd', + 'rsrc/js/application/diff/DiffChangeset.js' => 'b49b59d6', + 'rsrc/js/application/diff/DiffChangesetList.js' => '1f2e5265', 'rsrc/js/application/diff/DiffInline.js' => 'e83d28f3', 'rsrc/js/application/diff/behavior-preview-link.js' => '051c7832', 'rsrc/js/application/differential/behavior-comment-preview.js' => '51c5ad07', @@ -498,7 +498,7 @@ return array( 'rsrc/js/core/behavior-more.js' => 'a80d0378', 'rsrc/js/core/behavior-object-selector.js' => '77c1f0b0', 'rsrc/js/core/behavior-oncopy.js' => '2926fff2', - 'rsrc/js/core/behavior-phabricator-nav.js' => '947753e0', + 'rsrc/js/core/behavior-phabricator-nav.js' => '81144dfa', 'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => 'acd29eee', 'rsrc/js/core/behavior-read-only-warning.js' => 'ba158207', 'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b', @@ -657,7 +657,7 @@ return array( 'javelin-behavior-phabricator-keyboard-pager' => 'a8da01f0', 'javelin-behavior-phabricator-keyboard-shortcuts' => '01fca1f0', 'javelin-behavior-phabricator-line-linker' => '1499a8cb', - 'javelin-behavior-phabricator-nav' => '947753e0', + 'javelin-behavior-phabricator-nav' => '81144dfa', 'javelin-behavior-phabricator-notification-example' => '8ce821c5', 'javelin-behavior-phabricator-object-selector' => '77c1f0b0', 'javelin-behavior-phabricator-oncopy' => '2926fff2', @@ -775,8 +775,8 @@ return array( 'phabricator-darklog' => 'c8e1ffe3', 'phabricator-darkmessage' => 'c48cccdd', 'phabricator-dashboard-css' => 'fe5b1869', - 'phabricator-diff-changeset' => '99abf4cd', - 'phabricator-diff-changeset-list' => '3b77efdd', + 'phabricator-diff-changeset' => 'b49b59d6', + 'phabricator-diff-changeset-list' => '1f2e5265', 'phabricator-diff-inline' => 'e83d28f3', 'phabricator-drag-and-drop-file-upload' => '58dea2fa', 'phabricator-draggable-list' => 'bea6e7f4', @@ -784,12 +784,12 @@ return array( 'phabricator-favicon' => '1fe2510c', 'phabricator-feed-css' => 'ecd4ec57', 'phabricator-file-upload' => '680ea2c8', - 'phabricator-filetree-view-css' => 'ea5b30a9', + 'phabricator-filetree-view-css' => 'b912ad97', 'phabricator-flag-css' => 'bba8f811', 'phabricator-keyboard-shortcut' => '1ae869f2', 'phabricator-keyboard-shortcut-manager' => 'c19dd9b9', 'phabricator-main-menu-view' => '1802a242', - 'phabricator-nav-view-css' => 'faf6a6fc', + 'phabricator-nav-view-css' => '028126f6', 'phabricator-notification' => '008faf9c', 'phabricator-notification-css' => '457861ec', 'phabricator-notification-menu-css' => '10685bd4', @@ -1044,6 +1044,10 @@ return array( 'javelin-uri', 'javelin-routable', ), + '1f2e5265' => array( + 'javelin-install', + 'phuix-button-view', + ), '1f6794f6' => array( 'javelin-behavior', 'javelin-stratcom', @@ -1143,10 +1147,6 @@ return array( 'javelin-dom', 'javelin-magical-init', ), - '3b77efdd' => array( - 'javelin-install', - 'phuix-button-view', - ), '3cb0b2fc' => array( 'javelin-behavior', 'javelin-dom', @@ -1561,6 +1561,16 @@ return array( '7f243deb' => array( 'javelin-install', ), + '81144dfa' => array( + 'javelin-behavior', + 'javelin-behavior-device', + 'javelin-stratcom', + 'javelin-dom', + 'javelin-magical-init', + 'javelin-vector', + 'javelin-request', + 'javelin-util', + ), '834a1173' => array( 'javelin-behavior', 'javelin-scrollbar', @@ -1648,16 +1658,6 @@ return array( 'javelin-workflow', 'javelin-dom', ), - '947753e0' => array( - 'javelin-behavior', - 'javelin-behavior-device', - 'javelin-stratcom', - 'javelin-dom', - 'javelin-magical-init', - 'javelin-vector', - 'javelin-request', - 'javelin-util', - ), '949c0fe5' => array( 'javelin-install', ), @@ -1678,17 +1678,6 @@ return array( 'javelin-mask', 'phabricator-drag-and-drop-file-upload', ), - '99abf4cd' => array( - 'javelin-dom', - 'javelin-util', - 'javelin-stratcom', - 'javelin-install', - 'javelin-workflow', - 'javelin-router', - 'javelin-behavior-device', - 'javelin-vector', - 'phabricator-diff-inline', - ), '9a6dd75c' => array( 'javelin-behavior', 'javelin-stratcom', @@ -1837,6 +1826,17 @@ return array( 'b3e7d692' => array( 'javelin-install', ), + 'b49b59d6' => array( + 'javelin-dom', + 'javelin-util', + 'javelin-stratcom', + 'javelin-install', + 'javelin-workflow', + 'javelin-router', + 'javelin-behavior-device', + 'javelin-vector', + 'phabricator-diff-inline', + ), 'b59e1e96' => array( 'javelin-behavior', 'javelin-stratcom', diff --git a/src/applications/differential/view/DifferentialChangesetDetailView.php b/src/applications/differential/view/DifferentialChangesetDetailView.php index d4a13745dc..cb697c2e9d 100644 --- a/src/applications/differential/view/DifferentialChangesetDetailView.php +++ b/src/applications/differential/view/DifferentialChangesetDetailView.php @@ -206,6 +206,7 @@ final class DifferentialChangesetDetailView extends AphrontView { 'displayPath' => hsprintf('%s', $display_parts), 'path' => $display_filename, 'icon' => $display_icon, + 'treeNodeID' => 'tree-node-'.$changeset->getAnchorName(), ), 'class' => $class, 'id' => $id, diff --git a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php index 9781fb0d02..1f699be8eb 100644 --- a/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php +++ b/src/applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php @@ -96,11 +96,20 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { $icon = $data->newFileTreeIcon(); $classes[] = $data->getFileTreeClass(); + + $count = phutil_tag( + 'span', + array( + 'class' => 'filetree-progress-hint', + 'id' => 'tree-node-'.$data->getAnchorName(), + )); } else { $name .= '/'; $title = $path->getFullPath().'/'; $icon = id(new PHUIIconView()) ->setIcon('fa-folder-open blue'); + + $count = null; } $name_element = phutil_tag( @@ -110,6 +119,7 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { ), $name); + $filetree[] = javelin_tag( $href ? 'a' : 'span', array( @@ -118,7 +128,7 @@ final class DifferentialChangesetFileTreeSideNavBuilder extends Phobject { 'title' => $title, 'class' => implode(' ', $classes), ), - array($icon, $name_element)); + array($count, $icon, $name_element)); } $tree->destroy(); diff --git a/webroot/rsrc/css/aphront/phabricator-nav-view.css b/webroot/rsrc/css/aphront/phabricator-nav-view.css index e8081a55e6..f3320e3eae 100644 --- a/webroot/rsrc/css/aphront/phabricator-nav-view.css +++ b/webroot/rsrc/css/aphront/phabricator-nav-view.css @@ -44,7 +44,7 @@ position: fixed; top: 0; bottom: 0; - left: 205px; + left: 410px; width: 7px; cursor: col-resize; @@ -66,7 +66,7 @@ .device-desktop .phabricator-standard-page-body .has-drag-nav .phabricator-nav-content { - margin-left: 212px; + margin-left: 417px; } .device-desktop .phabricator-standard-page-body .has-drag-nav @@ -81,7 +81,7 @@ } .device-desktop .phui-navigation-shell .has-drag-nav .phabricator-nav-local { - width: 205px; + width: 410px; padding: 0; background: transparent; } diff --git a/webroot/rsrc/css/layout/phabricator-filetree-view.css b/webroot/rsrc/css/layout/phabricator-filetree-view.css index 21bbe9f0af..6497c37056 100644 --- a/webroot/rsrc/css/layout/phabricator-filetree-view.css +++ b/webroot/rsrc/css/layout/phabricator-filetree-view.css @@ -50,11 +50,6 @@ background-color: {$hovergrey}; } -.phabricator-filetree .phabricator-active-nav-focus { - background-color: {$hovergrey}; - border-left: 4px solid {$sky}; -} - .phabricator-filetree .filetree-added { background: {$sh-greenbackground}; } @@ -66,3 +61,31 @@ .phabricator-filetree .filetree-movecopy { background: {$sh-orangebackground}; } + +.phabricator-filetree .phabricator-active-nav-focus { + background-color: {$hovergrey}; + border-left: 4px solid {$sky}; +} + +.phabricator-filetree .filetree-progress-hint { + width: 24px; + margin-right: 6px; + display: inline-block; + padding: 0 4px; + border-radius: 4px; + font-size: smaller; + background: {$greybackground}; + text-align: center; + opacity: 0.5; +} + +.phabricator-filetree .filetree-comments-visible { + background: {$lightblue}; + opacity: 0.75; + color: {$darkgreytext}; +} + +.phabricator-filetree .filetree-comments-completed { + background: {$darkgreybackground}; + color: {$greytext}; +} diff --git a/webroot/rsrc/js/application/diff/DiffChangeset.js b/webroot/rsrc/js/application/diff/DiffChangeset.js index 72eeae294a..24d734573d 100644 --- a/webroot/rsrc/js/application/diff/DiffChangeset.js +++ b/webroot/rsrc/js/application/diff/DiffChangeset.js @@ -27,6 +27,7 @@ JX.install('DiffChangeset', { this._highlight = data.highlight; this._encoding = data.encoding; this._loaded = data.loaded; + this._treeNodeID = data.treeNodeID; this._leftID = data.left; this._rightID = data.right; @@ -62,6 +63,7 @@ JX.install('DiffChangeset', { _changesetList: null, _icon: null, + _treeNodeID: null, getLeftChangesetID: function() { return this._leftID; @@ -737,7 +739,8 @@ JX.install('DiffChangeset', { _rebuildAllInlines: function() { var rows = JX.DOM.scry(this._node, 'tr'); - for (var ii = 0; ii < rows.length; ii++) { + var ii; + for (ii = 0; ii < rows.length; ii++) { var row = rows[ii]; if (this._getRowType(row) != 'comment') { continue; @@ -749,6 +752,75 @@ JX.install('DiffChangeset', { } }, + redrawFileTree: function() { + var tree; + try { + tree = JX.$(this._treeNodeID); + } catch (e) { + return; + } + + var inlines = this._inlines; + var done = []; + var undone = []; + var inline; + + for (var ii = 0; ii < inlines.length; ii++) { + inline = inlines[ii]; + + if (inline.isDeleted()) { + continue; + } + + if (inline.isSynthetic()) { + continue; + } + + if (inline.isEditing()) { + continue; + } + + if (!inline.getID()) { + // These are new comments which have been cancelled, and do not + // count as anything. + continue; + } + + if (inline.isDraft()) { + continue; + } + + if (!inline.isDone()) { + undone.push(inline); + } else { + done.push(inline); + } + } + + var total = done.length + undone.length; + + var hint; + var is_visible; + var is_completed; + if (total) { + if (done.length) { + hint = [done.length, '/', total]; + } else { + hint = total; + } + is_visible = true; + is_completed = (done.length == total); + } else { + hint = '-'; + is_visible = false; + is_completed = false; + } + + JX.DOM.setContent(tree, hint); + JX.DOM.alterClass(tree, 'filetree-comments-visible', is_visible); + JX.DOM.alterClass(tree, 'filetree-comments-completed', is_completed); + }, + toggleVisibility: function() { this._visible = !this._visible; diff --git a/webroot/rsrc/js/application/diff/DiffChangesetList.js b/webroot/rsrc/js/application/diff/DiffChangesetList.js index ec0270ac12..e62d2f51dd 100644 --- a/webroot/rsrc/js/application/diff/DiffChangesetList.js +++ b/webroot/rsrc/js/application/diff/DiffChangesetList.js @@ -915,6 +915,11 @@ JX.install('DiffChangesetList', { this._bannerChangeset = null; this._redrawBanner(); + + var changesets = this._changesets; + for (var ii = 0; ii < changesets.length; ii++) { + changesets[ii].redrawFileTree(); + } }, _onscroll: function() { diff --git a/webroot/rsrc/js/core/behavior-phabricator-nav.js b/webroot/rsrc/js/core/behavior-phabricator-nav.js index e37680abf0..74909e447d 100644 --- a/webroot/rsrc/js/core/behavior-phabricator-nav.js +++ b/webroot/rsrc/js/core/behavior-phabricator-nav.js @@ -28,6 +28,10 @@ JX.behavior('phabricator-nav', function(config) { JX.enableDispatch(document.body, 'mousemove'); JX.DOM.listen(drag, 'mousedown', null, function(e) { + if (!e.isNormalMouseEvent()) { + return; + } + dragging = JX.$V(e); // Show the "col-resize" cursor on the whole document while we're From 7d4362690f1c8e616fd9ff6d917f676f163918f6 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:26:01 -0800 Subject: [PATCH 86/89] Fix transposed name/email in Mailgun adapter Summary: Ref T12677. This argument order was swapped. Test Plan: Will push/verify. Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19042 --- .../adapter/PhabricatorMailImplementationMailgunAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php index 12c54e0d6a..349dae2d27 100644 --- a/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php +++ b/src/applications/metamta/adapter/PhabricatorMailImplementationMailgunAdapter.php @@ -21,7 +21,7 @@ final class PhabricatorMailImplementationMailgunAdapter if (empty($this->params['reply-to'])) { $this->params['reply-to'] = array(); } - $this->params['reply-to'][] = $this->renderAddress($name, $email); + $this->params['reply-to'][] = $this->renderAddress($email, $name); return $this; } From 09b446b269f5cec0ba154872fddc354db44fd5e1 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:48:59 -0800 Subject: [PATCH 87/89] Don't run older mail setup checks if "cluster.mailers" is configured Summary: Ref T12677. Skip these checks if we're doing the new stuff. Also, allow priority to be unspecified. Test Plan: Will deploy. Maniphest Tasks: T12677 Differential Revision: https://secure.phabricator.com/D19043 --- src/applications/config/check/PhabricatorMailSetupCheck.php | 4 ++++ .../cluster/config/PhabricatorClusterMailersConfigType.php | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/applications/config/check/PhabricatorMailSetupCheck.php b/src/applications/config/check/PhabricatorMailSetupCheck.php index 2b8e4e12d5..b3b6143ad0 100644 --- a/src/applications/config/check/PhabricatorMailSetupCheck.php +++ b/src/applications/config/check/PhabricatorMailSetupCheck.php @@ -7,6 +7,10 @@ final class PhabricatorMailSetupCheck extends PhabricatorSetupCheck { } protected function executeChecks() { + if (PhabricatorEnv::getEnvConfig('cluster.mailers')) { + return; + } + $adapter = PhabricatorEnv::getEnvConfig('metamta.mail-adapter'); switch ($adapter) { diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php index 2a7550c419..b3b110298f 100644 --- a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -63,8 +63,8 @@ final class PhabricatorClusterMailersConfigType } $map[$key] = true; - $priority = idx($spec, 'priority', 0); - if ($priority <= 0) { + $priority = idx($spec, 'priority'); + if ($priority !== null && $priority <= 0) { throw $this->newException( pht( 'Mailer configuration ("%s") is invalid: priority must be '. From d45952344bd020b196c635b158fdcc905803865a Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:55:02 -0800 Subject: [PATCH 88/89] Use setOptions() to trigger mailer option validation, not validateOptions() --- .../cluster/config/PhabricatorClusterMailersConfigType.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php index b3b110298f..60547eea44 100644 --- a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -85,7 +85,7 @@ final class PhabricatorClusterMailersConfigType $options = idx($spec, 'options', array()); try { - id(clone $adapters[$type])->validateOptions($options); + id(clone $adapters[$type])->setOptions($options); } catch (Exception $ex) { throw $this->newException( pht( From 8de794d3c28a11f1a30e93e7bd74e71031de6ab5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 8 Feb 2018 17:58:14 -0800 Subject: [PATCH 89/89] Make optional options actually optional in cluster mailer config validation --- .../cluster/config/PhabricatorClusterMailersConfigType.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php index 60547eea44..03f30506bd 100644 --- a/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php +++ b/src/infrastructure/cluster/config/PhabricatorClusterMailersConfigType.php @@ -85,6 +85,8 @@ final class PhabricatorClusterMailersConfigType $options = idx($spec, 'options', array()); try { + $defaults = $adapters[$type]->newDefaultOptions(); + $options = $options + $defaults; id(clone $adapters[$type])->setOptions($options); } catch (Exception $ex) { throw $this->newException(