From 25e7b7d53cd5900722771615ce13bd2b3974b5b3 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Sat, 7 Dec 2013 13:37:18 +1100 Subject: [PATCH] Implement support for creating and updating fragments from ZIPs Summary: This implements support for creating and updating fragments from ZIP files. It allows you to upload a ZIP via the Files application, create a fragment from it, and have it recursively imported into Phragment. Updating that folder with another ZIP will recursively create, update and delete files as appropriate. The logic for creating and updating fragments from files has also been centralized into the PhragmentFragment class. Directories are also now supported; a directory fragment is simply a fragment that has no patches; thus a directory fragment can be converted to a file fragment by uploading a first patch for it. Test Plan: Uploaded ZIP files through the interface and saw all of the fragments get created and updated as expected. Reviewers: epriestley, #blessed_reviewers Reviewed By: epriestley CC: Korvin, epriestley, aran Maniphest Tasks: T4205 Differential Revision: https://secure.phabricator.com/D7729 --- .../sql/patches/20131206.phragmentnull.sql | 2 + .../controller/PhragmentBrowseController.php | 20 +- .../controller/PhragmentController.php | 51 +++-- .../controller/PhragmentCreateController.php | 21 +- .../controller/PhragmentUpdateController.php | 24 +-- .../query/PhragmentFragmentQuery.php | 1 - .../phragment/storage/PhragmentFragment.php | 202 ++++++++++++++++++ .../patch/PhabricatorBuiltinPatchList.php | 4 + 8 files changed, 270 insertions(+), 55 deletions(-) create mode 100644 resources/sql/patches/20131206.phragmentnull.sql diff --git a/resources/sql/patches/20131206.phragmentnull.sql b/resources/sql/patches/20131206.phragmentnull.sql new file mode 100644 index 0000000000..01dbd9bfe3 --- /dev/null +++ b/resources/sql/patches/20131206.phragmentnull.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_phragment.phragment_fragment +MODIFY latestVersionPHID VARCHAR(64) NULL; diff --git a/src/applications/phragment/controller/PhragmentBrowseController.php b/src/applications/phragment/controller/PhragmentBrowseController.php index 551a39b07e..adb4c0d796 100644 --- a/src/applications/phragment/controller/PhragmentBrowseController.php +++ b/src/applications/phragment/controller/PhragmentBrowseController.php @@ -57,14 +57,18 @@ final class PhragmentBrowseController extends PhragmentController { $item = id(new PHUIObjectItemView()); $item->setHeader($fragment->getName()); $item->setHref($this->getApplicationURI('/browse/'.$fragment->getPath())); - $item->addAttribute(pht( - 'Last Updated %s', - phabricator_datetime( - $fragment->getLatestVersion()->getDateCreated(), - $viewer))); - $item->addAttribute(pht( - 'Latest Version %s', - $fragment->getLatestVersion()->getSequence())); + if (!$fragment->isDirectory()) { + $item->addAttribute(pht( + 'Last Updated %s', + phabricator_datetime( + $fragment->getLatestVersion()->getDateCreated(), + $viewer))); + $item->addAttribute(pht( + 'Latest Version %s', + $fragment->getLatestVersion()->getSequence())); + } else { + $item->addAttribute('Directory'); + } $list->addItem($item); } diff --git a/src/applications/phragment/controller/PhragmentController.php b/src/applications/phragment/controller/PhragmentController.php index 22eab7fddb..d156745f4b 100644 --- a/src/applications/phragment/controller/PhragmentController.php +++ b/src/applications/phragment/controller/PhragmentController.php @@ -66,13 +66,16 @@ abstract class PhragmentController extends PhabricatorController { $this->loadHandles($phids); - $file = id(new PhabricatorFileQuery()) - ->setViewer($viewer) - ->withPHIDs(array($fragment->getLatestVersion()->getFilePHID())) - ->executeOne(); + $file = null; $file_uri = null; - if ($file !== null) { - $file_uri = $file->getBestURI(); + if (!$fragment->isDirectory()) { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($fragment->getLatestVersion()->getFilePHID())) + ->executeOne(); + if ($file !== null) { + $file_uri = $file->getBestURI(); + } } $header = id(new PHUIHeaderView()) @@ -96,12 +99,21 @@ abstract class PhragmentController extends PhabricatorController { ->setHref($this->getApplicationURI("zip/".$fragment->getPath())) ->setDisabled(false) // TODO: Policy ->setIcon('zip')); - $actions->addAction( - id(new PhabricatorActionView()) - ->setName(pht('Update Fragment')) - ->setHref($this->getApplicationURI("update/".$fragment->getPath())) - ->setDisabled(false) // TODO: Policy - ->setIcon('edit')); + if (!$fragment->isDirectory()) { + $actions->addAction( + id(new PhabricatorActionView()) + ->setName(pht('Update Fragment')) + ->setHref($this->getApplicationURI("update/".$fragment->getPath())) + ->setDisabled(false) // TODO: Policy + ->setIcon('edit')); + } else { + $actions->addAction( + id(new PhabricatorActionView()) + ->setName(pht('Convert to File')) + ->setHref($this->getApplicationURI("update/".$fragment->getPath())) + ->setDisabled(false) // TODO: Policy + ->setIcon('edit')); + } if ($is_history_view) { $actions->addAction( id(new PhabricatorActionView()) @@ -121,9 +133,18 @@ abstract class PhragmentController extends PhabricatorController { ->setObject($fragment) ->setActionList($actions); - $properties->addProperty( - pht('Latest Version'), - $this->renderHandlesForPHIDs(array($fragment->getLatestVersionPHID()))); + if (!$fragment->isDirectory()) { + $properties->addProperty( + pht('Type'), + pht('File')); + $properties->addProperty( + pht('Latest Version'), + $this->renderHandlesForPHIDs(array($fragment->getLatestVersionPHID()))); + } else { + $properties->addProperty( + pht('Type'), + pht('Directory')); + } return id(new PHUIObjectBoxView()) ->setHeader($header) diff --git a/src/applications/phragment/controller/PhragmentCreateController.php b/src/applications/phragment/controller/PhragmentCreateController.php index 388db986f0..8ae0ef1ccb 100644 --- a/src/applications/phragment/controller/PhragmentCreateController.php +++ b/src/applications/phragment/controller/PhragmentCreateController.php @@ -54,21 +54,12 @@ final class PhragmentCreateController extends PhragmentController { $depth = $parent->getDepth() + 1; } - $version = id(new PhragmentFragmentVersion()); - $version->setSequence(0); - $version->setFragmentPHID(''); // Can't set this yet... - $version->setFilePHID($file->getPHID()); - $version->save(); - - $fragment->setPath(trim($parent_path.'/'.$v_name, '/')); - $fragment->setDepth($depth); - $fragment->setLatestVersionPHID($version->getPHID()); - $fragment->setViewPolicy($v_viewpolicy); - $fragment->setEditPolicy($v_editpolicy); - $fragment->save(); - - $version->setFragmentPHID($fragment->getPHID()); - $version->save(); + PhragmentFragment::createFromFile( + $viewer, + $file, + trim($parent_path.'/'.$v_name, '/'), + $v_viewpolicy, + $v_editpolicy); return id(new AphrontRedirectResponse()) ->setURI('/phragment/browse/'.trim($parent_path.'/'.$v_name, '/')); diff --git a/src/applications/phragment/controller/PhragmentUpdateController.php b/src/applications/phragment/controller/PhragmentUpdateController.php index 14a7a5d685..ea9f3b6c59 100644 --- a/src/applications/phragment/controller/PhragmentUpdateController.php +++ b/src/applications/phragment/controller/PhragmentUpdateController.php @@ -31,22 +31,14 @@ final class PhragmentUpdateController extends PhragmentController { } if (!count($errors)) { - $existing = id(new PhragmentFragmentVersionQuery()) - ->setViewer($viewer) - ->withFragmentPHIDs(array($fragment->getPHID())) - ->execute(); - $sequence = count($existing); - - $fragment->openTransaction(); - $version = id(new PhragmentFragmentVersion()); - $version->setSequence($sequence); - $version->setFragmentPHID($fragment->getPHID()); - $version->setFilePHID($file->getPHID()); - $version->save(); - - $fragment->setLatestVersionPHID($version->getPHID()); - $fragment->save(); - $fragment->saveTransaction(); + // If the file is a ZIP archive (has application/zip mimetype) + // then we extract the zip and apply versions for each of the + // individual fragments, creating and deleting files as needed. + if ($file->getMimeType() === "application/zip") { + $fragment->updateFromZIP($viewer, $file); + } else { + $fragment->updateFromFile($viewer, $file); + } return id(new AphrontRedirectResponse()) ->setURI('/phragment/browse/'.$fragment->getPath()); diff --git a/src/applications/phragment/query/PhragmentFragmentQuery.php b/src/applications/phragment/query/PhragmentFragmentQuery.php index 15f5e75643..30dfa6c9e5 100644 --- a/src/applications/phragment/query/PhragmentFragmentQuery.php +++ b/src/applications/phragment/query/PhragmentFragmentQuery.php @@ -115,7 +115,6 @@ final class PhragmentFragmentQuery foreach ($page as $key => $fragment) { $version_phid = $fragment->getLatestVersionPHID(); if (empty($versions[$version_phid])) { - unset($page[$key]); continue; } $fragment->attachLatestVersion($versions[$version_phid]); diff --git a/src/applications/phragment/storage/PhragmentFragment.php b/src/applications/phragment/storage/PhragmentFragment.php index c7f5e3fb68..4efd76d083 100644 --- a/src/applications/phragment/storage/PhragmentFragment.php +++ b/src/applications/phragment/storage/PhragmentFragment.php @@ -38,7 +38,14 @@ final class PhragmentFragment extends PhragmentDAO return $this->file = $file; } + public function isDirectory() { + return $this->latestVersionPHID === null; + } + public function getLatestVersion() { + if ($this->latestVersionPHID === null) { + return null; + } return $this->assertAttached($this->latestVersion); } @@ -46,6 +53,201 @@ final class PhragmentFragment extends PhragmentDAO return $this->latestVersion = $version; } + +/* -( Updating ) --------------------------------------------------------- */ + + + /** + * Create a new fragment from a file. + */ + public static function createFromFile( + PhabricatorUser $viewer, + PhabricatorFile $file = null, + $path, + $view_policy, + $edit_policy) { + + $fragment = id(new PhragmentFragment()); + $fragment->setPath($path); + $fragment->setDepth(count(explode('/', $path))); + $fragment->setLatestVersionPHID(null); + $fragment->setViewPolicy($view_policy); + $fragment->setEditPolicy($edit_policy); + $fragment->save(); + + // Directory fragments have no versions associated with them, so we + // just return the fragment at this point. + if ($file === null) { + return $fragment; + } + + if ($file->getMimeType() === "application/zip") { + $fragment->updateFromZIP($viewer, $file); + } else { + $fragment->updateFromFile($viewer, $file); + } + + return $fragment; + } + + + /** + * Set the specified file as the next version for the fragment. + */ + public function updateFromFile( + PhabricatorUser $viewer, + PhabricatorFile $file) { + + $existing = id(new PhragmentFragmentVersionQuery()) + ->setViewer($viewer) + ->withFragmentPHIDs(array($this->getPHID())) + ->execute(); + $sequence = count($existing); + + $this->openTransaction(); + $version = id(new PhragmentFragmentVersion()); + $version->setSequence($sequence); + $version->setFragmentPHID($this->getPHID()); + $version->setFilePHID($file->getPHID()); + $version->save(); + + $this->setLatestVersionPHID($version->getPHID()); + $this->save(); + $this->saveTransaction(); + } + + /** + * Apply the specified ZIP archive onto the fragment, removing + * and creating fragments as needed. + */ + public function updateFromZIP( + PhabricatorUser $viewer, + PhabricatorFile $file) { + + if ($file->getMimeType() !== "application/zip") { + throw new Exception("File must have mimetype 'application/zip'"); + } + + // First apply the ZIP as normal. + $this->updateFromFile($viewer, $file); + + // Ensure we have ZIP support. + $zip = null; + try { + $zip = new ZipArchive(); + } catch (Exception $e) { + // The server doesn't have php5-zip, so we can't do recursive updates. + return; + } + + $temp = new TempFile(); + Filesystem::writeFile($temp, $file->loadFileData()); + if (!$zip->open($temp)) { + throw new Exception("Unable to open ZIP"); + } + + // Get all of the paths and their data from the ZIP. + $mappings = array(); + for ($i = 0; $i < $zip->numFiles; $i++) { + $path = trim($zip->getNameIndex($i), '/'); + $stream = $zip->getStream($path); + $data = null; + // If the stream is false, then it is a directory entry. We leave + // $data set to null for directories so we know not to create a + // version entry for them. + if ($stream !== false) { + $data = stream_get_contents($stream); + fclose($stream); + } + $mappings[$path] = $data; + } + + // Adjust the paths relative to this fragment so we can look existing + // fragments up in the DB. + $base_path = $this->getPath(); + $paths = array(); + foreach ($mappings as $p => $data) { + $paths[] = $base_path.'/'.$p; + } + + // FIXME: What happens when a child exists, but the current user + // can't see it. We're going to create a new child with the exact + // same path and then bad things will happen. + $children = id(new PhragmentFragmentQuery()) + ->setViewer($viewer) + ->needLatestVersion(true) + ->withPaths($paths) + ->execute(); + $children = mpull($children, null, 'getPath'); + + // Iterate over the existing fragments. + foreach ($children as $full_path => $child) { + $path = substr($full_path, strlen($base_path) + 1); + if (array_key_exists($path, $mappings)) { + if ($child->isDirectory() && $mappings[$path] === null) { + // Don't create a version entry for a directory + // (unless it's been converted into a file). + continue; + } + + // The file is being updated. + $file = PhabricatorFile::newFromFileData( + $mappings[$path], + array('name' => basename($path))); + $child->updateFromFile($viewer, $file); + } else { + // The file is being deleted. + $child->deleteFile($viewer); + } + } + + // Iterate over the mappings to find new files. + foreach ($mappings as $path => $data) { + if (!array_key_exists($base_path.'/'.$path, $children)) { + // The file is being created. If the data is null, + // then this is explicitly a directory being created. + $file = null; + if ($mappings[$path] !== null) { + $file = PhabricatorFile::newFromFileData( + $mappings[$path], + array('name' => basename($path))); + } + PhragmentFragment::createFromFile( + $viewer, + $file, + $base_path.'/'.$path, + $this->getViewPolicy(), + $this->getEditPolicy()); + } + } + } + + /** + * Delete the contents of the specified fragment. + */ + public function deleteFile(PhabricatorUser $viewer) { + $existing = id(new PhragmentFragmentVersionQuery()) + ->setViewer($viewer) + ->withFragmentPHIDs(array($this->getPHID())) + ->execute(); + $sequence = count($existing); + + $this->openTransaction(); + $version = id(new PhragmentFragmentVersion()); + $version->setSequence($sequence); + $version->setFragmentPHID($this->getPHID()); + $version->setFilePHID(null); + $version->save(); + + $this->setLatestVersionPHID($version->getPHID()); + $this->save(); + $this->saveTransaction(); + } + + +/* -( Policy Interface )--------------------------------------------------- */ + + public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, diff --git a/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php b/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php index 84b67a5323..7b2d30f0e5 100644 --- a/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php +++ b/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php @@ -1820,6 +1820,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList { 'type' => 'sql', 'name' => $this->getPatchPath('20131206.phragment.sql'), ), + '20131206.phragmentnull.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('20131206.phragmentnull.sql'), + ), ); } }