diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index cd892ee172..ef3dd661a7 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -475,6 +475,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryEditLocalController' => 'applications/diffusion/controller/DiffusionRepositoryEditLocalController.php', 'DiffusionRepositoryEditMainController' => 'applications/diffusion/controller/DiffusionRepositoryEditMainController.php', 'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php', + 'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php', 'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php', 'DiffusionRepositoryNewController' => 'applications/diffusion/controller/DiffusionRepositoryNewController.php', 'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php', @@ -3223,6 +3224,7 @@ phutil_register_library_map(array( 'DiffusionRepositoryEditLocalController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditMainController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController', + 'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryListController' => 'DiffusionController', 'DiffusionRepositoryNewController' => 'DiffusionController', 'DiffusionRepositoryRef' => 'Phobject', diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php index fff226e662..71f29ed4bf 100644 --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -92,6 +92,7 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication { 'delete/' => 'DiffusionRepositoryEditDeleteController', 'hosting/' => 'DiffusionRepositoryEditHostingController', '(?Pserve)/' => 'DiffusionRepositoryEditHostingController', + 'update/' => 'DiffusionRepositoryEditUpdateController', ), 'pathtree/(?P.*)' => 'DiffusionPathTreeController', 'mirror/' => array( diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php index a451aaf711..8cf00006f8 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php @@ -207,6 +207,14 @@ final class DiffusionRepositoryEditMainController ->setHref($this->getRepositoryControllerURI($repository, 'edit/basic/')); $view->addAction($edit); + $edit = id(new PhabricatorActionView()) + ->setIcon('fa-refresh') + ->setName(pht('Update Now')) + ->setWorkflow(true) + ->setHref( + $this->getRepositoryControllerURI($repository, 'edit/update/')); + $view->addAction($edit); + $activate = id(new PhabricatorActionView()) ->setHref( $this->getRepositoryControllerURI($repository, 'edit/activate/')) @@ -280,6 +288,10 @@ final class DiffusionRepositoryEditMainController pht('Status'), $this->buildRepositoryStatus($repository)); + $view->addProperty( + pht('Update Frequency'), + $this->buildRepositoryUpdateInterval($repository)); + $description = $repository->getDetail('description'); $view->addSectionHeader(pht('Description')); if (!strlen($description)) { @@ -942,14 +954,16 @@ final class DiffusionRepositoryEditMainController ->setNote($message->getParameter('message'))); return $view; case PhabricatorRepositoryStatusMessage::CODE_OKAY: + $ago = (PhabricatorTime::getNow() - $message->getEpoch()); $view->addItem( id(new PHUIStatusItemView()) ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') ->setTarget(pht('Updates OK')) ->setNote( pht( - 'Last updated %s.', - phabricator_datetime($message->getEpoch(), $viewer)))); + 'Last updated %s (%s ago).', + phabricator_datetime($message->getEpoch(), $viewer), + phutil_format_relative_time_detailed($ago)))); break; } } else { @@ -1020,12 +1034,34 @@ final class DiffusionRepositoryEditMainController id(new PHUIStatusItemView()) ->setIcon(PHUIStatusItemView::ICON_UP, 'indigo') ->setTarget(pht('Prioritized')) - ->setNote(pht('This repository will be updated soon.'))); + ->setNote(pht('This repository will be updated soon!'))); } return $view; } + private function buildRepositoryUpdateInterval( + PhabricatorRepository $repository) { + + $smart_wait = $repository->loadUpdateInterval(); + + $doc_href = PhabricatorEnv::getDoclink( + 'Diffusion User Guide: Repository Updates'); + + return array( + phutil_format_relative_time_detailed($smart_wait), + " \xC2\xB7 ", + phutil_tag( + 'a', + array( + 'href' => $doc_href, + 'target' => '_blank', + ), + pht('Learn More')), + ); + } + + private function buildMirrorActions( PhabricatorRepository $repository) { diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php new file mode 100644 index 0000000000..eb8115e0f3 --- /dev/null +++ b/src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php @@ -0,0 +1,75 @@ +getRequest(); + $viewer = $request->getUser(); + $drequest = $this->diffusionRequest; + $repository = $drequest->getRepository(); + + $repository = id(new PhabricatorRepositoryQuery()) + ->setViewer($viewer) + ->requireCapabilities( + array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + )) + ->withIDs(array($repository->getID())) + ->executeOne(); + if (!$repository) { + return new Aphront404Response(); + } + + $edit_uri = $this->getRepositoryControllerURI($repository, 'edit/'); + + if ($request->isFormPost()) { + $params = array( + 'callsigns' => array( + $repository->getCallsign(), + ), + ); + + id(new ConduitCall('diffusion.looksoon', $params)) + ->setUser($viewer) + ->execute(); + + return id(new AphrontRedirectResponse())->setURI($edit_uri); + } + + $doc_name = 'Diffusion User Guide: Repository Updates'; + $doc_href = PhabricatorEnv::getDoclink($doc_name); + $doc_link = phutil_tag( + 'a', + array( + 'href' => $doc_href, + 'target' => '_blank', + ), + $doc_name); + + return $this->newDialog() + ->setTitle(pht('Update Repository Now')) + ->appendParagraph( + pht( + 'Normally, Phabricator automatically updates repositories '. + 'based on how much time has elapsed since the last commit. '. + 'This helps reduce load if you have a large number of mostly '. + 'inactive repositories, which is common.')) + ->appendParagraph( + pht( + 'You can manually schedule an update for this repository. The '. + 'daemons will perform the update as soon as possible. This may '. + 'be helpful if you have just made a commit to a rarely used '. + 'repository.')) + ->appendParagraph( + pht( + 'To learn more about how Phabricator updates repositories, '. + 'read %s in the documentation.', + $doc_link)) + ->addCancelButton($edit_uri) + ->addSubmitButton(pht('Schedule Update')); + } + + +} diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php index ceadbe60bf..1cac11c74d 100644 --- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php +++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php @@ -345,55 +345,23 @@ final class PhabricatorRepositoryPullLocalDaemon phlog($stderr_msg); } - $sleep_for = (int)$repository->getDetail('pull-frequency', $min_sleep); + // For now, continue respecting this deprecated setting for raising the + // minimum pull frequency. + // TODO: Remove this some day once this code has been completely stable + // for a while. + $sleep_for = (int)$repository->getDetail('pull-frequency'); + $min_sleep = max($sleep_for, $min_sleep); - // Smart wait: pull rarely used repositories less frequently. Find the - // most recent commit which is older than the current time (this keeps us - // from spinning on repositories with a silly commit post-dated to some time - // in 2037), and adjust how frequently we pull based on how frequently this - // repository updates. + $smart_wait = $repository->loadUpdateInterval($min_sleep); - $table = id(new PhabricatorRepositoryCommit()); - $last_commit = queryfx_one( - $table->establishConnection('w'), - 'SELECT epoch FROM %T - WHERE repositoryID = %d AND epoch <= %d - ORDER BY epoch DESC LIMIT 1', - $table->getTableName(), - $repository->getID(), - time() + $min_sleep); - if ($last_commit) { - $time_since_commit = (time() + $min_sleep) - $last_commit['epoch']; + $this->log( + pht( + 'Based on activity in repository "%s", considering a wait of %s '. + 'seconds before update.', + $repository->getMonogram(), + new PhutilNumber($smart_wait))); - // Wait 0.5% of the time since the last commit before we pull. This gives - // us these wait times: - // - // 50 minutes or less: 15 seconds - // about 3 hours: 1 minute - // about 16 hours: 5 minutes - // about 2 days: 15 minutes - // 50 days or more: 6 hours - - $smart_wait = ($time_since_commit / 200); - $smart_wait = min($smart_wait, phutil_units('6 hours in seconds')); - - $this->log( - pht( - 'Last commit to repository "%s" was %s seconds ago; considering '. - 'a wait of %s seconds before update.', - $repository->getMonogram(), - new PhutilNumber($time_since_commit), - new PhutilNumber($smart_wait))); - - $smart_wait = max(15, $smart_wait); - $sleep_for = max($smart_wait, $sleep_for); - } - - if ($sleep_for < $min_sleep) { - $sleep_for = $min_sleep; - } - - return time() + $sleep_for; + return time() + $smart_wait; } diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 43587cb6d7..cc4fe75a37 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1319,6 +1319,69 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO } + /** + * Load the pull frequency for this repository, based on the time since the + * last activity. + * + * We pull rarely used repositories less frequently. This finds the most + * recent commit which is older than the current time (which prevents us from + * spinning on repositories with a silly commit post-dated to some time in + * 2037). We adjust the pull frequency based on when the most recent commit + * occurred. + * + * @param int The minimum update interval to use, in seconds. + * @return int Repository update interval, in seconds. + */ + public function loadUpdateInterval($minimum = 15) { + // If a repository is still importing, always pull it as frequently as + // possible. This prevents us from hanging for a long time at 99.9% when + // importing an inactive repository. + if ($this->isImporting()) { + return $minimum; + } + + $window_start = (PhabricatorTime::getNow() + $minimum); + + $table = id(new PhabricatorRepositoryCommit()); + $last_commit = queryfx_one( + $table->establishConnection('r'), + 'SELECT epoch FROM %T + WHERE repositoryID = %d AND epoch <= %d + ORDER BY epoch DESC LIMIT 1', + $table->getTableName(), + $this->getID(), + $window_start); + if ($last_commit) { + $time_since_commit = ($window_start - $last_commit['epoch']); + + $last_few_days = phutil_units('3 days in seconds'); + + if ($time_since_commit <= $last_few_days) { + // For repositories with activity in the recent past, we wait one + // extra second for every 10 minutes since the last commit. This + // shorter backoff is intended to handle weekends and other short + // breaks from development. + $smart_wait = ($time_since_commit / 600); + } else { + // For repositories without recent activity, we wait one extra second + // for every 4 minutes since the last commit. This longer backoff + // handles rarely used repositories, up to the maximum. + $smart_wait = ($time_since_commit / 240); + } + + // We'll never wait more than 6 hours to pull a repository. + $longest_wait = phutil_units('6 hours in seconds'); + $smart_wait = min($smart_wait, $longest_wait); + + $smart_wait = max($minimum, $smart_wait); + } else { + $smart_wait = $minimum; + } + + return $smart_wait; + } + + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/docs/user/userguide/diffusion_updates.diviner b/src/docs/user/userguide/diffusion_updates.diviner new file mode 100644 index 0000000000..2c5bb9a6f1 --- /dev/null +++ b/src/docs/user/userguide/diffusion_updates.diviner @@ -0,0 +1,103 @@ +@title Diffusion User Guide: Repository Updates +@group userguide + +Explains how Diffusion updates repositories to discover new changes. + +Overview +======== + +When Phabricator is configured to import repositories which are hosted +elsewhere, it needs to poll those repositories for changes. If it polls too +frequently, it can create too much load locally and on remote services. If it +polls too rarely, it may take a long time for commits to show up in the web +interface. + +This document describes the rules around polling and how to understand and +adjust the behavior. In general: + + - Phabricator chooses a default poll interval based on repository + activity. These intervals range from every 15 seconds (for active + repositories) to every 6 hours (for repositories with no commits in two + months). + - If you use `arc` to push commits, or you host repositories on Phabricator, + repositories automatically update after changes are pushed. + - If you don't use `arc` and your repository is hosted elsewhere, this + document describes ways you can make polling more responsive. + + +Default Behavior +================ + +By default, Phabricator determines how frequently to poll repositories by +examining how long it has been since the last commit. In most cases this is +fairly accurate and produces good behavior. In particular, it automatically +reduces the polling frequency for rarely-used repositories. This dramatically +reduces load for installs with a large number of inactive repositories, which +is common. + +For repositories with activity in the last 3 days, we wait 1 second for every +10 minutes without activity. The table below has some examples. + +| Time Since Commit | Poll Interval | +|-------------------|------------------| +| //Minimum// | 15 seconds | +| 6h | about 30 seconds | +| 12h | about 1 minute | +| 1 day | about 2 minutes | +| 2 days | about 5 minutes | +| 3 days | about 7 minutes | + +This means that you may need to wait about 2 minutes for the first commit to +be imported in the morning, and about 5 minutes after a long weekend, but other +commits to active repositories should usually be recognized in 30 seconds or +less. + +For repositories with no activity in the last 3 days, we wait longer between +updates (1 second for every 4 minutes without activity). The table below has +some examples. + +| Time Since Commit | Poll Interval | +|-------------------|------------------| +| 4 days | about 30 minutes | +| 7 days | about 45 minutes | +| 10 days | about 1 hour | +| 20 days | about 2 hours | +| 30 days | about 3 hours | +| //Maximum// | 6 hours | + +You can find the exact default poll frequency of a repository in +Diffusion > (Choose a Repository) > Edit Repository, under "Update Frequency". +You can also see the time when the repository was last updated in this +interface. + +Repositories that are currently importing are always updated at the minimum +update frequency so the import finishes as quickly as possible. + + +Triggering Repository Updates +============================= + +If you want Phabricator to update a repository more quickly than the default +update frequency (for example, because you just pushed a commit to it), you can +tell Phabricator that it should schedule an update as soon as possible. + +There are several ways to do this: + + - If you push changes with `arc land` or `arc commit`, this will be done + for you automatically. These commits should normally be recognized within + a few seconds. + - If your repository is hosted on Phabricator, this will also be done for you + automatically. + - You can schedule an update from the web interface, in Diffusion > + (Choose a Repository) > Edit Repository > Update Now. + - You can make a call to the Conduit API method `diffusion.looksoon`. This + hints to Phabricator that it should poll a repository as soon as it can. + All of the other mechanisms do this under the hood. + +In particular, you may be able to add a commit hook to your external repository +which calls `diffusion.looksoon`. This should make an external repository about +as responsive as a hosted repository. + +If a repository has an update scheduled, the Diffusion > (Choose a +Repository) > Edit Repository interface will show that the the repository is +prioritized and will be updated soon.