Surface task queue temporary failure rate in Daemon console

Summary: Fixes T3557. One thing which made T3557 kind of a mess was the lack of information about progress through temporary failures. Add a column which records a task's last failure time, and surface it in the console.

Test Plan: {F51277}

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T3557

Differential Revision: https://secure.phabricator.com/D6550

resources/sql/patches/20130723.taskstarttime.sql

@@ -0,0 +1,6 @@
+ALTER TABLE {$NAMESPACE}_worker.worker_activetask
+  ADD failureTime INT UNSIGNED;
+
+ALTER TABLE {$NAMESPACE}_worker.worker_activetask
+  ADD KEY `key_failuretime` (`failureTime`);
+

src/applications/daemon/controller/PhabricatorDaemonConsoleController.php

@@ -11,6 +11,10 @@ final class PhabricatorDaemonConsoleController
       'dateModified > %d',
       time() - (60 * 15));
 
+    $failed = id(new PhabricatorWorkerActiveTask())->loadAllWhere(
+      'failureTime > %d',
+      time() - (60 * 15));
+
     $completed_info = array();
     foreach ($completed as $completed_task) {
       $class = $completed_task->getTaskClass();
@@ -36,6 +40,14 @@ final class PhabricatorDaemonConsoleController
       );
     }
 
+    if ($failed) {
+      $rows[] = array(
+        phutil_tag('em', array(), pht('Temporary Failures')),
+        count($failed),
+        null,
+      );
+    }
+
     $completed_table = new AphrontTableView($rows);
     $completed_table->setNoDataString(
       pht('No tasks have completed in the last 15 minutes.'));
@@ -52,8 +64,10 @@ final class PhabricatorDaemonConsoleController
         'n',
       ));
 
+    $completed_header = id(new PhabricatorHeaderView())
+      ->setHeader(pht('Recently Completed Tasks (Last 15m)'));
+
     $completed_panel = new AphrontPanelView();
-    $completed_panel->setHeader(pht('Recently Completed Tasks (15m)'));
     $completed_panel->appendChild($completed_table);
     $completed_panel->setNoBackground();
 
@@ -149,10 +163,17 @@ final class PhabricatorDaemonConsoleController
     $queued_panel->appendChild($queued_table);
     $queued_panel->setNoBackground();
 
+    $crumbs = $this->buildApplicationCrumbs();
+    $crumbs->addCrumb(
+      id(new PhabricatorCrumbView())
+        ->setName(pht('Console')));
+
     $nav = $this->buildSideNavView();
     $nav->selectFilter('/');
     $nav->appendChild(
       array(
+        $crumbs,
+        $completed_header,
         $completed_panel,
         $daemon_header,
         $daemon_table,
@@ -164,6 +185,7 @@ final class PhabricatorDaemonConsoleController
       $nav,
       array(
         'title' => pht('Console'),
+        'dust' => true,
       ));
   }
 

src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php

@@ -2,6 +2,8 @@
 
 final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
 
+  protected $failureTime;
+
   private $serverTime;
   private $localTime;
 
@@ -132,6 +134,7 @@ final class PhabricatorWorkerActiveTask extends PhabricatorWorkerTask {
     } catch (Exception $ex) {
       $this->setExecutionException($ex);
       $this->setFailureCount($this->getFailureCount() + 1);
+      $this->setFailureTime(time());
 
       $retry = $worker->getWaitBeforeRetry($this);
       $retry = coalesce(

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -1471,6 +1471,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
         'type' => 'php',
         'name' => $this->getPatchPath('20130716.archivememberlessprojects.php'),
       ),
+      '20130723.taskstarttime.sql' => array(
+        'type' => 'sql',
+        'name' => $this->getPatchPath('20130723.taskstarttime.sql'),
+      ),
     );
   }
 }