archive/phabricator
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Allow Fact analysis of commits

Browse Source 
Summary: For immutable objects, just use the ID as a cursor.

Test Plan:
  - Analyzed commits from an empty cursor.
  - Checked that cursor was good.
  - Pulled some more commits.
  - Analyzed commits again, verified it only hit the new ones.
  - Verified the graph of "Count of CMIT" looked reasonable.

Reviewers: vrana

Reviewed By: vrana

CC: aran

Maniphest Tasks: T1866

Differential Revision: https://secure.phabricator.com/D3656
This commit is contained in:
epriestley
2012-10-08 13:27:06 -07:00
parent 4b6d3e1be9
commit cd9d78c107
3 changed files with 64 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/applications/fact/extract/PhabricatorFactUpdateIterator.php
View File

@@ -33,7 +33,6 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
public function __construct(LiskDAO $object) {
$this->set = new LiskDAOSet();
$this->object = $object->putInSet($this->set);
$this->position = '0:0';
}
public function setPosition($position) {
@@ -46,7 +45,11 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
}
protected function getCursorFromObject($object) {
return $object->getDateModified().':'.$object->getID();
if ($object->hasProperty('dateModified')) {
return $object->getDateModified().':'.$object->getID();
} else {
return $object->getID();
}
}
public function key() {
@@ -54,32 +57,50 @@ final class PhabricatorFactUpdateIterator extends PhutilBufferedIterator {
}
protected function loadPage() {
list($after_epoch, $after_id) = explode(':', $this->cursor);
$this->set->clearSet();
// NOTE: We ignore recent updates because once we process an update we'll
// never process rows behind it again. We need to read only rows which
// we're sure no new rows will be inserted behind. If we read a row that
// was updated on the current second, another update later on in this second
// could affect an object with a lower ID, and we'd skip that update. To
// avoid this, just ignore any rows which have been updated in the last few
// seconds. This also reduces the amount of work we need to do if an object
// is repeatedly updated; we will just look at the end state without
// processing the intermediate states. Finally, this gives us reasonable
// protections against clock skew between the machine the daemon is running
// on and any machines performing writes.
if ($this->object->hasProperty('dateModified')) {
if ($this->cursor) {
list($after_epoch, $after_id) = explode(':', $this->cursor);
} else {
$after_epoch = 0;
$after_id = 0;
}
$page = $this->object->loadAllWhere(
'((dateModified > %d) OR (dateModified = %d AND id > %d))
AND (dateModified < %d - %d)
ORDER BY dateModified ASC, id ASC LIMIT %d',
$after_epoch,
$after_epoch,
$after_id,
time(),
$this->ignoreUpdatesDuration,
$this->getPageSize());
// NOTE: We ignore recent updates because once we process an update we'll
// never process rows behind it again. We need to read only rows which
// we're sure no new rows will be inserted behind. If we read a row that
// was updated on the current second, another update later on in this
// second could affect an object with a lower ID, and we'd skip that
// update. To avoid this, just ignore any rows which have been updated in
// the last few seconds. This also reduces the amount of work we need to
// do if an object is repeatedly updated; we will just look at the end
// state without processing the intermediate states. Finally, this gives
// us reasonable protections against clock skew between the machine the
// daemon is running on and any machines performing writes.
$page = $this->object->loadAllWhere(
'((dateModified > %d) OR (dateModified = %d AND id > %d))
AND (dateModified < %d - %d)
ORDER BY dateModified ASC, id ASC LIMIT %d',
$after_epoch,
$after_epoch,
$after_id,
time(),
$this->ignoreUpdatesDuration,
$this->getPageSize());
} else {
if ($this->cursor) {
$after_id = $this->cursor;
} else {
$after_id = 0;
}
$page = $this->object->loadAllWhere(
'id > %d ORDER BY id ASC LIMIT %d',
$after_id,
$this->getPageSize());
}
if ($page) {
$this->cursor = $this->getCursorFromObject(end($page));