From ef1340bd3207c95f149309b47d0253fc788db595 Mon Sep 17 00:00:00 2001 From: Austin McKinley Date: Thu, 16 Apr 2020 13:57:12 -0700 Subject: [PATCH] Add Ferret support to Paste Summary: Ref PHI1292. Enable fulltext searchs in paste. Maybe this should only index a snippet instead of the entire content? Also updates table names in `PhabricatorPasteQuery`. Test Plan: Created some pastes, indexed them, searched for them. Reviewers: amckinley Subscribers: codeblock, Korvin, PHID-OPKG-gm6ozazyms6q6i22gyam Differential Revision: https://secure.phabricator.com/D20650 --- .../20200416.paste.01.ferret.doc.sql | 9 +++++ .../20200416.paste.02.ferret.field.sql | 8 ++++ .../20200416.paste.03.ferret.ngrams.sql | 5 +++ .../20200416.paste.04.ferret.cngrams.sql | 7 ++++ src/__phutil_library_map__.php | 6 +++ .../paste/editor/PhabricatorPasteEditor.php | 4 ++ .../engine/PhabricatorPasteFerretEngine.php | 18 +++++++++ .../engine/PhabricatorPasteFulltextEngine.php | 37 +++++++++++++++++++ .../paste/query/PhabricatorPasteQuery.php | 20 ++++++---- .../paste/storage/PhabricatorPaste.php | 19 +++++++++- 10 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 resources/sql/autopatches/20200416.paste.01.ferret.doc.sql create mode 100644 resources/sql/autopatches/20200416.paste.02.ferret.field.sql create mode 100644 resources/sql/autopatches/20200416.paste.03.ferret.ngrams.sql create mode 100644 resources/sql/autopatches/20200416.paste.04.ferret.cngrams.sql create mode 100644 src/applications/paste/engine/PhabricatorPasteFerretEngine.php create mode 100644 src/applications/paste/engine/PhabricatorPasteFulltextEngine.php diff --git a/resources/sql/autopatches/20200416.paste.01.ferret.doc.sql b/resources/sql/autopatches/20200416.paste.01.ferret.doc.sql new file mode 100644 index 0000000000..65b687f176 --- /dev/null +++ b/resources/sql/autopatches/20200416.paste.01.ferret.doc.sql @@ -0,0 +1,9 @@ +CREATE TABLE {$NAMESPACE}_paste.paste_paste_fdocument ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectPHID VARBINARY(64) NOT NULL, + isClosed BOOL NOT NULL, + authorPHID VARBINARY(64), + ownerPHID VARBINARY(64), + epochCreated INT UNSIGNED NOT NULL, + epochModified INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20200416.paste.02.ferret.field.sql b/resources/sql/autopatches/20200416.paste.02.ferret.field.sql new file mode 100644 index 0000000000..98a92ad9f0 --- /dev/null +++ b/resources/sql/autopatches/20200416.paste.02.ferret.field.sql @@ -0,0 +1,8 @@ +CREATE TABLE {$NAMESPACE}_paste.paste_paste_ffield ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + fieldKey VARCHAR(4) NOT NULL COLLATE {$COLLATE_TEXT}, + rawCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + termCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + normalCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20200416.paste.03.ferret.ngrams.sql b/resources/sql/autopatches/20200416.paste.03.ferret.ngrams.sql new file mode 100644 index 0000000000..be4c93b2a6 --- /dev/null +++ b/resources/sql/autopatches/20200416.paste.03.ferret.ngrams.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_paste.paste_paste_fngrams ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20200416.paste.04.ferret.cngrams.sql b/resources/sql/autopatches/20200416.paste.04.ferret.cngrams.sql new file mode 100644 index 0000000000..51b3c05027 --- /dev/null +++ b/resources/sql/autopatches/20200416.paste.04.ferret.cngrams.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_paste.paste_paste_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 840575911f..02f63c1c46 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -4072,7 +4072,9 @@ phutil_register_library_map(array( 'PhabricatorPasteEditController' => 'applications/paste/controller/PhabricatorPasteEditController.php', 'PhabricatorPasteEditEngine' => 'applications/paste/editor/PhabricatorPasteEditEngine.php', 'PhabricatorPasteEditor' => 'applications/paste/editor/PhabricatorPasteEditor.php', + 'PhabricatorPasteFerretEngine' => 'applications/paste/engine/PhabricatorPasteFerretEngine.php', 'PhabricatorPasteFilenameContextFreeGrammar' => 'applications/paste/lipsum/PhabricatorPasteFilenameContextFreeGrammar.php', + 'PhabricatorPasteFulltextEngine' => 'applications/paste/engine/PhabricatorPasteFulltextEngine.php', 'PhabricatorPasteLanguageTransaction' => 'applications/paste/xaction/PhabricatorPasteLanguageTransaction.php', 'PhabricatorPasteListController' => 'applications/paste/controller/PhabricatorPasteListController.php', 'PhabricatorPastePastePHIDType' => 'applications/paste/phid/PhabricatorPastePastePHIDType.php', @@ -10652,6 +10654,8 @@ phutil_register_library_map(array( 'PhabricatorApplicationTransactionInterface', 'PhabricatorSpacesInterface', 'PhabricatorConduitResultInterface', + 'PhabricatorFerretInterface', + 'PhabricatorFulltextInterface', ), 'PhabricatorPasteApplication' => 'PhabricatorApplication', 'PhabricatorPasteArchiveController' => 'PhabricatorPasteController', @@ -10662,7 +10666,9 @@ phutil_register_library_map(array( 'PhabricatorPasteEditController' => 'PhabricatorPasteController', 'PhabricatorPasteEditEngine' => 'PhabricatorEditEngine', 'PhabricatorPasteEditor' => 'PhabricatorApplicationTransactionEditor', + 'PhabricatorPasteFerretEngine' => 'PhabricatorFerretEngine', 'PhabricatorPasteFilenameContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhabricatorPasteFulltextEngine' => 'PhabricatorFulltextEngine', 'PhabricatorPasteLanguageTransaction' => 'PhabricatorPasteTransactionType', 'PhabricatorPasteListController' => 'PhabricatorPasteController', 'PhabricatorPastePastePHIDType' => 'PhabricatorPHIDType', diff --git a/src/applications/paste/editor/PhabricatorPasteEditor.php b/src/applications/paste/editor/PhabricatorPasteEditor.php index 76ade878c4..319ac444ee 100644 --- a/src/applications/paste/editor/PhabricatorPasteEditor.php +++ b/src/applications/paste/editor/PhabricatorPasteEditor.php @@ -94,4 +94,8 @@ final class PhabricatorPasteEditor return true; } + protected function supportsSearch() { + return true; + } + } diff --git a/src/applications/paste/engine/PhabricatorPasteFerretEngine.php b/src/applications/paste/engine/PhabricatorPasteFerretEngine.php new file mode 100644 index 0000000000..22057bb51f --- /dev/null +++ b/src/applications/paste/engine/PhabricatorPasteFerretEngine.php @@ -0,0 +1,18 @@ +setViewer($this->getViewer()) + ->withPHIDs(array($object->getPHID())) + ->needContent(true) + ->executeOne(); + + $document->setDocumentTitle($paste->getTitle()); + + $document->addRelationship( + $paste->isArchived() + ? PhabricatorSearchRelationship::RELATIONSHIP_CLOSED + : PhabricatorSearchRelationship::RELATIONSHIP_OPEN, + $paste->getPHID(), + PhabricatorPastePastePHIDType::TYPECONST, + PhabricatorTime::getNow()); + + $document->addField( + PhabricatorSearchDocumentFieldType::FIELD_BODY, + $paste->getContent()); + + $document->addRelationship( + PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR, + $paste->getAuthorPHID(), + PhabricatorPeopleUserPHIDType::TYPECONST, + $paste->getDateCreated()); + } + +} diff --git a/src/applications/paste/query/PhabricatorPasteQuery.php b/src/applications/paste/query/PhabricatorPasteQuery.php index 693c82780f..d90ef3b1d2 100644 --- a/src/applications/paste/query/PhabricatorPasteQuery.php +++ b/src/applications/paste/query/PhabricatorPasteQuery.php @@ -110,62 +110,66 @@ final class PhabricatorPasteQuery if ($this->ids !== null) { $where[] = qsprintf( $conn, - 'id IN (%Ld)', + 'paste.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, - 'phid IN (%Ls)', + 'paste.phid IN (%Ls)', $this->phids); } if ($this->authorPHIDs !== null) { $where[] = qsprintf( $conn, - 'authorPHID IN (%Ls)', + 'paste.authorPHID IN (%Ls)', $this->authorPHIDs); } if ($this->parentPHIDs !== null) { $where[] = qsprintf( $conn, - 'parentPHID IN (%Ls)', + 'paste.parentPHID IN (%Ls)', $this->parentPHIDs); } if ($this->languages !== null) { $where[] = qsprintf( $conn, - 'language IN (%Ls)', + 'paste.language IN (%Ls)', $this->languages); } if ($this->dateCreatedAfter !== null) { $where[] = qsprintf( $conn, - 'dateCreated >= %d', + 'paste.dateCreated >= %d', $this->dateCreatedAfter); } if ($this->dateCreatedBefore !== null) { $where[] = qsprintf( $conn, - 'dateCreated <= %d', + 'paste.dateCreated <= %d', $this->dateCreatedBefore); } if ($this->statuses !== null) { $where[] = qsprintf( $conn, - 'status IN (%Ls)', + 'paste.status IN (%Ls)', $this->statuses); } return $where; } + protected function getPrimaryTableAlias() { + return 'paste'; + } + private function getContentCacheKey(PhabricatorPaste $paste) { return implode( ':', diff --git a/src/applications/paste/storage/PhabricatorPaste.php b/src/applications/paste/storage/PhabricatorPaste.php index 971ec63f86..3d19a00230 100644 --- a/src/applications/paste/storage/PhabricatorPaste.php +++ b/src/applications/paste/storage/PhabricatorPaste.php @@ -11,7 +11,9 @@ final class PhabricatorPaste extends PhabricatorPasteDAO PhabricatorDestructibleInterface, PhabricatorApplicationTransactionInterface, PhabricatorSpacesInterface, - PhabricatorConduitResultInterface { + PhabricatorConduitResultInterface, + PhabricatorFerretInterface, + PhabricatorFulltextInterface { protected $title; protected $authorPHID; @@ -277,4 +279,19 @@ final class PhabricatorPaste extends PhabricatorPasteDAO ); } + +/* -( PhabricatorFerretInterface )----------------------------------------- */ + + + public function newFerretEngine() { + return new PhabricatorPasteFerretEngine(); + } + + +/* -( PhabricatorFulltextInterface )--------------------------------------- */ + + public function newFulltextEngine() { + return new PhabricatorPasteFulltextEngine(); + } + }