HomePhorge
Diffusion Phorge (master) Code

phorge/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
1720209f16c3master

Last Change
Raw File

PhabricatorFerretFulltextEngineExtension.php
View Options

<?php
final class PhabricatorFerretFulltextEngineExtension
extends PhabricatorFulltextEngineExtension {
const EXTENSIONKEY = 'ferret';
public function getExtensionName() {
return pht('Ferret Fulltext Engine');
}
public function shouldIndexFulltextObject($object) {
return ($object instanceof PhabricatorFerretInterface);
}
public function indexFulltextObject(
$object,
PhabricatorSearchAbstractDocument $document) {
$phid = $document->getPHID();
$engine = $object->newFerretEngine();
$is_closed = 0;
$author_phid = null;
$owner_phid = null;
foreach ($document->getRelationshipData() as $relationship) {
list($related_type, $related_phid) = $relationship;
switch ($related_type) {
case PhabricatorSearchRelationship::RELATIONSHIP_OPEN:
$is_closed = 0;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_CLOSED:
$is_closed = 1;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_OWNER:
$owner_phid = $related_phid;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED:
$owner_phid = null;
break;
case PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR:
$author_phid = $related_phid;
break;
}
}
$stemmer = $engine->newStemmer();
// Copy all of the "title" and "body" fields to create new "core" fields.
// This allows users to search "in title or body" with the "core:" prefix.
$document_fields = $document->getFieldData();
$virtual_fields = array();
foreach ($document_fields as $field) {
$virtual_fields[] = $field;
list($key, $raw_corpus) = $field;
switch ($key) {
case PhabricatorSearchDocumentFieldType::FIELD_TITLE:
case PhabricatorSearchDocumentFieldType::FIELD_BODY:
$virtual_fields[] = array(
PhabricatorSearchDocumentFieldType::FIELD_CORE,
$raw_corpus,
);
break;
}
$virtual_fields[] = array(
PhabricatorSearchDocumentFieldType::FIELD_ALL,
$raw_corpus,
);
}
$empty_template = array(
'raw' => array(),
'term' => array(),
'normal' => array(),
);
$ferret_corpus_map = array();
foreach ($virtual_fields as $field) {
list($key, $raw_corpus) = $field;
if (!strlen($raw_corpus)) {
continue;
}
$term_corpus = $engine->newTermsCorpus($raw_corpus);
$normal_corpus = $stemmer->stemCorpus($raw_corpus);
$normal_corpus = $engine->newTermsCorpus($normal_corpus);
if (!isset($ferret_corpus_map[$key])) {
$ferret_corpus_map[$key] = $empty_template;
}
$ferret_corpus_map[$key]['raw'][] = $raw_corpus;
$ferret_corpus_map[$key]['term'][] = $term_corpus;
$ferret_corpus_map[$key]['normal'][] = $normal_corpus;
}
$ferret_fields = array();
$ngrams_source = array();
foreach ($ferret_corpus_map as $key => $fields) {
$raw_corpus = $fields['raw'];
$raw_corpus = implode("\n", $raw_corpus);
if (strlen($raw_corpus)) {
$ngrams_source[] = $raw_corpus;
}
$normal_corpus = $fields['normal'];
$normal_corpus = implode("\n", $normal_corpus);
if (strlen($normal_corpus)) {
$ngrams_source[] = $normal_corpus;
}
$term_corpus = $fields['term'];
$term_corpus = implode("\n", $term_corpus);
if (strlen($term_corpus)) {
$ngrams_source[] = $term_corpus;
}
$ferret_fields[] = array(
'fieldKey' => $key,
'rawCorpus' => $raw_corpus,
'termCorpus' => $term_corpus,
'normalCorpus' => $normal_corpus,
);
}
$ngrams_source = implode("\n", $ngrams_source);
$ngram_engine = new PhabricatorSearchNgramEngine();
$ngrams = $ngram_engine->getTermNgramsFromString($ngrams_source);
$conn = $object->establishConnection('w');
if ($ngrams) {
$common = queryfx_all(
$conn,
'SELECT ngram FROM %T WHERE ngram IN (%Ls)',
$engine->getCommonNgramsTableName(),
$ngrams);
$common = ipull($common, 'ngram', 'ngram');
foreach ($ngrams as $key => $ngram) {
if (isset($common[$ngram])) {
unset($ngrams[$key]);
continue;
}
// NOTE: MySQL discards trailing whitespace in CHAR(X) columns.
$trimmed_ngram = rtrim($ngram, ' ');
if (isset($common[$trimmed_ngram])) {
unset($ngrams[$key]);
continue;
}
}
}
$object->openTransaction();
try {
// See T13587. If this document already exists in the index, we try to
// update the existing rows to avoid leaving the ngrams table heavily
// fragmented.
$old_document = queryfx_one(
$conn,
'SELECT id FROM %T WHERE objectPHID = %s',
$engine->getDocumentTableName(),
$object->getPHID());
if ($old_document) {
$old_document_id = (int)$old_document['id'];
} else {
$old_document_id = null;
}
if ($old_document_id === null) {
queryfx(
$conn,
'INSERT INTO %T (objectPHID, isClosed, epochCreated, epochModified,
authorPHID, ownerPHID) VALUES (%s, %d, %d, %d, %ns, %ns)',
$engine->getDocumentTableName(),
$object->getPHID(),
$is_closed,
$document->getDocumentCreated(),
$document->getDocumentModified(),
$author_phid,
$owner_phid);
$document_id = $conn->getInsertID();
$is_new = true;
} else {
$document_id = $old_document_id;
queryfx(
$conn,
'UPDATE %T
SET
isClosed = %d,
epochCreated = %d,
epochModified = %d,
authorPHID = %ns,
ownerPHID = %ns
WHERE id = %d',
$engine->getDocumentTableName(),
$is_closed,
$document->getDocumentCreated(),
$document->getDocumentModified(),
$author_phid,
$owner_phid,
$document_id);
$is_new = false;
}
$this->updateStoredFields(
$conn,
$is_new,
$document_id,
$engine,
$ferret_fields);
$this->updateStoredNgrams(
$conn,
$is_new,
$document_id,
$engine,
$ngrams);
} catch (Exception $ex) {
$object->killTransaction();
throw $ex;
} catch (Throwable $ex) {
$object->killTransaction();
throw $ex;
}
$object->saveTransaction();
}
private function updateStoredFields(
AphrontDatabaseConnection $conn,
$is_new,
$document_id,
PhabricatorFerretEngine $engine,
$new_fields) {
if (!$is_new) {
$old_fields = queryfx_all(
$conn,
'SELECT * FROM %T WHERE documentID = %d',
$engine->getFieldTableName(),
$document_id);
} else {
$old_fields = array();
}
$old_fields = ipull($old_fields, null, 'fieldKey');
$new_fields = ipull($new_fields, null, 'fieldKey');
$delete_rows = array();
$insert_rows = array();
$update_rows = array();
foreach ($old_fields as $field_key => $old_field) {
if (!isset($new_fields[$field_key])) {
$delete_rows[] = $old_field;
}
}
$compare_keys = array(
'rawCorpus',
'termCorpus',
'normalCorpus',
);
foreach ($new_fields as $field_key => $new_field) {
if (!isset($old_fields[$field_key])) {
$insert_rows[] = $new_field;
continue;
}
$old_field = $old_fields[$field_key];
$same_row = true;
foreach ($compare_keys as $compare_key) {
if ($old_field[$compare_key] !== $new_field[$compare_key]) {
$same_row = false;
break;
}
}
if ($same_row) {
continue;
}
$new_field['id'] = $old_field['id'];
$update_rows[] = $new_field;
}
if ($delete_rows) {
queryfx(
$conn,
'DELETE FROM %T WHERE id IN (%Ld)',
$engine->getFieldTableName(),
ipull($delete_rows, 'id'));
}
foreach ($update_rows as $update_row) {
queryfx(
$conn,
'UPDATE %T
SET
rawCorpus = %s,
termCorpus = %s,
normalCorpus = %s
WHERE id = %d',
$engine->getFieldTableName(),
$update_row['rawCorpus'],
$update_row['termCorpus'],
$update_row['normalCorpus'],
$update_row['id']);
}
foreach ($insert_rows as $insert_row) {
queryfx(
$conn,
'INSERT INTO %T (documentID, fieldKey, rawCorpus, termCorpus,
normalCorpus) VALUES (%d, %s, %s, %s, %s)',
$engine->getFieldTableName(),
$document_id,
$insert_row['fieldKey'],
$insert_row['rawCorpus'],
$insert_row['termCorpus'],
$insert_row['normalCorpus']);
}
}
private function updateStoredNgrams(
AphrontDatabaseConnection $conn,
$is_new,
$document_id,
PhabricatorFerretEngine $engine,
$new_ngrams) {
if ($is_new) {
$old_ngrams = array();
} else {
$old_ngrams = queryfx_all(
$conn,
'SELECT id, ngram FROM %T WHERE documentID = %d',
$engine->getNgramsTableName(),
$document_id);
}
$old_ngrams = ipull($old_ngrams, 'id', 'ngram');
$new_ngrams = array_fuse($new_ngrams);
$delete_ids = array();
$insert_ngrams = array();
// NOTE: MySQL discards trailing whitespace in CHAR(X) columns.
foreach ($old_ngrams as $ngram => $id) {
if (isset($new_ngrams[$ngram])) {
continue;
}
$untrimmed_ngram = $ngram.' ';
if (isset($new_ngrams[$untrimmed_ngram])) {
continue;
}
$delete_ids[] = $id;
}
foreach ($new_ngrams as $ngram) {
if (isset($old_ngrams[$ngram])) {
continue;
}
$trimmed_ngram = rtrim($ngram, ' ');
if (isset($old_ngrams[$trimmed_ngram])) {
continue;
}
$insert_ngrams[] = $ngram;
}
if ($delete_ids) {
$sql = array();
foreach ($delete_ids as $id) {
$sql[] = qsprintf(
$conn,
'%d',
$id);
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'DELETE FROM %T WHERE id IN (%LQ)',
$engine->getNgramsTableName(),
$chunk);
}
}
if ($insert_ngrams) {
$sql = array();
foreach ($insert_ngrams as $ngram) {
$sql[] = qsprintf(
$conn,
'(%d, %s)',
$document_id,
$ngram);
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'INSERT INTO %T (documentID, ngram) VALUES %LQ',
$engine->getNgramsTableName(),
$chunk);
}
}
}
public function newFerretSearchFunctions() {
return array(
id(new FerretConfigurableSearchFunction())
->setFerretFunctionName('all')
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_ALL),
id(new FerretConfigurableSearchFunction())
->setFerretFunctionName('title')
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_TITLE),
id(new FerretConfigurableSearchFunction())
->setFerretFunctionName('body')
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_BODY),
id(new FerretConfigurableSearchFunction())
->setFerretFunctionName('core')
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_CORE),
id(new FerretConfigurableSearchFunction())
->setFerretFunctionName('comment')
->setFerretFieldKey(PhabricatorSearchDocumentFieldType::FIELD_COMMENT),
);
}
}

Owner Packages

  • No Owners