Commit 30ceb454 authored by xorti's avatar xorti

[NEW] Add support to use MySQL search index as ElasticSearch index fallback

parent bde2f3cc
Pipeline #63106911 failed with stages
in 29 minutes and 35 seconds
......@@ -60,9 +60,11 @@ class Services_Search_Controller
return [
'title' => tr('Rebuild Index'),
'stat' => $stat,
'stat' => $stat['default'],
'search_engine' => $engine,
'search_version' => $version,
'fallback_search_set' => $unifiedsearchlib->getFallbackIndexEngine() != null,
'fallback_search_indexed' => ! empty($stat['fallback']),
'queue_count' => $unifiedsearchlib->getQueueCount(),
'execution_time' => FormatterHelper::formatTime($timer->stop()),
'memory_usage' => FormatterHelper::formatMemory(memory_get_usage()),
......@@ -151,7 +153,7 @@ class Services_Search_Controller
];
if ($item['object_type'] == 'trackeritem') {
$transformed['status_icon'] = smarty_function_tracker_item_status_icon(['item' => $item['object_id']], $smarty->getEmptyInternalTemplate());
}
}
return $transformed;
});
......
......@@ -110,10 +110,17 @@ class IndexRebuildCommand extends Command
if ($result) {
if (! $cron) {
$output->writeln("Indexed");
foreach ($result as $key => $val) {
foreach ($result['default'] as $key => $val) {
$output->writeln(" $key: $val");
}
$output->writeln('Rebuilding index done');
if (\TikiLib::lib('unifiedsearch')->getFallbackIndexEngine()) {
if (! empty($result['fallback'])) {
$output->writeln('Fallback index was also rebuilt');
} else {
$output->writeln('<comment>Fallback index was not rebuilt</comment>');
}
}
$output->writeln('Execution time: ' . FormatterHelper::formatTime($timer->stop()));
$output->writeln('Current Memory usage: ' . FormatterHelper::formatMemory(memory_get_usage()));
$output->writeln('Memory peak usage before indexing: ' . FormatterHelper::formatMemory($memory_peak_usage_before));
......
......@@ -261,6 +261,12 @@ function prefs_unified_list()
'description' => tra('The maximum number of fields per search index in Elasticsearch version 5.x and above'),
'default' => '1000',
],
'unified_elastic_mysql_search_fallback' => [
'name' => tra('Use MySQL Full-Text Search (fallback)'),
'type' => 'flag',
'description' => tra('In case of Elasticsearch is active and unavailable, use MySQL Full-Text Search as fallback'),
'default' => 'n',
],
'unified_mysql_index_current' => [
'name' => tra('MySQL full-text search current index'),
'description' => tra('A new index is created upon rebuilding, and the old one is then destroyed. This setting enables seeing the currently active index.'),
......
......@@ -690,9 +690,16 @@ class PreferencesLib
return $out;
}
public function rebuildIndex()
/**
* @param bool $fallback Rebuild fallback search index
* @return Search_Index_Interface|\ZendSearch\Lucene\SearchIndexInterface|null
* @throws Exception
*/
public function rebuildIndex($fallback = false)
{
$index = TikiLib::lib('unifiedsearch')->getIndex('preference');
global $prefs;
$index = TikiLib::lib('unifiedsearch')->getIndex('preference', ! $fallback);
$index->destroy();
$typeFactory = $index->getTypeFactory();
......@@ -719,7 +726,13 @@ class PreferencesLib
}
}
// Rebuild fallback index
if (! $fallback && $fallbackEngine = TikiLib::lib('unifiedsearch')->getFallbackIndexEngine()) {
$defaultEngine = $prefs['unified_engine'];
$prefs['unified_engine'] = $fallbackEngine;
$this->rebuildIndex(true);
$prefs['unified_engine'] = $defaultEngine;
}
return $index;
}
......
......@@ -161,9 +161,11 @@ class UnifiedSearchLib
/**
* @param int $loggit 0=no logging, 1=log to Search_Indexer.log, 2=log to Search_Indexer_console.log
* @return array
* @param bool $fallback If the fallback index is being rebuild
* @return array|bool
* @throws Exception
*/
function rebuild($loggit = 0)
public function rebuild($loggit = 0, $fallback = false)
{
global $prefs;
switch ($prefs['unified_engine']) {
......@@ -224,7 +226,6 @@ class UnifiedSearchLib
die('Unsupported');
}
// Build in -new
TikiLib::lib('queue')->clear(self::INCREMENT_QUEUE);
$tikilib = TikiLib::lib('tiki');
......@@ -251,6 +252,9 @@ class UnifiedSearchLib
Feedback::error(tr('The search index could not be rebuilt.') . '<br />' . $e->getMessage());
}
$stats = [];
$stats['default'] = $stat;
// Force destruction to clear locks
if ($indexer) {
$indexer->clearSources();
......@@ -279,6 +283,7 @@ class UnifiedSearchLib
case 'elastic':
$oldIndex = null; // assignAlias will handle the clean-up
$tikilib->set_preference('unified_elastic_index_current', $indexName);
$connection->assignAlias($aliasName, $indexName);
break;
......@@ -297,6 +302,19 @@ class UnifiedSearchLib
}
}
if ($fallback) {
// Fallback index was rebuilt. Proceed with default index operations
return $stats['default'];
}
// Rebuild mysql as fallback for elasticsearch engine
if (! $fallback && $fallbackEngine = TikiLib::lib('unifiedsearch')->getFallbackIndexEngine()) {
$defaultEngine = $prefs['unified_engine'];
$prefs['unified_engine'] = $fallbackEngine;
$stats['fallback'] = $this->rebuild($loggit, true);
$prefs['unified_engine'] = $defaultEngine;
}
// Process the documents updated while we were processing the update
$this->processUpdateQueue(1000);
......@@ -308,7 +326,7 @@ class UnifiedSearchLib
$this->isRebuildingNow = false;
$access->preventRedirect(false);
return $stat;
return $stats;
}
/**
......@@ -346,9 +364,12 @@ class UnifiedSearchLib
/**
* Get the index location depending on $tikidomain for multi-tiki
*
* @return string path to index directory
* @param string $indexType
* @param string $engine If not set, it uses default unified search engine
* @return string path to index directory
* @throws Exception
*/
private function getIndexLocation($indexType = 'data')
private function getIndexLocation($indexType = 'data', $engine = null)
{
global $prefs, $tikidomain;
$mapping = [
......@@ -368,7 +389,7 @@ class UnifiedSearchLib
],
];
$engine = $prefs['unified_engine'];
$engine = $engine ?: $prefs['unified_engine'];
if (isset($mapping[$engine][$indexType])) {
$index = $mapping[$engine][$indexType];
......@@ -692,11 +713,11 @@ class UnifiedSearchLib
/**
* @return Search_Index_Interface
*/
function getIndex($indexType = 'data')
function getIndex($indexType = 'data', $useCache = true)
{
global $prefs, $tiki_p_admin;
if (isset($this->indices[$indexType])) {
if (isset($this->indices[$indexType]) && $useCache) {
return $this->indices[$indexType];
}
......@@ -706,36 +727,41 @@ class UnifiedSearchLib
$writeMode = true;
}
switch ($prefs['unified_engine']) {
case 'lucene':
ZendSearch\Lucene\Lucene::setTermsPerQueryLimit($prefs['unified_lucene_terms_limit']);
$index = new Search_Lucene_Index($this->getIndexLocation($indexType), $prefs['language'], $prefs['unified_lucene_highlight'] == 'y');
$index->setCache(TikiLib::lib('cache'));
$index->setMaxResults($prefs['unified_lucene_max_result']);
$index->setResultSetLimit($prefs['unified_lucene_max_resultset_limit']);
$engine = $prefs['unified_engine'];
$fallbackMySQL = false;
return $index;
case 'elastic':
$index = $this->getIndexLocation($indexType);
if (empty($index)) {
break;
}
if ($engine == 'lucene') {
ZendSearch\Lucene\Lucene::setTermsPerQueryLimit($prefs['unified_lucene_terms_limit']);
$index = new Search_Lucene_Index($this->getIndexLocation($indexType), $prefs['language'], $prefs['unified_lucene_highlight'] == 'y');
$index->setCache(TikiLib::lib('cache'));
$index->setMaxResults($prefs['unified_lucene_max_result']);
$index->setResultSetLimit($prefs['unified_lucene_max_resultset_limit']);
$connection = $this->getElasticConnection($writeMode);
return $index;
}
if ($engine == 'elastic' && $index = $this->getIndexLocation($indexType)) {
$connection = $this->getElasticConnection($writeMode);
if ($connection->getStatus()->status === 200) {
$index = new Search_Elastic_Index($connection, $index);
$index->setCamelCaseEnabled($prefs['unified_elastic_camel_case'] == 'y');
$index->setPossessiveStemmerEnabled($prefs['unified_elastic_possessive_stemmer'] == 'y');
$index->setFacetCount($prefs['search_facet_default_amount']);
$this->indices[$indexType] = $index;
return $index;
case 'mysql':
$index = $this->getIndexLocation($indexType);
if (empty($index)) {
break;
}
}
$index = new Search_MySql_Index(TikiDb::get(), $index);
return $index;
if ($prefs['unified_elastic_mysql_search_fallback'] === 'y') {
$fallbackMySQL = true;
Feedback::warning(['mes' => tr('Unable to connect to the main search index, MySQL full-text search used, the search results might not be accurate')]);
$prefs['unified_incremental_update'] = 'n';
}
}
if (($engine == 'mysql' || $fallbackMySQL) && $index = $this->getIndexLocation($indexType, 'mysql')) {
$index = new Search_MySql_Index(TikiDb::get(), $index);
return $index;
}
// Do nothing, provide a fake index.
......@@ -1113,7 +1139,7 @@ class UnifiedSearchLib
$range = explode(',', $range);
if (count($range) > 2) {
$facet->addRange($range[1], $range[0], $range[2]);
} else if (count($range) > 1) {
} elseif (count($range) > 1) {
$facet->addRange($range[1], $range[0]);
}
}
......@@ -1328,4 +1354,20 @@ class UnifiedSearchLib
$logName = $prefs['tmpDir'] . (substr($prefs['tmpDir'], -1) === '/' ? '' : '/') . $logName . '.log';
return $logName;
}
/**
* Return the fallback search engine name
*
* @return string|null
*/
public function getFallbackIndexEngine()
{
global $prefs;
if ($prefs['unified_engine'] == 'elastic' && $prefs['unified_elastic_mysql_search_fallback'] === 'y') {
return 'mysql';
}
return null;
}
}
......@@ -96,6 +96,10 @@
{preference name="unified_elastic_index_current"}
{preference name="unified_elastic_field_limit"}
{preference name="unified_relation_object_indexing"}
{remarksbox type=tip title="{tr}About Use MySQL Full-Text Search as fallback{/tr}"}
{tr}Elasticsearch is a tiki external service. You should set at least a daily full index rebuild to keep the MySQL index updated in case of Elastic unavailability{/tr}.
{/remarksbox}
{preference name="unified_elastic_mysql_search_fallback"}
</div>
{preference name="unified_lucene_default_operator"}
......
......@@ -25,6 +25,17 @@
{/foreach}
</ul>
{/remarksbox}
{if $fallback_search_set}
{if $fallback_search_indexed}
{remarksbox type='feedback' title="{tr}Fallback search engine{/tr}"}
<p>{tr}Fallback search index was rebuild.{/tr}</p>
{/remarksbox}
{else}
{remarksbox type='error' title="{tr}Fallback search engine{/tr}"}
<p>{tr}Fallback search index was not rebuilt.{/tr}</p>
{/remarksbox}
{/if}
{/if}
{remarksbox type='feedback' title="{tr}Execution Statistics{/tr}"}
<ul>
<li>{tr}Execution time:{/tr} {$execution_time}</li>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment