Commit c2671941 authored by Victor Emanouilov's avatar Victor Emanouilov
Browse files

[FIX] manticore: deal with 256 full-text field limit in manticore - get...

[FIX] manticore: deal with 256 full-text field limit in manticore - get snapshot of whole index fields and their types, use preference to prefer some fields in the full-text index, cut the indexing at 255 fields and add the rest as string attributes which allow the slower regex search
parent eba7723e
Pipeline #647925381 canceled with stages
......@@ -112,6 +112,17 @@ class Search_ContentSource_ActivityStreamSource implements Search_ContentSource_
return array_merge(['event_type', 'modification_date', 'clear_list', 'date'], array_keys($mapping));
}
public function getProvidedFieldTypes()
{
$mapping = $this->lib->getMapping();
return array_merge([
'event_type' => 'identifier',
'modification_date' => 'timestamp',
'clear_list' => 'multivalue',
'date' => 'timestamp'
], $mapping);
}
public function getGlobalFields()
{
return [
......
......@@ -68,7 +68,7 @@ class Search_ContentSource_ArticleSource implements Search_ContentSource_Interfa
'description' => $typeFactory->plaintext($article['heading']),
'sitetitle' => $typeFactory->plaintext($sitetitle),
'siteurl' => $typeFactory->plaintext($siteurl),
'siteurl' => $typeFactory->plaintext($siteurl),
'topic_id' => $typeFactory->identifier($article['topicId']),
'topic_name' => $typeFactory->plaintext($topic_name),
......@@ -101,7 +101,7 @@ class Search_ContentSource_ArticleSource implements Search_ContentSource_Interfa
'date',
'sitetitle',
'siteurl',
'siteurl',
'topic_id',
'topic_name',
......@@ -120,6 +120,37 @@ class Search_ContentSource_ArticleSource implements Search_ContentSource_Interfa
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'contributors' => 'multivalue',
'description' => 'plaintext',
'date' => 'timestamp',
'sitetitle' => 'plaintext',
'siteurl' => 'plaintext',
'topic_id' => 'identifier',
'topic_name' => 'plaintext',
'article_content' => 'wikitext',
'article_type' => 'identifier',
'article_topline' => 'wikitext',
'article_subtitle' => 'wikitext',
'article_author' => 'plaintext',
'article_linkto' => 'plaintext',
'view_permission' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
'published' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -77,6 +77,26 @@ class Search_ContentSource_BlogPostSource implements Search_ContentSource_Interf
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'blog_id' => 'identifier',
'blog_excerpt' => 'wikitext',
'blog_content' => 'wikitext',
'view_permission' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -114,6 +114,34 @@ class Search_ContentSource_CalendarItemSource implements Search_ContentSource_In
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'description' => 'plaintext',
'calendar_id' => 'identifier',
'start_date' => 'timestamp',
'end_date' => 'timestamp',
'priority' => 'numeric',
'status' => 'numeric',
'status_text' => 'identifier',
'url' => 'identifier',
'recurrence_id' => 'identifier',
'view_permission' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
'trackeritems' => 'multivalue',
];
}
public function getGlobalFields()
{
return [
......
......@@ -58,12 +58,27 @@ class Search_ContentSource_CalendarSource implements Search_ContentSource_Interf
'personal',
'user',
'searchable',
'view_permission',
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'description' => 'plaintext',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'personal' => 'identifier',
'user' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -56,6 +56,19 @@ class Search_ContentSource_CategorySource implements Search_ContentSource_Interf
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'description' => 'plaintext',
'path' => 'sortable',
'searchable' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -120,6 +120,27 @@ class Search_ContentSource_CommentSource implements Search_ContentSource_Interfa
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'url' => 'identifier',
'comment_content' => 'wikitext',
'parent_thread_id' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
'view_permission' => 'identifier',
'global_view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -58,12 +58,26 @@ class Search_ContentSource_FileGallerySource implements Search_ContentSource_Int
'gallery_id',
'searchable',
'view_permission',
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'description' => 'plaintext',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'gallery_id' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -135,6 +135,43 @@ class Search_ContentSource_FileSource implements Search_ContentSource_Interface,
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'title_unstemmed' => 'simpletext',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'description' => 'plaintext',
'filename' => 'identifier',
'filetype' => 'sortable',
'filesize' => 'plaintext',
'hits' => 'numeric',
'gallery_id' => 'identifier',
'file_comment' => 'plaintext',
'file_content' => 'plaintext',
'ocr_content' => 'plaintext',
'view_permission' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
'email_subject' => 'plaintext',
'email_from' => 'plaintext',
'email_sender' => 'plaintext',
'email_recipient' => 'plaintext',
'email_date' => 'timestamp',
'email_content_type' => 'plaintext',
'email_body' => 'plainmediumtext',
'email_plaintext' => 'plainmediumtext',
'email_html' => 'plainmediumtext',
];
}
public function getGlobalFields()
{
return [
......
......@@ -172,7 +172,8 @@ class Search_ContentSource_ForumPostSource implements Search_ContentSource_Inter
'root_thread_id',
'parent_contributors',
'hits',
'type',
'thread_type',
'reply_count',
'locked',
'lastpost_title',
......@@ -182,7 +183,45 @@ class Search_ContentSource_ForumPostSource implements Search_ContentSource_Inter
'lastpost_post_snippet',
'lastpost_hits',
'lastpost_thread_id',
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'language' => 'identifier',
'creation_date' => 'timestamp',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'post_content' => 'wikitext',
'post_author' => 'wikitext',
'post_snippet' => 'plaintext',
'forum_id' => 'identifier',
'forum_section' => 'identifier',
'forum_title' => 'sortable',
'parent_thread_id' => 'identifier',
'view_permission' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
'root_thread_id' => 'identifier',
'parent_contributors' => 'multivalue',
'hits' => 'numeric',
'thread_type' => 'identifier',
'reply_count' => 'numeric',
'locked' => 'identifier',
'lastpost_title' => 'sortable',
'lastpost_modification_date' => 'timestamp',
'lastpost_contributors' => 'multivalue',
'lastpost_post_content' => 'wikitext',
'lastpost_post_snippet' => 'plaintext',
'lastpost_hits' => 'numeric',
'lastpost_thread_id' => 'identifier',
];
}
......
......@@ -56,12 +56,25 @@ class Search_ContentSource_ForumSource implements Search_ContentSource_Interface
'forum_section',
'searchable',
'view_permission',
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'creation_date' => 'timestamp',
'date' => 'timestamp',
'description' => 'plaintext',
'language' => 'identifier',
'forum_section' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -51,6 +51,18 @@ class Search_ContentSource_GoalEventSource implements Search_ContentSource_Inter
return ['event_type', 'modification_date', 'user', 'goal_groups', 'target', 'date'];
}
public function getProvidedFieldTypes()
{
return [
'event_type' => 'identifier',
'modification_date' => 'timestamp',
'user' => 'identifier',
'goal_groups' => 'multivalue',
'target' => 'identifier',
'date' => 'timestamp'
];
}
public function getGlobalFields()
{
return [
......
......@@ -54,6 +54,18 @@ class Search_ContentSource_GroupSource implements Search_ContentSource_Interface
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'description' => 'plaintext',
'searchable' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -28,6 +28,13 @@ interface Search_ContentSource_Interface
*/
public function getProvidedFields();
/**
* Returns an array containing the list of field names that can be provided
* by the content source and their respective search types.
*/
public function getProvidedFieldTypes();
/**
* Returns an array containing the list of field names that must be included
* in the global content.
......
......@@ -83,6 +83,21 @@ class Search_ContentSource_SheetSource implements Search_ContentSource_Interface
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'description' => 'sortable',
'modification_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'sheet_content' => 'plaintext',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -58,6 +58,17 @@ class Search_ContentSource_Static implements Search_ContentSource_Interface
return array_keys($this->typeMap);
}
public function getProvidedFieldTypes()
{
$out = [];
foreach ($this->typeMap as $key => $value) {
$out[$key] = $value;
}
return $out;
}
public function getGlobalFields()
{
return array_fill_keys(array_keys($this->typeMap), true);
......
......@@ -76,6 +76,7 @@ class Search_ContentSource_TrackerFieldSource implements Search_ContentSource_In
'description',
'tracker_id',
'tracker_name',
'position',
'searchable',
......@@ -83,6 +84,21 @@ class Search_ContentSource_TrackerFieldSource implements Search_ContentSource_In
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'description' => 'plaintext',
'tracker_id' => 'identifier',
'tracker_name' => 'sortable',
'position' => 'numeric',
'searchable' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -155,6 +155,37 @@ class Search_ContentSource_TrackerItemSource implements Search_ContentSource_Int
return array_unique($data);
}
public function getProvidedFieldTypes()
{
static $data;
if (is_array($data)) {
return $data;
}
$data = [
'title' => 'sortable',
'language' => 'identifier',
'modification_date' => 'timestamp',
'creation_date' => 'timestamp',
'date' => 'timestamp',
'contributors' => 'multivalue',
'tracker_status' => 'identifier',
'tracker_id' => 'identifier',
'view_permission' => 'identifier',
'parent_object_id' => 'identifier',
'parent_object_type' => 'identifier',
];
foreach ($this->getAllIndexableHandlers() as $handler) {
$data = array_merge($data, $handler->getProvidedFieldTypes());
}
return $data;
}
public function getGlobalFields()
{
static $data;
......
......@@ -60,6 +60,21 @@ class Search_ContentSource_TrackerSource implements Search_ContentSource_Interfa
];
}
public function getProvidedFieldTypes()
{
return [
'title' => 'sortable',
'modification_date' => 'timestamp',
'creation_date' => 'timestamp',
'date' => 'timestamp',
'description' => 'plaintext',
'searchable' => 'identifier',
'view_permission' => 'identifier',
];
}
public function getGlobalFields()
{
return [
......
......@@ -176,6 +176,44 @@ class Search_ContentSource_UserSource implements Search_ContentSource_Interface
return array_unique($data);
}
public function getProvidedFieldTypes()
{
static $data;
if (is_array($data)) {
return $data;
}
$data = [
'title' => 'sortable',
'title_unstemmed' => 'simpletext',
'creation_date' => 'timestamp',
'date' => 'timestamp',
'wiki_content' => 'wikitext',
'user_country' => 'sortable',
'user_gender' => 'sortable',
'user_homepage' => 'sortable',
'user_realName' => 'sortable',
'user_allowmsgs' => 'sortable',
'user_language' => 'multivalue',
'user_style' => 'sortable',
'user_page' => 'sortable',
'geo_located' => 'identifier',
'geo_location' => 'identifier',
'searchable' => 'identifier',
'groups' => 'multivalue',
];