Skip to content
Snippets Groups Projects
Commit fe605235 authored by Valery Sizov's avatar Valery Sizov
Browse files

ES: all searches are done but it reqiores lots of refactoring

parent c0f6df43
No related branches found
No related tags found
1 merge request!109Elasticsearch integration
Hide whitespace changes
Inline Side-by-side
Showing
with 447 additions and 340 deletions
Gemfile
+ 5
− 0
View file @ fe605235
......@@ -91,6 +91,11 @@ gem "six", '~> 0.2.0'
# Seed data
gem "seed-fu", '~> 2.3.5'
# Search
gem 'elasticsearch-model'
gem 'elasticsearch-rails'
gem 'gitlab-elasticsearch-git', require: "elasticsearch/git"
# Markdown and HTML processing
gem 'html-pipeline', '~> 1.11.0'
gem 'task_list', '~> 1.0.2', require: 'task_list/railtie'
......
app/controllers/search_controller.rb
+ 34
− 45
View file @ fe605235
......@@ -4,56 +4,45 @@ class SearchController < ApplicationController
layout 'search'
def show
# return if params[:search].nil? || params[:search].blank?
return if params[:search].nil? || params[:search].blank?
# @search_term = params[:search]
@search_term = params[:search]
# if params[:project_id].present?
# @project = Project.find_by(id: params[:project_id])
# @project = nil unless can?(current_user, :download_code, @project)
# end
# if params[:group_id].present?
# @group = Group.find_by(id: params[:group_id])
# @group = nil unless can?(current_user, :read_group, @group)
# end
# @scope = params[:scope]
# @show_snippets = params[:snippets].eql? 'true'
# @search_results =
# if @project
# unless %w(blobs notes issues merge_requests milestones wiki_blobs
# commits).include?(@scope)
# @scope = 'blobs'
# end
# Search::ProjectService.new(@project, current_user, params).execute
# elsif @show_snippets
# unless %w(snippet_blobs snippet_titles).include?(@scope)
# @scope = 'snippet_blobs'
# end
# Search::SnippetService.new(current_user, params).execute
# else
# unless %w(projects issues merge_requests milestones).include?(@scope)
# @scope = 'projects'
# end
# Search::GlobalService.new(current_user, params).execute
# end
# @objects = @search_results.objects(@scope, params[:page])
@group = Group.find_by(id: params[:group_id]) if params[:group_id].present?
if params[:project_id].present?
@project = Project.find_by(id: params[:project_id])
@project = nil unless can?(current_user, :download_code, @project)
end
if project
return access_denied! unless can?(current_user, :download_code, project)
@search_results = SearchService.new(current_user, params).project_search(project)
else
@search_results = SearchService.new(current_user, params).global_search
if params[:group_id].present?
@group = Group.find_by(id: params[:group_id])
@group = nil unless can?(current_user, :read_group, @group)
end
@search_results = SearchDecorator.new(@search_results, params[:type])
@scope = params[:scope]
@show_snippets = params[:snippets].eql? 'true'
@search_results =
if @project
unless %w(blobs notes issues merge_requests milestones wiki_blobs
commits).include?(@scope)
@scope = 'blobs'
end
Search::ProjectService.new(@project, current_user, params).execute
elsif @show_snippets
unless %w(snippet_blobs snippet_titles).include?(@scope)
@scope = 'snippet_blobs'
end
Search::SnippetService.new(current_user, params).execute
else
unless %w(projects issues merge_requests milestones).include?(@scope)
@scope = 'projects'
end
Search::GlobalService.new(current_user, params).execute
end
@objects = @search_results.objects(@scope, params[:page])
end
def autocomplete
......
app/elastic/application_search.rb
+ 33
− 40
View file @ fe605235
......@@ -4,52 +4,49 @@ module ApplicationSearch
included do
include Elasticsearch::Model
# $ host git-elasticsearch-1.production.infra.home
# git-elasticsearch-1.production.infra.home has address 10.40.56.23
self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-')
settings \
index: {
query: {
default_field: :name
},
analysis: {
:analyzer => {
:index_analyzer => {
type: "custom",
tokenizer: "ngram_tokenizer",
filter: %w(lowercase asciifolding name_ngrams)
},
:search_analyzer => {
type: "custom",
tokenizer: "standard",
filter: %w(lowercase asciifolding )
}
},
tokenizer: {
ngram_tokenizer: {
type: "NGram",
min_gram: 1,
max_gram: 20,
token_chars: %w(letter digit connector_punctuation punctuation)
}
query: {
default_field: :name
},
filter: {
name_ngrams: {
type: "NGram",
max_gram: 20,
min_gram: 1
analysis: {
:analyzer => {
:my_analyzer => {
type: "custom",
tokenizer: "ngram_tokenizer",
filter: %w(lowercase asciifolding name_ngrams)
},
:search_analyzer => {
type: "custom",
tokenizer: "standard",
filter: %w(lowercase asciifolding)
}
},
tokenizer: {
ngram_tokenizer: {
type: "nGram",
min_gram: 1,
max_gram: 20,
token_chars: %w(letter digit connector_punctuation punctuation)
}
},
filter: {
name_ngrams: {
type: "nGram",
max_gram: 20,
min_gram: 1
}
}
}
}
}
after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index, self.class.to_s, self.id) }, on: :create
after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update
after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy
after_touch lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
after_commit ->{ ElasticIndexerWorker.perform_async(:index, self.class.to_s, self.id) }, on: :create
after_commit ->{ ElasticIndexerWorker.perform_async(:update, self.class.to_s, self.id) }, on: :update
after_commit ->{ ElasticIndexerWorker.perform_async(:delete, self.class.to_s, self.id) }, on: :destroy
end
module ClassMethods
......@@ -59,11 +56,7 @@ def highlight_options(fields)
memo
end
{
pre_tags: ["gitlabelasticsearch→"],
post_tags: ["←gitlabelasticsearch"],
fields: es_fields
}
{ fields: es_fields }
end
end
end
app/elastic/issues_search.rb
+ 11
− 46
View file @ fe605235
......@@ -8,49 +8,33 @@ module IssuesSearch
indexes :id, type: :integer, index: :not_analyzed
indexes :iid, type: :integer, index: :not_analyzed
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :state, type: :string
indexes :project_id, type: :integer, index: :not_analyzed
indexes :author_id, type: :integer, index: :not_analyzed
#indexes :assignee_id, type: :integer, index: :not_analyzed
indexes :project, type: :nested
indexes :author, type: :nested
#indexes :assignee, type: :nested
indexes :title_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :date, index: :not_analyzed
indexes :created_at_sort, type: :string, index: :not_analyzed
end
def as_indexed_json(options = {})
as_json(
include: {
project: { only: :id },
author: { only: :id },
#assignee: { only: :id }
author: { only: :id }
}
).merge({
title_sort: title.downcase,
updated_at_sort: updated_at,
created_at_sort: created_at
})
).merge({ updated_at_sort: updated_at })
end
def self.search(query, page: 1, per: 20, options: {})
page ||= 1
if options[:in].blank?
options[:in] = %w(title^2 description)
else
options[:in].push(%w(title^2 description) - options[:in])
end
def self.elastic_search(query, options: {})
options[:in] = %w(title^2 description)
query_hash = {
query: {
filtered: {
......@@ -62,9 +46,7 @@ def self.search(query, page: 1, per: 20, options: {})
}
},
},
},
size: per,
from: per * (page.to_i - 1)
}
}
if query.blank?
......@@ -81,30 +63,13 @@ def self.search(query, page: 1, per: 20, options: {})
}
end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ title_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
if options[:highlight]
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
end
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
self.__elasticsearch__.search(query_hash)
end
end
......
app/elastic/merge_requests_search.rb
+ 12
− 56
View file @ fe605235
......@@ -8,10 +8,10 @@ module MergeRequestsSearch
indexes :id, type: :integer, index: :not_analyzed
indexes :iid, type: :integer, index: :not_analyzed
indexes :target_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :source_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :target_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :source_branch, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :state, type: :string
......@@ -20,15 +20,11 @@ module MergeRequestsSearch
indexes :source_project_id, type: :integer, index: :not_analyzed
indexes :target_project_id, type: :integer, index: :not_analyzed
indexes :author_id, type: :integer, index: :not_analyzed
#indexes :assignee_id, type: :integer, index: :not_analyzed
indexes :source_project, type: :nested
indexes :target_project, type: :nested
indexes :author, type: :nested
#indexes :assignee, type: :nested
indexes :title_sort, type: :string, index: 'not_analyzed'
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end
......@@ -37,25 +33,13 @@ def as_indexed_json(options = {})
include: {
source_project: { only: :id },
target_project: { only: :id },
author: { only: :id },
#assignee: { only: :id }
author: { only: :id }
}
).merge({
title_sort: title.downcase,
updated_at_sort: updated_at,
created_at_sort: created_at
})
).merge({ updated_at_sort: updated_at })
end
def self.search(query, page: 1, per: 20, options: {})
page ||= 1
if options[:in].blank?
options[:in] = %w(title^2 description)
else
options[:in].push(%w(title^2 description) - options[:in])
end
def self.elastic_search(query, options: {})
options[:in] = %w(title^2 description)
query_hash = {
query: {
......@@ -68,18 +52,7 @@ def self.search(query, page: 1, per: 20, options: {})
}
},
},
},
facets: {
targetProjectFacet: {
terms: {
field: :target_project_id,
all_terms: true,
size: Project.count
}
}
},
size: per,
from: per * (page.to_i - 1)
}
}
if query.blank?
......@@ -101,30 +74,13 @@ def self.search(query, page: 1, per: 20, options: {})
}
end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ title_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash)
end
end
......
app/elastic/groups_search.rb → app/elastic/milestones_search.rb
+ 63
− 0
View file @ fe605235
module GroupsSearch
module MilestonesSearch
extend ActiveSupport::Concern
included do
......@@ -6,33 +6,20 @@ module GroupsSearch
mappings do
indexes :id, type: :integer
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :path, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :project_id, type: :integer, index: :not_analyzed
indexes :created_at, type: :date
indexes :name_sort, type: :string, index: :not_analyzed
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end
def as_indexed_json(options = {})
as_json.merge({
name_sort: name.downcase,
created_at_sort: created_at,
updated_at_sort: updated_at
})
as_json.merge({ updated_at_sort: updated_at })
end
def self.search(query, page: 1, per: 20, options: {})
page ||= 1
if options[:in].blank?
options[:in] = %w(name^10 path^5)
else
options[:in].push(%w(name^10 path^5) - options[:in])
end
def self.elastic_search(query, options: {})
options[:in] = %w(title^2 description)
query_hash = {
query: {
......@@ -45,9 +32,7 @@ def self.search(query, page: 1, per: 20, options: {})
}
},
},
},
size: per,
from: per * (page.to_i - 1)
}
}
if query.blank?
......@@ -55,42 +40,22 @@ def self.search(query, page: 1, per: 20, options: {})
query_hash[:track_scores] = true
end
if options[:gids]
if options[:project_ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
ids: {
values: options[:gids]
terms: {
project_id: [options[:project_ids]].flatten
}
}
end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ name_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
#query_hash[:sort] = [
#{ name_sort: { order: :asc, mode: :min }},
#:_score
#]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash)
end
......
app/elastic/notes_search.rb 0 → 100644
+ 55
− 0
View file @ fe605235
module NotesSearch
extend ActiveSupport::Concern
included do
include ApplicationSearch
mappings do
indexes :id, type: :integer
indexes :note, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :project_id, type: :integer, index: :not_analyzed
indexes :created_at, type: :date
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end
def as_indexed_json(options = {})
as_json.merge({ updated_at_sort: updated_at })
end
def self.elastic_search(query, options: {})
options[:in] = ["note"]
query_hash = {
query: {
filtered: {
query: {match: {note: query}},
},
}
}
if query.blank?
query_hash[:query][:filtered][:query] = { match_all: {}}
query_hash[:track_scores] = true
end
if options[:project_ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
terms: {
project_id: [options[:project_ids]].flatten
}
}
end
query_hash[:sort] = [
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash)
end
end
end
app/elastic/projects_search.rb
+ 16
− 97
View file @ fe605235
......@@ -7,11 +7,11 @@ module ProjectsSearch
mappings do
indexes :id, type: :integer, index: 'not_analyzed'
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :path, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :name_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :path_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :path, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :name_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :path_with_namespace, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :description, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :namespace_id, type: :integer, index: 'not_analyzed'
......@@ -20,49 +20,17 @@ module ProjectsSearch
indexes :visibility_level, type: :integer, index: 'not_analyzed'
indexes :last_activity_at, type: :date
indexes :last_pushed_at, type: :date
indexes :owners, type: :nested
indexes :masters, type: :nested
indexes :developers, type: :nested
indexes :reporters, type: :nested
indexes :guests, type: :nested
indexes :categories, type: :nested
indexes :name_with_namespace_sort, type: :string, index: 'not_analyzed'
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end
def as_indexed_json(options={})
as_json(
include: {
owners: { only: :id },
masters: { only: :id },
developers: { only: :id },
reporters: { only: :id },
guests: { only: :id },
categories: { only: :name}
}
).merge({
def as_indexed_json(options = {})
as_json.merge({
name_with_namespace: name_with_namespace,
name_with_namespace_sort: name_with_namespace.downcase,
path_with_namespace: path_with_namespace,
updated_at_sort: updated_at,
created_at_sort: created_at
path_with_namespace: path_with_namespace
})
end
def self.search(query, page: 1, per: 20, options: {})
page ||= 1
if options[:in].blank?
options[:in] = %w(name^10 name_with_namespace^2 path_with_namespace path^9)
else
options[:in].push(%w(name^10 name_with_namespace^2 path_with_namespace path^9) - options[:in])
end
def self.elastic_search(query, options: {})
options[:in] = %w(name^10 name_with_namespace^2 path_with_namespace path^9)
query_hash = {
query: {
......@@ -76,25 +44,6 @@ def self.search(query, page: 1, per: 20, options: {})
},
},
},
facets: {
namespaceFacet: {
terms: {
field: :namespace_id,
all_terms: true,
size: Namespace.count
}
},
categoryFacet: {
terms: {
field: "categories.name",
all_terms: true,
# FIXME. Remove to_a
size: Project.category_counts.to_a.count
}
}
},
size: per,
from: per * (page.to_i - 1)
}
if query.blank?
......@@ -135,18 +84,6 @@ def self.search(query, page: 1, per: 20, options: {})
}
end
if options[:category]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
nested: {
path: :categories,
filter: {
term: { "categories.name" => options[:category] }
}
}
}
end
if options[:non_archived]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
......@@ -169,9 +106,9 @@ def self.search(query, page: 1, per: 20, options: {})
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
nested: {
path: :owners,
path: :owner,
filter: {
term: { "owners.id" => options[:owner_id] }
term: { "owner.id" => options[:owner_id] }
}
}
}
......@@ -186,28 +123,10 @@ def self.search(query, page: 1, per: 20, options: {})
}
end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ name_with_namespace_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
:_score
]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
query_hash[:sort] = [:_score]
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash)
end
......
app/elastic/repositories_search.rb
+ 2
− 8
View file @ fe605235
......@@ -23,14 +23,8 @@ def self.import
Project.find_each do |project|
if project.repository.exists? && !project.repository.empty?
begin
project.repository.index_commits
rescue
end
begin
project.repository.index_blobs
rescue
end
project.repository.index_commits
project.repository.index_blobs
end
end
end
......
app/elastic/snippets_search.rb 0 → 100644
+ 106
− 0
View file @ fe605235
module SnippetsSearch
extend ActiveSupport::Concern
included do
include ApplicationSearch
mappings do
indexes :id, type: :integer, index: :not_analyzed
indexes :title, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :file_name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :content, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :created_at, type: :date
indexes :updated_at, type: :date
indexes :state, type: :string
indexes :project_id, type: :integer, index: :not_analyzed
indexes :author_id, type: :integer, index: :not_analyzed
indexes :project, type: :nested
indexes :author, type: :nested
indexes :updated_at_sort, type: :date, index: :not_analyzed
end
def as_indexed_json(options = {})
as_json(
include: {
project: { only: :id },
author: { only: :id }
}
)
end
def self.elastic_search(query, options: {})
options[:in] = %w(title file_name)
query_hash = {
query: {
filtered: {
query: {
multi_match: {
fields: options[:in],
query: "#{query}",
operator: :and
}
},
},
}
}
if query.blank?
query_hash[:query][:filtered][:query] = { match_all: {}}
query_hash[:track_scores] = true
end
if options[:ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
terms: {
id: [options[:ids]].flatten
}
}
end
query_hash[:sort] = [
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
self.__elasticsearch__.search(query_hash)
end
def self.elastic_search_code(query, options: {})
options[:in] = %w(title file_name)
query_hash = {
query: {
filtered: {
query: {match: {content: query}},
},
}
}
if options[:ids]
query_hash[:query][:filtered][:filter] ||= { and: [] }
query_hash[:query][:filtered][:filter][:and] << {
terms: {
id: [options[:ids]].flatten
}
}
end
query_hash[:sort] = [
{ updated_at_sort: { order: :desc, mode: :min } },
:_score
]
query_hash[:highlight] = { fields: options[:in].inject({}) { |a, o| a[o.to_sym] = {} } }
self.__elasticsearch__.search(query_hash)
end
end
end
app/elastic/users_search.rb
+ 11
− 45
View file @ fe605235
......@@ -6,21 +6,17 @@ module UsersSearch
mappings do
indexes :id, type: :integer
indexes :email, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :username, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :email, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :name, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :username, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :bio, type: :string
indexes :skype, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :skype, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :linkedin, type: :string
indexes :twitter, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, index_analyzer: :index_analyzer
indexes :twitter, type: :string, index_options: 'offsets', search_analyzer: :search_analyzer, analyzer: :my_analyzer
indexes :state, type: :string
indexes :website_url, type: :string
indexes :created_at, type: :date
indexes :admin, type: :boolean
indexes :name_sort, type: :string, index: 'not_analyzed'
indexes :created_at_sort, type: :string, index: 'not_analyzed'
indexes :updated_at_sort, type: :string, index: 'not_analyzed'
end
def as_indexed_json(options = {})
......@@ -31,16 +27,8 @@ def as_indexed_json(options = {})
})
end
def self.search(query, page: 1, per: 20, options: {})
page ||= 1
per ||= 20
if options[:in].blank?
options[:in] = %w(name^3 username^2 email)
else
options[:in].push(%w(name^3 username^2 email) - options[:in])
end
def self.elastic_search(query, options: {})
options[:in] = %w(name^3 username^2 email)
query_hash = {
query: {
......@@ -53,9 +41,7 @@ def self.search(query, page: 1, per: 20, options: {})
}
},
},
},
size: per,
from: per * (page.to_i - 1)
}
}
if query.blank?
......@@ -81,30 +67,10 @@ def self.search(query, page: 1, per: 20, options: {})
}
end
options[:order] = :default if options[:order].blank?
order = case options[:order].to_sym
when :newest
{ created_at_sort: { order: :asc, mode: :min } }
when :oldest
{ created_at_sort: { order: :desc, mode: :min } }
when :recently_updated
{ updated_at_sort: { order: :asc, mode: :min } }
when :last_updated
{ updated_at_sort: { order: :desc, mode: :min } }
else
{ name_sort: { order: :asc, mode: :min } }
end
query_hash[:sort] = [
order,
:_score
]
if options[:highlight]
query_hash[:highlight] = highlight_options(options[:in])
end
query_hash[:sort] = [:_score]
query_hash[:highlight] = highlight_options(options[:in])
self.__elasticsearch__.search(query_hash)
end
end
......
app/elastic/wiki_repositories_search.rb 0 → 100644
+ 31
− 0
View file @ fe605235
module WikiRepositoriesSearch
extend ActiveSupport::Concern
included do
include Elasticsearch::Git::Repository
self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
def repository_id
"wiki_#{project.id}"
end
def self.repositories_count
Project.where(wiki_enabled: true).count
end
def client_for_indexing
self.__elasticsearch__.client
end
def self.import
ProjectWiki.__elasticsearch__.create_index! force: true
Project.where(wiki_enabled: true).find_each do |project|
unless project.wiki.empty?
project.wiki.index_blobs
end
end
end
end
end
app/models/group.rb
+ 0
− 1
View file @ fe605235
......@@ -19,7 +19,6 @@
class Group < Namespace
include Gitlab::ConfigHelper
include Referable
include GroupsSearch
has_many :group_members, dependent: :destroy, as: :source, class_name: 'GroupMember'
alias_method :members, :group_members
......
app/models/milestone.rb
+ 1
− 0
View file @ fe605235
......@@ -24,6 +24,7 @@ class Milestone < ActiveRecord::Base
include Sortable
include Referable
include StripAttribute
include MilestonesSearch
belongs_to :project
has_many :issues
......
app/models/note.rb
+ 1
− 0
View file @ fe605235
......@@ -26,6 +26,7 @@ class Note < ActiveRecord::Base
include Gitlab::CurrentSettings
include Participable
include Mentionable
include NotesSearch
default_value_for :system, false
......
app/models/project.rb
+ 4
− 0
View file @ fe605235
......@@ -943,6 +943,10 @@ def create_wiki
false
end
def wiki
ProjectWiki.new(self, self.owner)
end
def reference_issue_tracker?
default_issues_tracker? || jira_tracker_active?
end
......
app/models/project_wiki.rb
+ 3
− 0
View file @ fe605235
class ProjectWiki
include Gitlab::ShellAdapter
include WikiRepositoriesSearch
MARKUPS = {
'Markdown' => :md,
......@@ -12,6 +13,8 @@ class CouldNotCreateWikiError < StandardError; end
# Returns a string describing what went wrong after
# an operation fails.
attr_reader :error_message
attr_reader :project
def initialize(project, user = nil)
@project = project
......
app/models/repository.rb
+ 53
− 1
View file @ fe605235
......@@ -108,6 +108,12 @@ def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset =
commits
end
def find_commits_by_message_with_elastic(query)
project.repository.search(query, type: :commit)[:commits][:results].map do |result|
commit result["_source"]["commit"]["sha"]
end
end
def find_branch(name)
raw_repository.branches.find { |branch| branch.name == name }
end
......@@ -660,7 +666,53 @@ def search_files(query, ref)
Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
end
def parse_search_result(result)
def parse_search_result(result, elastic = false)
if elastic
parse_search_result_from_elastic(result)
else
parse_search_result_from_grep(result)
end
end
def parse_search_result_from_elastic(result)
ref = result["_source"]["blob"]["oid"]
filename = result["_source"]["blob"]["path"]
content = result["_source"]["blob"]["content"]
total_lines = content.lines.size
term = result["highlight"]["blob.content"][0].match(/gitlabelasticsearch→(.*)←gitlabelasticsearch/)[1]
found_line_number = 0
content.each_line.each_with_index do |line, index|
if line.include?(term)
found_line_number = index
break
end
end
from = if found_line_number >= 2
found_line_number - 2
else
found_line_number
end
to = if (total_lines - found_line_number) > 3
found_line_number + 2
else
found_line_number
end
data = content.lines[from..to]
OpenStruct.new(
filename: filename,
ref: ref,
startline: from + 1,
data: data.join
)
end
def parse_search_result_from_grep(result)
ref = nil
filename = nil
startline = 0
......
app/models/snippet.rb
+ 1
− 0
View file @ fe605235
......@@ -21,6 +21,7 @@ class Snippet < ActiveRecord::Base
include Participable
include Referable
include Sortable
include SnippetsSearch
default_value_for :visibility_level, Snippet::PRIVATE
......
app/services/search/global_service.rb
+ 5
− 1
View file @ fe605235
......@@ -12,7 +12,11 @@ def execute
projects = projects.in_namespace(group.id) if group
project_ids = projects.pluck(:id)
Gitlab::SearchResults.new(project_ids, params[:search])
if Gitlab.config.elasticsearch.enabled
Gitlab::Elastic::SearchResults.new(project_ids, params[:search])
else
Gitlab::SearchResults.new(project_ids, params[:search])
end
end
end
end
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment