Commit 6486cb01 authored by Sean McGivern's avatar Sean McGivern

Merge branch '1052-advanced-search-syntax' into 'master'

Support advanced search queries using elasticsearch

Closes #2037

See merge request !1770
parents ebe5fef5 c557ab54
Pipeline #8017437 failed with stages
in 119 minutes and 46 seconds
......@@ -113,7 +113,7 @@ gem 'seed-fu', '~> 2.3.5'
gem 'elasticsearch-model', '~> 0.1.9'
gem 'elasticsearch-rails', '~> 0.1.9'
gem 'elasticsearch-api', '5.0.3'
gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
gem 'gitlab-elasticsearch-git', '1.2.0', require: "elasticsearch/git"
gem 'aws-sdk'
gem 'faraday_middleware-aws-signers-v4'
......
......@@ -287,7 +287,7 @@ GEM
mime-types (>= 1.19)
rugged (>= 0.23.0b)
github-markup (1.4.0)
gitlab-elasticsearch-git (1.1.1)
gitlab-elasticsearch-git (1.2.0)
activemodel (~> 4.2)
activesupport (~> 4.2)
charlock_holmes (~> 0.7)
......@@ -949,7 +949,7 @@ DEPENDENCIES
gemojione (~> 3.0)
gitaly (~> 0.5.0)
github-linguist (~> 4.7.0)
gitlab-elasticsearch-git (= 1.1.1)
gitlab-elasticsearch-git (= 1.2.0)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-license (~> 1.0)
gitlab-markup (~> 1.5.1)
......
......@@ -117,10 +117,10 @@ module Elastic
query: {
bool: {
must: [{
multi_match: {
simple_query_string: {
fields: fields,
query: query,
operator: :and
default_operator: :and
}
}]
}
......
......@@ -50,14 +50,7 @@ module Elastic
def self.elastic_search(query, options: {})
options[:in] = ['note']
query_hash = {
query: {
bool: {
must: [{ match: { note: query } }],
},
}
}
query_hash = basic_query_hash(%w[note], query)
query_hash = project_ids_filter(query_hash, options)
query_hash = confidentiality_filter(query_hash, options[:current_user])
......
......@@ -52,23 +52,9 @@ module Elastic
end
def self.elastic_search_code(query, options: {})
query_hash = {
query: {
bool: {
must: [{ match: { content: query } }]
}
}
}
query_hash = basic_query_hash(%w(content), query)
query_hash = filter(query_hash, options[:user])
query_hash[:sort] = [
{ updated_at: { order: :desc } },
:_score
]
query_hash[:highlight] = { fields: { content: {} } }
self.__elasticsearch__.search(query_hash)
end
......
......@@ -13,3 +13,7 @@
- unless params[:snippets].eql? 'true'
= render 'filter' if current_user
= button_tag "Search", class: "btn btn-success btn-search"
- if current_application_settings.elasticsearch_search?
.help-block
= link_to 'Advanced search functionality', help_page_path('user/search/advanced-search-syntax.md'), target: '_blank'
is enabled.
---
title: Support advanced search queries using elasticsearch
merge_request: 1770
author:
## Advanced search syntax
If your site administrator has enabled [Elasticsearch integration](../../integration/elasticsearch.md)
then some advanced search functionality is available.
Full details can be found in the
[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html#_simple_query_string_syntax)
but here's a quick guide:
* Searches look for all the words in a query, in any order - e.g.: searching
issues for `display bug` will return all issues matching both those words, in any order.
* To find the exact term, use double quotes: `"display bug"`
* To find bugs not mentioning display, use `-`: `bug -display`
* To find a bug in display or sound, use `|`: `bug display | sound`
* To group terms together, use parentheses: `bug | (display +sound)`
* To match a partial word, use `*`: `bug find_by_*`
* To find a term containing one of these symbols, use `\`: `argument \-last`
......@@ -10,7 +10,7 @@ module Gitlab
def initialize(current_user, query, limit_project_ids, public_and_internal_projects = true)
@current_user = current_user
@limit_project_ids = limit_project_ids
@query = Shellwords.shellescape(query) if query.present?
@query = query
@public_and_internal_projects = public_and_internal_projects
end
......
......@@ -15,19 +15,19 @@ describe Issue, elastic: true do
it "searches issues" do
Sidekiq::Testing.inline! do
create :issue, title: 'bla-bla term', project: project
create :issue, description: 'bla-bla term', project: project
create :issue, title: 'bla-bla term1', project: project
create :issue, description: 'bla-bla term2', project: project
create :issue, project: project
# The issue I have no access to
create :issue, title: 'bla-bla term'
create :issue, title: 'bla-bla term3'
Gitlab::Elastic::Helper.refresh_index
end
options = { project_ids: [project.id] }
expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
end
it "returns json with all needed elements" do
......
......@@ -15,19 +15,19 @@ describe MergeRequest, elastic: true do
project = create :project
Sidekiq::Testing.inline! do
create :merge_request, title: 'bla-bla term', source_project: project
create :merge_request, description: 'term in description', source_project: project, target_branch: "feature2"
create :merge_request, title: 'bla-bla term1', source_project: project
create :merge_request, description: 'term2 in description', source_project: project, target_branch: "feature2"
create :merge_request, source_project: project, target_branch: "feature3"
# The merge request you have no access to
create :merge_request, title: 'also with term'
create :merge_request, title: 'also with term3'
Gitlab::Elastic::Helper.refresh_index
end
options = { project_ids: [project.id] }
expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
expect(described_class.elastic_search('term1 | term2 | term3', options: options).total_count).to eq(2)
end
it "returns json with all needed elements" do
......
......@@ -15,19 +15,19 @@ describe Milestone, elastic: true do
project = create :empty_project
Sidekiq::Testing.inline! do
create :milestone, title: 'bla-bla term', project: project
create :milestone, description: 'bla-bla term', project: project
create :milestone, title: 'bla-bla term1', project: project
create :milestone, description: 'bla-bla term2', project: project
create :milestone, project: project
# The milestone you have no access to
create :milestone, title: 'bla-bla term'
create :milestone, title: 'bla-bla term3'
Gitlab::Elastic::Helper.refresh_index
end
options = { project_ids: [project.id] }
expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
end
it "returns json with all needed elements" do
......
......@@ -15,18 +15,18 @@ describe Note, elastic: true do
issue = create :issue
Sidekiq::Testing.inline! do
create :note, note: 'bla-bla term', project: issue.project
create :note, note: 'bla-bla term1', project: issue.project
create :note, project: issue.project
# The note in the project you have no access to
create :note, note: 'bla-bla term'
create :note, note: 'bla-bla term2'
Gitlab::Elastic::Helper.refresh_index
end
options = { project_ids: [issue.project.id] }
expect(described_class.elastic_search('term', options: options).total_count).to eq(1)
expect(described_class.elastic_search('term1 | term2', options: options).total_count).to eq(1)
end
it "indexes && searches diff notes" do
......
......@@ -15,8 +15,8 @@ describe Project, elastic: true do
project_ids = []
Sidekiq::Testing.inline! do
project = create :empty_project, name: 'test'
project1 = create :empty_project, path: 'test1'
project = create :empty_project, name: 'test1'
project1 = create :empty_project, path: 'test2'
project2 = create :empty_project
create :empty_project, path: 'someone_elses_project'
project_ids += [project.id, project1.id, project2.id]
......@@ -24,8 +24,9 @@ describe Project, elastic: true do
Gitlab::Elastic::Helper.refresh_index
end
expect(described_class.elastic_search('test', options: { project_ids: project_ids }).total_count).to eq(1)
expect(described_class.elastic_search('test1', options: { project_ids: project_ids }).total_count).to eq(1)
expect(described_class.elastic_search('test2', options: { project_ids: project_ids }).total_count).to eq(1)
expect(described_class.elastic_search('test*', options: { project_ids: project_ids }).total_count).to eq(2)
expect(described_class.elastic_search('someone_elses_project', options: { project_ids: project_ids }).total_count).to eq(0)
end
......
......@@ -15,12 +15,14 @@ describe ProjectWiki, elastic: true do
project = create :empty_project
Sidekiq::Testing.inline! do
project.wiki.create_page("index_page", "Bla bla")
project.wiki.create_page("index_page", "Bla bla term1")
project.wiki.create_page("omega_page", "Bla bla term2")
project.wiki.index_blobs
Gitlab::Elastic::Helper.refresh_index
end
expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
expect(project.wiki.search('term1', type: :blob)[:blobs][:total_count]).to eq(1)
expect(project.wiki.search('term1 | term2', type: :blob)[:blobs][:total_count]).to eq(2)
end
end
......@@ -22,6 +22,7 @@ describe Repository, elastic: true do
end
expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
expect(project.repository.search('def | popen')[:blobs][:total_count] > 1).to be_truthy
expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)
end
......
......@@ -19,9 +19,9 @@ describe Snippet, elastic: true do
let!(:internal_snippet) { create(:snippet, :internal, content: 'password: XXX') }
let!(:private_snippet) { create(:snippet, :private, content: 'password: XXX', author: author) }
let!(:project_public_snippet) { create(:snippet, :public, project: project, content: 'password: XXX') }
let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: XXX') }
let!(:project_private_snippet) { create(:snippet, :private, project: project, content: 'password: XXX') }
let!(:project_public_snippet) { create(:snippet, :public, project: project, content: 'password: 123') }
let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: 456') }
let!(:project_private_snippet) { create(:snippet, :private, project: project, content: 'password: 789') }
before do
Gitlab::Elastic::Helper.refresh_index
......@@ -60,6 +60,16 @@ describe Snippet, elastic: true do
expect(result.records).to match_array [public_snippet, internal_snippet, private_snippet]
end
it 'supports advanced search syntax' do
member = create(:user)
project.add_reporter(member)
result = described_class.elastic_search_code('password +(123 | 789)', options: { user: member })
expect(result.total_count).to eq(2)
expect(result.records).to match_array [project_public_snippet, project_private_snippet]
end
[:admin, :auditor].each do |user_type|
it "returns all snippets for #{user_type}" do
superuser = create(user_type)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment