Use traversal_ids in note confidentiality searches
What does this MR do and why?
This MR replaces the confidentiality checks for notes search queries that go to Elasticsearch. The main part of the change is to move from a list of authorized projects to the optimized traversal_ids + project_id list (already used by authorization in this query).
In some large groups, the authorized project list is > 50_000 which leads to Elasticsearch errors because the max terms count limit is 65_536.
AI Summary
This change improves how GitLab's search feature handles privacy and confidentiality for notes (comments) on issues.
Previously, the system used a simpler method to check if users could see confidential notes, which could be slow when searching across many projects. The new implementation adds a more efficient filtering system that uses "traversal IDs" - a faster way to check user permissions across project hierarchies.
The update introduces a new confidentiality filter specifically for notes that considers two levels of privacy: whether the note itself is confidential, and whether the issue it's attached to is confidential. A user can see a note if: the issue isn't confidential and the note isn't confidential, OR they're the author/assignee of a confidential issue, OR they have proper project access permissions.
This change is controlled by a feature flag, so it can be gradually rolled out and easily disabled if issues arise. The old filtering method remains as a fallback. The new system should make searches faster, especially when looking through large numbers of projects and groups, while maintaining the same security and privacy protections.
References
Related to #589754
Screenshots or screen recordings
| Before | After |
|---|---|
How to set up and validate locally
For testing the coverage of all permutations of confidentiality, I'm relying heavily upon confidentiality specs in:
- ee/spec/services/ee/search/global_service_notes_on_XXXX_visibility_spec.rb
- ee/spec/services/ee/search/group_service_notes_on_XXXX_visibility_spec.rb
- ee/spec/services/ee/search/project_service_notes_on_XXXX_visibility_spec.rb
These specs exist for notes on notable types currently indexed (issues, merge requests, snippets and commits). Each uses a slightly different permission gate based on the feature access level requirements. Issues uses an additional gate based on issue confidentiality.
Manually testing this, you can:
- enable elasticsearch on gdk and index the instance (must NOT be in SaaS mode)
- turn on the performance bar to see the ES queries
- with a non-admin user, run global and group level searches for comments
- look at the ES queries with the flag on and off
before
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "note:match:search_terms",
"fields": [
"note"
],
"query": "*",
"lenient": true,
"default_operator": "and"
}
}
],
"should": [],
"filter": [
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "137-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "139-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "140-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:issues_access_level:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:merge_requests_access_level:enabled_or_private",
"merge_requests_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "137-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "139-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "140-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:snippets_access_level:enabled_or_private",
"snippets_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:repository_access_level:enabled_or_private",
"repository_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:issues_access_level:enabled_or_private",
"issues_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31,
36
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:merge_requests_access_level:enabled_or_private",
"merge_requests_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:snippets_access_level:enabled_or_private",
"snippets_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31,
36
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:repository_access_level:enabled_or_private",
"repository_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31
]
}
}
]
}
},
{
"bool": {
"should": [
{
"terms": {
"_name": "filters:permissions:global:issues_access_level:enabled",
"issues_access_level": [
20
]
}
},
{
"terms": {
"_name": "filters:permissions:global:merge_requests_access_level:enabled",
"merge_requests_access_level": [
20
]
}
},
{
"terms": {
"_name": "filters:permissions:global:snippets_access_level:enabled",
"snippets_access_level": [
20
]
}
},
{
"terms": {
"_name": "filters:permissions:global:repository_access_level:enabled",
"repository_access_level": [
20
]
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:visibility_level:public_and_internal",
"visibility_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"bool": {
"must": [
{
"bool": {
"_name": "note:confidentiality:issue:not_confidential",
"should": [
{
"bool": {
"must_not": [
{
"exists": {
"field": "issue"
}
}
]
}
},
{
"term": {
"issue.confidential": false
}
}
]
}
},
{
"bool": {
"_name": "note:confidentiality:not_confidential",
"should": [
{
"bool": {
"must_not": [
{
"exists": {
"field": "confidential"
}
}
]
}
},
{
"term": {
"confidential": false
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"bool": {
"_name": "note:confidentiality:issue:confidential",
"should": {
"term": {
"issue.confidential": true
}
}
}
},
{
"bool": {
"_name": "note:confidentiality:not_confidential",
"should": [
{
"bool": {
"must_not": [
{
"exists": {
"field": "confidential"
}
}
]
}
},
{
"term": {
"confidential": false
}
}
]
}
},
{
"bool": {
"_name": "note:confidentiality:user:issue_author:issue_assignee:project_membership",
"should": [
{
"term": {
"issue.author_id": 66
}
},
{
"term": {
"issue.assignee_id": 66
}
},
{
"terms": {
"project_id": [
25,
26,
27,
31,
33
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"bool": {
"_name": "note:confidentiality:confidential",
"should": {
"term": {
"confidential": true
}
}
}
},
{
"bool": {
"_name": "note:confidentiality:user:project_membership",
"should": {
"terms": {
"_name": "note:confidentiality:project:membership:id",
"project_id": [
25,
26,
27,
31,
33
]
}
}
}
}
]
}
}
]
}
},
{
"bool": {
"_name": "note:archived:non_archived",
"should": [
{
"bool": {
"filter": {
"term": {
"archived": {
"value": false
}
}
}
}
},
{
"bool": {
"must_not": {
"exists": {
"field": "archived"
}
}
}
}
]
}
}
]
}
},
"highlight": {
"fields": {
"note": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}
</details>
<details>
<summary>after</summary>
```json
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "note:match:search_terms",
"fields": [
"note"
],
"query": "*",
"lenient": true,
"default_operator": "and"
}
}
],
"should": [],
"filter": [
{
"bool": {
"should": [
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "137-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "139-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "140-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:issues_access_level:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:merge_requests_access_level:enabled_or_private",
"merge_requests_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "137-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "139-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "140-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:snippets_access_level:enabled_or_private",
"snippets_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "131-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:permissions:global:private_access:ancestry_filter:descendants",
"value": "107-"
}
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:repository_access_level:enabled_or_private",
"repository_access_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:issues_access_level:enabled_or_private",
"issues_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31,
36
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:merge_requests_access_level:enabled_or_private",
"merge_requests_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:snippets_access_level:enabled_or_private",
"snippets_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31,
36
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"_name": "filters:permissions:global:private_access:repository_access_level:enabled_or_private",
"repository_access_level": [
20,
10
]
}
},
{
"terms": {
"_name": "filters:permissions:global:private_access:project:member",
"project_id": [
31
]
}
}
]
}
},
{
"bool": {
"should": [
{
"terms": {
"_name": "filters:permissions:global:issues_access_level:enabled",
"issues_access_level": [
20
]
}
},
{
"terms": {
"_name": "filters:permissions:global:merge_requests_access_level:enabled",
"merge_requests_access_level": [
20
]
}
},
{
"terms": {
"_name": "filters:permissions:global:snippets_access_level:enabled",
"snippets_access_level": [
20
]
}
},
{
"terms": {
"_name": "filters:permissions:global:repository_access_level:enabled",
"repository_access_level": [
20
]
}
}
],
"filter": [
{
"terms": {
"_name": "filters:permissions:global:visibility_level:public_and_internal",
"visibility_level": [
20,
10
]
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"bool": {
"filter": [
{
"bool": {
"_name": "filters:confidentiality:notes:not_on_issue_or_not_confidential",
"should": [
{
"bool": {
"_name": "filters:confidentiality:notes:not_on_issue",
"must_not": [
{
"exists": {
"field": "issue"
}
}
]
}
},
{
"term": {
"issue.confidential": {
"_name": "filters:confidentiality:notes:non_confidential_issue",
"value": false
}
}
}
]
}
},
{
"bool": {
"_name": "filters:confidentiality:notes:not_confidential",
"should": [
{
"bool": {
"must_not": [
{
"exists": {
"field": "confidential"
}
}
]
}
},
{
"term": {
"confidential": false
}
}
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"term": {
"issue.confidential": {
"value": true,
"_name": "filters:confidentiality:notes:issue:confidential"
}
}
},
{
"bool": {
"_name": "filters:confidentiality:notes:not_confidential",
"should": [
{
"bool": {
"must_not": [
{
"exists": {
"field": "confidential"
}
}
]
}
},
{
"term": {
"confidential": false
}
}
]
}
},
{
"bool": {
"should": [
{
"term": {
"issue.author_id": {
"_name": "filters:confidentiality:notes:confidential:as_author",
"value": 66
}
}
},
{
"term": {
"issue.assignee_id": {
"_name": "filters:confidentiality:notes:confidential:as_assignee",
"value": 66
}
}
},
{
"terms": {
"_name": "filters:confidentiality:notes:private:project:member",
"project_id": [
31
]
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:confidentiality:notes:private:ancestry_filter:descendants",
"value": "107-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:confidentiality:notes:private:ancestry_filter:descendants",
"value": "131-"
}
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
}
]
}
},
{
"bool": {
"should": [
{
"terms": {
"_name": "filters:confidentiality:notes:private:project:member",
"project_id": [
31
]
}
},
{
"bool": {
"should": [
{
"prefix": {
"traversal_ids": {
"_name": "filters:confidentiality:notes:private:ancestry_filter:descendants",
"value": "107-"
}
}
},
{
"prefix": {
"traversal_ids": {
"_name": "filters:confidentiality:notes:private:ancestry_filter:descendants",
"value": "131-"
}
}
}
],
"minimum_should_match": 1
}
}
],
"filter": [
{
"term": {
"confidential": {
"_name": "filters:confidentiality:notes:confidential",
"value": true
}
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"_name": "note:archived:non_archived",
"should": [
{
"bool": {
"filter": {
"term": {
"archived": {
"value": false
}
}
}
}
},
{
"bool": {
"must_not": {
"exists": {
"field": "archived"
}
}
}
}
]
}
}
]
}
},
"highlight": {
"fields": {
"note": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}MR acceptance checklist
Evaluate this MR against the MR acceptance checklist. It helps you analyze changes to reduce risks in quality, performance, reliability, security, and maintainability.
Edited by Terri Chu