Remove issue project joins for group/project scoped searches
What does this MR do?
Related to #273195 (closed)
Change the Group and Project scoped issues searches to use the project_id field on the issue document instead of the joined project parent in the Elasticsearch query for project permissions check.
Global searches will continue to use the join in the query until further de-normalization work can be completed.
I chose not to add specs because the permissions functionality already has specs that cover issues.
Elasticsearch Query comparison
Before
Note: queries taken from Prod
Project scoped issues Elasticsearch query
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"title^2",
"description"
],
"query": "test",
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:issue",
"value": "issue"
}
}
},
{
"has_parent": {
"_name": "issue:authorized:project",
"parent_type": "project",
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"terms": {
"_name": "issue:authorized:project:membership:id",
"id": [
278964
]
}
},
{
"terms": {
"_name": "issue:authorized:project:issues:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
]
}
}
]
}
}
}
}
]
}
},
"highlight": {
"fields": {
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}Group scoped issues Elasticsearch query
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"title^2",
"description"
],
"query": "test",
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:issue",
"value": "issue"
}
}
},
{
"has_parent": {
"_name": "issue:authorized:project",
"parent_type": "project",
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"terms": {
"_name": "issue:authorized:project:membership:id",
"id": [
13083,
13764,
14022,
14289,
19776,
20086,
20699,
23081,
27468,
27470,
32732,
36743,
74823,
83282,
95156,
116212,
145205,
150440,
227582,
250324,
250833,
278964,
387896,
430285,
443787,
444821,
480929,
554859,
593728,
684698,
730448,
734943,
766015,
818896,
876090,
928825,
931715,
998792,
1075790,
1254421,
1265999,
1329047,
1379171,
1441932,
1470839,
1507906,
1777822,
1794617,
1911766,
1990920,
2009901,
2127625,
2317465,
2337675,
2383700,
2651596,
2670515,
2725567,
2890326,
2903741,
2953390,
3010986,
3010998,
3094319,
3101096,
3180151,
3188754,
3195199,
3362933,
3395064,
3430480,
3466815,
3541392,
3588247,
3601513,
3605985,
3626648,
3631141,
3651684,
3662668,
3698388,
3787977,
3825482,
3828396,
3842996,
3843116,
3844141,
3871132,
3871556,
3885956,
3885980,
3933206,
3933372,
3991945,
4108541,
4149988,
4157550,
4176070,
4262096,
4456656,
4492386,
4534254,
4542834,
4562632,
4685354,
4793514,
4870361,
4875494,
4904157,
4921652,
4949400,
5019307,
5064907,
5191294,
5261717,
5279538,
5351075,
5408708,
5417931,
5420758,
5444181,
5456231,
5457651,
5457755,
5465687,
5467143,
5467277,
5474112,
5486671,
5509547,
5635189,
5647182,
5677844,
5727416,
5737157,
5772881,
5777179,
5777819,
5777853,
5777918,
5777939,
5777952,
5777954,
5777976,
5778037,
5778044,
5778056,
5778074,
5778093,
5778095,
5828233,
5849726,
5851774,
5945915,
5981322,
5994307,
5996549,
6043225,
6043227,
6057676,
6123459,
6124209,
6135689,
6143038,
6185025,
6250233,
6251062,
6299390,
6299406,
6309715,
6310158,
6329679,
6367157,
6374580,
6374588,
6374596,
6379312,
6388398,
6390880,
6457868,
6523803,
6523862,
6938270,
7071043,
7071551,
7243435,
7453181,
7523614,
7524061,
7527683,
7540679,
7542725,
7560116,
7560137,
7602162,
7650596,
7688358,
7745145,
7750063,
7750843,
7776928,
7792567,
7937396,
7951712,
7963964,
8057910,
8226179,
8262564,
8343284,
8362868,
8368324,
8368695,
8368700,
8375261,
8377576,
8427052,
8671969,
8987255,
9049889,
9134345,
9149724,
9172481,
9184510,
9223256,
9223291,
9237774,
9281155,
9353893,
9359704,
9438583,
9492132,
9538485,
9628392,
9628760,
9628831,
9712018,
9870184,
9927571,
9986071,
10024275,
10071001,
10071132,
10304488,
10400718,
10506825,
10523039,
10586771,
10614162,
10619765,
10633620,
10747057,
10748426,
10833541,
10947320,
10947578,
10953870,
10973541,
11015994,
11047304,
11067315,
11080193,
11173751,
11196060,
11203076,
11229385,
11261137,
11316755,
11317506,
11322136,
11373038,
11380952,
11432834,
11446522,
11449672,
11511606,
11520423,
11533294,
11574953,
11688089,
11745002,
11758351,
11869718,
11880131,
11915984,
11925674,
11925677,
11925682,
11925685,
11925689,
11925692,
11925695,
11925697,
11925699,
11925701,
11925704,
11925705,
11925708,
11925710,
11925713,
11997420,
12006272,
12051231,
12051824,
12075545,
12104213,
12106293,
12138769,
12193672,
12193723,
12233892,
12270872,
12271982,
12271993,
12271995,
12271997,
12272000,
12272001,
12272002,
12272003,
12272004,
12272005,
12272006,
12272007,
12272008,
12272009,
12272010,
12272011,
12272014,
12272015,
12272016,
12272018,
12272019,
12272024,
12272025,
12272030,
12272032,
12272040,
12272049,
12272051,
12272053,
12272056,
12272064,
12272066,
12272071,
12272093,
12272095,
12272119,
12272164,
12272174,
12272200,
12272586,
12272605,
12273972,
12273973,
12273975,
12273978,
12273983,
12273988,
12273995,
12273999,
12274019,
12274036,
12274047,
12274048,
12274049,
12274053,
12274058,
12274059,
12274061,
12274062,
12274065,
12274066,
12274067,
12274068,
12274070,
12274076,
12274078,
12274079,
12274081,
12274089,
12274094,
12274099,
12274104,
12274109,
12274122,
12274127,
12274139,
12274166,
12274192,
12274200,
12274218,
12274295,
12425329,
12446167,
12450931,
12483816,
12507547,
12558964,
12609591,
12609592,
12690061,
12724255,
12959103,
13015884,
13017629,
13017904,
13017951,
13017970,
13017973,
13017990,
13018004,
13018017,
13053017,
13084505,
13130853,
13142177,
13284652,
13327609,
13331704,
13348998,
13362575,
13368043,
13437682,
13453461,
13454093,
13467157,
13473820,
13490286,
13584096,
13589474,
13589490,
13590021,
13626558,
13664986,
13672316,
13672681,
13673582,
13673632,
13673686,
13673757,
13673934,
13674085,
13674136,
13675927,
13675979,
13676002,
13676032,
13676096,
13676142,
13676188,
13676249,
13676309,
13676350,
13682597,
13683636,
13740954,
13798059,
13815397,
13822214,
13823180,
13824926,
13831684,
13842214,
13842876,
13942461,
13954053,
13997108,
14018700,
14071101,
14097509,
14130725,
14132361,
14146123,
14158369,
14213955,
14217204,
14235835,
14264161,
14290884,
14292404,
14356474,
14372596,
14378900,
14392833,
14399034,
14402567,
14404642,
14442567,
14446830,
14477200,
14520336,
14579026,
14645774,
14694517,
14707715,
14720460,
14771920,
14785608,
14810991,
14921629,
15019630,
15032634,
15065439,
15101703,
15112583,
15114652,
15126989,
15158038,
15161313,
15182494,
15191824,
15206937,
15207036,
15234689,
15291320,
15297693,
15333630,
15363819,
15393051,
15412762,
15420077,
15445353,
15445883,
15447440,
15461649,
15486630,
15499610,
15502498,
15572444,
15642544,
15662771,
15665075,
15667093,
15677708,
15685912,
15687385,
15704782,
15710326,
15720150,
15769471,
15769622,
15769659,
15815706,
15816248,
15821588,
15822964,
15828516,
15841984,
15844807,
15875037,
15926775,
15943672,
15962657,
16110032,
16233736,
16239404,
16240616,
16247504,
16257114,
16288111,
16297627,
16302012,
16311413,
16379995,
16466244,
16470734,
16488249,
16505341,
16549277,
16552062,
16573099,
16590122,
16597098,
16603968,
16613287,
16613296,
16622651,
16694332,
16700143,
16706750,
16732333,
16816732,
16819118,
16819125,
16819131,
16819134,
16819140,
16819151,
16819248,
16842968,
16863027,
16873105,
16883353,
16913647,
17019611,
17022380,
17023897,
17031626,
17035255,
17039597,
17043690,
17050975,
17113313,
17120026,
17130955,
17133307,
17134833,
17136273,
17151529,
17166102,
17169260,
17176376,
17177253,
17192985,
17245095,
17253921,
17257555,
17271570,
17281958,
17311004,
17325482,
17334694,
17345914,
17370206,
17370850,
17370917,
17371314,
17372321,
17373998,
17375085,
17375753,
17392353,
17407852,
17410444,
17421565,
17429946,
17433833,
17446351,
17450826,
17471714,
17522813,
17523881,
17530059,
17545989,
17584993,
17616536,
17657269,
17661412,
17661947,
17719928,
17720083,
17738687,
17743501,
17747896,
17753367,
17802074,
17816825,
17821569,
17823896,
17842971,
17843116,
17863546,
17875927,
17882005,
17885954,
17894202,
17930014,
17960932,
17966464,
17967450,
17967459,
17971433,
17978213,
17987891,
17990319,
17992812,
17994706,
18021647,
18026144,
18042963,
18060348,
18113582,
18140840,
18147276,
18147374,
18147399,
18156681,
18157270,
18161573,
18163072,
18163099,
18179066,
18208929,
18250019,
18256369,
18256375,
18256379,
18256394,
18283598,
18284159,
18298876,
18305763,
18307741,
18307889,
18308312,
18313773,
18315938,
18321485,
18331927,
18342819,
18346158,
18347703,
18347716,
18347753,
18347757,
18347776,
18347780,
18347796,
18347808,
18347809,
18347819,
18347820,
18347830,
18347831,
18347833,
18347834,
18347835,
18347836,
18347837,
18347844,
18347848,
18347863,
18347890,
18347935,
18347953,
18348379,
18348449,
18374998,
18406509,
18439881,
18464659,
18464813,
18466096,
18480370,
18594323,
18594390,
18641703,
18650560,
18663049,
18668507,
18716754,
18719477,
18741849,
18759998,
18760252,
18766720,
18766749,
18770944,
18776356,
18789638,
18792504,
18847064,
18855948,
18860383,
18863350,
18867949,
18875792,
18877555,
18880248,
18893322,
18897919,
18899486,
18899500,
18904005,
18905432,
18912995,
18920956,
18922546,
18925373,
18943563,
18943607,
18976211,
18977208,
18991900,
19019153,
19028203,
19031064,
19042106,
19047934,
19145311,
19155669,
19192813,
19209142,
19245461,
19253018,
19253326,
19256834,
19265543,
19279270,
19290493,
19290857,
19310217,
19315261,
19402176,
19413101,
19416940,
19438691,
19461377,
19483656,
19486292,
19533665,
19536993,
19551890,
19568296,
19586313,
19594758,
19611351,
19617580,
19655424,
19657914,
19666238,
19668251,
19716327,
19720958,
19722963,
19797387,
19800404,
19815415,
19843335,
19861191,
19871264,
19871573,
19904939,
19922160,
19926831,
19928869,
19935960,
19936172,
19936363,
19936368,
19941935,
19969997,
19971311,
19974437,
19985677,
19993243,
20009186,
20014237,
20017176,
20059805,
20081132,
20083979,
20084708,
20085049,
20102452,
20107267,
20110067,
20140116,
20145500,
20172887,
20172997,
20190582,
20204282,
20209910,
20233854,
20234539,
20307753,
20326496,
20352787,
20360386,
20376798,
20466430,
20468480,
20498312,
20510065,
20516845,
20623750,
20630561,
20632461,
20632468,
20632493,
20632503,
20632621,
20632627,
20662496,
20666273,
20744986,
20747799,
20773921,
20776072,
20778930,
20785013,
20791019,
20802859,
20803052,
20839085,
20850906,
20867992,
20904766,
20916876,
20921195,
20973262,
21005471,
21029182,
21055344,
21055409,
21055520,
21055609,
21055767,
21138460,
21186073,
21198535,
21250701,
21286161,
21297458,
21324924,
21332563,
21332670,
21333175,
21351796,
21368827,
21375917,
21426890,
21439066,
21454570,
21479276,
21479995,
21483460,
21486653,
21501997,
21503580,
21513228,
21525059,
21531004,
21565866,
21573386,
21573444,
21582428,
21591386,
21607123,
21628863,
21635031,
21649753,
21731412,
21731462,
21740473,
21751536,
21751817,
21764500,
21796485,
21808150,
21819002,
21824450,
21877288,
21877506,
21877574,
21939352,
21949381,
21958988,
21967079,
21967100,
22031371,
22046463,
22057235
]
}
},
{
"terms": {
"_name": "issue:authorized:project:issues:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
]
}
}
]
}
}
}
}
]
}
},
"highlight": {
"fields": {
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}After
Note: queries taken from gdk
Project scoped issues Elasticsearch query
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"title^2",
"description"
],
"query": "test",
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:issue",
"value": "issue"
}
}
},
{
"terms": {
"_name": "issue:authorized:project",
"project_id": [
10,
11,
12
]
}
}
]
}
},
"highlight": {
"fields": {
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}Group scoped issues Elasticsearch query
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "project:match:search_terms",
"fields": [
"name^10",
"name_with_namespace^2",
"path_with_namespace",
"path^9",
"description"
],
"query": "test",
"default_operator": "and"
}
}
],
"filter": [
{
"terms": {
"_name": "doc:is_a:project",
"type": [
"project"
]
}
},
{
"bool": {
"should": [
{
"terms": {
"_name": "project:membership:id",
"id": [
12,
11,
10
]
}
}
]
}
}
]
}
},
"highlight": {
"fields": {
"name": {},
"name_with_namespace": {},
"path_with_namespace": {},
"path": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}Performance Results
Run with options:
- against gdk
- using GPT and setting up data using GPT Data Generator
- 20s_10rps.json
- tests: api_v4_search_groups.js, api_v4_search_projects.js
Before
█ API - Project Search
✗ { endpoint:issues }...........: avg=6828.39ms min=628.19ms med=7532.61ms max=10785.30ms p(90)=10680.80ms p(95)=10716.72ms █ API - Group Search
✗ { endpoint:issues }...........: avg=6997.89ms min=795.78ms med=7261.47ms max=12179.80ms p(90)=11921.46ms p(95)=12146.75msAfter
█ API - Project Search
✗ { endpoint:issues }...........: avg=6248.58ms min=453.14ms med=6853.24ms max=10710.59ms p(90)=10187.38ms iterations......................: 25 0.894139/s █ API - Group Search
✗ { endpoint:issues }...........: avg=6502.02ms min=535.69ms med=6788.94ms max=11603.92ms p(90)=10710.43ms p(95)=11080.89msScreenshots (strongly suggested)
Does this MR meet the acceptance criteria?
Conformity
-
Changelog entry - [-] Documentation (if required)
-
Code review guidelines -
Merge request performance guidelines -
Style guides - [-] Database guides
-
Separation of EE specific content
Availability and Testing
-
Review and add/update tests for this feature/bug. Consider all test levels. See the Test Planning Process. -
Tested in all supported browsers -
Informed Infrastructure department of a default or new setting change, if applicable per definition of done
Security
If this MR contains changes to processing or storing of credentials or tokens, authorization and authentication methods and other items described in the security review guidelines:
-
Label as security and @ mention @gitlab-com/gl-security/appsec -
The MR includes necessary changes to maintain consistency between UI, API, email, or other methods -
Security reports checked/validated by a reviewer from the AppSec team
Edited by Terri Chu