Remove issue project joins for group/project scoped searches
The issue data stored in Elasticsearch already contains the project_id
field in the document
This should allow project and group scoped searches to use the project_id
instead of doing a join on the parent field.
We expect this to reduce the complexity of the group/project scoped searches and improve performance.
Example project scoped issues Elasticsearch query generated
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"title^2",
"description"
],
"query": "test",
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:issue",
"value": "issue"
}
}
},
{
"has_parent": {
"_name": "issue:authorized:project",
"parent_type": "project",
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"terms": {
"_name": "issue:authorized:project:membership:id",
"id": [
278964
]
}
},
{
"terms": {
"_name": "issue:authorized:project:issues:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
]
}
}
]
}
}
}
}
]
}
},
"highlight": {
"fields": {
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}
Example group scoped issues Elasticsearch query generated
{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"title^2",
"description"
],
"query": "test",
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:issue",
"value": "issue"
}
}
},
{
"has_parent": {
"_name": "issue:authorized:project",
"parent_type": "project",
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"terms": {
"_name": "issue:authorized:project:membership:id",
"id": [
13083,
13764,
14022,
14289,
19776,
20086,
20699,
23081,
27468,
27470,
32732,
36743,
74823,
83282,
95156,
116212,
145205,
150440,
227582,
250324,
250833,
278964,
387896,
430285,
443787,
444821,
480929,
554859,
593728,
684698,
730448,
734943,
766015,
818896,
876090,
928825,
931715,
998792,
1075790,
1254421,
1265999,
1329047,
1379171,
1441932,
1470839,
1507906,
1777822,
1794617,
1911766,
1990920,
2009901,
2127625,
2317465,
2337675,
2383700,
2651596,
2670515,
2725567,
2890326,
2903741,
2953390,
3010986,
3010998,
3094319,
3101096,
3180151,
3188754,
3195199,
3362933,
3395064,
3430480,
3466815,
3541392,
3588247,
3601513,
3605985,
3626648,
3631141,
3651684,
3662668,
3698388,
3787977,
3825482,
3828396,
3842996,
3843116,
3844141,
3871132,
3871556,
3885956,
3885980,
3933206,
3933372,
3991945,
4108541,
4149988,
4157550,
4176070,
4262096,
4456656,
4492386,
4534254,
4542834,
4562632,
4685354,
4793514,
4870361,
4875494,
4904157,
4921652,
4949400,
5019307,
5064907,
5191294,
5261717,
5279538,
5351075,
5408708,
5417931,
5420758,
5444181,
5456231,
5457651,
5457755,
5465687,
5467143,
5467277,
5474112,
5486671,
5509547,
5635189,
5647182,
5677844,
5727416,
5737157,
5772881,
5777179,
5777819,
5777853,
5777918,
5777939,
5777952,
5777954,
5777976,
5778037,
5778044,
5778056,
5778074,
5778093,
5778095,
5828233,
5849726,
5851774,
5945915,
5981322,
5994307,
5996549,
6043225,
6043227,
6057676,
6123459,
6124209,
6135689,
6143038,
6185025,
6250233,
6251062,
6299390,
6299406,
6309715,
6310158,
6329679,
6367157,
6374580,
6374588,
6374596,
6379312,
6388398,
6390880,
6457868,
6523803,
6523862,
6938270,
7071043,
7071551,
7243435,
7453181,
7523614,
7524061,
7527683,
7540679,
7542725,
7560116,
7560137,
7602162,
7650596,
7688358,
7745145,
7750063,
7750843,
7776928,
7792567,
7937396,
7951712,
7963964,
8057910,
8226179,
8262564,
8343284,
8362868,
8368324,
8368695,
8368700,
8375261,
8377576,
8427052,
8671969,
8987255,
9049889,
9134345,
9149724,
9172481,
9184510,
9223256,
9223291,
9237774,
9281155,
9353893,
9359704,
9438583,
9492132,
9538485,
9628392,
9628760,
9628831,
9712018,
9870184,
9927571,
9986071,
10024275,
10071001,
10071132,
10304488,
10400718,
10506825,
10523039,
10586771,
10614162,
10619765,
10633620,
10747057,
10748426,
10833541,
10947320,
10947578,
10953870,
10973541,
11015994,
11047304,
11067315,
11080193,
11173751,
11196060,
11203076,
11229385,
11261137,
11316755,
11317506,
11322136,
11373038,
11380952,
11432834,
11446522,
11449672,
11511606,
11520423,
11533294,
11574953,
11688089,
11745002,
11758351,
11869718,
11880131,
11915984,
11925674,
11925677,
11925682,
11925685,
11925689,
11925692,
11925695,
11925697,
11925699,
11925701,
11925704,
11925705,
11925708,
11925710,
11925713,
11997420,
12006272,
12051231,
12051824,
12075545,
12104213,
12106293,
12138769,
12193672,
12193723,
12233892,
12270872,
12271982,
12271993,
12271995,
12271997,
12272000,
12272001,
12272002,
12272003,
12272004,
12272005,
12272006,
12272007,
12272008,
12272009,
12272010,
12272011,
12272014,
12272015,
12272016,
12272018,
12272019,
12272024,
12272025,
12272030,
12272032,
12272040,
12272049,
12272051,
12272053,
12272056,
12272064,
12272066,
12272071,
12272093,
12272095,
12272119,
12272164,
12272174,
12272200,
12272586,
12272605,
12273972,
12273973,
12273975,
12273978,
12273983,
12273988,
12273995,
12273999,
12274019,
12274036,
12274047,
12274048,
12274049,
12274053,
12274058,
12274059,
12274061,
12274062,
12274065,
12274066,
12274067,
12274068,
12274070,
12274076,
12274078,
12274079,
12274081,
12274089,
12274094,
12274099,
12274104,
12274109,
12274122,
12274127,
12274139,
12274166,
12274192,
12274200,
12274218,
12274295,
12425329,
12446167,
12450931,
12483816,
12507547,
12558964,
12609591,
12609592,
12690061,
12724255,
12959103,
13015884,
13017629,
13017904,
13017951,
13017970,
13017973,
13017990,
13018004,
13018017,
13053017,
13084505,
13130853,
13142177,
13284652,
13327609,
13331704,
13348998,
13362575,
13368043,
13437682,
13453461,
13454093,
13467157,
13473820,
13490286,
13584096,
13589474,
13589490,
13590021,
13626558,
13664986,
13672316,
13672681,
13673582,
13673632,
13673686,
13673757,
13673934,
13674085,
13674136,
13675927,
13675979,
13676002,
13676032,
13676096,
13676142,
13676188,
13676249,
13676309,
13676350,
13682597,
13683636,
13740954,
13798059,
13815397,
13822214,
13823180,
13824926,
13831684,
13842214,
13842876,
13942461,
13954053,
13997108,
14018700,
14071101,
14097509,
14130725,
14132361,
14146123,
14158369,
14213955,
14217204,
14235835,
14264161,
14290884,
14292404,
14356474,
14372596,
14378900,
14392833,
14399034,
14402567,
14404642,
14442567,
14446830,
14477200,
14520336,
14579026,
14645774,
14694517,
14707715,
14720460,
14771920,
14785608,
14810991,
14921629,
15019630,
15032634,
15065439,
15101703,
15112583,
15114652,
15126989,
15158038,
15161313,
15182494,
15191824,
15206937,
15207036,
15234689,
15291320,
15297693,
15333630,
15363819,
15393051,
15412762,
15420077,
15445353,
15445883,
15447440,
15461649,
15486630,
15499610,
15502498,
15572444,
15642544,
15662771,
15665075,
15667093,
15677708,
15685912,
15687385,
15704782,
15710326,
15720150,
15769471,
15769622,
15769659,
15815706,
15816248,
15821588,
15822964,
15828516,
15841984,
15844807,
15875037,
15926775,
15943672,
15962657,
16110032,
16233736,
16239404,
16240616,
16247504,
16257114,
16288111,
16297627,
16302012,
16311413,
16379995,
16466244,
16470734,
16488249,
16505341,
16549277,
16552062,
16573099,
16590122,
16597098,
16603968,
16613287,
16613296,
16622651,
16694332,
16700143,
16706750,
16732333,
16816732,
16819118,
16819125,
16819131,
16819134,
16819140,
16819151,
16819248,
16842968,
16863027,
16873105,
16883353,
16913647,
17019611,
17022380,
17023897,
17031626,
17035255,
17039597,
17043690,
17050975,
17113313,
17120026,
17130955,
17133307,
17134833,
17136273,
17151529,
17166102,
17169260,
17176376,
17177253,
17192985,
17245095,
17253921,
17257555,
17271570,
17281958,
17311004,
17325482,
17334694,
17345914,
17370206,
17370850,
17370917,
17371314,
17372321,
17373998,
17375085,
17375753,
17392353,
17407852,
17410444,
17421565,
17429946,
17433833,
17446351,
17450826,
17471714,
17522813,
17523881,
17530059,
17545989,
17584993,
17616536,
17657269,
17661412,
17661947,
17719928,
17720083,
17738687,
17743501,
17747896,
17753367,
17802074,
17816825,
17821569,
17823896,
17842971,
17843116,
17863546,
17875927,
17882005,
17885954,
17894202,
17930014,
17960932,
17966464,
17967450,
17967459,
17971433,
17978213,
17987891,
17990319,
17992812,
17994706,
18021647,
18026144,
18042963,
18060348,
18113582,
18140840,
18147276,
18147374,
18147399,
18156681,
18157270,
18161573,
18163072,
18163099,
18179066,
18208929,
18250019,
18256369,
18256375,
18256379,
18256394,
18283598,
18284159,
18298876,
18305763,
18307741,
18307889,
18308312,
18313773,
18315938,
18321485,
18331927,
18342819,
18346158,
18347703,
18347716,
18347753,
18347757,
18347776,
18347780,
18347796,
18347808,
18347809,
18347819,
18347820,
18347830,
18347831,
18347833,
18347834,
18347835,
18347836,
18347837,
18347844,
18347848,
18347863,
18347890,
18347935,
18347953,
18348379,
18348449,
18374998,
18406509,
18439881,
18464659,
18464813,
18466096,
18480370,
18594323,
18594390,
18641703,
18650560,
18663049,
18668507,
18716754,
18719477,
18741849,
18759998,
18760252,
18766720,
18766749,
18770944,
18776356,
18789638,
18792504,
18847064,
18855948,
18860383,
18863350,
18867949,
18875792,
18877555,
18880248,
18893322,
18897919,
18899486,
18899500,
18904005,
18905432,
18912995,
18920956,
18922546,
18925373,
18943563,
18943607,
18976211,
18977208,
18991900,
19019153,
19028203,
19031064,
19042106,
19047934,
19145311,
19155669,
19192813,
19209142,
19245461,
19253018,
19253326,
19256834,
19265543,
19279270,
19290493,
19290857,
19310217,
19315261,
19402176,
19413101,
19416940,
19438691,
19461377,
19483656,
19486292,
19533665,
19536993,
19551890,
19568296,
19586313,
19594758,
19611351,
19617580,
19655424,
19657914,
19666238,
19668251,
19716327,
19720958,
19722963,
19797387,
19800404,
19815415,
19843335,
19861191,
19871264,
19871573,
19904939,
19922160,
19926831,
19928869,
19935960,
19936172,
19936363,
19936368,
19941935,
19969997,
19971311,
19974437,
19985677,
19993243,
20009186,
20014237,
20017176,
20059805,
20081132,
20083979,
20084708,
20085049,
20102452,
20107267,
20110067,
20140116,
20145500,
20172887,
20172997,
20190582,
20204282,
20209910,
20233854,
20234539,
20307753,
20326496,
20352787,
20360386,
20376798,
20466430,
20468480,
20498312,
20510065,
20516845,
20623750,
20630561,
20632461,
20632468,
20632493,
20632503,
20632621,
20632627,
20662496,
20666273,
20744986,
20747799,
20773921,
20776072,
20778930,
20785013,
20791019,
20802859,
20803052,
20839085,
20850906,
20867992,
20904766,
20916876,
20921195,
20973262,
21005471,
21029182,
21055344,
21055409,
21055520,
21055609,
21055767,
21138460,
21186073,
21198535,
21250701,
21286161,
21297458,
21324924,
21332563,
21332670,
21333175,
21351796,
21368827,
21375917,
21426890,
21439066,
21454570,
21479276,
21479995,
21483460,
21486653,
21501997,
21503580,
21513228,
21525059,
21531004,
21565866,
21573386,
21573444,
21582428,
21591386,
21607123,
21628863,
21635031,
21649753,
21731412,
21731462,
21740473,
21751536,
21751817,
21764500,
21796485,
21808150,
21819002,
21824450,
21877288,
21877506,
21877574,
21939352,
21949381,
21958988,
21967079,
21967100,
22031371,
22046463,
22057235
]
}
},
{
"terms": {
"_name": "issue:authorized:project:issues:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
]
}
}
]
}
}
}
}
]
}
},
"highlight": {
"fields": {
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}
Epic order
- Remove joins from the Elasticsearch query for project/group scoped issues search
- De-normalize the issue permission data. <-- will include migration and can be done in multiple MRs
- Remove joins from the Elasticsearch query for globally scoped issues search
- Migration to copy issue data to new index
- Migration to remove issue data from combined index **
- Rake task changes to support new index **
- Backend changes for supporting new indexes during Zero Downtime Re-indexing **
- Admin UI changes to support new index (any time after new index created/data migrated to it)
** must be released in conjunction with the Migration to copy issue data to new index
Edited by Terri Chu