Skip to content

Advanced Search: Optimize group-level searches for Issue Scope

Proposal

If you take a look at this group search, you can see that we pass a lot of project ids to elasticsearch query, which makes it slow. I believe that we should start thinking about optimizing group-level searches by introducing group hierarchy into our documents in order to allow efficient filtering.

click to see the query
{
   "query":{
      "bool":{
         "must":[
            {
               "simple_query_string":{
                  "_name":"issue:match:search_terms",
                  "fields":[
                     "iid^3",
                     "title^2",
                     "description"
                  ],
                  "query":"test",
                  "lenient":true,
                  "default_operator":"and"
               }
            }
         ],
         "filter":[
            {
               "term":{
                  "type":{
                     "_name":"doc:is_a:issue",
                     "value":"issue"
                  }
               }
            },
            {
               "terms":{
                  "_name":"issue:authorized:project",
                  "project_id":[
                     13083,
                     13764,
                     14022,
                     14289,
                     19776,
                     20086,
                     20699,
                     23081,
                     27468,
                     27470,
                     32732,
                     36743,
                     74823,
                     83282,
                     95156,
                     116212,
                     145205,
                     150440,
                     227582,
                     250324,
                     250833,
                     278964,
                     387896,
                     430285,
                     443787,
                     444821,
                     554859,
                     593728,
                     684698,
                     730448,
                     734943,
                     766015,
                     818896,
                     876090,
                     928825,
                     931715,
                     998792,
                     1075790,
                     1254421,
                     1265999,
                     1329047,
                     1379171,
                     1441932,
                     1470839,
                     1507906,
                     1777822,
                     1794617,
                     1911766,
                     1990920,
                     2009901,
                     2127625,
                     2317465,
                     2337675,
                     2383700,
                     2651596,
                     2670515,
                     2725567,
                     2890326,
                     2903741,
                     2953390,
                     3010986,
                     3010998,
                     3094319,
                     3362933,
                     3395064,
                     3430480,
                     3466815,
                     3588247,
                     3605985,
                     3631141,
                     3651684,
                     3662668,
                     3698388,
                     3828396,
                     3842996,
                     3843116,
                     3844141,
                     3871132,
                     3871556,
                     3885956,
                     3885980,
                     3933206,
                     3933372,
                     3991945,
                     4108541,
                     4149988,
                     4157550,
                     4176070,
                     4262096,
                     4456656,
                     4492386,
                     4534254,
                     4542834,
                     4562632,
                     4685354,
                     4793514,
                     4870361,
                     4875494,
                     4904157,
                     4921652,
                     4949400,
                     5019307,
                     5064907,
                     5191294,
                     5261717,
                     5279538,
                     5351075,
                     5408708,
                     5417931,
                     5420758,
                     5444181,
                     5456231,
                     5457651,
                     5457755,
                     5465687,
                     5467143,
                     5467277,
                     5474112,
                     5509547,
                     5635189,
                     5647182,
                     5677844,
                     5727416,
                     5737157,
                     5772881,
                     5777179,
                     5777819,
                     5777853,
                     5777918,
                     5777939,
                     5777952,
                     5777954,
                     5777976,
                     5778037,
                     5778044,
                     5778056,
                     5778074,
                     5778093,
                     5778095,
                     5828233,
                     5849726,
                     5851774,
                     5945915,
                     5981322,
                     5994307,
                     5996549,
                     6043227,
                     6057676,
                     6143038,
                     6185025,
                     6237085,
                     6237088,
                     6237091,
                     6250233,
                     6329679,
                     6374580,
                     6374588,
                     6374596,
                     6388398,
                     6392811,
                     6457868,
                     6523803,
                     6523862,
                     6535935,
                     6578235,
                     7071043,
                     7071551,
                     7145793,
                     7243435,
                     7453181,
                     7523614,
                     7524061,
                     7540679,
                     7560116,
                     7560137,
                     7602162,
                     7660893,
                     7688358,
                     7745145,
                     7750063,
                     7776928,
                     7783667,
                     7792567,
                     7937396,
                     7951712,
                     7963964,
                     8057910,
                     8262564,
                     8343284,
                     8362868,
                     8368695,
                     8368700,
                     8377576,
                     8671969,
                     8987255,
                     9049889,
                     9134345,
                     9149724,
                     9172481,
                     9184510,
                     9223256,
                     9223291,
                     9237774,
                     9353893,
                     9359704,
                     9438583,
                     9492132,
                     9870184,
                     9927571,
                     10024275,
                     10071001,
                     10071132,
                     10304488,
                     10400718,
                     10476326,
                     10506825,
                     10523039,
                     10586771,
                     10614162,
                     10619765,
                     10633620,
                     10747057,
                     10748426,
                     10833541,
                     10861561,
                     10947320,
                     10947578,
                     10953870,
                     10973541,
                     11015994,
                     11047304,
                     11067315,
                     11080193,
                     11139665,
                     11173751,
                     11196060,
                     11203076,
                     11229385,
                     11261137,
                     11316755,
                     11317506,
                     11322136,
                     11373038,
                     11380952,
                     11432834,
                     11446522,
                     11449672,
                     11511606,
                     11520423,
                     11533294,
                     11574953,
                     11625232,
                     11688089,
                     11745002,
                     11758351,
                     11869718,
                     11880131,
                     11915984,
                     11925674,
                     11925677,
                     11925682,
                     11925685,
                     11925689,
                     11925692,
                     11925695,
                     11925697,
                     11925699,
                     11925701,
                     11925704,
                     11925705,
                     11925708,
                     11925710,
                     11925713,
                     11997420,
                     12006272,
                     12051824,
                     12075545,
                     12104213,
                     12106293,
                     12138769,
                     12193672,
                     12193723,
                     12233892,
                     12270872,
                     12271982,
                     12271993,
                     12271995,
                     12271997,
                     12272000,
                     12272001,
                     12272002,
                     12272003,
                     12272004,
                     12272005,
                     12272006,
                     12272007,
                     12272008,
                     12272009,
                     12272010,
                     12272011,
                     12272014,
                     12272015,
                     12272016,
                     12272018,
                     12272019,
                     12272024,
                     12272025,
                     12272030,
                     12272032,
                     12272040,
                     12272049,
                     12272051,
                     12272053,
                     12272056,
                     12272064,
                     12272066,
                     12272071,
                     12272093,
                     12272095,
                     12272119,
                     12272164,
                     12272174,
                     12272200,
                     12272586,
                     12272605,
                     12273972,
                     12273973,
                     12273975,
                     12273978,
                     12273983,
                     12273988,
                     12273995,
                     12273999,
                     12274019,
                     12274036,
                     12274047,
                     12274048,
                     12274049,
                     12274053,
                     12274058,
                     12274059,
                     12274061,
                     12274062,
                     12274065,
                     12274066,
                     12274067,
                     12274068,
                     12274070,
                     12274076,
                     12274078,
                     12274079,
                     12274081,
                     12274089,
                     12274094,
                     12274099,
                     12274104,
                     12274109,
                     12274122,
                     12274127,
                     12274139,
                     12274166,
                     12274192,
                     12274200,
                     12274218,
                     12274295,
                     12425329,
                     12446167,
                     12450931,
                     12483816,
                     12507547,
                     12558964,
                     12690061,
                     12959103,
                     13015884,
                     13017629,
                     13017904,
                     13017938,
                     13017951,
                     13017970,
                     13017973,
                     13017990,
                     13018004,
                     13018017,
                     13053017,
                     13084505,
                     13130853,
                     13142177,
                     13284652,
                     13327609,
                     13331704,
                     13348998,
                     13362575,
                     13368043,
                     13437682,
                     13453461,
                     13454093,
                     13467157,
                     13473820,
                     13490286,
                     13584096,
                     13589474,
                     13589490,
                     13590021,
                     13626558,
                     13664986,
                     13672316,
                     13672681,
                     13673582,
                     13673632,
                     13673686,
                     13673757,
                     13673934,
                     13674085,
                     13674136,
                     13675927,
                     13675979,
                     13676002,
                     13676032,
                     13676096,
                     13676142,
                     13676188,
                     13676249,
                     13676309,
                     13676350,
                     13682597,
                     13683636,
                     13740954,
                     13798059,
                     13815397,
                     13822214,
                     13823180,
                     13824926,
                     13831684,
                     13842214,
                     13842876,
                     13942461,
                     13954053,
                     13997108,
                     14018700,
                     14071101,
                     14097509,
                     14130725,
                     14132361,
                     14146123,
                     14158369,
                     14213955,
                     14217204,
                     14235835,
                     14290884,
                     14292404,
                     14356474,
                     14372596,
                     14378900,
                     14392833,
                     14399034,
                     14402567,
                     14404642,
                     14442567,
                     14446830,
                     14477200,
                     14520336,
                     14579026,
                     14645774,
                     14694517,
                     14707715,
                     14720460,
                     14771920,
                     14785608,
                     14810991,
                     14853550,
                     14921629,
                     15019630,
                     15032634,
                     15065439,
                     15101703,
                     15112583,
                     15114652,
                     15126989,
                     15158038,
                     15161313,
                     15182494,
                     15191824,
                     15206937,
                     15207036,
                     15208888,
                     15232184,
                     15291320,
                     15297693,
                     15363819,
                     15393051,
                     15412762,
                     15420077,
                     15445353,
                     15445883,
                     15447440,
                     15461649,
                     15486630,
                     15499610,
                     15502498,
                     15572444,
                     15642544,
                     15662771,
                     15665075,
                     15667093,
                     15677708,
                     15685819,
                     15685912,
                     15704782,
                     15710326,
                     15720150,
                     15769471,
                     15769622,
                     15769659,
                     15815706,
                     15816248,
                     15821588,
                     15822964,
                     15828516,
                     15841984,
                     15844807,
                     15875037,
                     15926775,
                     15943672,
                     15962657,
                     16110032,
                     16233736,
                     16239404,
                     16240616,
                     16247504,
                     16257114,
                     16288111,
                     16297627,
                     16302012,
                     16379995,
                     16466244,
                     16470734,
                     16488249,
                     16505341,
                     16549277,
                     16552062,
                     16573099,
                     16590122,
                     16597098,
                     16603968,
                     16613287,
                     16613296,
                     16622651,
                     16683102,
                     16694332,
                     16700143,
                     16706750,
                     16732333,
                     16816732,
                     16819118,
                     16819125,
                     16819131,
                     16819134,
                     16819140,
                     16819151,
                     16819248,
                     16842968,
                     16863027,
                     16873105,
                     16913647,
                     17019611,
                     17022380,
                     17023897,
                     17031626,
                     17035255,
                     17039597,
                     17043690,
                     17050975,
                     17113313,
                     17120026,
                     17130955,
                     17133307,
                     17134833,
                     17136273,
                     17151529,
                     17166102,
                     17169260,
                     17176376,
                     17177253,
                     17192985,
                     17245095,
                     17253921,
                     17257555,
                     17271570,
                     17281958,
                     17311004,
                     17325482,
                     17334694,
                     17345914,
                     17349153,
                     17370206,
                     17370850,
                     17370917,
                     17371314,
                     17372321,
                     17373998,
                     17375753,
                     17392353,
                     17407852,
                     17410444,
                     17421565,
                     17429946,
                     17433833,
                     17450826,
                     17471714,
                     17522813,
                     17523881,
                     17530059,
                     17545989,
                     17584993,
                     17616536,
                     17657269,
                     17661412,
                     17661947,
                     17719928,
                     17720083,
                     17738687,
                     17743501,
                     17747896,
                     17753367,
                     17802074,
                     17816825,
                     17821569,
                     17823896,
                     17842971,
                     17843116,
                     17863546,
                     17875927,
                     17882005,
                     17885954,
                     17894202,
                     17930014,
                     17960932,
                     17966464,
                     17967450,
                     17967459,
                     17971433,
                     17978213,
                     17987891,
                     17990319,
                     17992812,
                     17994706,
                     18021647,
                     18026144,
                     18042963,
                     18060348,
                     18113582,
                     18140840,
                     18147276,
                     18147374,
                     18147399,
                     18156681,
                     18157270,
                     18161573,
                     18163072,
                     18163099,
                     18179066,
                     18208929,
                     18250019,
                     18256369,
                     18256375,
                     18256379,
                     18256394,
                     18283598,
                     18284159,
                     18298876,
                     18305763,
                     18307741,
                     18307889,
                     18308312,
                     18313773,
                     18315938,
                     18321485,
                     18331927,
                     18342819,
                     18346158,
                     18347703,
                     18347716,
                     18347753,
                     18347757,
                     18347776,
                     18347780,
                     18347796,
                     18347808,
                     18347809,
                     18347819,
                     18347820,
                     18347830,
                     18347831,
                     18347833,
                     18347834,
                     18347835,
                     18347836,
                     18347837,
                     18347844,
                     18347848,
                     18347863,
                     18347890,
                     18347935,
                     18347953,
                     18348379,
                     18348449,
                     18374998,
                     18406509,
                     18439881,
                     18464659,
                     18464813,
                     18466096,
                     18480370,
                     18594323,
                     18594390,
                     18641703,
                     18650560,
                     18663049,
                     18668507,
                     18716754,
                     18719477,
                     18741849,
                     18759998,
                     18760252,
                     18766720,
                     18766749,
                     18770944,
                     18776356,
                     18789638,
                     18792504,
                     18847064,
                     18855948,
                     18860383,
                     18863350,
                     18875792,
                     18877555,
                     18880248,
                     18893322,
                     18897919,
                     18899486,
                     18899500,
                     18905432,
                     18912995,
                     18922546,
                     18925373,
                     18943563,
                     18943607,
                     18976211,
                     18977208,
                     18991900,
                     19019153,
                     19028203,
                     19031064,
                     19042106,
                     19043830,
                     19047934,
                     19145311,
                     19155669,
                     19192813,
                     19209142,
                     19245461,
                     19253018,
                     19253326,
                     19256834,
                     19265543,
                     19279270,
                     19290493,
                     19290857,
                     19310217,
                     19315261,
                     19402176,
                     19413101,
                     19416940,
                     19438691,
                     19461377,
                     19483656,
                     19486292,
                     19533665,
                     19536993,
                     19551890,
                     19568296,
                     19586313,
                     19594758,
                     19611351,
                     19617580,
                     19655424,
                     19657914,
                     19666238,
                     19668251,
                     19716327,
                     19720958,
                     19722963,
                     19797387,
                     19800404,
                     19815415,
                     19843335,
                     19861191,
                     19871264,
                     19871573,
                     19890267,
                     19904939,
                     19922160,
                     19926831,
                     19928869,
                     19935960,
                     19936172,
                     19936363,
                     19936368,
                     19941935,
                     19969997,
                     19971311,
                     19974437,
                     19985677,
                     19993243,
                     20009186,
                     20014237,
                     20017176,
                     20059805,
                     20081132,
                     20083979,
                     20084708,
                     20085049,
                     20102452,
                     20107267,
                     20110067,
                     20140116,
                     20145500,
                     20172887,
                     20172997,
                     20190582,
                     20204282,
                     20209910,
                     20233854,
                     20234539,
                     20307753,
                     20326496,
                     20352787,
                     20360386,
                     20376798,
                     20466430,
                     20468480,
                     20498312,
                     20510065,
                     20516845,
                     20623750,
                     20630561,
                     20662496,
                     20666273,
                     20744986,
                     20747799,
                     20773921,
                     20776072,
                     20778930,
                     20785013,
                     20791019,
                     20802859,
                     20803052,
                     20839085,
                     20850906,
                     20867992,
                     20904766,
                     20916876,
                     20921195,
                     20973262,
                     21005471,
                     21029182,
                     21055344,
                     21055409,
                     21055520,
                     21055609,
                     21055767,
                     21138460,
                     21146917,
                     21186073,
                     21198535,
                     21250701,
                     21286161,
                     21297458,
                     21324924,
                     21332563,
                     21333175,
                     21368827,
                     21375917,
                     21426890,
                     21439066,
                     21454570,
                     21479276,
                     21479995,
                     21483460,
                     21486653,
                     21501997,
                     21503580,
                     21513228,
                     21525059,
                     21531004,
                     21565866,
                     21573444,
                     21582428,
                     21607123,
                     21628863,
                     21635031,
                     21649753,
                     21731412,
                     21731462,
                     21733688,
                     21740473,
                     21751536,
                     21751817,
                     21764500,
                     21796485,
                     21808150,
                     21819002,
                     21824450,
                     21877506,
                     21877574,
                     21939352,
                     21949381,
                     21958988,
                     21967079,
                     21967100,
                     22024501,
                     22031371,
                     22046463,
                     22057235,
                     22070439,
                     22111360,
                     22130851,
                     22133520,
                     22200396,
                     22200599,
                     22201268,
                     22201414,
                     22249364,
                     22304113,
                     22342780,
                     22351703,
                     22363055,
                     22382594,
                     22408078,
                     22409005,
                     22413016,
                     22413375,
                     22414284,
                     22418898,
                     22432227,
                     22433156,
                     22438483,
                     22487050,
                     22488218,
                     22488569,
                     22517790,
                     22540162,
                     22564042,
                     22568832,
                     22574075,
                     22592843,
                     22637103,
                     22679171,
                     22734535,
                     22785135,
                     22795840,
                     22806496,
                     22807429,
                     22810929,
                     22810940,
                     22811022,
                     22847718,
                     22848448,
                     22850480,
                     22874770,
                     22877515,
                     22899058,
                     22907418,
                     22954041,
                     22955441,
                     22956868,
                     22958987,
                     22963764,
                     22971355,
                     22975621,
                     22976387,
                     22976557,
                     22978009,
                     22978925,
                     22979053,
                     22979589,
                     22979778,
                     22979980,
                     22980130,
                     22991406,
                     23011180,
                     23022199,
                     23036920,
                     23043633,
                     23105702,
                     23125600,
                     23141735,
                     23156104,
                     23171098,
                     23174714,
                     23176166,
                     23203100,
                     23203118,
                     23221705,
                     23221789,
                     23344605,
                     23345837,
                     23365123,
                     23455079,
                     23457285,
                     23465101,
                     23492503,
                     23494511,
                     23495190,
                     23508645,
                     23510942,
                     23511999,
                     23512541,
                     23517830,
                     23526204,
                     23544263,
                     23545481,
                     23586678,
                     23591572,
                     23597027,
                     23597850,
                     23599364,
                     23601234,
                     23617987,
                     23627092,
                     23690656,
                     23698027,
                     23729229,
                     23730491,
                     23740056,
                     23772241,
                     23809347,
                     23821898,
                     23854438,
                     23858731,
                     23862566,
                     23916841,
                     23928229,
                     23944319,
                     23966461,
                     23992030,
                     23992434,
                     23992616,
                     23995504,
                     23996573,
                     23999955,
                     24007929,
                     24015272,
                     24055623,
                     24075172,
                     24111720,
                     24112267,
                     24112430,
                     24124293,
                     24127174,
                     24134396,
                     24189018,
                     24189296,
                     24200638,
                     24225194,
                     24249085,
                     24257614,
                     24296909,
                     24297895,
                     24300893,
                     24339834,
                     24345542,
                     24349523,
                     24351182,
                     24351793,
                     24386141,
                     24434775,
                     24436506,
                     24451242,
                     24467497,
                     24470059,
                     24501533,
                     24519238,
                     24615307,
                     24671550,
                     24673064,
                     24693199,
                     24703104,
                     24703162,
                     24763979,
                     24800312,
                     24873101,
                     24907469,
                     24964122,
                     24964171,
                     25031023,
                     25033712,
                     25036868,
                     25037358,
                     25049492,
                     25158783,
                     25162318,
                     25205782,
                     25208804,
                     25215287,
                     25216298,
                     25247374,
                     25252998,
                     25312761,
                     25336653,
                     25338304,
                     25375648,
                     25386283,
                     25398815,
                     25402115,
                     25415323,
                     25416006,
                     25417540,
                     25418438,
                     25425413,
                     25429985,
                     25443214,
                     25495696,
                     25527940,
                     25534115,
                     25545088,
                     25560556,
                     25567144,
                     25582529,
                     25582530,
                     25582531,
                     25582532,
                     25582534,
                     25582535,
                     25582562,
                     25691329,
                     25837735,
                     25838334,
                     25839455,
                     25840330,
                     25840667,
                     25843190,
                     25847700,
                     25851920,
                     25861038,
                     25891188,
                     25896760,
                     25944737,
                     25969582,
                     25981420,
                     25995980,
                     26045778,
                     26074395,
                     26074460,
                     26074605,
                     26074844,
                     26076251,
                     26077297,
                     26079419,
                     26130408,
                     26220949,
                     26250592,
                     26394287,
                     26421322,
                     26434779,
                     26435679,
                     26438354,
                     26543618,
                     26555236,
                     26566881,
                     26573787,
                     26576566,
                     26579657,
                     26584477,
                     26615930,
                     26615960,
                     26616051,
                     26616061,
                     26616069,
                     26630862,
                     26638991,
                     26698878,
                     26699071,
                     26699200,
                     26710947,
                     26723025,
                     26815390,
                     26821765,
                     26853758,
                     26893673,
                     26903700,
                     26903728,
                     26909873,
                     26913409,
                     27038823,
                     27040581,
                     27051800,
                     27060148,
                     27069846,
                     27079314,
                     27097450,
                     27209445,
                     27209463,
                     27213646,
                     27214210,
                     27219974,
                     27220296,
                     27221011,
                     27226337,
                     27227767,
                     27229445,
                     27244047,
                     27260823,
                     27261025,
                     27269145,
                     27269166,
                     27286937,
                     27326269,
                     27327788,
                     27424635,
                     27428957,
                     27455878,
                     27456355,
                     27456679,
                     27456706,
                     27456810,
                     27457647,
                     27484836,
                     27484846,
                     27484871,
                     27490309,
                     27494514,
                     27517755,
                     27520435,
                     27523122,
                     27526487,
                     27540367,
                     27540387,
                     27540393,
                     27540443,
                     27540450,
                     27540464,
                     27545927,
                     27546552,
                     27577530,
                     27605238,
                     27622671,
                     27650153,
                     27651631,
                     27656943,
                     27668507,
                     27681689,
                     27686541,
                     27779007,
                     27792539,
                     27812280,
                     27819859,
                     27821122,
                     27821148,
                     27822211,
                     27831771,
                     27918529,
                     27919014,
                     27945995,
                     27958807,
                     27973277,
                     27987151,
                     28003746,
                     28011075,
                     28022549,
                     28024541,
                     28025891
                  ]
               }
            }
         ]
      }
   },
   "highlight":{
      "fields":{
         "iid":{
            
         },
         "title":{
            
         },
         "description":{
            
         }
      },
      "number_of_fragments":0,
      "pre_tags":[
         "gitlabelasticsearch→"
      ],
      "post_tags":[
         "←gitlabelasticsearch"
      ]
   }
}

I think we can take inspiration from the ancestry gem and from Rapid Action: Linear namespace queries. Ideally, we should be able to filter documents by a group of any level of inheritance, not only by top level ones.

Downsides

The downside of this approach is that we'd need to update documents if the project they belong to is moved


cc @terrichu @JohnMcGuire @changzhengliu

Release Notes

Searching across groups that have a large number of projects can be slow. This is because there was a check for each project individually.

Initially enabled in Gitlab 14.6, we now use inheritance to make these searches perform twice as fast.

https://docs.gitlab.com/ee/user/search/advanced_search.html#global-search-scopes

Edited by John McGuire