Reduce/Increase throughtput based on monitoring segments
juarezr opened this issue · 0 comments
juarezr commented
Reducing server indexing overhead
Indexing could start failing when solrcopy would be used to aggressively.
For improving this one could look a data in the segments handler in admin API.
The strategy can be monitoring the size and age of merge candidate segments.
Example
The address http://localhost:8983/solr/corename/admin/segments?wt=json would return a json response like:
{
"responseHeader":{
"status":0,
"QTime":3
},
"segments":{
"_dnc":{
"name":"_dnc",
"delCount":0,
"sizeInBytes":3720192411,
"size":1352743,
"age":"2020-05-15T04:08:23.499Z",
"source":"merge",
"version":"6.6.6"
},
"_d4q":{
"name":"_d4q",
"delCount":538,
"sizeInBytes":694162377,
"size":214347,
"age":"2020-05-15T03:51:56.820Z",
"source":"merge",
"version":"6.6.6"
},
"_eju":{
"name":"_eju",
"delCount":6070,
"sizeInBytes":855393299,
"size":265206,
"age":"2020-05-15T04:57:48.340Z",
"source":"merge",
"version":"6.6.6"
},
"_dsx":{
"name":"_dsx",
"delCount":0,
"sizeInBytes":503792200,
"size":149989,
"age":"2020-05-15T04:13:45.125Z",
"source":"merge",
"version":"6.6.6"
},
"_e8s":{
"name":"_e8s",
"delCount":47776,
"sizeInBytes":648225090,
"size":196282,
"age":"2020-05-15T04:27:55.925Z",
"source":"merge",
"version":"6.6.6"
},
"_e43":{
"name":"_e43",
"delCount":0,
"sizeInBytes":520414059,
"size":155389,
"age":"2020-05-15T04:23:38.009Z",
"source":"merge",
"version":"6.6.6"
},
"_fef":{
"name":"_fef",
"delCount":0,
"sizeInBytes":725333779,
"size":222873,
"age":"2020-05-15T05:24:41.320Z",
"source":"merge",
"version":"6.6.6"
},
"_es7":{
"name":"_es7",
"delCount":0,
"sizeInBytes":513762350,
"size":159011,
"age":"2020-05-15T05:05:08.497Z",
"source":"merge",
"version":"6.6.6"
},
"_f0u":{
"name":"_f0u",
"delCount":0,
"sizeInBytes":573389084,
"size":172592,
"age":"2020-05-15T05:12:38.196Z",
"source":"merge",
"version":"6.6.6"
},
"_g0p":{
"name":"_g0p",
"delCount":45940,
"sizeInBytes":1024868184,
"size":322497,
"age":"2020-05-15T05:44:38.097Z",
"source":"merge",
"version":"6.6.6"
},
"_f8w":{
"name":"_f8w",
"delCount":0,
"sizeInBytes":497816681,
"size":147568,
"age":"2020-05-15T05:19:41.182Z",
"source":"merge",
"version":"6.6.6"
},
"_gaz":{
"name":"_gaz",
"delCount":0,
"sizeInBytes":593980446,
"size":180392,
"age":"2020-05-15T05:52:56.207Z",
"source":"merge",
"version":"6.6.6"
},
"_fkn":{
"name":"_fkn",
"delCount":3275,
"sizeInBytes":12405668,
"size":3451,
"age":"2020-05-15T05:29:46.512Z",
"source":"merge",
"version":"6.6.6",
"mergeCandidate":true
},
"_g42":{
"name":"_g42",
"delCount":123,
"sizeInBytes":133100032,
"size":37831,
"age":"2020-05-15T05:47:16.496Z",
"source":"merge",
"version":"6.6.6"
},
"_gf8":{
"name":"_gf8",
"delCount":0,
"sizeInBytes":276107099,
"size":79162,
"age":"2020-05-15T05:56:58.199Z",
"source":"merge",
"version":"6.6.6"
},
"_gp6":{
"name":"_gp6",
"delCount":0,
"sizeInBytes":101241802,
"size":29038,
"age":"2020-05-15T06:06:45.908Z",
"source":"merge",
"version":"6.6.6"
},
"_gkg":{
"name":"_gkg",
"delCount":0,
"sizeInBytes":593948603,
"size":179423,
"age":"2020-05-15T06:02:00.224Z",
"source":"merge",
"version":"6.6.6"
},
"_gw6":{
"name":"_gw6",
"delCount":0,
"sizeInBytes":227529468,
"size":65338,
"age":"2020-05-15T06:13:29.840Z",
"source":"merge",
"version":"6.6.6"
},
"_grn":{
"name":"_grn",
"delCount":0,
"sizeInBytes":549726114,
"size":167534,
"age":"2020-05-15T06:09:19.418Z",
"source":"merge",
"version":"6.6.6"
},
"_h1e":{
"name":"_h1e",
"delCount":0,
"sizeInBytes":521633536,
"size":156034,
"age":"2020-05-15T06:19:10.628Z",
"source":"merge",
"version":"6.6.6"
},
"_gv2":{
"name":"_gv2",
"delCount":0,
"sizeInBytes":78213126,
"size":22363,
"age":"2020-05-15T06:12:37.908Z",
"source":"merge",
"version":"6.6.6"
},
"_gxh":{
"name":"_gxh",
"delCount":0,
"sizeInBytes":96406063,
"size":26971,
"age":"2020-05-15T06:15:24.332Z",
"source":"merge",
"version":"6.6.6"
},
"_h6p":{
"name":"_h6p",
"delCount":0,
"sizeInBytes":171039601,
"size":48299,
"age":"2020-05-15T06:23:30.900Z",
"source":"merge",
"version":"6.6.6"
},
"_h20":{
"name":"_h20",
"delCount":0,
"sizeInBytes":65716159,
"size":19190,
"age":"2020-05-15T06:19:40.164Z",
"source":"merge",
"version":"6.6.6"
},
"_h3n":{
"name":"_h3n",
"delCount":0,
"sizeInBytes":84534197,
"size":24092,
"age":"2020-05-15T06:21:01.420Z",
"source":"merge",
"version":"6.6.6"
},
"_h34":{
"name":"_h34",
"delCount":0,
"sizeInBytes":43293118,
"size":12481,
"age":"2020-05-15T06:20:35.132Z",
"source":"merge",
"version":"6.6.6"
},
"_h52":{
"name":"_h52",
"delCount":0,
"sizeInBytes":59819241,
"size":17372,
"age":"2020-05-15T06:22:06.211Z",
"source":"merge",
"version":"6.6.6"
},
"_h5v":{
"name":"_h5v",
"delCount":0,
"sizeInBytes":72945656,
"size":21005,
"age":"2020-05-15T06:22:53.226Z",
"source":"merge",
"version":"6.6.6"
},
"_h7j":{
"name":"_h7j",
"delCount":0,
"sizeInBytes":71065724,
"size":20723,
"age":"2020-05-15T06:24:14.704Z",
"source":"merge",
"version":"6.6.6"
},
"_h6q":{
"name":"_h6q",
"delCount":0,
"sizeInBytes":2253642,
"size":654,
"age":"2020-05-15T06:23:39.117Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h6s":{
"name":"_h6s",
"delCount":0,
"sizeInBytes":4712345,
"size":1316,
"age":"2020-05-15T06:23:48.008Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h78":{
"name":"_h78",
"delCount":0,
"sizeInBytes":3324666,
"size":910,
"age":"2020-05-15T06:24:07.306Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h7s":{
"name":"_h7s",
"delCount":0,
"sizeInBytes":4712917,
"size":1312,
"age":"2020-05-15T06:24:33.932Z",
"source":"merge",
"version":"6.6.6",
"mergeCandidate":true
},
"_h7i":{
"name":"_h7i",
"delCount":0,
"sizeInBytes":4906686,
"size":1389,
"age":"2020-05-15T06:24:22.213Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h7k":{
"name":"_h7k",
"delCount":0,
"sizeInBytes":3548433,
"size":1027,
"age":"2020-05-15T06:24:21.420Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h7h":{
"name":"_h7h",
"delCount":0,
"sizeInBytes":4420269,
"size":1299,
"age":"2020-05-15T06:24:22.029Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h7q":{
"name":"_h7q",
"delCount":0,
"sizeInBytes":8349820,
"size":2400,
"age":"2020-05-15T06:24:33.616Z",
"source":"flush",
"version":"6.6.6"
},
"_h7p":{
"name":"_h7p",
"delCount":0,
"sizeInBytes":8634659,
"size":2468,
"age":"2020-05-15T06:24:33.823Z",
"source":"flush",
"version":"6.6.6"
},
"_h7r":{
"name":"_h7r",
"delCount":0,
"sizeInBytes":390310,
"size":109,
"age":"2020-05-15T06:24:34.421Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
},
"_h7t":{
"name":"_h7t",
"delCount":0,
"sizeInBytes":4729144,
"size":1395,
"age":"2020-05-15T06:24:37.886Z",
"source":"flush",
"version":"6.6.6",
"mergeCandidate":true
}
}
}