Goal is to evaluate how different workload distribution methods affect performance given some fixed computation and network requirements:
- in a single thread
- using
multiprocessing.Pool - using Ray's
multiprocessing.Poolimplementation - using a custom Ray actor setup
Multiprocessing pool: [[0.04492474, 0.04238272, 0.04784584, 0.04459476, 0.05346537, 0.06565928, 0.07287812, 0.08336544, 0.13213754, 0.16105366, 0.19780564, 0.30675626, 0.36134315, 0.54445148], [0.0454607 , 0.03886008, 0.04453325, 0.05459309, 0.06282043, 0.06533766, 0.07240486, 0.10128665, 0.16491675, 0.20055127, 0.28423715, 0.36882401, 0.49012971, 0.7327683 ], [0.04231381, 0.05783367, 0.05498075, 0.04701138, 0.07089901, 0.08144498, 0.09674454, 0.11706495, 0.21060395, 0.26519608, 0.35307312, 0.48864007, 0.68731689, 0.99208999], [0.0588429 , 0.04970169, 0.05882001, 0.0684669 , 0.07380533, 0.0899024 , 0.12802601, 0.15234852, 0.27199149, 0.36800599, 0.49551105, 0.66220665, 0.98180985, 1.2447741 ], [0.05257511, 0.06329751, 0.07515287, 0.0786345 , 0.11459041, 0.13859439, 0.15368557, 0.21233201, 0.35741186, 0.46255183, 0.6642251 , 0.88907647, 1.23525095, 1.701689 ], [0.06078625, 0.07025814, 0.08156061, 0.08540368, 0.11931705, 0.1850276 , 0.20164061, 0.25054717, 0.49747944, 0.64528704, 1.00463271, 1.26604748, 1.80036545, 2.38028908], [0.08103085, 0.10792756, 0.11786199, 0.10396528, 0.16566682, 0.20191669, 0.26952815, 0.33948731, 0.64801478, 0.88565254, 1.30033636, 1.73595572, 2.45759439, 3.66698241], [0.09899592, 0.09346485, 0.13981724, 0.13460064, 0.20990205, 0.25768304, 0.34873486, 0.48069549, 0.96133471, 1.22038388, 1.77332401, 2.48457313, 3.44868898, 4.77722502], [0.12353778, 0.11626267, 0.17848587, 0.18513489, 0.28442669, 0.34495616, 0.48540521, 0.77531266, 1.34630728, 1.75374937, 2.4773128 , 3.48887801, 4.75541139, 6.72015262], [0.16551876, 0.15247726, 0.23179007, 0.23388553, 0.45177794, 0.45279074, 0.67265415, 1.36667871, 1.8672986 , 2.46142173, 3.47302198, 4.75764132, 6.76331353, 9.55171895],]
Ray pool with rebuilding: [[ 3.86187172, 1.43593073, 1.56066418, 1.43641257, 1.57331777, 1.66063213, 1.9224174 , 1.98327708, 2.01592922, 2.07054281, 2.15626264, 2.12706971, 2.37636852, 2.55979991], [ 2.1704855 , 1.94201708, 2.02236986, 1.89938211, 1.92793941, 1.93268514, 1.97738647, 2.00774384, 2.05265498, 2.02959275, 2.16907716, 2.23836708, 2.48108244, 2.47682738], [ 1.95827985, 1.93218446, 1.95214987, 1.93423891, 2.17605424, 1.91429782, 1.97714329, 2.03083444, 2.06886744, 2.13868284, 2.36233568, 2.44510818, 2.55957413, 2.800915 ], [ 1.90684867, 2.05536699, 2.02550817, 2.06039047, 1.94777894, 2.02290726, 2.12895656, 2.07276464, 2.18771338, 2.21055031, 2.49727416, 2.60650301, 2.82676506, 3.15734696], [ 1.96772599, 1.95183921, 1.96566057, 2.11640167, 2.06356788, 2.02426934, 2.0394547 , 2.11201 , 2.39506269, 2.30564308, 2.51150775, 3.01635814, 3.19985008, 3.8066361 ], [ 1.92989516, 1.98947358, 2.06262708, 2.03270841, 2.12024498, 2.12409258, 2.22118258, 2.17194629, 2.44113898, 2.62941432, 2.80101109, 3.19376349, 3.70752144, 4.61329103], [ 1.96316576, 1.94596219, 2.12336588, 2.06006813, 2.13382506, 2.34898448, 2.26039267, 2.47412276, 2.57652211, 2.86585665, 3.38182664, 3.79624248, 4.35974503, 5.53970242], [ 1.99922752, 2.0350132 , 2.01550412, 2.10461593, 2.36578584, 2.24502087, 2.41250944, 2.60055685, 2.9881351 , 3.22304702, 3.78542495, 4.47209573, 5.86903358, 6.98800945], [ 2.10648227, 2.18973541, 2.11866879, 2.16567707, 2.28700733, 2.53633356, 2.6309402 , 2.96972489, 3.23696947, 3.84590697, 4.66606879, 5.52548075, 7.14732361, 9.24953818], [ 2.08288813, 2.15141273, 2.26907086, 2.22668076, 2.49679399, 2.47671819, 2.86353421, 3.27859354, 3.94310713, 4.51965666, 5.63697672, 7.05841088, 9.05403161, 12.36632204],]
Ray pool without rebuilding: [[7.63106346e-03, 8.32557678e-03, 9.85813141e-03, 1.12354755e-02, 1.82373524e-02, 2.13024616e-02, 3.88369560e-02, 4.49922085e-02, 5.18727303e-02, 9.23600197e-02, 1.17223024e-01, 1.59385204e-01, 2.25480556e-01, 3.26858759e-01], [9.39202309e-03, 9.95039940e-03, 1.54960155e-02, 1.78275108e-02, 3.00111771e-02, 3.08687687e-02, 4.64587212e-02, 5.69529533e-02, 9.36126709e-02, 1.16930962e-01, 1.59743786e-01, 2.24760532e-01, 3.26281548e-01, 4.32718754e-01], [1.49846077e-02, 1.25334263e-02, 1.75509453e-02, 1.77168846e-02, 3.41494083e-02, 9.74442959e-02, 7.39138126e-02, 9.41848755e-02, 1.32635117e-01, 1.55031919e-01, 2.24286556e-01, 3.36545944e-01, 4.47077751e-01, 7.05618143e-01], [2.65917778e-02, 2.15222836e-02, 5.03671169e-02, 4.17685509e-02, 7.08889961e-02, 1.01963520e-01, 1.19611979e-01, 1.87098503e-01, 2.48819828e-01, 3.67422581e-01, 5.35342455e-01, 6.51599884e-01, 8.90594959e-01, 1.23173428e+00], [3.35059166e-02, 3.65400314e-02, 6.78393841e-02, 6.58054352e-02, 8.87494087e-02, 1.25490189e-01, 1.91085339e-01, 2.39352226e-01, 3.43223810e-01, 4.95273590e-01, 6.34335041e-01, 8.83843660e-01, 1.27510691e+00, 1.77129722e+00], [5.02901077e-02, 5.46870232e-02, 7.67621994e-02, 8.18161964e-02, 1.34880066e-01, 2.13716507e-01, 2.92918921e-01, 3.46671581e-01, 5.69765806e-01, 7.63123274e-01, 9.65455532e-01, 1.28749394e+00, 1.75700426e+00, 2.50405574e+00], [8.57729912e-02, 6.98630810e-02, 1.31803274e-01, 1.02919102e-01, 1.96664333e-01, 2.35895872e-01, 3.54160786e-01, 4.59807158e-01, 6.82631016e-01, 9.20553923e-01, 1.36505985e+00, 1.90080070e+00, 2.52802396e+00, 3.48489475e+00], [9.63568687e-02, 1.05616808e-01, 1.62790537e-01, 1.53912067e-01, 2.66423702e-01, 3.22328091e-01, 5.01246691e-01, 6.38187408e-01, 9.82694864e-01, 1.28940535e+00, 1.78382277e+00, 2.55014086e+00, 3.51435447e+00, 5.03458333e+00], [1.27902746e-01, 1.33184910e-01, 2.13215113e-01, 2.17139721e-01, 3.81304502e-01, 4.63495016e-01, 6.58889055e-01, 9.20259476e-01, 1.26067734e+00, 1.76155186e+00, 2.54900360e+00, 3.59667993e+00, 4.95295548e+00, 6.98493934e+00], [2.32776403e-01, 1.82781458e-01, 2.92403460e-01, 2.86549330e-01, 5.03504992e-01, 6.55026913e-01, 9.27602291e-01, 1.27044010e+00, 1.81196213e+00, 2.44112253e+00, 3.64125633e+00, 4.90925956e+00, 6.95644736e+00, 9.96895194e+00]]
- As much as 30% performance difference depending on how we schedule jobs
- likely due to cooling issues