Use only protobuf for encoding
tebeka opened this issue · 0 comments
tebeka commented
Currently we're using msgpack
in the HTTP client/server and protobuf
in the gRPC client/server. It'll make the code much simpler if we'll use just one form of encoding. And since gRPC it's probably best to use protobuf
.
Also from a small test, protobuf
is about 10 times faster in encoding and 5 times faster decoding.
Here's the output of the below benchmark:
go test -bench . _t/encode_test.go
goos: linux
goarch: amd64
BenchmarkMsgpackEncode-4 200 9794262 ns/op
BenchmarkMsgpackDecode-4 200 9143863 ns/op
BenchmarkProtobufEncode-4 2000 1259038 ns/op
BenchmarkProtobufDecode-4 1000 1797280 ns/op
PASS
ok command-line-arguments 10.431s
encode_test.go
package main
import (
"bytes"
"fmt"
"math/rand"
"testing"
"time"
"github.com/golang/protobuf/proto"
"github.com/v3io/frames"
"github.com/v3io/frames/grpc"
"github.com/v3io/frames/pb"
)
var (
random = rand.New(rand.NewSource(time.Now().Unix()))
testFrame frames.Frame
pbTestFrame *pb.Frame
)
func randFrame(size int) (frames.Frame, error) {
var (
columns []frames.Column
col frames.Column
err error
)
bools := make([]bool, size)
for i := range bools {
if random.Float64() < 0.5 {
bools[i] = true
}
}
col, err = frames.NewSliceColumn("bools", bools)
if err != nil {
return nil, err
}
columns = append(columns, col)
col, err = floatCol("floats", size)
if err != nil {
return nil, err
}
columns = append(columns, col)
ints := make([]int64, size)
for i := range ints {
ints[i] = random.Int63()
}
col, err = frames.NewSliceColumn("ints", ints)
if err != nil {
return nil, err
}
columns = append(columns, col)
strings := make([]string, size)
for i := range strings {
strings[i] = fmt.Sprintf("val-%d", i)
}
col, err = frames.NewSliceColumn("strings", strings)
if err != nil {
return nil, err
}
columns = append(columns, col)
times := make([]time.Time, size)
for i := range times {
times[i] = time.Now().Add(time.Duration(i) * time.Second)
}
col, err = frames.NewSliceColumn("times", times)
if err != nil {
return nil, err
}
columns = append(columns, col)
return frames.NewFrame(columns, nil, nil)
}
func init() {
var err error
testFrame, err = randFrame(13893)
if err != nil {
panic(err)
}
pbTestFrame, err = grpc.FrameToPB(testFrame)
if err != nil {
panic(err)
}
}
func floatCol(name string, size int) (frames.Column, error) {
floats := make([]float64, size)
for i := range floats {
floats[i] = random.Float64()
}
return frames.NewSliceColumn(name, floats)
}
func BenchmarkMsgpackEncode(b *testing.B) {
b.StopTimer()
var buf bytes.Buffer
enc := frames.NewEncoder(&buf)
b.StartTimer()
for i := 0; i < b.N; i++ {
if err := enc.Encode(testFrame); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkMsgpackDecode(b *testing.B) {
b.StopTimer()
var buf bytes.Buffer
enc := frames.NewEncoder(&buf)
if err := enc.Encode(testFrame); err != nil {
b.Fatal(err)
}
r := bytes.NewReader(buf.Bytes())
dec := frames.NewDecoder(r)
b.StartTimer()
for i := 0; i < b.N; i++ {
_, err := dec.DecodeFrame()
if err != nil {
b.Fatal(err)
}
if _, err := r.Seek(0, 0); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkProtobufEncode(b *testing.B) {
b.StopTimer()
var buf bytes.Buffer
b.StartTimer()
for i := 0; i < b.N; i++ {
data, err := proto.Marshal(pbTestFrame)
if err != nil {
b.Fatal(err)
}
buf.Write(data)
}
}
func BenchmarkProtobufDecode(b *testing.B) {
b.StopTimer()
data, err := proto.Marshal(pbTestFrame)
if err != nil {
b.Fatal(err)
}
b.StartTimer()
frame := &pb.Frame{}
for i := 0; i < b.N; i++ {
if err := proto.Unmarshal(data, frame); err != nil {
b.Fatal(err)
}
}
}