v3io/frames

Use only protobuf for encoding

tebeka opened this issue · 0 comments

Currently we're using msgpack in the HTTP client/server and protobuf in the gRPC client/server. It'll make the code much simpler if we'll use just one form of encoding. And since gRPC it's probably best to use protobuf.

Also from a small test, protobuf is about 10 times faster in encoding and 5 times faster decoding.
Here's the output of the below benchmark:

 go test -bench . _t/encode_test.go
goos: linux
goarch: amd64
BenchmarkMsgpackEncode-4    	     200	   9794262 ns/op
BenchmarkMsgpackDecode-4    	     200	   9143863 ns/op
BenchmarkProtobufEncode-4   	    2000	   1259038 ns/op
BenchmarkProtobufDecode-4   	    1000	   1797280 ns/op
PASS
ok  	command-line-arguments	10.431s

encode_test.go

package main

import (
	"bytes"
	"fmt"
	"math/rand"
	"testing"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/v3io/frames"
	"github.com/v3io/frames/grpc"
	"github.com/v3io/frames/pb"
)

var (
	random      = rand.New(rand.NewSource(time.Now().Unix()))
	testFrame   frames.Frame
	pbTestFrame *pb.Frame
)

func randFrame(size int) (frames.Frame, error) {
	var (
		columns []frames.Column
		col     frames.Column
		err     error
	)

	bools := make([]bool, size)
	for i := range bools {
		if random.Float64() < 0.5 {
			bools[i] = true
		}
	}
	col, err = frames.NewSliceColumn("bools", bools)
	if err != nil {
		return nil, err
	}
	columns = append(columns, col)

	col, err = floatCol("floats", size)
	if err != nil {
		return nil, err
	}
	columns = append(columns, col)

	ints := make([]int64, size)
	for i := range ints {
		ints[i] = random.Int63()
	}
	col, err = frames.NewSliceColumn("ints", ints)
	if err != nil {
		return nil, err
	}
	columns = append(columns, col)

	strings := make([]string, size)
	for i := range strings {
		strings[i] = fmt.Sprintf("val-%d", i)
	}
	col, err = frames.NewSliceColumn("strings", strings)
	if err != nil {
		return nil, err
	}
	columns = append(columns, col)

	times := make([]time.Time, size)
	for i := range times {
		times[i] = time.Now().Add(time.Duration(i) * time.Second)
	}
	col, err = frames.NewSliceColumn("times", times)
	if err != nil {
		return nil, err
	}
	columns = append(columns, col)

	return frames.NewFrame(columns, nil, nil)
}

func init() {
	var err error
	testFrame, err = randFrame(13893)
	if err != nil {
		panic(err)
	}

	pbTestFrame, err = grpc.FrameToPB(testFrame)
	if err != nil {
		panic(err)
	}

}

func floatCol(name string, size int) (frames.Column, error) {
	floats := make([]float64, size)
	for i := range floats {
		floats[i] = random.Float64()
	}

	return frames.NewSliceColumn(name, floats)
}

func BenchmarkMsgpackEncode(b *testing.B) {
	b.StopTimer()
	var buf bytes.Buffer
	enc := frames.NewEncoder(&buf)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		if err := enc.Encode(testFrame); err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkMsgpackDecode(b *testing.B) {
	b.StopTimer()
	var buf bytes.Buffer
	enc := frames.NewEncoder(&buf)
	if err := enc.Encode(testFrame); err != nil {
		b.Fatal(err)
	}
	r := bytes.NewReader(buf.Bytes())
	dec := frames.NewDecoder(r)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_, err := dec.DecodeFrame()
		if err != nil {
			b.Fatal(err)
		}
		if _, err := r.Seek(0, 0); err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkProtobufEncode(b *testing.B) {
	b.StopTimer()
	var buf bytes.Buffer
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		data, err := proto.Marshal(pbTestFrame)
		if err != nil {
			b.Fatal(err)
		}
		buf.Write(data)
	}
}

func BenchmarkProtobufDecode(b *testing.B) {
	b.StopTimer()
	data, err := proto.Marshal(pbTestFrame)
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()
	frame := &pb.Frame{}
	for i := 0; i < b.N; i++ {
		if err := proto.Unmarshal(data, frame); err != nil {
			b.Fatal(err)
		}
	}
}