Golang short tips & tricks
This list of short golang code tips & tricks will help keep collected knowledge in one place.
- 41 - Telegram bot
- 40 - Distributed consensus
- 39 - Public private struct fields and funcs
- 38 - Marshal Unmarshal to byte slice using gob
- 37 - Time series on leveldb
- 36 - Custom type marshal unmarshal json
- 35 - Test specific functions
- 34 - File path exists
- 33 - Table driven tests
- 32 - Work with consul
- 31 - Do not need any web framework
- 30 - Heart beat and lead election with etcd
- 29 - Partial json read
- 28 - Interact with etcd with http.Request
- 27 - Go-style concurrency in C
- 26 - Go channels are slow
- 25 - Avoid conversions with hidden alloc copy
- 24 - Slice shuffle
- 23 - Defer is slow
- 22 - Any number of args
- 21 - Test bench http request handler
- 20 - Use Atomics or GOMAXPROCS=1
- 19 - Chunked HTTP response with flusher
- 18 - Use for range for channels
- 17 - Use context API
- 16 - Go routines syncronization
- 15 - Time interval measurement
- 14 - Benchmark switch vs else if
- 13 - Use ASM in Go Code
- 12 - JSON with unknown structure
- 11 - Websocket over HTTP2
- 10 - HTTP2
- 9 - Error handling
- 8 - Memory management with pools of objects
- 7 - Sort slice of time periods
- 6 - Fast http server
- 5 - Close channel to notify many
- 4 - Is channel closed?
- 3 - Http request/response with close notify and timeout
- 2 - Import packages
- 1 - Map
- 0 - Slices
2016-06-04 by @beyondns
Simple echo Telegram bot no external dependencies
package main
import (
"bytes"
"encoding/json"
"errors"
"io/ioutil"
"log"
"net/http"
"net/url"
"strconv"
"time"
)
// https://core.telegram.org/bots/api
const (
API = "https://api.telegram.org/bot"
TOKEN = <YOUR TOKEN HERE>
)
type Chat struct {
Id int `json:"id"`
}
type From struct {
Id int `json:"id"`
}
type Message struct {
Id int `json:"message_id"`
From From `json:"from"`
Chat Chat `json:"chat"`
Text string `json:"text"`
Date uint32 `json:"date"`
}
type Update struct {
Id int `json:"update_id"`
Msg Message `json:"message"`
}
type Result struct {
Ok bool `json:"ok"`
Res []Update `json:"result"`
}
/*
{
"ok": true,
"result": [{
"update_id": 215439784,
"message": {
"message_id": 21,
"from": {
"id": 209816615,
"first_name": "Black",
"last_name": "Ninja",
"username": "blackninja3k"
},
"chat": {
"id": 209816615,
"first_name": "Black",
"last_name": "Ninja",
"username": "blackninja3k",
"type": "private"
},
"date": 1459938757,
"text": "hello"
}
}, {
"update_id": 215439785,
"message": {
"message_id": 22,
"from": {
"id": 209816615,
"first_name": "Black",
"last_name": "Ninja",
"username": "blackninja3k"
},
"chat": {
"id": 209816615,
"first_name": "Black",
"last_name": "Ninja",
"username": "blackninja3k",
"type": "private"
},
"date": 1459939461,
"text": "pooooop"
}
}]
}
*/
func HandleUpdate(u *Update) {
log.Printf("upd:%v", *u)
msgDate := time.Unix(int64(u.Msg.Date), 0)
if time.Since(msgDate).Minutes() < 1 {
SendMessage(u.Msg.Chat.Id, "u:"+u.Msg.Text)
}
}
func main() {
log.Printf("Me:%s", Me())
var lastID = 0
for {
res := Updates()
if res.Ok {
for _, u := range res.Res {
if u.Id > lastID {
lastID = u.Id
HandleUpdate(&u)
}
}
}
time.Sleep(time.Second * 2)
}
}
func Me() string {
st, d, err := httpRequest("GET", API+TOKEN+"/getMe", nil, time.Second*15)
if err != nil {
log.Fatal(err)
}
if st != http.StatusOK {
log.Fatalf("status %d", st)
}
return string(d)
}
func Updates() Result {
st, d, err := httpRequest("GET", API+TOKEN+"/getUpdates", nil, time.Second*15)
if err != nil {
log.Fatal(err)
}
if st != http.StatusOK {
log.Fatalf("status %d", st)
}
res := Result{}
if len(d) > 0 {
//log.Printf(string(d))
err := json.Unmarshal(d, &res)
if err != nil {
log.Fatal(err)
}
}
return res
}
func SendMessage(chat_id int, text string) string {
v := url.Values{}
v.Add("chat_id", strconv.Itoa(chat_id))
v.Add("text", text)
st, d, err := httpRequest("GET", API+TOKEN+"/sendMessage?"+v.Encode(),
nil, time.Second*15)
if err != nil {
log.Fatal(err)
}
if st != http.StatusOK {
log.Fatalf("status %d", st)
}
return string(d)
}
func httpRequest(meth, u string, data []byte,
timeLimit time.Duration) (int, []byte, error) {
tr := &http.Transport{}
client := &http.Client{Transport: tr}
c := make(chan error, 1)
var respStatus int
var respBody []byte
req, err := http.NewRequest(meth, u, bytes.NewBuffer(data))
if err != nil {
return 0, nil, err
}
go func() {
resp, err := client.Do(req)
if err != nil {
goto E
}
respStatus = resp.StatusCode
respBody, err = ioutil.ReadAll(resp.Body)
E:
c <- err
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
}()
select {
case <-time.After(timeLimit):
tr.CancelRequest(req)
log.Printf("Request timeout")
<-c // Wait for goroutine to return.
return 0, nil, errors.New("request time out")
case err := <-c:
if err != nil {
log.Printf("Error in request goroutine %v", err)
return 0, nil, err
}
return respStatus, respBody, nil
}
}
2016-01-04 by @beyondns
Distributed consensus is a common shared state or logic support equal across multiple nodes.
1-stage (semi or one-many) consensus: send data to nodes, get responses, find consensus on one node.
2-stage (full or many-many) consensus: send data to nodes, nodes send all-to-all, find consensus on each node.
package main
import (
"flag"
"log"
"net/http"
"fmt"
"strings"
"time"
"bytes"
"errors"
"io/ioutil"
"sync"
)
var (
Nodes []string
)
func sysHandler1(w http.ResponseWriter, r *http.Request) {
d, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("bad reguest, error %v", err), http.StatusBadRequest)
return
}
log.Printf("sysHandler1: %s",string(d))
w.WriteHeader(http.StatusOK)
}
func sysHandler2(w http.ResponseWriter, r *http.Request) {
d, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("bad reguest, error %v", err), http.StatusBadRequest)
return
}
log.Printf("sysHandler2: %s",string(d))
if localConsensus(d,"1",w,r){
w.WriteHeader(http.StatusOK)
}
w.WriteHeader(http.StatusOK)
}
func apiHandler(w http.ResponseWriter, r *http.Request) {
d, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("bad reguest, error %v", err), http.StatusBadRequest)
return
}
log.Printf("apiHandler: %s",string(d))
if localConsensus(d,"2",w,r){
w.WriteHeader(http.StatusOK)
}
}
func localConsensus(d []byte, st string,
w http.ResponseWriter, r *http.Request) bool{
if len(d)>0{
ok,err:=SendDataToWorld(d,st)
if err!=nil {
http.Error(w, fmt.Sprintf("SendDataToWorld error %v", err), http.StatusBadRequest)
return false
}
if !ok {
http.Error(w, "no consensus", http.StatusBadRequest)
return false
}
}
return true
}
func main() {
nodes := flag.String("nodes", "127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379", "nodes host0:port0,host1:port1,..")
api := flag.String("api", "127.0.0.1:12379", "api host:port")
flag.Parse()
Nodes=strings.Split(*nodes,",")
log.Printf("Nodes %v",Nodes)
http.HandleFunc("/api", apiHandler)
http.HandleFunc("/sys1", sysHandler1)
http.HandleFunc("/sys2", sysHandler2)
log.Fatal(http.ListenAndServe(*api, nil))
}
func SendDataToWorld(data []byte, st string) (bool,error){
var l = len(Nodes)
var wg sync.WaitGroup
wg.Add(l)
var rd [][]byte = make([][]byte, l)
var rs []int = make([]int, l)
var re []bool = make([]bool, l)
for i, u := range Nodes {
go func(i int, u string) {
var e error
rs[i],rd[i],e=httpRequest("POST",
"http://"+u+"/sys"+st,data,time.Second*15)
re[i]=(e==nil)
wg.Done()
}(i, u)
}
wg.Wait()
log.Printf("%v %v %v",rs,rd,re)
return findConsensus(rs,rd,re)
}
func httpRequest(meth, u string, data []byte,
timeLimit time.Duration) (int, []byte, error) {
tr := &http.Transport{}
client := &http.Client{Transport: tr}
c := make(chan error, 1)
var respStatus int
var respBody []byte
req, err := http.NewRequest(meth, u, bytes.NewBuffer(data))
if err != nil {
return 0, nil, err
}
go func() {
resp, err := client.Do(req)
if err != nil {
goto E
}
respStatus = resp.StatusCode
respBody, err = ioutil.ReadAll(resp.Body)
E:
c <- err
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
}()
select {
case <-time.After(timeLimit):
tr.CancelRequest(req)
log.Printf("Request timeout")
<-c // Wait for goroutine to return.
return 0, nil, errors.New("request time out")
case err := <-c:
if err != nil {
log.Printf("Error in request goroutine %v", err)
return 0, nil, err
}
return respStatus, respBody, nil
}
}
func findConsensus(rs []int,rd [][]byte,re []bool)(bool,error){
if len(rs) != len(rd) || len(rs) == 0 || len(rd) == 0 {
panic(fmt.Sprintf("FindConsensus error papams: %v,%v", rs, rd))
}
ok:=0
n:=0
for i, _ := range rs{
if !re[i]{
continue
}
n++
if rs[i] == http.StatusOK{
ok++
}
}
log.Printf("find consensus %d : %d",ok,n)
// 51% ok
if float32(ok)>float32(n)/float32(2){
return true,nil
}
return false,nil
}
2016-26-03 by @beyondns
Go has specific approach of definition private/public package fields and funcs. All declarations within package is visible (different os files).
But from other packages only names with 1st capital letter is visible (public)
lib/l.go
package mylib
import (
"fmt"
)
const (
Const1 = "c1"
const2 = "c2"
)
var (
Var1 = 1
var2 = 2
)
type Mydata struct{
X int // Public
y int // private
}
func (md *Mydata) DoAPI(){
fmt.Println("DoAPI")
}
func (md *Mydata) doAPI(){
fmt.Println("doAPI")
}
main.go
package main
import(
"./lib"
"fmt"
)
func main(){
x:=mylib.Mydata{
X:1,
//y:2, // error private
}
x.DoAPI()
// x.doAPI() // error private
fmt.Println(x.X)
// fmt.Print(x.y) // error private
fmt.Println(mylib.Const1)
// fmt.Println(mylib.const2) // error private
fmt.Println(mylib.Var1)
// fmt.Println(mylib.var2) // error private
}
2016-25-03 by @beyondns
import (
"bytes"
"encoding/gob"
)
func gobMarshal(v interface{}) ([]byte, error) {
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(v); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func gobUnmarshal(data []byte, v interface{}) error {
return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
}
2016-22-03 by @beyondns
Use timestamp as a key in keyvalue storage, leveldb or any other. Just iterate over specific period of time.
import (
"github.com/syndtr/goleveldb/leveldb"
)
var (
bcdb *leveldb.DB
)
func init(){
var err error
bcdb, err = leveldb.OpenFile("./db/bc.db", nil)
if err!=nil{
xlog.Error(err)
return
}
xlog.Notice("db init ok")
}
Test
func TestDbTS(t *testing.T) {
err := bcdb.Put([]byte("t0"),[]byte("boom"), nil)
if err!=nil{
t.Error(err)
}
var timelabel int64
batch := new(leveldb.Batch)
for i:=0;i<16;i++{
if i==8{
timelabel=time.Now().UnixNano()
}
batch.Put([]byte(fmt.Sprintf("t%d",time.Now().UnixNano())),
[]byte(fmt.Sprintf("v%d",i)))
}
err = bcdb.Write(batch, nil)
if err!=nil{
t.Error(err)
}
iter := bcdb.NewIterator(nil, nil)
for ok := iter.Seek([]byte(fmt.Sprintf("t%d",timelabel))); ok; ok = iter.Next() {
key,value := iter.Key(),iter.Value()
ks:=string(key)
if ks[0] == 't'{
ks=ks[1:]
}
ki, err := strconv.ParseInt(ks, 10, 64)
if err!=nil{
xlog.Error(err)
}
xlog.Debugf("key time : %s | value: %s\n", time.Unix(0,ki), value)
}
iter.Release()
err = iter.Error()
if err!=nil{
t.Error(err)
}
}
2016-21-03 by @beyondns
Json marshalling can be defined for custom type by implementing MarshalJSON/UnmarshalJSON methods.
type ByteSlice []byte
func (s *ByteSlice) MarshalJSON() ([]byte, error) {
return []byte(`"`+hex.EncodeToString(*s)+`"`), nil
}
func (s *ByteSlice) UnmarshalJSON(data []byte) error {
d:=string(data)
if d[0]=='"'{d=d[1:]}
l:=len(d)
if d[l-1]=='"'{d=d[:l-1]}
var err error
*s,err=hex.DecodeString(d)
return err
}
Testing
type ByteSliceHolder struct {
Bin ByteSlice `json:"bin"`
}
func TestByteSliceJSON(t *testing.T) {
bsh := &ByteSliceHolder{}
_, err := fmt.Sscanf("082372739127341723ab", "%x", &bsh.Bin)
if err != nil {
t.Fatal("fmt.Sscanf failed")
}
bin, err := json.Marshal(bsh)
if err != nil {
t.Errorf("json.Marshal failed ", err)
}
//xlog.Debugf("%s", bin)
bsh2 := &ByteSliceHolder{}
err = json.Unmarshal(bin, &bsh2)
if err != nil {
t.Errorf("json.Marshal failed ", err)
}
if !bytes.Equal(bsh.Bin,bsh2.Bin){
t.Errorf("json.Marshal failed ", err)
}
}
2016-21-03 by @beyondns
To test TestFunc1
go test -run Func1
run argument is a regex, "Func$" test all funcs with "Func" at beginning.
2016-15-03 by @beyondns
if _, err := os.Stat(filepath); os.IsNotExist(err) {
// doesn't exist
}
if _, err := os.Stat(filepath); err == nil {
// exists
}
2016-15-03 by @beyondns
Table driven tests very simple but efficient. An example shamelessly stolen from advanced-testing-with-go
package main
import (
"testing"
)
func TestAdd(t *testing.T) {
cases := []struct{ A, B, Sum int }{
{1, 1, 2},
{1, -1, 0},
{1, 0, 1},
{0, 0, 0},
{3, 2, 1}, // error!!!
}
for _, c := range cases {
a := c.A + c.B
e := c.Sum
if a != e {
t.Errorf("%d + %d = %d, expected %d", c.A, c.B, a, e)
}
}
}
--- FAIL: TestAdd (0.00s)
tdt_test.go:20: 3 + 2 = 5, expected 1
2016-14-03 by @beyondns
Consul has its own client but just use net/http
consul agent -dev -advertise=127.0.0.1
package main
import (
"errors"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"encoding/json"
)
// https://www.consul.io/docs/agent/http/kv.html
var (
consulKV = "http://127.0.0.1:8500/v1/kv/"
)
func httpRequest(meth, u, val string, timeLimit time.Duration) (int, []byte, error) {
tr := &http.Transport{}
client := &http.Client{Transport: tr}
c := make(chan error, 1)
var respStatus int
var respBody []byte
req, err := http.NewRequest(meth, u, strings.NewReader(val))
if err != nil {
return 0, nil, err
}
go func() {
resp, err := client.Do(req)
if err != nil {
goto E
}
respStatus = resp.StatusCode
respBody, err = ioutil.ReadAll(resp.Body)
E:
c <- err
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
}()
select {
case <-time.After(timeLimit):
tr.CancelRequest(req)
log.Printf("Request timeout")
<-c // Wait for goroutine to return.
return 0, nil, errors.New("request time out")
case err := <-c:
if err != nil {
log.Printf("Error in request goroutine %v", err)
return 0, nil, err
}
return respStatus, respBody, nil
}
}
//curl -v -XPUT http://127.0.0.1:8500/v1/kv/boom -d foo
func consulSet(k, v string) (int, []byte, error) {
return httpRequest("PUT", consulKV+k, v, time.Second)
}
//curl -v http://127.0.0.1:8500/v1/kv/boom
func consulGet(k string) (int, []byte, error) {
return httpRequest("GET", consulKV+k, "", time.Second)
}
type KVPair struct {
Key string
CreateIndex uint64
ModifyIndex uint64
LockIndex uint64
Flags uint64
Value []byte
Session string
}
func main() {
s, d, err := consulSet("foo", "hero")
if err != nil {
log.Fatalf("Set error %v", err)
}
log.Printf("set %d %s", s, string(d))
s, d, err = consulGet("foo")
if err != nil {
log.Fatalf("Get error %v", err)
}
if s!=200{
log.Fatalf("Get status %d", s)
}
var kv []KVPair
json.Unmarshal(d,&kv)
if len(kv)==0{
log.Fatalf("len(kv)==0")
}
log.Printf("get %d %s %s", s, string(d), kv[0].Value)
}
2016/03/15 20:49:54 set 200 true
2016/03/15 20:49:54 get 200 [{"LockIndex":0,"Key":"foo","Flags":0,"Value":"aGVybw==","CreateIndex":7,"ModifyIndex":252}] hero