Socrata Open Data API (SODA) GET client for Golang
For SODA docs see http://dev.socrata.com
This is a simple client for get requests only. The client provides basic structs for querying and filtering the database. Although all operations are supported most are just provided as a string to allow for all kind of complex queries.
Just go get it
go get -u github.com/SebastiaanKlippert/go-soda
The default GetRequest struct is not safe for use in multiple goroutines, create one for each goroutine or use the OffsetGetRequest.
The OffsetGetRequest is a wrapper around the GetRequest and provides an easy offset counter to get loads of data. It can be shared by multiple goroutines to get your data a lot faster.
For each GetRequest you can request metadata (using a separate API call). The metadata contains info about the dataset like creation and update times, licensing info and advanced column info.
sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
metadata, err := sodareq.Metadata.Get()See the test file for more examples.
func QuerySample() {
sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
//count all records
count, err := sodareq.Count()
if err != nil {
log.Fatal(err)
}
fmt.Println(count)
//get dataset last updated time
modified, err := sodareq.Modified()
if err != nil {
log.Fatal(err)
}
fmt.Println(modified)
//list all fields/columns
fields, err := sodareq.Fields()
if err != nil {
log.Fatal(err)
}
fmt.Println(fields)
//get some JSON data using a complex query
sodareq.Format = "json"
sodareq.Query.Select = []string{"farm_name", "category", "item", "zipcode"}
sodareq.Query.Where = `lower(farm_name) like '%sun%farm%' AND (item in('Radishes',
'Cucumbers') OR lower(item) like '%flower%')`
sodareq.Query.Limit = 1000
sodareq.Query.AddOrder("farm_name", soda.DirAsc)
sodareq.Query.AddOrder("category", soda.DirDesc)
//count this result first
querycount, err := sodareq.Count()
if err != nil {
log.Fatal(err)
}
fmt.Println(querycount)
//get the results
resp, err := sodareq.Get()
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
rawresp, _ := ioutil.ReadAll(resp.Body)
fmt.Println(string(rawresp))
}
func JSONSample() {
sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
//get some JSON data
sodareq.Format = "json"
sodareq.Filters["item"] = "Radishes"
sodareq.Query.Limit = 10
resp, err := sodareq.Get()
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
results := make([]map[string]interface{}, 0)
err = json.NewDecoder(resp.Body).Decode(&results)
if err != nil {
log.Fatal(err)
}
//Process data here
for _, r := range results {
fmt.Println(r["farm_name"], r["item"])
}
}
func CSVSample() {
sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
sodareq.Format = "csv"
sodareq.Filters["item"] = "Radishes"
sodareq.Query.Limit = 10
resp, err := sodareq.Get()
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
//Process data here
csvreader := csv.NewReader(resp.Body)
for {
record, err := csvreader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
fmt.Println(record)
}
}Get all data in batches of 2000 rows using 4 goroutines
func GetAllData() error {
gr := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
gr.Format = "json"
gr.Query.AddOrder("zipcode", soda.DirAsc)
ogr, err := soda.NewOffsetGetRequest(gr)
if err != nil {
return err
}
for i := 0; i < 4; i++ {
ogr.Add(1)
go func() {
defer ogr.Done()
for {
resp, err := ogr.Next(2000)
if err == soda.ErrDone {
break
}
if err != nil {
log.Fatal(err)
}
results := make([]map[string]interface{}, 0)
err = json.NewDecoder(resp.Body).Decode(&results)
resp.Body.Close()
if err != nil {
log.Fatal(err)
}
//Process your data
}
}()
}
ogr.Wait()
return nil
}