Seamlessly save and load protocol buffers to and from BigQuery using Go.
This library provides add-ons to cloud.google.com/bigquery for first-class protobuf support using protobuf reflection.
$ go get -u go.einride.tech/protobuf-bigqueryBigQuery schema inference for arbitrary protobuf messages.
func ExampleInferSchema() {
msg := &library.Book{}
schema := protobq.InferSchema(msg)
expected := bigquery.Schema{
{Name: "name", Type: bigquery.StringFieldType},
{Name: "author", Type: bigquery.StringFieldType},
{Name: "title", Type: bigquery.StringFieldType},
{Name: "read", Type: bigquery.BooleanFieldType},
}
fmt.Println(cmp.Equal(expected, schema))
// Output: true
}An implementation of bigquery.ValueSaver that saves arbitrary protobuf messages to BigQuery.
func ExampleMessageSaver() {
ctx := context.Background()
// Write protobuf messages to a BigQuery table.
projectID := flag.String("project", "", "BigQuery project to write to.")
datasetID := flag.String("dataset", "", "BigQuery dataset to write to.")
tableID := flag.String("table", "", "BigQuery table to write to.")
create := flag.Bool("create", false, "Flag indicating whether to create the table.")
flag.Parse()
// Connect to BigQuery.
client, err := bigquery.NewClient(ctx, *projectID)
if err != nil {
panic(err) // TODO: Handle error.
}
table := client.Dataset(*datasetID).Table(*tableID)
// Create the table by inferring the BigQuery schema from the protobuf schema.
if *create {
if err := table.Create(ctx, &bigquery.TableMetadata{
Schema: protobq.InferSchema(&publicv1.FilmLocation{}),
}); err != nil {
panic(err) // TODO: Handle error.
}
}
// Insert the protobuf messages.
inserter := table.Inserter()
for i, filmLocation := range []*publicv1.FilmLocation{
{Title: "Dark Passage", ReleaseYear: 1947, Locations: "Filbert Steps"},
{Title: "D.O.A", ReleaseYear: 1950, Locations: "Union Square"},
{Title: "Flower Drum Song", ReleaseYear: 1961, Locations: "Chinatown"},
} {
if err := inserter.Put(ctx, &protobq.MessageSaver{
Message: filmLocation,
InsertID: strconv.Itoa(i), // include an optional insert ID
}); err != nil {
panic(err) // TODO: Handle error.
}
}
}An implementation of bigquery.ValueLoader that loads arbitrary protobuf messages from BigQuery.
func ExampleMessageLoader() {
ctx := context.Background()
// Read from the public "film locations" BigQuery dataset into a proto message.
const (
project = "bigquery-public-data"
dataset = "san_francisco_film_locations"
table = "film_locations"
)
// Connect to BigQuery.
client, err := bigquery.NewClient(ctx, project)
if err != nil {
panic(err) // TODO: Handle error.
}
// Load BigQuery rows into a FilmLocation message.
messageLoader := &protobq.MessageLoader{
Message: &publicv1.FilmLocation{},
}
// Iterate rows in table.
rowIterator := client.Dataset(dataset).Table(table).Read(ctx)
for {
// Load next row into the FilmLocation message.
if err := rowIterator.Next(messageLoader); err != nil {
if errors.Is(err, iterator.Done) {
break
}
panic(err) // TODO: Handle error.
}
// Print the message.
fmt.Println(prototext.Format(messageLoader.Message))
}
}| Protobuf | BigQuery |
|---|---|
| google.protobuf.Timestamp | TIMESTAMP |
| google.protobuf.Duration | FLOAT (seconds) |
| google.protobuf.DoubleValue | FLOAT |
| google.protobuf.FloatValue | FLOAT |
| google.protobuf.Int32Value | INTEGER |
| google.protobuf.Int64Value | INTEGER |
| google.protobuf.Uint32Value | INTEGER |
| google.protobuf.Uint64Value | INTEGER |
| google.protobuf.BoolValue | BOOLEAN |
| google.protobuf.StringValue | STRING |
| google.protobuf.BytesValue | BYTES |
| google.protobuf.StructValue | STRING (JSON) |
| Protobuf | BigQuery |
|---|---|
| google.type.Date | DATE |
| google.type.DateTime | RECORD (or DATETIME) |
| google.type.LatLng | GEOGRAPHY |
| google.type.TimeOfDay | TIME |