Break backward compatibility
Opened this issue · 1 comments
zeminzhou commented
Test code
package main
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/xitongsys/parquet-go-source/local"
"github.com/xitongsys/parquet-go/reader"
"github.com/xitongsys/parquet-go/writer"
)
func TestParquetVariousTypes(t *testing.T) {
type Test struct {
Date int32 `parquet:"name=date, type=INT32, convertedtype=date"`
TimeMillis int32 `parquet:"name=timemillis, type=INT32, convertedtype=TIME_MILLIS"`
TimeMicros int64 `parquet:"name=timemicros, type=INT64, convertedtype=TIME_MICROS"`
TimestampMillis int64 `parquet:"name=timestampmillis, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
TimestampMicros int64 `parquet:"name=timestampmicros, type=INT64, convertedtype=TIMESTAMP_MICROS"`
Decimal1 int32 `parquet:"name=decimal1, type=INT32, convertedtype=DECIMAL, scale=2, precision=9"`
Decimal2 int32 `parquet:"name=decimal2, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"`
Decimal3 int64 `parquet:"name=decimal3, type=INT64, convertedtype=DECIMAL, scale=2, precision=18"`
Decimal6 int32 `parquet:"name=decimal6, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"`
}
// prepare data
name := "test123.parquet"
fw, err := local.NewLocalFileWriter(name)
require.NoError(t, err)
test := &Test{}
writer0, err := writer.NewParquetWriter(fw, test, 2)
require.NoError(t, err)
v := &Test{
Date: 18564, // 2020-10-29
TimeMillis: 62775123, // 17:26:15.123
TimeMicros: 62775123456, // 17:26:15.123
TimestampMillis: 1603963672356, // 2020-10-29T09:27:52.356Z
TimestampMicros: 1603963672356956, // 2020-10-29T09:27:52.356956Z
Decimal1: -12345678, // -123456.78
Decimal2: 456, // 0.0456
Decimal3: 123456789012345678, // 1234567890123456.78
Decimal6: -1, // -0.0001
}
require.NoError(t, writer0.Write(v))
require.NoError(t, writer0.WriteStop())
require.NoError(t, fw.Close())
fr, err := local.NewLocalFileReader(name)
require.NoError(t, err)
reader, err := reader.NewParquetReader(fr, nil, 2)
require.NoError(t, err)
for _, c := range reader.SchemaHandler.SchemaElements {
if c != nil && c.LogicalType != nil {
if c.LogicalType.IsSetTIMESTAMP() {
require.True(t, c.LogicalType.TIMESTAMP.GetIsAdjustedToUTC())
} else if c.LogicalType.IsSetTIME() {
require.True(t, c.LogicalType.TIME.GetIsAdjustedToUTC())
}
}
}
}
From document, the GetIsAdjustedToUTC
from return true. https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
zeminzhou commented
/cc @xitongsys