xitongsys/parquet-go

Break backward compatibility

Opened this issue · 1 comments

Test code

package main

import (
        "testing"

        "github.com/stretchr/testify/require"
        "github.com/xitongsys/parquet-go-source/local"
        "github.com/xitongsys/parquet-go/reader"
        "github.com/xitongsys/parquet-go/writer"
)

func TestParquetVariousTypes(t *testing.T) {
        type Test struct {
                Date            int32 `parquet:"name=date, type=INT32, convertedtype=date"`
                TimeMillis      int32 `parquet:"name=timemillis, type=INT32, convertedtype=TIME_MILLIS"`
                TimeMicros      int64 `parquet:"name=timemicros, type=INT64, convertedtype=TIME_MICROS"`
                TimestampMillis int64 `parquet:"name=timestampmillis, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
                TimestampMicros int64 `parquet:"name=timestampmicros, type=INT64, convertedtype=TIMESTAMP_MICROS"`

                Decimal1 int32 `parquet:"name=decimal1, type=INT32, convertedtype=DECIMAL, scale=2, precision=9"`
                Decimal2 int32 `parquet:"name=decimal2, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"`
                Decimal3 int64 `parquet:"name=decimal3, type=INT64, convertedtype=DECIMAL, scale=2, precision=18"`
                Decimal6 int32 `parquet:"name=decimal6, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"`
        }

        // prepare data
        name := "test123.parquet"
        fw, err := local.NewLocalFileWriter(name)
        require.NoError(t, err)
        test := &Test{}
        writer0, err := writer.NewParquetWriter(fw, test, 2)
        require.NoError(t, err)

        v := &Test{
                Date:            18564,              // 2020-10-29
                TimeMillis:      62775123,           // 17:26:15.123
                TimeMicros:      62775123456,        // 17:26:15.123
                TimestampMillis: 1603963672356,      // 2020-10-29T09:27:52.356Z
                TimestampMicros: 1603963672356956,   // 2020-10-29T09:27:52.356956Z
                Decimal1:        -12345678,          // -123456.78
                Decimal2:        456,                // 0.0456
                Decimal3:        123456789012345678, // 1234567890123456.78
                Decimal6:        -1,                 // -0.0001
        }
        require.NoError(t, writer0.Write(v))
        require.NoError(t, writer0.WriteStop())
        require.NoError(t, fw.Close())

        fr, err := local.NewLocalFileReader(name)
        require.NoError(t, err)
        reader, err := reader.NewParquetReader(fr, nil, 2)
        require.NoError(t, err)

        for _, c := range reader.SchemaHandler.SchemaElements {
                if c != nil && c.LogicalType != nil {
                        if c.LogicalType.IsSetTIMESTAMP() {
                                require.True(t, c.LogicalType.TIMESTAMP.GetIsAdjustedToUTC())
                        } else if c.LogicalType.IsSetTIME() {
                                require.True(t, c.LogicalType.TIME.GetIsAdjustedToUTC())
                        }
                }
        }
}

From document, the GetIsAdjustedToUTC from return true. https://github.com/apache/parquet-format/blob/master/LogicalTypes.md