From 8d6f16ef479e6b661214c4d93cf30cdb6cbee138 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Sun, 15 Dec 2024 22:23:57 -0800 Subject: [PATCH 1/6] String converters. --- lib/typing/converters/converters.go | 33 +++++++++++++++++++++++++++++ lib/typing/values/string.go | 19 +++++++++++------ 2 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 lib/typing/converters/converters.go diff --git a/lib/typing/converters/converters.go b/lib/typing/converters/converters.go new file mode 100644 index 000000000..a01558b73 --- /dev/null +++ b/lib/typing/converters/converters.go @@ -0,0 +1,33 @@ +package converters + +import ( + "fmt" + + "github.com/artie-labs/transfer/lib/typing" + + "github.com/artie-labs/transfer/lib/typing/ext" +) + +type StringConverter interface { + Convert(value any) (string, error) +} + +func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { + switch kd.Kind { + case typing.Date.Kind: + return DateStringConverter{}, nil + } + + return nil, nil +} + +type DateStringConverter struct{} + +func (DateStringConverter) Convert(value any) (string, error) { + _time, err := ext.ParseDateFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as date, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.PostgresDateFormat), nil +} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 544be3a9e..472371ac3 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/artie-labs/transfer/lib/typing/converters" + "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" @@ -36,14 +38,17 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return "", fmt.Errorf("colVal is nil") } - switch colKind.Kind { - case typing.Date.Kind: - _time, err := ext.ParseDateFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as date, colVal: '%v', err: %w", colVal, err) - } + sv, err := converters.GetStringConverter(colKind) + if err != nil { + return "", fmt.Errorf("failed to get string converter: %w", err) + } + + // Prioritize converters + if sv != nil { + return sv.Convert(colVal) + } - return _time.Format(ext.PostgresDateFormat), nil + switch colKind.Kind { case typing.Time.Kind: _time, err := ext.ParseTimeFromAny(colVal) if err != nil { From 580ec7b260415f2e845f24da6640f3d6254d985d Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Sun, 15 Dec 2024 22:28:59 -0800 Subject: [PATCH 2/6] Clean up. --- lib/typing/converters/converters.go | 33 --------- lib/typing/converters/string_converter.go | 86 +++++++++++++++++++++++ lib/typing/values/string.go | 31 -------- 3 files changed, 86 insertions(+), 64 deletions(-) delete mode 100644 lib/typing/converters/converters.go create mode 100644 lib/typing/converters/string_converter.go diff --git a/lib/typing/converters/converters.go b/lib/typing/converters/converters.go deleted file mode 100644 index a01558b73..000000000 --- a/lib/typing/converters/converters.go +++ /dev/null @@ -1,33 +0,0 @@ -package converters - -import ( - "fmt" - - "github.com/artie-labs/transfer/lib/typing" - - "github.com/artie-labs/transfer/lib/typing/ext" -) - -type StringConverter interface { - Convert(value any) (string, error) -} - -func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { - switch kd.Kind { - case typing.Date.Kind: - return DateStringConverter{}, nil - } - - return nil, nil -} - -type DateStringConverter struct{} - -func (DateStringConverter) Convert(value any) (string, error) { - _time, err := ext.ParseDateFromAny(value) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as date, colVal: '%v', err: %w", value, err) - } - - return _time.Format(ext.PostgresDateFormat), nil -} diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go new file mode 100644 index 000000000..83ecaafc5 --- /dev/null +++ b/lib/typing/converters/string_converter.go @@ -0,0 +1,86 @@ +package converters + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/artie-labs/transfer/lib/typing" + "github.com/artie-labs/transfer/lib/typing/ext" +) + +type StringConverter interface { + Convert(value any) (string, error) +} + +func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { + switch kd.Kind { + case typing.Date.Kind: + return DateConverter{}, nil + case typing.Time.Kind: + return TimeConverter{}, nil + case typing.TimestampNTZ.Kind: + return TimestampNTZConverter{}, nil + case typing.TimestampTZ.Kind: + return TimestampTZConverter{}, nil + case typing.Array.Kind: + return ArrayConverter{}, nil + } + + return nil, nil +} + +type DateConverter struct{} + +func (DateConverter) Convert(value any) (string, error) { + _time, err := ext.ParseDateFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as date, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.PostgresDateFormat), nil +} + +type TimeConverter struct{} + +func (TimeConverter) Convert(value any) (string, error) { + _time, err := ext.ParseTimeFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as time, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.PostgresTimeFormatNoTZ), nil +} + +type TimestampNTZConverter struct{} + +func (TimestampNTZConverter) Convert(value any) (string, error) { + _time, err := ext.ParseTimestampNTZFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as timestampNTZ, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.RFC3339NoTZ), nil +} + +type TimestampTZConverter struct{} + +func (TimestampTZConverter) Convert(value any) (string, error) { + _time, err := ext.ParseTimestampTZFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as timestampTZ, colVal: '%v', err: %w", value, err) + } + + return _time.Format(time.RFC3339Nano), nil +} + +type ArrayConverter struct{} + +func (ArrayConverter) Convert(value any) (string, error) { + colValBytes, err := json.Marshal(value) + if err != nil { + return "", err + } + + return string(colValBytes), nil +} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 472371ac3..5d24e4398 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -6,7 +6,6 @@ import ( "reflect" "strconv" "strings" - "time" "github.com/artie-labs/transfer/lib/typing/converters" @@ -14,7 +13,6 @@ import ( "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/decimal" - "github.com/artie-labs/transfer/lib/typing/ext" ) func Float64ToString(value float64) string { @@ -43,33 +41,11 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return "", fmt.Errorf("failed to get string converter: %w", err) } - // Prioritize converters if sv != nil { return sv.Convert(colVal) } switch colKind.Kind { - case typing.Time.Kind: - _time, err := ext.ParseTimeFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as time, colVal: '%v', err: %w", colVal, err) - } - - return _time.Format(ext.PostgresTimeFormatNoTZ), nil - case typing.TimestampNTZ.Kind: - _time, err := ext.ParseTimestampNTZFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as timestampNTZ, colVal: '%v', err: %w", colVal, err) - } - - return _time.Format(ext.RFC3339NoTZ), nil - case typing.TimestampTZ.Kind: - _time, err := ext.ParseTimestampTZFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as timestampTZ, colVal: '%v', err: %w", colVal, err) - } - - return _time.Format(time.RFC3339Nano), nil case typing.String.Kind: isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice _, isMap := colVal.(map[string]any) @@ -101,13 +77,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return string(colValBytes), nil } } - case typing.Array.Kind: - colValBytes, err := json.Marshal(colVal) - if err != nil { - return "", err - } - - return string(colValBytes), nil case typing.Float.Kind: switch parsedVal := colVal.(type) { case float32: From a0bd0b0429c738234be5034587777be56efbb567 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Sun, 15 Dec 2024 22:32:08 -0800 Subject: [PATCH 3/6] Adding TODO. --- lib/typing/converters/string_converter.go | 1 + lib/typing/values/string.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index 83ecaafc5..573276b92 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -27,6 +27,7 @@ func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { return ArrayConverter{}, nil } + // TODO: Return an error when all the types are implemented. return nil, nil } diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 5d24e4398..30c294eed 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -7,11 +7,10 @@ import ( "strconv" "strings" - "github.com/artie-labs/transfer/lib/typing/converters" - "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" + "github.com/artie-labs/transfer/lib/typing/converters" "github.com/artie-labs/transfer/lib/typing/decimal" ) @@ -45,6 +44,8 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return sv.Convert(colVal) } + // TODO: Move all of this into converter function + switch colKind.Kind { case typing.String.Kind: isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice From 39a530cdcead4feb0be33c85fe2e43ef82ec769d Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Dec 2024 21:16:20 -0800 Subject: [PATCH 4/6] Imports. --- lib/typing/converters/string_converter.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index ab4f70e70..edb27ab47 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -6,7 +6,6 @@ import ( "time" "github.com/artie-labs/transfer/lib/config/constants" - "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/ext" ) From de3270cf1a70f6114f84ce857ed3514b6f152f27 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Dec 2024 21:58:02 -0800 Subject: [PATCH 5/6] Clean up. --- clients/mssql/values.go | 4 +- clients/snowflake/snowflake_test.go | 6 +- lib/typing/converters/string_converter.go | 61 +++++++++++++++++ .../converters/string_converter_test.go | 10 ++- lib/typing/converters/util.go | 19 ++++++ lib/typing/converters/util_test.go | 68 +++++++++++++++++++ lib/typing/values/string.go | 48 ------------- lib/typing/values/string_test.go | 61 ----------------- 8 files changed, 162 insertions(+), 115 deletions(-) create mode 100644 lib/typing/converters/util.go create mode 100644 lib/typing/converters/util_test.go diff --git a/clients/mssql/values.go b/clients/mssql/values.go index b15b3d64a..c4dd4b8e9 100644 --- a/clients/mssql/values.go +++ b/clients/mssql/values.go @@ -10,9 +10,9 @@ import ( "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/columns" + "github.com/artie-labs/transfer/lib/typing/converters" "github.com/artie-labs/transfer/lib/typing/decimal" "github.com/artie-labs/transfer/lib/typing/ext" - "github.com/artie-labs/transfer/lib/typing/values" ) func parseValue(colVal any, colKind columns.Column) (any, error) { @@ -22,7 +22,7 @@ func parseValue(colVal any, colKind columns.Column) (any, error) { boolVal, isOk := colVal.(bool) if isOk { - colVal = values.BooleanToBit(boolVal) + colVal = converters.BooleanToBit(boolVal) } colValString := fmt.Sprint(colVal) diff --git a/clients/snowflake/snowflake_test.go b/clients/snowflake/snowflake_test.go index 5c072f2a0..b71985e9f 100644 --- a/clients/snowflake/snowflake_test.go +++ b/clients/snowflake/snowflake_test.go @@ -103,7 +103,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeReestablishAuth() { for i := 0; i < 5; i++ { rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{ - "id": fmt.Sprintf("pk-%d", i), + "id": i, "created_at": time.Now().Format(time.RFC3339Nano), "name": fmt.Sprintf("Robin-%d", i), } @@ -146,7 +146,7 @@ func (s *SnowflakeTestSuite) TestExecuteMerge() { for i := 0; i < 5; i++ { rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{ - "id": fmt.Sprintf("pk-%d", i), + "id": i, "created_at": time.Now().Format(time.RFC3339Nano), "name": fmt.Sprintf("Robin-%d", i), } @@ -211,7 +211,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeDeletionFlagRemoval() { rowsData := make(map[string]map[string]any) for i := 0; i < 5; i++ { rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{ - "id": fmt.Sprintf("pk-%d", i), + "id": i, "created_at": time.Now().Format(time.RFC3339Nano), "name": fmt.Sprintf("Robin-%d", i), } diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index edb27ab47..4bc21758d 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "github.com/artie-labs/transfer/lib/typing/decimal" + "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/ext" @@ -16,6 +18,7 @@ type StringConverter interface { func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { switch kd.Kind { + // Time case typing.Date.Kind: return DateConverter{}, nil case typing.Time.Kind: @@ -26,6 +29,13 @@ func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { return TimestampTZConverter{}, nil case typing.Array.Kind: return ArrayConverter{}, nil + // Numbers + case typing.EDecimal.Kind: + return DecimalConverter{}, nil + case typing.Integer.Kind: + return IntegerConverter{}, nil + case typing.Float.Kind: + return FloatConverter{}, nil } // TODO: Return an error when all the types are implemented. @@ -94,3 +104,54 @@ func (ArrayConverter) Convert(value any) (string, error) { return string(colValBytes), nil } + +type IntegerConverter struct{} + +func (IntegerConverter) Convert(value any) (string, error) { + switch parsedVal := value.(type) { + case float32: + return Float32ToString(parsedVal), nil + case float64: + return Float64ToString(parsedVal), nil + case bool: + return fmt.Sprint(BooleanToBit(parsedVal)), nil + case int, int8, int16, int32, int64: + return fmt.Sprint(parsedVal), nil + default: + return "", fmt.Errorf("unexpected value: '%v', type: %T", value, value) + } +} + +type FloatConverter struct{} + +func (FloatConverter) Convert(value any) (string, error) { + switch parsedVal := value.(type) { + case float32: + return Float32ToString(parsedVal), nil + case float64: + return Float64ToString(parsedVal), nil + case int, int8, int16, int32, int64: + return fmt.Sprint(parsedVal), nil + default: + return "", fmt.Errorf("unexpected value: '%v', type: %T", value, value) + } +} + +type DecimalConverter struct{} + +func (DecimalConverter) Convert(value any) (string, error) { + switch castedColVal := value.(type) { + case float32: + return Float32ToString(castedColVal), nil + case float64: + return Float64ToString(castedColVal), nil + case int64, int32: + return fmt.Sprint(castedColVal), nil + case string: + return castedColVal, nil + case *decimal.Decimal: + return castedColVal.String(), nil + default: + return "", fmt.Errorf("unexpected value: '%v' type: %T", value, value) + } +} diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go index 62f046cba..731b40a91 100644 --- a/lib/typing/converters/string_converter_test.go +++ b/lib/typing/converters/string_converter_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestArrayConverter(t *testing.T) { +func TestArrayConverter_Convert(t *testing.T) { // Array { // Normal arrays @@ -22,3 +22,11 @@ func TestArrayConverter(t *testing.T) { assert.Equal(t, `["__debezium_unavailable_value"]`, val) } } + +func TestIntegerConverter_Convert(t *testing.T) { + for _, val := range []any{42, int8(42), int16(42), int32(42), int64(42)} { + parsedVal, err := IntegerConverter{}.Convert(val) + assert.NoError(t, err) + assert.Equal(t, "42", parsedVal) + } +} diff --git a/lib/typing/converters/util.go b/lib/typing/converters/util.go new file mode 100644 index 000000000..b8444889c --- /dev/null +++ b/lib/typing/converters/util.go @@ -0,0 +1,19 @@ +package converters + +import "strconv" + +func Float64ToString(value float64) string { + return strconv.FormatFloat(value, 'f', -1, 64) +} + +func Float32ToString(value float32) string { + return strconv.FormatFloat(float64(value), 'f', -1, 32) +} + +func BooleanToBit(val bool) int { + if val { + return 1 + } else { + return 0 + } +} diff --git a/lib/typing/converters/util_test.go b/lib/typing/converters/util_test.go new file mode 100644 index 000000000..47472745f --- /dev/null +++ b/lib/typing/converters/util_test.go @@ -0,0 +1,68 @@ +package converters + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBooleanToBit(t *testing.T) { + assert.Equal(t, 1, BooleanToBit(true)) + assert.Equal(t, 0, BooleanToBit(false)) +} + +func TestFloat32ToString(t *testing.T) { + type ioPair struct { + input float32 + output string + } + + ioPairs := []ioPair{ + {123.456, "123.456"}, + {0.0, "0"}, + {-1.0, "-1"}, + {1.0, "1"}, + {340282350000000000000000000000000000000, "340282350000000000000000000000000000000"}, + {math.MaxFloat32, "340282350000000000000000000000000000000"}, + {0.000000000000000000000000000000000000000000001, "0.000000000000000000000000000000000000000000001"}, + {-340282350000000000000000000000000000000, "-340282350000000000000000000000000000000"}, + {1.401298464324817070923729583289916131280e-45, "0.000000000000000000000000000000000000000000001"}, + {1.17549435e-38, "0.000000000000000000000000000000000000011754944"}, + {-1.17549435e-38, "-0.000000000000000000000000000000000000011754944"}, + {2.71828, "2.71828"}, + {-2.71828, "-2.71828"}, + {3.14159, "3.14159"}, + {-3.14159, "-3.14159"}, + } + + for _, pair := range ioPairs { + assert.Equal(t, pair.output, Float32ToString(pair.input), pair.input) + } +} + +func TestFloat64ToString(t *testing.T) { + type ioPair struct { + input float64 + output string + } + + ioPairs := []ioPair{ + {123.456, "123.456"}, + {0.0, "0"}, + {-1.0, "-1"}, + {1.0, "1"}, + {1.7976931348623157e+308, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {math.MaxFloat64, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {4.9406564584124654e-324, "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"}, + {-1.7976931348623157e+308, "-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {2.718281828459045, "2.718281828459045"}, + {-2.718281828459045, "-2.718281828459045"}, + {3.141592653589793, "3.141592653589793"}, + {-3.141592653589793, "-3.141592653589793"}, + } + + for _, pair := range ioPairs { + assert.Equal(t, pair.output, Float64ToString(pair.input), pair.input) + } +} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 30c294eed..f6b123773 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -4,32 +4,14 @@ import ( "encoding/json" "fmt" "reflect" - "strconv" "strings" "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/converters" - "github.com/artie-labs/transfer/lib/typing/decimal" ) -func Float64ToString(value float64) string { - return strconv.FormatFloat(value, 'f', -1, 64) -} - -func Float32ToString(value float32) string { - return strconv.FormatFloat(float64(value), 'f', -1, 32) -} - -func BooleanToBit(val bool) int { - if val { - return 1 - } else { - return 0 - } -} - func ToString(colVal any, colKind typing.KindDetails) (string, error) { if colVal == nil { return "", fmt.Errorf("colVal is nil") @@ -45,7 +27,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { } // TODO: Move all of this into converter function - switch colKind.Kind { case typing.String.Kind: isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice @@ -78,35 +59,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return string(colValBytes), nil } } - case typing.Float.Kind: - switch parsedVal := colVal.(type) { - case float32: - return Float32ToString(parsedVal), nil - case float64: - return Float64ToString(parsedVal), nil - } - case typing.Integer.Kind: - switch parsedVal := colVal.(type) { - case float32: - return Float32ToString(parsedVal), nil - case float64: - return Float64ToString(parsedVal), nil - case bool: - return fmt.Sprint(BooleanToBit(parsedVal)), nil - } - case typing.EDecimal.Kind: - switch castedColVal := colVal.(type) { - // It's okay if it's not a *decimal.Decimal, so long as it's a float or string. - // By having the flexibility of handling both *decimal.Decimal and float64/float32/string values within the same batch will increase our ability for data digestion. - case int64, int32, float64, float32: - return fmt.Sprint(castedColVal), nil - case string: - return castedColVal, nil - case *decimal.Decimal: - return castedColVal.String(), nil - } - - return "", fmt.Errorf("unexpected colVal type: %T", colVal) } return fmt.Sprint(colVal), nil diff --git a/lib/typing/values/string_test.go b/lib/typing/values/string_test.go index 5ad48a2af..20c3cd5f3 100644 --- a/lib/typing/values/string_test.go +++ b/lib/typing/values/string_test.go @@ -1,7 +1,6 @@ package values import ( - "math" "testing" "time" @@ -14,66 +13,6 @@ import ( "github.com/artie-labs/transfer/lib/typing/ext" ) -func TestFloat32ToString(t *testing.T) { - type ioPair struct { - input float32 - output string - } - - ioPairs := []ioPair{ - {123.456, "123.456"}, - {0.0, "0"}, - {-1.0, "-1"}, - {1.0, "1"}, - {340282350000000000000000000000000000000, "340282350000000000000000000000000000000"}, - {math.MaxFloat32, "340282350000000000000000000000000000000"}, - {0.000000000000000000000000000000000000000000001, "0.000000000000000000000000000000000000000000001"}, - {-340282350000000000000000000000000000000, "-340282350000000000000000000000000000000"}, - {1.401298464324817070923729583289916131280e-45, "0.000000000000000000000000000000000000000000001"}, - {1.17549435e-38, "0.000000000000000000000000000000000000011754944"}, - {-1.17549435e-38, "-0.000000000000000000000000000000000000011754944"}, - {2.71828, "2.71828"}, - {-2.71828, "-2.71828"}, - {3.14159, "3.14159"}, - {-3.14159, "-3.14159"}, - } - - for _, pair := range ioPairs { - assert.Equal(t, pair.output, Float32ToString(pair.input), pair.input) - } -} - -func TestFloat64ToString(t *testing.T) { - type ioPair struct { - input float64 - output string - } - - ioPairs := []ioPair{ - {123.456, "123.456"}, - {0.0, "0"}, - {-1.0, "-1"}, - {1.0, "1"}, - {1.7976931348623157e+308, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {math.MaxFloat64, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {4.9406564584124654e-324, "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"}, - {-1.7976931348623157e+308, "-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {2.718281828459045, "2.718281828459045"}, - {-2.718281828459045, "-2.718281828459045"}, - {3.141592653589793, "3.141592653589793"}, - {-3.141592653589793, "-3.141592653589793"}, - } - - for _, pair := range ioPairs { - assert.Equal(t, pair.output, Float64ToString(pair.input), pair.input) - } -} - -func TestBooleanToBit(t *testing.T) { - assert.Equal(t, 1, BooleanToBit(true)) - assert.Equal(t, 0, BooleanToBit(false)) -} - func TestToString(t *testing.T) { { // Nil value From 4cb0b9199244a0cf1b9a6150d6a996cdf522131a Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Dec 2024 22:04:37 -0800 Subject: [PATCH 6/6] Clean up. --- lib/typing/converters/string_converter.go | 2 +- .../converters/string_converter_test.go | 81 ++++++++++++++++++- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index 4bc21758d..0f2f8cd01 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -145,7 +145,7 @@ func (DecimalConverter) Convert(value any) (string, error) { return Float32ToString(castedColVal), nil case float64: return Float64ToString(castedColVal), nil - case int64, int32: + case int, int8, int16, int32, int64: return fmt.Sprint(castedColVal), nil case string: return castedColVal, nil diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go index 731b40a91..2bf7d0789 100644 --- a/lib/typing/converters/string_converter_test.go +++ b/lib/typing/converters/string_converter_test.go @@ -3,6 +3,10 @@ package converters import ( "testing" + "github.com/artie-labs/transfer/lib/numbers" + + "github.com/artie-labs/transfer/lib/typing/decimal" + "github.com/artie-labs/transfer/lib/config/constants" "github.com/stretchr/testify/assert" ) @@ -23,10 +27,81 @@ func TestArrayConverter_Convert(t *testing.T) { } } +func TestFloatConverter_Convert(t *testing.T) { + { + // Unexpected type + _, err := FloatConverter{}.Convert("foo") + assert.ErrorContains(t, err, `unexpected value: 'foo', type: string`) + } + { + // Float32 + val, err := FloatConverter{}.Convert(float32(123.45)) + assert.NoError(t, err) + assert.Equal(t, "123.45", val) + } + { + // Float64 + val, err := FloatConverter{}.Convert(float64(123.45)) + assert.NoError(t, err) + assert.Equal(t, "123.45", val) + } + { + // Integers + for _, input := range []any{42, int8(42), int16(42), int32(42), int64(42), float32(42), float64(42)} { + val, err := FloatConverter{}.Convert(input) + assert.NoError(t, err) + assert.Equal(t, "42", val) + } + } +} + func TestIntegerConverter_Convert(t *testing.T) { - for _, val := range []any{42, int8(42), int16(42), int32(42), int64(42)} { - parsedVal, err := IntegerConverter{}.Convert(val) + { + // Various numbers + for _, val := range []any{42, int8(42), int16(42), int32(42), int64(42), float32(42), float64(42)} { + parsedVal, err := IntegerConverter{}.Convert(val) + assert.NoError(t, err) + assert.Equal(t, "42", parsedVal) + } + } + { + // Booleans + { + // True + val, err := IntegerConverter{}.Convert(true) + assert.NoError(t, err) + assert.Equal(t, "1", val) + } + { + // False + val, err := IntegerConverter{}.Convert(false) + assert.NoError(t, err) + assert.Equal(t, "0", val) + } + } +} + +func TestDecimalConverter_Convert(t *testing.T) { + { + // Extended decimal + val, err := DecimalConverter{}.Convert(decimal.NewDecimal(numbers.MustParseDecimal("123.45"))) assert.NoError(t, err) - assert.Equal(t, "42", parsedVal) + assert.Equal(t, "123.45", val) + } + { + // Floats + for _, input := range []any{float32(123.45), float64(123.45)} { + val, err := DecimalConverter{}.Convert(input) + assert.NoError(t, err) + assert.Equal(t, "123.45", val) + } + } + { + // Integers + for _, input := range []any{42, int8(42), int16(42), int32(42), int64(42), float32(42), float64(42)} { + val, err := DecimalConverter{}.Convert(input) + assert.NoError(t, err) + assert.Equal(t, "42", val) + } } }