From d591eb9909e6a923c478e672273e62406778f3ad Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 13:41:01 +0200 Subject: [PATCH 01/15] chore: add SAP HANA driver dependency --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 96f1394c47..0c1ac81de5 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.8.3 github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 github.com/DATA-DOG/go-sqlmock v1.4.1 + github.com/SAP/go-hdb v0.14.1 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01 github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0 diff --git a/go.sum b/go.sum index ea64f80e98..3f95eeccce 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITg github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/SAP/go-hdb v0.14.1 h1:hkw4ozGZ/i4eak7ZuGkY5e0hxiXFdNUBNhr4AvZVNFE= +github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= From 87a15361d348698b998d8df35f6d0661b51bb84c Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 13:41:54 +0200 Subject: [PATCH 02/15] feat: add SAP HANA db support --- stdlib/sql/from.go | 2 + stdlib/sql/from_private_test.go | 8 ++ stdlib/sql/hdb.go | 212 ++++++++++++++++++++++++++++++++ stdlib/sql/source_validator.go | 6 + stdlib/sql/to.go | 14 ++- 5 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 stdlib/sql/hdb.go diff --git a/stdlib/sql/from.go b/stdlib/sql/from.go index 975f540925..4f9043b81a 100644 --- a/stdlib/sql/from.go +++ b/stdlib/sql/from.go @@ -127,6 +127,8 @@ func createFromSQLSource(prSpec plan.ProcedureSpec, dsid execute.DatasetID, a ex newRowReader = NewAwsAthenaRowReader case "bigquery": newRowReader = NewBigQueryRowReader + case "hdb": + newRowReader = NewHdbRowReader default: return nil, errors.Newf(codes.Invalid, "sql driver %s not supported", spec.DriverName) } diff --git a/stdlib/sql/from_private_test.go b/stdlib/sql/from_private_test.go index 61bc13f25d..323e43cdef 100644 --- a/stdlib/sql/from_private_test.go +++ b/stdlib/sql/from_private_test.go @@ -81,6 +81,14 @@ func TestFromSqlUrlValidation(t *testing.T) { Query: "", }, ErrMsg: "", + }, { + Name: "ok hdb", + Spec: &FromSQLProcedureSpec{ + DriverName: "hdb", + DataSourceName: "hdb://user:password@localhost:39013", + Query: "", + }, + ErrMsg: "", }, { Name: "invalid driver", Spec: &FromSQLProcedureSpec{ diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go new file mode 100644 index 0000000000..3ed2f8d865 --- /dev/null +++ b/stdlib/sql/hdb.go @@ -0,0 +1,212 @@ +package sql + +import ( + "database/sql" + "fmt" + "math/big" + "strings" + "time" + + _ "github.com/SAP/go-hdb/driver" + hdb "github.com/SAP/go-hdb/driver" + "github.com/influxdata/flux" + "github.com/influxdata/flux/codes" + "github.com/influxdata/flux/execute" + "github.com/influxdata/flux/internal/errors" + "github.com/influxdata/flux/values" +) + +// SAP HANA DB support. +// Notes: +// * the last version compatible with Go 1.12 is v0.14.1. In v0.14.3, they started using 1.13 type sql.NullTime +// and added `go 1.14` directive to go.mod (https://github.com/SAP/go-hdb/releases/tag/v0.14.3) +// * HDB returns BOOLEAN as TINYINT +// * TIMESTAMP does not TZ info stored, but: +// - it is "strongly discouraged" to store data in local time zone: https://blogs.sap.com/2018/03/28/trouble-with-time/ +// - more on timestamps in HDB: https://help.sap.com/viewer/f1b440ded6144a54ada97ff95dac7adf/2.4/en-US/a394f75dcbe64b42b7a887231af8f15f.html +// - the driver does is not capable of converting date- or time-formatted strings to time.Time when inserting to DATE or TIME columns +// Therefore TIMESTAMP is mapped to TTime and vice-versa here. +// * the hdb driver is rather strict, eg. does not convert date- or time-formatted string values to time.Time, +// or float64 to Decimal on its own and just throws "unsupported conversion" error + +type HdbRowReader struct { + Cursor *sql.Rows + columns []interface{} + columnTypes []flux.ColType + columnNames []string + sqlTypes []*sql.ColumnType + NextFunc func() bool + CloseFunc func() error +} + +// Next prepares HdbRowReader to return rows +func (m *HdbRowReader) Next() bool { + if m.NextFunc != nil { + return m.NextFunc() + } + next := m.Cursor.Next() + if next { + columnNames, err := m.Cursor.Columns() + if err != nil { + return false + } + m.columns = make([]interface{}, len(columnNames)) + columnPointers := make([]interface{}, len(columnNames)) + for i := 0; i < len(columnNames); i++ { + columnPointers[i] = &m.columns[i] + } + if err := m.Cursor.Scan(columnPointers...); err != nil { + return false + } + } + return next +} + +func (m *HdbRowReader) GetNextRow() ([]values.Value, error) { + row := make([]values.Value, len(m.columns)) + for i, column := range m.columns { + switch value := column.(type) { + case bool, int64, float64, string: + row[i] = values.New(value) + case time.Time: + // DATE, TIME types get scanned to time.Time by the driver too, + // but they have no counterpart in Flux therefore will be represented as string + switch m.sqlTypes[i].DatabaseTypeName() { + case "DATE": + row[i] = values.NewString(value.Format(layoutDate)) + case "TIME": + row[i] = values.NewString(value.Format(layoutTime)) + default: + row[i] = values.NewTime(values.ConvertTime(value)) + } + case []uint8: + switch m.columnTypes[i] { + case flux.TFloat: + var out hdb.Decimal + err := out.Scan(value) + if err != nil { + return nil, err + } + newFloat, _ := (*big.Rat)(&out).Float64() + row[i] = values.NewFloat(newFloat) + default: + row[i] = values.NewString(string(value)) + } + case nil: + row[i] = values.NewNull(flux.SemanticType(m.columnTypes[i])) + default: + execute.PanicUnknownType(flux.TInvalid) + } + } + return row, nil +} + +func (m *HdbRowReader) InitColumnNames(names []string) { + m.columnNames = names +} + +func (m *HdbRowReader) InitColumnTypes(types []*sql.ColumnType) { + fluxTypes := make([]flux.ColType, len(types)) + for i := 0; i < len(types); i++ { + switch types[i].DatabaseTypeName() { + case "TINYINT", "SMALLINT", "INTEGER", "BIGINT": + fluxTypes[i] = flux.TInt + case "REAL", "DOUBLE", "DECIMAL": + fluxTypes[i] = flux.TFloat + case "TIMESTAMP": // not exactly correct (see Notes) + fluxTypes[i] = flux.TTime + default: + fluxTypes[i] = flux.TString + } + } + m.columnTypes = fluxTypes + m.sqlTypes = types +} + +func (m *HdbRowReader) ColumnNames() []string { + return m.columnNames +} + +func (m *HdbRowReader) ColumnTypes() []flux.ColType { + return m.columnTypes +} + +func (m *HdbRowReader) SetColumnTypes(types []flux.ColType) { + m.columnTypes = types +} + +func (m *HdbRowReader) SetColumns(i []interface{}) { + m.columns = i +} + +func (m *HdbRowReader) Close() error { + if m.CloseFunc != nil { + return m.CloseFunc() + } + if err := m.Cursor.Err(); err != nil { + return err + } + return m.Cursor.Close() +} + +func NewHdbRowReader(r *sql.Rows) (execute.RowReader, error) { + reader := &HdbRowReader{ + Cursor: r, + } + cols, err := r.Columns() + if err != nil { + return nil, err + } + reader.InitColumnNames(cols) + + types, err := r.ColumnTypes() + if err != nil { + return nil, err + } + reader.InitColumnTypes(types) + + return reader, nil +} + +var fluxToHdb = map[flux.ColType]string{ + flux.TFloat: "DOUBLE", + flux.TInt: "BIGINT", + flux.TString: "NVARCHAR(5000)", // 5000 is the max + flux.TBool: "BOOLEAN", + flux.TTime: "TIMESTAMP", // not exactly correct (see Notes) +} + +// HdbTranslateColumn translates flux colTypes into their corresponding SAP HANA column type +func HdbColumnTranslateFunc() translationFunc { + return func(f flux.ColType, colName string) (string, error) { + s, found := fluxToHdb[f] + if !found { + return "", errors.Newf(codes.Internal, "SAP HANA does not support column type %s", f.String()) + } + return colName + " " + s, nil + } +} + +// Template for conditional query by table existence check +var hdbDoIfTableNotExistsTemplate = `DO +BEGIN + DECLARE X_EXISTS INT = 0; + SELECT COUNT(*) INTO X_EXISTS FROM TABLES %s; + IF :X_EXISTS = 0 + THEN + %s; + END IF; +END; +` + +// Adds SAP HANA specific table existence check to CREATE TABLE statement +func hdbAddIfNotExist(table string, query string) string { + var where string + parts := strings.SplitN(table, ".", 2) + if len(parts) == 2 { // fully-qualified table name + where = fmt.Sprintf("WHERE SCHEMA_NAME=UPPER('%s') AND TABLE_NAME=UPPER('%s')", parts[0], parts[1]) + } else { // table in user default schema + where = fmt.Sprintf("WHERE TABLE_NAME=UPPER('%s')", table) + } + return fmt.Sprintf(hdbDoIfTableNotExistsTemplate, where, query) +} diff --git a/stdlib/sql/source_validator.go b/stdlib/sql/source_validator.go index 3e07526f99..7a0a1c9316 100644 --- a/stdlib/sql/source_validator.go +++ b/stdlib/sql/source_validator.go @@ -102,6 +102,12 @@ func validateDataSource(validator url.Validator, driverName string, dataSourceNa Host: cfg.ProjectID, Path: cfg.Location, } + case "hdb": // SAP HANA + // an example is: hdb://user:password@host:port + u, err = neturl.Parse(dataSourceName) + if err != nil { + return errors.Newf(codes.Invalid, "invalid data source url: %v", err) + } default: return errors.Newf(codes.Invalid, "sql driver %s not supported", driverName) } diff --git a/stdlib/sql/to.go b/stdlib/sql/to.go index d6b7659b1d..5022259f8e 100644 --- a/stdlib/sql/to.go +++ b/stdlib/sql/to.go @@ -275,6 +275,8 @@ func getTranslationFunc(driverName string) (func() translationFunc, error) { return nil, errors.Newf(codes.Invalid, "writing is not supported for %s", driverName) case "bigquery": return BigQueryColumnTranslateFunc, nil + case "hdb": + return HdbColumnTranslateFunc, nil default: return nil, errors.Newf(codes.Internal, "invalid driverName: %s", driverName) } @@ -335,10 +337,14 @@ func CreateInsertComponents(t *ToSQLTransformation, tbl flux.Table) (colNames [] if t.spec.Spec.DriverName != "sqlmock" { var q string - if !isMssqlDriver(t.spec.Spec.DriverName) { - q = fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) - } else { // SQL Server does not support IF NOT EXIST + if isMssqlDriver(t.spec.Spec.DriverName) { // SQL Server does not support IF NOT EXIST q = fmt.Sprintf("IF OBJECT_ID('%s', 'U') IS NULL BEGIN CREATE TABLE %s (%s) END", t.spec.Spec.Table, t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) + } else if t.spec.Spec.DriverName == "hdb" { // SAP HANA does not support IF NOT EXIST + q = fmt.Sprintf("CREATE TABLE %s (%s)", t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) + q = hdbAddIfNotExist(t.spec.Spec.Table, q) + batchSize = 1 // SAP HANA does not support INSERT/UPDATE batching via a single SQL command + } else { + q = fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) } _, err = t.tx.Exec(q) if err != nil { @@ -395,7 +401,7 @@ func CreateInsertComponents(t *ToSQLTransformation, tbl flux.Table) (colNames [] return err } - if i != 0 && i%batchSize == 0 { + if (i != 0 && i%batchSize == 0) || (batchSize == 1) { // create "mini batches" of values - each one represents a single db.Exec to SQL valArgsArray = append(valArgsArray, valueArgs) valStringArray = append(valStringArray, valueStrings) From aa0f8d6ccb164f94ffd1ac6b1a93174e223a946d Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 14:00:14 +0200 Subject: [PATCH 03/15] test: add HDB test --- stdlib/sql/sql_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/stdlib/sql/sql_test.go b/stdlib/sql/sql_test.go index aba8f6e8ce..2872fb99b0 100644 --- a/stdlib/sql/sql_test.go +++ b/stdlib/sql/sql_test.go @@ -586,6 +586,67 @@ func TestBigQueryParsing(t *testing.T) { } } +func TestHdbParsing(t *testing.T) { + // here we want to build a mocked representation of what's in our MySql db, and then run our RowReader over it, then verify that the results + // are as expected. + // NOTE: no meaningful test for reading bools, because the DB doesn't support them, and we already know that we can read INT types + testCases := []struct { + name string + columnName string + data *sql.Rows + want [][]values.Value + }{ + { + name: "ints", + columnName: "_int", + data: mockRowsToSQLRows(sqlmock.NewRows([]string{"column"}).AddRow(int64(6)).AddRow(int64(1)).AddRow(int64(643)).AddRow(int64(42)).AddRow(int64(1283))), + want: [][]values.Value{{values.NewInt(6)}, {values.NewInt(1)}, {values.NewInt(643)}, {values.NewInt(42)}, {values.NewInt(1283)}}, + }, + { + name: "floats", + columnName: "_float", + data: mockRowsToSQLRows(sqlmock.NewRows([]string{"column"}).AddRow(float64(6)).AddRow(float64(1)).AddRow(float64(643)).AddRow(float64(42)).AddRow(float64(1283))), + want: [][]values.Value{{values.NewFloat(6)}, {values.NewFloat(1)}, {values.NewFloat(643)}, {values.NewFloat(42)}, {values.NewFloat(1283)}}, + }, + { + name: "strings", + columnName: "_string", + data: mockRowsToSQLRows(sqlmock.NewRows([]string{"column"}).AddRow(string("6")).AddRow(string("1")).AddRow(string("643")).AddRow(string("42")).AddRow(string("1283"))), + want: [][]values.Value{{values.NewString("6")}, {values.NewString("1")}, {values.NewString("643")}, {values.NewString("42")}, {values.NewString("1283")}}, + }, + { + name: "timestamp", + columnName: "timestamp", + data: mockRowsToSQLRows(sqlmock.NewRows([]string{"column"}).AddRow(createTestTimes()[0].(time.Time)).AddRow(createTestTimes()[1].(time.Time)).AddRow(createTestTimes()[2].(time.Time)).AddRow(createTestTimes()[3].(time.Time)).AddRow(createTestTimes()[4].(time.Time))), + want: [][]values.Value{ + {values.NewTime(values.ConvertTime(createTestTimes()[0].(time.Time)))}, + {values.NewTime(values.ConvertTime(createTestTimes()[1].(time.Time)))}, + {values.NewTime(values.ConvertTime(createTestTimes()[2].(time.Time)))}, + {values.NewTime(values.ConvertTime(createTestTimes()[3].(time.Time)))}, + {values.NewTime(values.ConvertTime(createTestTimes()[4].(time.Time)))}, + }, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + + TestReader, err := NewHdbRowReader(tc.data) + if !cmp.Equal(nil, err) { + t.Fatalf("unexpected result -want/+got\n\n%s\n\n", cmp.Diff(nil, err)) + } + i := 0 + for TestReader.Next() { + row, _ := TestReader.GetNextRow() + if !cmp.Equal(tc.want[i], row) { + t.Fatalf("unexpected result -want/+got\n\n%s\n\n", cmp.Diff(tc.want[i], row)) + } + i++ + } + }) + } +} + func createTestTimes() []interface{} { str := []string{"2019-06-03 13:59:00", "2019-06-03 13:59:01", From d5d7fbe9df1daa87934b40a15ab5064044fa34d9 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 14:23:44 +0200 Subject: [PATCH 04/15] style: remove code comment --- stdlib/sql/hdb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 3ed2f8d865..ba74480b83 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -24,7 +24,6 @@ import ( // * TIMESTAMP does not TZ info stored, but: // - it is "strongly discouraged" to store data in local time zone: https://blogs.sap.com/2018/03/28/trouble-with-time/ // - more on timestamps in HDB: https://help.sap.com/viewer/f1b440ded6144a54ada97ff95dac7adf/2.4/en-US/a394f75dcbe64b42b7a887231af8f15f.html -// - the driver does is not capable of converting date- or time-formatted strings to time.Time when inserting to DATE or TIME columns // Therefore TIMESTAMP is mapped to TTime and vice-versa here. // * the hdb driver is rather strict, eg. does not convert date- or time-formatted string values to time.Time, // or float64 to Decimal on its own and just throws "unsupported conversion" error From fbc88b573c28d92cd05442a51f5d34506c3d603e Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 14:36:49 +0200 Subject: [PATCH 05/15] test: add HDB test --- stdlib/sql/to_privates_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/stdlib/sql/to_privates_test.go b/stdlib/sql/to_privates_test.go index 6ea1c03368..924c6d222e 100644 --- a/stdlib/sql/to_privates_test.go +++ b/stdlib/sql/to_privates_test.go @@ -335,3 +335,33 @@ func TestBigQueryTranslation(t *testing.T) { t.Fail() } } + +func TestHdbTranslation(t *testing.T) { + hdbTypeTranslations := map[string]flux.ColType{ + "DOUBLE": flux.TFloat, + "BIGINT": flux.TInt, + "NVARCHAR(5000)": flux.TString, + "TIMESTAMP": flux.TTime, + "BOOLEAN": flux.TBool, + } + + columnLabel := "apples" + // verify that valid returns expected happiness for hdb + sqlT, err := getTranslationFunc("hdb") + if !cmp.Equal(nil, err) { + t.Log(cmp.Diff(nil, err)) + t.Fail() + } + + for dbTypeString, fluxType := range hdbTypeTranslations { + v, err := sqlT()(fluxType, columnLabel) + if !cmp.Equal(nil, err) { + t.Log(cmp.Diff(nil, err)) + t.Fail() + } + if !cmp.Equal(columnLabel+" "+dbTypeString, v) { + t.Log(cmp.Diff(columnLabel+" "+dbTypeString, v)) + t.Fail() + } + } +} \ No newline at end of file From 65440867efc9dea518da54e51cc94e7a9c21e230 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 17:20:29 +0200 Subject: [PATCH 06/15] test: add more tests --- stdlib/sql/to_privates_test.go | 27 +++++++++++++++++++++++++++ stdlib/sql/to_test.go | 9 +++++++++ 2 files changed, 36 insertions(+) diff --git a/stdlib/sql/to_privates_test.go b/stdlib/sql/to_privates_test.go index 924c6d222e..1255f6ee42 100644 --- a/stdlib/sql/to_privates_test.go +++ b/stdlib/sql/to_privates_test.go @@ -90,6 +90,21 @@ func TestTranslationDriverReturn(t *testing.T) { t.Fail() } + // verify that valid returns expected error for AWS Athena (yes, error) + expectedErr := errors.Newf(codes.Invalid, "writing is not supported for awsathena") + _, err = getTranslationFunc("awsathena") + if !cmp.Equal(expectedErr, err) { + t.Log(cmp.Diff(expectedErr, err)) + t.Fail() + } + + // verify that valid returns expected happiness for SAP HANA + _, err = getTranslationFunc("hdb") + if !cmp.Equal(nil, err) { + t.Log(cmp.Diff(nil, err)) + t.Fail() + } + } func TestSqliteTranslation(t *testing.T) { @@ -364,4 +379,16 @@ func TestHdbTranslation(t *testing.T) { t.Fail() } } + + // test no match + var _unsupportedType flux.ColType = 666 + _, err = sqlT()(_unsupportedType, columnLabel) + if cmp.Equal(nil, err) { + t.Log(cmp.Diff(nil, err)) + t.Fail() + } + if !cmp.Equal("SAP HANA does not support column type unknown", err.Error()) { + t.Log(cmp.Diff("SAP HANA does not support column type unknown", err.Error())) + t.Fail() + } } \ No newline at end of file diff --git a/stdlib/sql/to_test.go b/stdlib/sql/to_test.go index 1a382ab50c..248d5c62c2 100644 --- a/stdlib/sql/to_test.go +++ b/stdlib/sql/to_test.go @@ -334,6 +334,15 @@ func TestToSql_NewTransformation(t *testing.T) { }, }, WantErr: "connection refused", + }, { + Name: "ok hdb", + Spec: &fsql.ToSQLProcedureSpec{ + Spec: &fsql.ToSQLOpSpec{ + DriverName: "hdb", + DataSourceName: "hdb://user:password@localhost:39013", + }, + }, + WantErr: "connection refused", }, { Name: "invalid driver", Spec: &fsql.ToSQLProcedureSpec{ From 0eb6d75f60ed5ff5457f3b878687578096b6daec Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 11 Aug 2020 21:11:27 +0200 Subject: [PATCH 07/15] test: add HDB code test --- stdlib/sql/hdb_test.go | 17 +++++++++++++++++ stdlib/sql/sql_test.go | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 stdlib/sql/hdb_test.go diff --git a/stdlib/sql/hdb_test.go b/stdlib/sql/hdb_test.go new file mode 100644 index 0000000000..4b89f06c91 --- /dev/null +++ b/stdlib/sql/hdb_test.go @@ -0,0 +1,17 @@ +package sql + +import ( + "strings" + "testing" +) + +func TestHdb_IfNoExist(t *testing.T) { + q := hdbAddIfNotExist("stores.orders_copy", "CREATE TABLE stores.orders_copy (ORDER_ID BIGINT)") + if !strings.HasPrefix(q, "DO") || !strings.Contains(q, "SCHEMA_NAME") || !strings.Contains(q, "TABLE_NAME") { + t.Fail() + } + q = hdbAddIfNotExist("orders_copy", "CREATE TABLE orders_copy (ORDER_ID BIGINT)") + if !strings.HasPrefix(q, "DO") || strings.Contains(q, "SCHEMA_NAME") || !strings.Contains(q, "TABLE_NAME") { + t.Fail() + } +} diff --git a/stdlib/sql/sql_test.go b/stdlib/sql/sql_test.go index 2872fb99b0..feb0f37396 100644 --- a/stdlib/sql/sql_test.go +++ b/stdlib/sql/sql_test.go @@ -616,7 +616,7 @@ func TestHdbParsing(t *testing.T) { }, { name: "timestamp", - columnName: "timestamp", + columnName: "_timestamp", data: mockRowsToSQLRows(sqlmock.NewRows([]string{"column"}).AddRow(createTestTimes()[0].(time.Time)).AddRow(createTestTimes()[1].(time.Time)).AddRow(createTestTimes()[2].(time.Time)).AddRow(createTestTimes()[3].(time.Time)).AddRow(createTestTimes()[4].(time.Time))), want: [][]values.Value{ {values.NewTime(values.ConvertTime(createTestTimes()[0].(time.Time)))}, From c0c4ffa1888fb77c283bbd5033c36b8f38a67b64 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 23 Sep 2020 15:28:52 +0200 Subject: [PATCH 08/15] fix: try to prevent SQL injection --- stdlib/sql/hdb.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index ba74480b83..4e8269daed 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -180,7 +180,7 @@ func HdbColumnTranslateFunc() translationFunc { return func(f flux.ColType, colName string) (string, error) { s, found := fluxToHdb[f] if !found { - return "", errors.Newf(codes.Internal, "SAP HANA does not support column type %s", f.String()) + return "", errors.Newf(codes.Invalid, "SAP HANA does not support column type %s", f.String()) } return colName + " " + s, nil } @@ -189,6 +189,8 @@ func HdbColumnTranslateFunc() translationFunc { // Template for conditional query by table existence check var hdbDoIfTableNotExistsTemplate = `DO BEGIN + DECLARE SCHEMA_NAME NVARCHAR(%d) = '%s'; + DECLARE TABLE_NAME NVARCHAR(%d) = '%s'; DECLARE X_EXISTS INT = 0; SELECT COUNT(*) INTO X_EXISTS FROM TABLES %s; IF :X_EXISTS = 0 @@ -198,14 +200,26 @@ BEGIN END; ` -// Adds SAP HANA specific table existence check to CREATE TABLE statement +// Adds SAP HANA specific table existence check to CREATE TABLE statement. func hdbAddIfNotExist(table string, query string) string { var where string + var args []interface{} parts := strings.SplitN(table, ".", 2) if len(parts) == 2 { // fully-qualified table name - where = fmt.Sprintf("WHERE SCHEMA_NAME=UPPER('%s') AND TABLE_NAME=UPPER('%s')", parts[0], parts[1]) + where = "WHERE SCHEMA_NAME=UPPER(:SCHEMA_NAME) AND TABLE_NAME=UPPER(:TABLE_NAME)" + args = append(args, len(parts[0])) + args = append(args, parts[0]) + args = append(args, len(parts[1])) + args = append(args, parts[1]) } else { // table in user default schema - where = fmt.Sprintf("WHERE TABLE_NAME=UPPER('%s')", table) + where = "WHERE TABLE_NAME=UPPER(:TABLE_NAME)" + args = append(args, len("default")) + args = append(args, "default") + args = append(args, len(table)) + args = append(args, table) } - return fmt.Sprintf(hdbDoIfTableNotExistsTemplate, where, query) + args = append(args, where) + args = append(args, query) + + return fmt.Sprintf(hdbDoIfTableNotExistsTemplate, args...) } From 6009e1e2411c95a44e994ec28dccbd75d647a966 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 23 Sep 2020 15:29:27 +0200 Subject: [PATCH 09/15] test: update to match template changes --- stdlib/sql/hdb_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stdlib/sql/hdb_test.go b/stdlib/sql/hdb_test.go index 4b89f06c91..f400c21c99 100644 --- a/stdlib/sql/hdb_test.go +++ b/stdlib/sql/hdb_test.go @@ -6,12 +6,14 @@ import ( ) func TestHdb_IfNoExist(t *testing.T) { + // test table with fqn q := hdbAddIfNotExist("stores.orders_copy", "CREATE TABLE stores.orders_copy (ORDER_ID BIGINT)") - if !strings.HasPrefix(q, "DO") || !strings.Contains(q, "SCHEMA_NAME") || !strings.Contains(q, "TABLE_NAME") { + if !strings.HasPrefix(q, "DO") || !strings.Contains(q, "(:SCHEMA_NAME)") || !strings.Contains(q, "(:TABLE_NAME)") { t.Fail() } + // test table in default schema q = hdbAddIfNotExist("orders_copy", "CREATE TABLE orders_copy (ORDER_ID BIGINT)") - if !strings.HasPrefix(q, "DO") || strings.Contains(q, "SCHEMA_NAME") || !strings.Contains(q, "TABLE_NAME") { + if !strings.HasPrefix(q, "DO") || strings.Contains(q, "(:SCHEMA_NAME)") || !strings.Contains(q, "(:TABLE_NAME)") { t.Fail() } } From ab18c20351b04b9c63a70bed2f7c100bfc3c37af Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 23 Sep 2020 17:57:47 +0200 Subject: [PATCH 10/15] style: template formatting --- stdlib/sql/hdb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 4e8269daed..0246dfeb69 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -189,8 +189,8 @@ func HdbColumnTranslateFunc() translationFunc { // Template for conditional query by table existence check var hdbDoIfTableNotExistsTemplate = `DO BEGIN - DECLARE SCHEMA_NAME NVARCHAR(%d) = '%s'; - DECLARE TABLE_NAME NVARCHAR(%d) = '%s'; + DECLARE SCHEMA_NAME NVARCHAR(%d) = '%s'; + DECLARE TABLE_NAME NVARCHAR(%d) = '%s'; DECLARE X_EXISTS INT = 0; SELECT COUNT(*) INTO X_EXISTS FROM TABLES %s; IF :X_EXISTS = 0 From 6b6f8f9e20f2e6e57a2d7ab54ff16524445eca1e Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 23 Sep 2020 19:27:35 +0200 Subject: [PATCH 11/15] fix: return error if column is BINARY --- stdlib/sql/hdb.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 0246dfeb69..47d1c0b325 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -88,7 +88,11 @@ func (m *HdbRowReader) GetNextRow() ([]values.Value, error) { } newFloat, _ := (*big.Rat)(&out).Float64() row[i] = values.NewFloat(newFloat) - default: + default: // flux.TString + switch m.sqlTypes[i].DatabaseTypeName() { + case "BINARY", "VARBINARY": + return nil, errors.Newf(codes.Invalid, "Flux does not support column type %s", m.sqlTypes[i].DatabaseTypeName()) + } row[i] = values.NewString(string(value)) } case nil: From 68fd9c1b871d8ebe77af07db5f1d951f8e37e316 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 6 Oct 2020 14:44:09 +0200 Subject: [PATCH 12/15] fix: escape names with quotes to help prevent SQL injection --- stdlib/sql/hdb.go | 26 +++++++++++++++++++++----- stdlib/sql/to.go | 6 ++++-- stdlib/sql/to_privates_test.go | 6 ++++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 47d1c0b325..301e57bfa6 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "math/big" + "strconv" "strings" "time" @@ -27,6 +28,8 @@ import ( // Therefore TIMESTAMP is mapped to TTime and vice-versa here. // * the hdb driver is rather strict, eg. does not convert date- or time-formatted string values to time.Time, // or float64 to Decimal on its own and just throws "unsupported conversion" error +// * Per naming conventions rules (https://documentation.sas.com/?cdcId=pgmsascdc&cdcVersion=9.4_3.5&docsetId=acreldb&docsetTarget=p1k98908uh9ovsn1jwzl3jg05exr.htm&locale=en), +// `sql.to` target table and column names are assumed in / converted to uppercase. type HdbRowReader struct { Cursor *sql.Rows @@ -179,14 +182,15 @@ var fluxToHdb = map[flux.ColType]string{ flux.TTime: "TIMESTAMP", // not exactly correct (see Notes) } -// HdbTranslateColumn translates flux colTypes into their corresponding SAP HANA column type +// HdbTranslateColumn translates flux colTypes into their corresponding SAP HANA column type for CREATE TABLE statement func HdbColumnTranslateFunc() translationFunc { return func(f flux.ColType, colName string) (string, error) { s, found := fluxToHdb[f] if !found { return "", errors.Newf(codes.Invalid, "SAP HANA does not support column type %s", f.String()) } - return colName + " " + s, nil + // quote the column name for safety also convert to uppercase per HDB naming conventions + return hdbEscapeName(colName, true) + " " + s, nil } } @@ -204,19 +208,19 @@ BEGIN END; ` -// Adds SAP HANA specific table existence check to CREATE TABLE statement. +// hdbAddIfNotExist adds SAP HANA specific table existence check to CREATE TABLE statement. func hdbAddIfNotExist(table string, query string) string { var where string var args []interface{} parts := strings.SplitN(table, ".", 2) if len(parts) == 2 { // fully-qualified table name - where = "WHERE SCHEMA_NAME=UPPER(:SCHEMA_NAME) AND TABLE_NAME=UPPER(:TABLE_NAME)" + where = "WHERE SCHEMA_NAME=ESCAPE_DOUBLE_QUOTES(UPPER(:SCHEMA_NAME)) AND TABLE_NAME=ESCAPE_DOUBLE_QUOTES(UPPER(:TABLE_NAME))" args = append(args, len(parts[0])) args = append(args, parts[0]) args = append(args, len(parts[1])) args = append(args, parts[1]) } else { // table in user default schema - where = "WHERE TABLE_NAME=UPPER(:TABLE_NAME)" + where = "WHERE TABLE_NAME=ESCAPE_DOUBLE_QUOTES(UPPER(:TABLE_NAME))" args = append(args, len("default")) args = append(args, "default") args = append(args, len(table)) @@ -227,3 +231,15 @@ func hdbAddIfNotExist(table string, query string) string { return fmt.Sprintf(hdbDoIfTableNotExistsTemplate, args...) } + +// hdbEscapeName escapes name in double quotes and convert it to uppercase per HDB naming conventions +func hdbEscapeName(name string, toUpper bool) string { + parts := strings.Split(name, ".") + for i, _ := range parts { + if toUpper { + parts[i] = strings.ToUpper(parts[i]) + } + parts[i] = strconv.Quote(strings.Trim(parts[i], "\"")) + } + return strings.Join(parts, ".") +} diff --git a/stdlib/sql/to.go b/stdlib/sql/to.go index 5022259f8e..981a57b9ec 100644 --- a/stdlib/sql/to.go +++ b/stdlib/sql/to.go @@ -340,9 +340,11 @@ func CreateInsertComponents(t *ToSQLTransformation, tbl flux.Table) (colNames [] if isMssqlDriver(t.spec.Spec.DriverName) { // SQL Server does not support IF NOT EXIST q = fmt.Sprintf("IF OBJECT_ID('%s', 'U') IS NULL BEGIN CREATE TABLE %s (%s) END", t.spec.Spec.Table, t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) } else if t.spec.Spec.DriverName == "hdb" { // SAP HANA does not support IF NOT EXIST - q = fmt.Sprintf("CREATE TABLE %s (%s)", t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) + // wrap CREATE TABLE statement with HDB-specific "if not exists" SQLScript check + q = fmt.Sprintf("CREATE TABLE %s (%s)", hdbEscapeName(t.spec.Spec.Table, true), strings.Join(newSQLTableCols, ",")) q = hdbAddIfNotExist(t.spec.Spec.Table, q) - batchSize = 1 // SAP HANA does not support INSERT/UPDATE batching via a single SQL command + // SAP HANA does not support INSERT/UPDATE batching via a single SQL command + batchSize = 1 } else { q = fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", t.spec.Spec.Table, strings.Join(newSQLTableCols, ",")) } diff --git a/stdlib/sql/to_privates_test.go b/stdlib/sql/to_privates_test.go index 1255f6ee42..8dd3ac7c0f 100644 --- a/stdlib/sql/to_privates_test.go +++ b/stdlib/sql/to_privates_test.go @@ -1,6 +1,8 @@ package sql import ( + "strconv" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -374,8 +376,8 @@ func TestHdbTranslation(t *testing.T) { t.Log(cmp.Diff(nil, err)) t.Fail() } - if !cmp.Equal(columnLabel+" "+dbTypeString, v) { - t.Log(cmp.Diff(columnLabel+" "+dbTypeString, v)) + if !cmp.Equal(strconv.Quote(strings.ToUpper(columnLabel))+" "+dbTypeString, v) { + t.Log(cmp.Diff(strconv.Quote(strings.ToUpper(columnLabel))+" "+dbTypeString, v)) t.Fail() } } From 3bd8ac63308de296f2e91989c6107d3bdedabe6c Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 6 Oct 2020 15:10:00 +0200 Subject: [PATCH 13/15] fix: S1005 linter check error --- stdlib/sql/hdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 301e57bfa6..88d97ab0d9 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -235,7 +235,7 @@ func hdbAddIfNotExist(table string, query string) string { // hdbEscapeName escapes name in double quotes and convert it to uppercase per HDB naming conventions func hdbEscapeName(name string, toUpper bool) string { parts := strings.Split(name, ".") - for i, _ := range parts { + for i := range parts { if toUpper { parts[i] = strings.ToUpper(parts[i]) } From 40a14095ff394f6d4a7e7c72682b33954a2046a4 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 6 Oct 2020 15:44:24 +0200 Subject: [PATCH 14/15] refactor: table name uppercase in Go code --- stdlib/sql/hdb.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 88d97ab0d9..929a99e55a 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -182,7 +182,7 @@ var fluxToHdb = map[flux.ColType]string{ flux.TTime: "TIMESTAMP", // not exactly correct (see Notes) } -// HdbTranslateColumn translates flux colTypes into their corresponding SAP HANA column type for CREATE TABLE statement +// HdbTranslateColumn translates flux colTypes into their corresponding SAP HANA column type func HdbColumnTranslateFunc() translationFunc { return func(f flux.ColType, colName string) (string, error) { s, found := fluxToHdb[f] @@ -212,15 +212,15 @@ END; func hdbAddIfNotExist(table string, query string) string { var where string var args []interface{} - parts := strings.SplitN(table, ".", 2) + parts := strings.SplitN(strings.ToUpper(table), ".", 2) // schema and table name assumed uppercase in HDB by default (see Notes) if len(parts) == 2 { // fully-qualified table name - where = "WHERE SCHEMA_NAME=ESCAPE_DOUBLE_QUOTES(UPPER(:SCHEMA_NAME)) AND TABLE_NAME=ESCAPE_DOUBLE_QUOTES(UPPER(:TABLE_NAME))" + where = "WHERE SCHEMA_NAME=ESCAPE_DOUBLE_QUOTES(:SCHEMA_NAME) AND TABLE_NAME=ESCAPE_DOUBLE_QUOTES(:TABLE_NAME)" args = append(args, len(parts[0])) args = append(args, parts[0]) args = append(args, len(parts[1])) args = append(args, parts[1]) } else { // table in user default schema - where = "WHERE TABLE_NAME=ESCAPE_DOUBLE_QUOTES(UPPER(:TABLE_NAME))" + where = "WHERE TABLE_NAME=ESCAPE_DOUBLE_QUOTES(:TABLE_NAME)" args = append(args, len("default")) args = append(args, "default") args = append(args, len(table)) From 1c094866a47f5bba409bdf03b2062c24e2503903 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Tue, 6 Oct 2020 15:56:21 +0200 Subject: [PATCH 15/15] style: go fmt --- stdlib/sql/hdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/sql/hdb.go b/stdlib/sql/hdb.go index 929a99e55a..d6eb6e43f6 100644 --- a/stdlib/sql/hdb.go +++ b/stdlib/sql/hdb.go @@ -213,7 +213,7 @@ func hdbAddIfNotExist(table string, query string) string { var where string var args []interface{} parts := strings.SplitN(strings.ToUpper(table), ".", 2) // schema and table name assumed uppercase in HDB by default (see Notes) - if len(parts) == 2 { // fully-qualified table name + if len(parts) == 2 { // fully-qualified table name where = "WHERE SCHEMA_NAME=ESCAPE_DOUBLE_QUOTES(:SCHEMA_NAME) AND TABLE_NAME=ESCAPE_DOUBLE_QUOTES(:TABLE_NAME)" args = append(args, len(parts[0])) args = append(args, parts[0])