Skip to content

Commit eef8515

Browse files
Patryk Jatczakpiotrczarnas
Patryk Jatczak
authored and
piotrczarnas
committed
Merged PR 2803: databricks sensors fixed, jdbc Driver is loaded manually
Related work items: #12408
2 parents 7835a01 + e3c3687 commit eef8515

17 files changed

+99
-53
lines changed

‎dqops/src/integration-test/java/com/dqops/databricks/connectors/DatabricksSourceConnectionIntegrationTests.java

+5-20
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,16 @@
1515
*/
1616
package com.dqops.databricks.connectors;
1717

18-
import com.dqops.connectors.*;
18+
import com.dqops.connectors.ConnectionProvider;
19+
import com.dqops.connectors.ConnectionProviderRegistryObjectMother;
20+
import com.dqops.connectors.ProviderType;
21+
import com.dqops.connectors.SourceTableModel;
1922
import com.dqops.connectors.databricks.DatabricksConnectionSpecObjectMother;
2023
import com.dqops.connectors.databricks.DatabricksSourceConnection;
2124
import com.dqops.core.secrets.SecretValueLookupContext;
2225
import com.dqops.core.secrets.SecretValueProviderObjectMother;
23-
import com.dqops.metadata.sources.ConnectionSpec;
2426
import com.dqops.databricks.BaseDatabricksIntegrationTest;
27+
import com.dqops.metadata.sources.ConnectionSpec;
2528
import com.dqops.sampledata.IntegrationTestSampleDataObjectMother;
2629
import com.dqops.sampledata.SampleCsvFileNames;
2730
import com.dqops.sampledata.SampleTableMetadata;
@@ -33,7 +36,6 @@
3336
import org.springframework.boot.test.context.SpringBootTest;
3437

3538
import java.util.List;
36-
import java.util.Objects;
3739

3840
@SpringBootTest
3941
public class DatabricksSourceConnectionIntegrationTests extends BaseDatabricksIntegrationTest {
@@ -60,23 +62,6 @@ void open_whenCalled_thenJustReturns() {
6062
this.sut.open(this.secretValueLookupContext);
6163
}
6264

63-
@Test
64-
void listSchemas_whenSchemasPresent_thenReturnsKnownSchemas() {
65-
this.sut.open(this.secretValueLookupContext);
66-
List<SourceSchemaModel> schemas = this.sut.listSchemas();
67-
68-
Assertions.assertEquals(1, schemas.size());
69-
Assertions.assertTrue(schemas.stream().anyMatch(m -> Objects.equals(m.getSchemaName(), DatabricksConnectionSpecObjectMother.getSchemaName())));
70-
}
71-
72-
@Test
73-
void listTables_whenDefaultSchemaListed_thenReturnsTables() {
74-
this.sut.open(this.secretValueLookupContext);
75-
List<SourceTableModel> tables = this.sut.listTables(DatabricksConnectionSpecObjectMother.getSchemaName(), null, 300, secretValueLookupContext);
76-
77-
Assertions.assertTrue(tables.size() == 0);
78-
}
79-
8065
@Test
8166
void listTables_whenUsedTableNameContainsFilter_thenReturnTableThatMatchFilter() {
8267
SampleTableMetadata sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/accuracy/DatabricksColumnAccuracyTotalMaxMatchPercentSensorParametersSpecIntegrationTest.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.dqops.sampledata.SampleTableMetadata;
3131
import com.dqops.sampledata.SampleTableMetadataObjectMother;
3232
import com.dqops.sensors.column.accuracy.ColumnAccuracyTotalMaxMatchPercentSensorParametersSpec;
33+
import com.dqops.testutils.ValueConverter;
3334
import org.junit.jupiter.api.Assertions;
3435
import org.junit.jupiter.api.BeforeEach;
3536
import org.junit.jupiter.api.Test;
@@ -78,7 +79,7 @@ void runSensor_onNullDataInPrimaryTable_thenReturnsValues() {
7879
Assertions.assertEquals(1, resultTable.rowCount());
7980
Assertions.assertEquals("expected_value", resultTable.column(0).name());
8081
Assertions.assertEquals("actual_value", resultTable.column(1).name());
81-
Assertions.assertEquals(1L, resultTable.column(0).get(0));
82+
Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0)));
8283
Assertions.assertEquals(null, resultTable.column(1).get(0));
8384
}
8485

@@ -104,7 +105,7 @@ void runSensor_onNullDataInForeignTable_thenReturnsValues() {
104105
Assertions.assertEquals("expected_value", resultTable.column(0).name());
105106
Assertions.assertEquals("actual_value", resultTable.column(1).name());
106107
Assertions.assertEquals(null, resultTable.column(0).get(0));
107-
Assertions.assertEquals(1L, resultTable.column(1).get(0));
108+
Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(1).get(0)));
108109
}
109110

110111
@Test

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/accuracy/DatabricksColumnAccuracyTotalMinMatchPercentSensorParametersSpecIntegrationTest.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.dqops.sampledata.SampleTableMetadata;
3131
import com.dqops.sampledata.SampleTableMetadataObjectMother;
3232
import com.dqops.sensors.column.accuracy.ColumnAccuracyTotalMinMatchPercentSensorParametersSpec;
33+
import com.dqops.testutils.ValueConverter;
3334
import org.junit.jupiter.api.Assertions;
3435
import org.junit.jupiter.api.BeforeEach;
3536
import org.junit.jupiter.api.Test;
@@ -77,7 +78,7 @@ void runSensor_onNullDataInPrimaryTable_thenReturnsValues() {
7778
Assertions.assertEquals(1, resultTable.rowCount());
7879
Assertions.assertEquals("expected_value", resultTable.column(0).name());
7980
Assertions.assertEquals("actual_value", resultTable.column(1).name());
80-
Assertions.assertEquals(0L, resultTable.column(0).get(0));
81+
Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0)));
8182
Assertions.assertEquals(null, resultTable.column(1).get(0));
8283
}
8384

@@ -103,7 +104,7 @@ void runSensor_onNullDataInForeignTable_thenReturnsValues() {
103104
Assertions.assertEquals("expected_value", resultTable.column(0).name());
104105
Assertions.assertEquals("actual_value", resultTable.column(1).name());
105106
Assertions.assertEquals(null, resultTable.column(0).get(0));
106-
Assertions.assertEquals(0L, resultTable.column(1).get(0));
107+
Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(1).get(0)));
107108
}
108109

109110
@Test

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() {
9191
Table resultTable = sensorResult.getResultTable();
9292
Assertions.assertEquals(1, resultTable.rowCount());
9393
Assertions.assertEquals("actual_value", resultTable.column(0).name());
94-
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)));
94+
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
9595
}
9696

9797
@Test
@@ -104,7 +104,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() {
104104
Table resultTable = sensorResult.getResultTable();
105105
Assertions.assertEquals(1, resultTable.rowCount());
106106
Assertions.assertEquals("actual_value", resultTable.column(0).name());
107-
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)));
107+
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
108108
}
109109

110110
@Test
@@ -117,7 +117,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() {
117117
Table resultTable = sensorResult.getResultTable();
118118
Assertions.assertEquals(1, resultTable.rowCount());
119119
Assertions.assertEquals("actual_value", resultTable.column(0).name());
120-
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)));
120+
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
121121
}
122122

123123
@Test
@@ -130,7 +130,7 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() {
130130
Table resultTable = sensorResult.getResultTable();
131131
Assertions.assertEquals(25, resultTable.rowCount());
132132
Assertions.assertEquals("actual_value", resultTable.column(0).name());
133-
Assertions.assertEquals(16.666, ValueConverter.toDouble(resultTable.column(0).get(0)));
133+
Assertions.assertEquals(16.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
134134
}
135135

136136
@Test
@@ -143,7 +143,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() {
143143
Table resultTable = sensorResult.getResultTable();
144144
Assertions.assertEquals(1, resultTable.rowCount());
145145
Assertions.assertEquals("actual_value", resultTable.column(0).name());
146-
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)));
146+
Assertions.assertEquals(41.379, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
147147
}
148148

149149
@Test

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericSampleStddevSensorParametersSpecIntegrationTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() {
8787
Table resultTable = sensorResult.getResultTable();
8888
Assertions.assertEquals(1, resultTable.rowCount());
8989
Assertions.assertEquals("actual_value", resultTable.column(0).name());
90-
Assertions.assertEquals(109.0493466280289, resultTable.column(0).get(0));
90+
Assertions.assertEquals(109.0493466280289, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
9191
}
9292

9393
@Test
@@ -100,7 +100,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() {
100100
Table resultTable = sensorResult.getResultTable();
101101
Assertions.assertEquals(1, resultTable.rowCount());
102102
Assertions.assertEquals("actual_value", resultTable.column(0).name());
103-
Assertions.assertEquals(109.0493466280289, resultTable.column(0).get(0));
103+
Assertions.assertEquals(109.0493466280289, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
104104
}
105105

106106
@Test
@@ -113,7 +113,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() {
113113
Table resultTable = sensorResult.getResultTable();
114114
Assertions.assertEquals(1, resultTable.rowCount());
115115
Assertions.assertEquals("actual_value", resultTable.column(0).name());
116-
Assertions.assertEquals(109.0493466280289, resultTable.column(0).get(0));
116+
Assertions.assertEquals(109.0493466280289, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001);
117117
}
118118

119119
@Test

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() {
157157

158158
@Test
159159
void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() {
160-
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$");
160+
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+[.][A-Za-z]{2,4}$");
161161
SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForErrorSampling(
162162
sampleTableMetadata, "email", this.checkSpec);
163163

@@ -176,7 +176,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR
176176

177177
@Test
178178
void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() {
179-
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$");
179+
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+[.][A-Za-z]{2,4}$");
180180
sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true);
181181
sampleTableMetadata.getTableSpec().getColumns().getAt(2).setId(true);
182182

@@ -204,7 +204,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the
204204

205205
@Test
206206
void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() {
207-
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$");
207+
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+[.][A-Za-z]{2,4}$");
208208
DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{
209209
setLevel1(new DataGroupingDimensionSpec() {{
210210
setSource(DataGroupingDimensionSource.column_value);

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextsMatchingRegexPercentSensorParametersSpecIntegrationTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() {
157157

158158
@Test
159159
void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() {
160-
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$");
160+
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+[.][A-Za-z]{2,4}$");
161161
SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForErrorSampling(
162162
sampleTableMetadata, "email", this.checkSpec);
163163

@@ -176,7 +176,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR
176176

177177
@Test
178178
void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() {
179-
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$");
179+
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+[.][A-Za-z]{2,4}$");
180180
sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true);
181181
sampleTableMetadata.getTableSpec().getColumns().getAt(2).setId(true);
182182

@@ -204,7 +204,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the
204204

205205
@Test
206206
void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() {
207-
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$");
207+
this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+[.][A-Za-z]{2,4}$");
208208
DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{
209209
setLevel1(new DataGroupingDimensionSpec() {{
210210
setSource(DataGroupingDimensionSource.column_value);

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() {
104104
Table resultTable = sensorResult.getResultTable();
105105
Assertions.assertEquals(1, resultTable.rowCount());
106106
Assertions.assertEquals("actual_value", resultTable.column(0).name());
107-
Assertions.assertEquals((double)33.333, (double) resultTable.column(0).get(0), 0.001);
107+
Assertions.assertEquals(37.03704071044922, (double) resultTable.column(0).get(0), 0.001);
108108
}
109109

110110
@Test
@@ -117,7 +117,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() {
117117
Table resultTable = sensorResult.getResultTable();
118118
Assertions.assertEquals(1, resultTable.rowCount());
119119
Assertions.assertEquals("actual_value", resultTable.column(0).name());
120-
Assertions.assertEquals((double)33.333, (double) resultTable.column(0).get(0), 0.001);
120+
Assertions.assertEquals(37.03704071044922, (double) resultTable.column(0).get(0), 0.001);
121121
}
122122

123123
@Test
@@ -143,7 +143,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() {
143143
Table resultTable = sensorResult.getResultTable();
144144
Assertions.assertEquals(1, resultTable.rowCount());
145145
Assertions.assertEquals("actual_value", resultTable.column(0).name());
146-
Assertions.assertEquals((double) 33.333, (double) resultTable.column(0).get(0), 0.001);
146+
Assertions.assertEquals(37.03704071044922, (double) resultTable.column(0).get(0), 0.001);
147147
}
148148

149149
@Test

‎dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ void runSensor_onNullData_thenReturnsValues() {
8484
@Test
8585
void runSensor_whenSensorExecutedProfiling_thenReturnsValues() {
8686
SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck(
87-
sampleTableMetadata, "email_ok", this.checkSpec);
87+
sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec);
8888

8989
SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters);
9090

‎dqops/src/main/java/com/dqops/connectors/databricks/DatabricksParametersSpec.java

+21
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ public class DatabricksParametersSpec extends BaseProviderParametersSpec
5959
@JsonPropertyDescription("Databricks access token the warehouse. Supports also a ${DATABRICKS_ACCESS_TOKEN} configuration with a custom environment variable.")
6060
private String accessToken;
6161

62+
@CommandLine.Option(names = {"--databricks-initialization-sql"}, description = "Custom SQL that is executed after connecting to Databricks.")
63+
@JsonPropertyDescription("Custom SQL that is executed after connecting to Databricks.")
64+
private String initializationSql;
65+
6266
@CommandLine.Option(names = {"-D"}, description = "Databricks additional properties that are added to the JDBC connection string")
6367
@JsonPropertyDescription("A dictionary of custom JDBC parameters that are added to the JDBC connection string, a key/value dictionary.")
6468
@JsonInclude(JsonInclude.Include.NON_EMPTY)
@@ -184,6 +188,23 @@ public void setAccessToken(String accessToken) {
184188
this.accessToken = accessToken;
185189
}
186190

191+
/**
192+
* Returns an initialization SQL that is executed after opening the connection.
193+
* @return
194+
*/
195+
public String getInitializationSql() {
196+
return initializationSql;
197+
}
198+
199+
/**
200+
* Sets an SQL query that is executed after opening a connection.
201+
* @param initializationSql Initialization sql.
202+
*/
203+
public void setInitializationSql(String initializationSql) {
204+
this.setDirtyIf(!Objects.equals(this.initializationSql, initializationSql));
205+
this.initializationSql = initializationSql;
206+
}
207+
187208
/**
188209
* Returns a key/value map of additional properties that are included in the JDBC connection string.
189210
* @return Key/value dictionary of additional JDBC properties.

‎dqops/src/main/java/com/dqops/connectors/databricks/DatabricksSourceConnection.java

+21
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,34 @@ public DatabricksSourceConnection(JdbcConnectionPool jdbcConnectionPool,
6060
super(jdbcConnectionPool, secretValueProvider, databricksConnectionProvider);
6161
}
6262

63+
/**
64+
* Opens a connection before it can be used for executing any statements.
65+
* @param secretValueLookupContext Secret value lookup context used to access shared credentials.
66+
*/
67+
@Override
68+
public void open(SecretValueLookupContext secretValueLookupContext) {
69+
super.open(secretValueLookupContext);
70+
71+
DatabricksParametersSpec databricksParametersSpec = this.getConnectionSpec().getDatabricks();
72+
if (!Strings.isNullOrEmpty(databricksParametersSpec.getInitializationSql())) {
73+
this.executeCommand(databricksParametersSpec.getInitializationSql(), JobCancellationToken.createDummyJobCancellationToken());
74+
}
75+
}
76+
6377
/**
6478
* Creates a hikari connection pool config for the connection specification.
6579
* @param secretValueLookupContext Secret value lookup context used to find shared credentials that could be used in the connection names.
6680
* @return Hikari config.
6781
*/
6882
@Override
6983
public HikariConfig createHikariConfig(SecretValueLookupContext secretValueLookupContext) {
84+
85+
try {
86+
Class.forName("com.databricks.client.jdbc.Driver");
87+
} catch (ClassNotFoundException e) {
88+
throw new RuntimeException(e);
89+
}
90+
7091
HikariConfig hikariConfig = new HikariConfig();
7192
ConnectionSpec connectionSpec = this.getConnectionSpec();
7293
DatabricksParametersSpec databricksParametersSpec = connectionSpec.getDatabricks();

‎dqops/src/test/java/com/dqops/connectors/databricks/DatabricksConnectionSpecObjectMother.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package com.dqops.connectors.databricks;
1717

1818
import com.dqops.connectors.ProviderType;
19+
import com.dqops.core.secrets.DevelopmentCredentialsSecretNames;
1920
import com.dqops.core.secrets.SecretValueLookupContext;
2021
import com.dqops.core.secrets.SecretValueProviderImpl;
2122
import com.dqops.metadata.sources.ConnectionSpec;
@@ -51,12 +52,13 @@ public static ConnectionSpec create() {
5152

5253
setDatabricks(new DatabricksParametersSpec()
5354
{{
54-
setHost(secretValueProvider.expandValue("${DATABRICKS_HOST}", secretValueLookupContext));
55+
setHost(DevelopmentCredentialsSecretNames.AZURE_DATABRICKS_HOST);
5556
setPort(String.valueOf(DATABRICKS_PORT));
5657
setProperties(Map.of(
57-
"HttpPath", secretValueProvider.expandValue("${DATABRICKS_HTTP_PATH}", secretValueLookupContext),
58-
"PWD", secretValueProvider.expandValue("${DATABRICKS_ACCESS_TOKEN}", secretValueLookupContext)
58+
"HttpPath", DevelopmentCredentialsSecretNames.AZURE_DATABRICKS_HTTP_PATH,
59+
"PWD", DevelopmentCredentialsSecretNames.AZURE_DATABRICKS_ACCESS_TOKEN
5960
));
61+
setInitializationSql("SET ANSI_MODE=FALSE");
6062
}});
6163
}};
6264
return connectionSpec;

‎dqops/src/test/java/com/dqops/core/secrets/DevelopmentCredentialsSecretNames.java

+15
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,19 @@ public class DevelopmentCredentialsSecretNames {
134134
*/
135135
public static final String GCS_DQO_AI_TESTING_INTEROPERABILITY_SECRET = "${sm://gcs-dqo-ai-testing-interoperability-secret}";
136136

137+
/**
138+
* Property to retrieve the Azure Databricks host
139+
*/
140+
public static final String AZURE_DATABRICKS_HOST = "${sm://azure-databricks-host}";
141+
142+
/**
143+
* Property to retrieve the Azure Databricks http path
144+
*/
145+
public static final String AZURE_DATABRICKS_HTTP_PATH = "${sm://azure-databricks-http-path}";
146+
147+
/**
148+
* Property to retrieve the Azure Databricks access token
149+
*/
150+
public static final String AZURE_DATABRICKS_ACCESS_TOKEN = "${sm://azure-databricks-access-token}";
151+
137152
}

0 commit comments

Comments
 (0)