From efa3d7ee5c34eea8a4027b88616df51ee535fca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20=C5=9Awi=C4=99cki?= Date: Tue, 6 Jul 2021 20:52:20 +0200 Subject: [PATCH] Add sql to generic --- build.sbt | 9 ++- demo/docker/nussknacker/nussknacker.conf | 21 +++++- docs/components/Sql.md | 69 +++++++++++++++++++ .../sql/db/pool/HikariDataSourceFactory.scala | 2 + .../sql/service/DatabaseQueryEnricher.scala | 1 + 5 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 docs/components/Sql.md diff --git a/build.sbt b/build.sbt index 76c9e5356a8..e1ec55eb756 100644 --- a/build.sbt +++ b/build.sbt @@ -341,13 +341,15 @@ lazy val dist = { (assembly in Compile) in generic, (assembly in Compile) in flinkProcessManager, (assembly in Compile) in engineStandalone, - (assembly in Compile) in openapi + (assembly in Compile) in openapi, + (assembly in Compile) in sql, ).value, mappings in Universal ++= Seq( (crossTarget in generic).value / "genericModel.jar" -> "model/genericModel.jar", (crossTarget in flinkProcessManager).value / s"nussknacker-flink-manager.jar" -> "managers/nussknacker-flink-manager.jar", (crossTarget in engineStandalone).value / s"nussknacker-standalone-manager.jar" -> "managers/nussknacker-standalone-manager.jar", - (crossTarget in openapi).value / "openapi.jar" -> "components/openapi.jar" + (crossTarget in openapi).value / "openapi.jar" -> "components/openapi.jar", + (crossTarget in sql).value / "sql.jar" -> "model/sql.jar" ), /* //FIXME: figure out how to filter out only for .tgz, not for docker mappings in Universal := { @@ -946,6 +948,9 @@ lazy val sql = (project in component("sql")). configs(IntegrationTest). settings(commonSettings). settings(Defaults.itSettings). + settings(commonSettings). + settings(assemblySampleSettings("sql.jar"): _*). + settings(publishAssemblySettings: _*). settings( name := "nussknacker-sql", libraryDependencies ++= Seq( diff --git a/demo/docker/nussknacker/nussknacker.conf b/demo/docker/nussknacker/nussknacker.conf index 6a7cb79347c..25a125d5750 100644 --- a/demo/docker/nussknacker/nussknacker.conf +++ b/demo/docker/nussknacker/nussknacker.conf @@ -1,12 +1,31 @@ #This configuration auguments and overrides configuration in docker image #Here we configure OpenAPI based enricher, which is implemented by python service in customerservice { + sqlEnricherDbPool { + driverClassName: "org.hsqldb.jdbc.JDBCDriver" + url: "jdbc:hsqldb:file:/opt/nussknacker/storage/db;sql.syntax_ora=true" + username: "sa" + password: "" + } + processTypes.streaming.modelConfig { - classPath: ["model/genericModel.jar", "components/openapi.jar"] + classPath: ["model/genericModel.jar", "components/openapi.jar", "model/sql.jar",] components.openAPI { url: "http://customerservice:5000/swagger" rootUrl: "http://customerservice:5000" categories: ["Default"] } + + components.databaseEnricher { + categories: ["Default"] + config: { + databaseQueryEnrichers: [ + { name: "db-query", dbPool: ${sqlEnricherDbPool} } + ] + databaseLookupEnrichers: [ + { name: "db-lookup", dbPool: ${sqlEnricherDbPool} } + ] + } + } } } diff --git a/docs/components/Sql.md b/docs/components/Sql.md new file mode 100644 index 00000000000..a9bdeeaaac6 --- /dev/null +++ b/docs/components/Sql.md @@ -0,0 +1,69 @@ +Overview +======== + +Nussknacker `Sql` enricher can connect to SQL databases with HikariCP JDBC connection pool. + +It supports: + +- real time database lookup - a simplified mode where you can select from table filtering for a specified key. +- generic DDM/DDL query enricher +- both `databaseQueryEnricher` as well as `databaseLookupEnricher` can cache queries results +- you can specify cache TTL (Time To Live) duration via `Cache TTL` property +- for `databaseQueryEnricher` you can specify `Result Strategy` + - `Result set` - for retrieving whole query result set + - `Single result` for retrieving single value + +Configuration +============= + +Sample configuration: + +You have to configure database connection pool you will be using in your sql enricher + +``` +myDatabasePool { + driverClassName: ${dbDriver} + url: ${myDatabaseUrl} + username: ${myDatabaseUser} + password: ${myDatabasePassword} + timeout: ${dbConnectionTimeout} + initialSize: ${dbInitialPoolSize} + maxTotal: ${dbMaxPoolSize} +} +``` + +| Parameter | Required | Default | Description | +| ---------- | -------- | ------- | ----------- | +| url | true | | URL with your database resource | +| username | true | | Authentication username | +| password | true | | Authentication password | +| driverClassName | true | | Database driver class name | +| timeout | false | 30s | Connection timeout | +| maxTotal | false | 10 | Maximum pool size | +| initialSize | false | 0 | Minimum idle size | + +Next you have to configure component itself. + +You can have multiple databaseQueryEnrichers and databaseLookupEnrichers for multiple various database connections. You +can also specify only one of them. + +``` +components { + yourUniqueComponentName: { + type: databaseEnricher #this defines your component type + config: { + databaseQueryEnrichers: [ + { name: "myDatabaseQuery", dbPool: ${myDatabasePool} } + ] + databaseLookupEnrichers: [ + { name: "myDatabaseLookup", dbPool: ${myDatabasePool} } + ] + } + } +} +``` + +| Parameter | Required | Default | Description | +| ---------- | -------- | ------- | ----------- | +| databaseQueryEnrichers | true | | List of database query enrichers components | +| databaseLookupEnrichers | true | | List of database lookup components | diff --git a/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/db/pool/HikariDataSourceFactory.scala b/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/db/pool/HikariDataSourceFactory.scala index a61d7200a2c..571836e019f 100644 --- a/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/db/pool/HikariDataSourceFactory.scala +++ b/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/db/pool/HikariDataSourceFactory.scala @@ -9,7 +9,9 @@ object HikariDataSourceFactory { hikariConf.setJdbcUrl(conf.url) hikariConf.setUsername(conf.username) hikariConf.setPassword(conf.password) + hikariConf.setMinimumIdle(conf.initialSize) hikariConf.setMaximumPoolSize(conf.maxTotal) + hikariConf.setConnectionTimeout(conf.timeout.toMillis) hikariConf.setDriverClassName(conf.driverClassName) conf.connectionProperties.foreach { case (name, value) => hikariConf.addDataSourceProperty(name, value) diff --git a/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseQueryEnricher.scala b/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseQueryEnricher.scala index 6a30eefd401..ae75ddc3b64 100644 --- a/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseQueryEnricher.scala +++ b/engine/components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseQueryEnricher.scala @@ -58,6 +58,7 @@ TODO: */ class DatabaseQueryEnricher(val dbPoolConfig: DBPoolConfig) extends EagerService with Lifecycle with SingleInputGenericNodeTransformation[ServiceInvoker] { + import DatabaseQueryEnricher._ override type State = TransformationState