diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 219901e4cb7..275b380fada 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -267,7 +267,7 @@ jobs: CYPRESS_BASE_URL: http://localhost:8083 shell: bash run: | - docker run -d -p 8083:8080 -e CONFIG_FORCE_scenarioTypes_streaming_engineConfig_type=stub -e NUSSKNACKER_CONFIG_FILE=/opt/nussknacker/conf/dev-application.conf --name nussknacker_e2e_fe touk/nussknacker:$NUSSKNACKER_VERSION + docker run -d -p 8083:8080 -e CONFIG_FORCE_scenarioTypes_streaming_deploymentConfig_type=stub -e NUSSKNACKER_CONFIG_FILE=/opt/nussknacker/conf/dev-application.conf --name nussknacker_e2e_fe touk/nussknacker:$NUSSKNACKER_VERSION cd ui/client npx wait-on $CYPRESS_BASE_URL && npm run test:e2e docker kill nussknacker_e2e_fe diff --git a/docs/Changelog.md b/docs/Changelog.md index 6683e8f5742..0eb383ce5cb 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -58,7 +58,10 @@ Nussknacker versions * [#1357](https://github.com/TouK/nussknacker/pull/1357) Add run mode to nodes to be able to determine if we are inside e.g. test process run. Run mode is can be declared as a dependency in generic node transformations. Nodes created via `@MethodToInvoke` can declare `RunMode` as an implicit parameter. `RunMode` is also available in `FlinkCustomNodeContext`. -* [#1911](https://github.com/TouK/nussknacker/pull/1911) Rename `process` to `scenario`, `subprocess` to `fragment` in messages at backend and some test cases names +* Various naming changes: + * [#1917](https://github.com/TouK/nussknacker/pull/1917) configuration of `engineConfig` to `deploymentConfig` + * [#1911](https://github.com/TouK/nussknacker/pull/1911) Rename `process` to `scenario`, `subprocess` to `fragment` in messages at backend and some test cases names + 0.3.1 (not released yet) ------------------------ diff --git a/docs/Configuration.md b/docs/Configuration.md index 95c0cb64e93..6cb25cab8f6 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -31,7 +31,7 @@ attachmentsPath: "/tmp/touk/esp-frontend/attachments" scenarioTypes { streaming { - engineConfig { + deploymentConfig { type: "flinkStreaming" restUrl: "http://localhost:8081" parallelism: 4 diff --git a/docs/MigrationGuide.md b/docs/MigrationGuide.md index 9c36dad6dda..52a2958b693 100644 --- a/docs/MigrationGuide.md +++ b/docs/MigrationGuide.md @@ -182,6 +182,8 @@ that will be hidden before parameter's evaluation - Removed: `getClusterClient` from `FlinkMiniClusterHolder` interface, because of flink compatibility at Flink 1.9 - Renamed: `FlinkStreamingProcessRegistrar` to `FlinkProcessManager` * [#1303](https://github.com/TouK/nussknacker/pull/1303) TypedObjectTypingResult has a new field: additionalInfo +* Various naming changes: + * [#1917](https://github.com/TouK/nussknacker/pull/1917) configuration of `engineConfig` to `deploymentConfig` ## In version 0.2.0 @@ -336,7 +338,7 @@ Be aware that we are using avro 1.9.2 instead of default Flink's 1.8.2 (for java ``` processTypes { "type e.g. streaming" { - engineConfig { + deploymentConfig { type: "flinkStreaming" PUT HERE PROPERTIES OF flinkConfig FROM OLD CONFIG } @@ -378,7 +380,7 @@ Be aware that we are using avro 1.9.2 instead of default Flink's 1.8.2 (for java * [#588](https://github.com/TouK/nussknacker/pull/588) [#882](https://github.com/TouK/nussknacker/pull/882) `FlinkSource` API changed, current implementation is now `BasicFlinkSource` * [#839](https://github.com/TouK/nussknacker/pull/839) [#882](https://github.com/TouK/nussknacker/pull/882) `FlinkSink` API changed, current implementation is now `BasicFlinkSink` * [#841](https://github.com/TouK/nussknacker/pull/841) `ProcessConfigCreator` API changed; note that currently all process objects are invoked with `ProcessObjectDependencies` as a parameter. The APIs of `KafkaSinkFactory`, `KafkaSourceFactory`, and all their implementations were changed. `Config` is available as property of `ProcessObjectDependencies` instance. -* [#863](https://github.com/TouK/nussknacker/pull/863) `restUrl` in `engineConfig` need to be preceded with protocol. Host with port only is not allowed anymore. +* [#863](https://github.com/TouK/nussknacker/pull/863) `restUrl` in `deploymentConfig` need to be preceded with protocol. Host with port only is not allowed anymore. * Rename `grafanaSettings` to `metricsSettings` in configuration. ## In version 0.0.12 diff --git a/engine/flink/management/sample/src/main/resources/defaultModelConfig.conf b/engine/flink/management/sample/src/main/resources/defaultModelConfig.conf index c2ac20257e9..8e3f35c8fde 100644 --- a/engine/flink/management/sample/src/main/resources/defaultModelConfig.conf +++ b/engine/flink/management/sample/src/main/resources/defaultModelConfig.conf @@ -83,7 +83,7 @@ additionalPropertiesConfig { ] } numberOfThreads: { - label: "Number of theards" + label: "Number of threads" editor: { type: "FixedValuesParameterEditor", possibleValues: [ diff --git a/engine/flink/management/src/it/resources/application.conf b/engine/flink/management/src/it/resources/application.conf index 0cda4b7b556..ebe5e73b954 100644 --- a/engine/flink/management/src/it/resources/application.conf +++ b/engine/flink/management/src/it/resources/application.conf @@ -1,5 +1,5 @@ -engineConfig { +deploymentConfig { type: "test" restUrl: "http://localhost:8081" queryableStateProxyUrl: "localhost:9069" diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala index e40d14a35df..6b47a399cf5 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala @@ -99,7 +99,7 @@ trait DockerTest extends DockerTestKit with ExtremelyPatientScalaFutures with La } def config: Config = ConfigFactory.load() - .withValue("engineConfig.restUrl", fromAnyRef(s"http://${jobManagerContainer.getIpAddresses().futureValue.head}:$FlinkJobManagerRestPort")) + .withValue("deploymentConfig.restUrl", fromAnyRef(s"http://${jobManagerContainer.getIpAddresses().futureValue.head}:$FlinkJobManagerRestPort")) .withValue("modelConfig.classPath", ConfigValueFactory.fromIterable(Collections.singletonList(classPath))) .withFallback(additionalConfig) diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingProcessManagerProvider.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingProcessManagerProvider.scala index bac3b611adc..f71c23b2541 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingProcessManagerProvider.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingProcessManagerProvider.scala @@ -43,6 +43,6 @@ object FlinkStreamingProcessManagerProvider { def defaultProcessManager(config: Config): ProcessManager = { val typeConfig = ProcessingTypeConfig.read(config) - new FlinkStreamingProcessManagerProvider().createProcessManager(typeConfig.toModelData, typeConfig.engineConfig) + new FlinkStreamingProcessManagerProvider().createProcessManager(typeConfig.toModelData, typeConfig.deploymentConfig) } } \ No newline at end of file diff --git a/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/ProcessManagerProvider.scala b/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/ProcessManagerProvider.scala index aefb97dc1a0..d5d50aea7b9 100644 --- a/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/ProcessManagerProvider.scala +++ b/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/ProcessManagerProvider.scala @@ -42,16 +42,16 @@ object ProcessingTypeConfig { def read(config: Config): ProcessingTypeConfig = ProcessingTypeConfig( - config.getString("engineConfig.type"), + config.getString("deploymentConfig.type"), config.as[List[URL]]("modelConfig.classPath"), - config.getConfig("engineConfig"), + config.getConfig("deploymentConfig"), config.getConfig("modelConfig") ) } case class ProcessingTypeConfig(engineType: String, classPath: List[URL], - engineConfig: Config, + deploymentConfig: Config, modelConfig: Config) { def toModelData: ModelData = ModelData(modelConfig, ModelClassLoader(classPath)) @@ -75,7 +75,7 @@ object ProcessingTypeData { def createProcessingTypeData(processManagerProvider: ProcessManagerProvider, processTypeConfig: ProcessingTypeConfig): ProcessingTypeData = { val modelData = processTypeConfig.toModelData - val managerConfig = processTypeConfig.engineConfig + val managerConfig = processTypeConfig.deploymentConfig createProcessingTypeData(processManagerProvider, modelData, managerConfig) } } diff --git a/nussknacker-dist/src/universal/conf/application.conf b/nussknacker-dist/src/universal/conf/application.conf index a3cb5e587af..a3deac48e63 100644 --- a/nussknacker-dist/src/universal/conf/application.conf +++ b/nussknacker-dist/src/universal/conf/application.conf @@ -10,7 +10,7 @@ categoriesConfig: { scenarioTypes { "streaming": { - engineConfig: ${flinkEngineConfig} + deploymentConfig: ${flinkDeploymentConfig} modelConfig: { classPath: ["model/genericModel.jar"] classPath: ${?MODEL_CLASS_PATH} @@ -22,7 +22,7 @@ scenarioTypes { } } -flinkEngineConfig { +flinkDeploymentConfig { jobManagerTimeout: 1m type: "flinkStreaming" restUrl: "http://jobmanager:8081" diff --git a/nussknacker-dist/src/universal/conf/dev-application.conf b/nussknacker-dist/src/universal/conf/dev-application.conf index bd93c791987..597f424636e 100644 --- a/nussknacker-dist/src/universal/conf/dev-application.conf +++ b/nussknacker-dist/src/universal/conf/dev-application.conf @@ -22,7 +22,7 @@ standaloneModelDir: ${?STANDALONE_MODEL_DIR} standaloneManagmentUrl: "http://localhost:8070" standaloneManagmentUrl: ${?STANDALONE_MANAGMENT_URL} -flinkEngineConfig { +flinkDeploymentConfig { type: "flinkStreaming" restUrl: "http://jobmanager:8081" restUrl: ${?FLINK_REST_URL} @@ -32,7 +32,7 @@ flinkEngineConfig { scenarioTypes { "streaming": { - engineConfig: ${flinkEngineConfig} + deploymentConfig: ${flinkDeploymentConfig} modelConfig: { classPath: [ ${managementModelDir}"/managementSample.jar" ] rocksDB: { @@ -41,7 +41,7 @@ scenarioTypes { } } "streaming-generic": { - engineConfig: ${flinkEngineConfig} + deploymentConfig: ${flinkDeploymentConfig} modelConfig: { classPath: [ ${genericModelDir}"/genericModel.jar" ] rocksDB: { @@ -50,7 +50,7 @@ scenarioTypes { } } "request-response": { - engineConfig { + deploymentConfig { type: "requestResponseStandalone" managementUrl: ${standaloneManagmentUrl} } diff --git a/ui/client/components/graph/node-modal/NodeDetailsContent.js b/ui/client/components/graph/node-modal/NodeDetailsContent.js index 1eb4c525519..1923be774bd 100644 --- a/ui/client/components/graph/node-modal/NodeDetailsContent.js +++ b/ui/client/components/graph/node-modal/NodeDetailsContent.js @@ -1,5 +1,5 @@ import classNames from "classnames" -import _ from "lodash" +import _, {sortBy} from "lodash" import React from "react" import {connect} from "react-redux" import {v4 as uuid4} from "uuid" @@ -435,7 +435,8 @@ export class NodeDetailsContent extends React.Component { null, "query-path", )] - const additionalFields = Object.entries(this.props.additionalPropertiesConfig).map( + //we sort by name, to have predictable order of properties (should be replaced by defining order in configuration) + const additionalFields = sortBy(Object.entries(this.props.additionalPropertiesConfig), e => e[0]).map( ([propName, propConfig]) => ( { cy.contains(/^properties/i).should("be.enabled").click() cy.get("[data-testid=node-modal]").should("be.visible").find("input").within(inputs => { cy.wrap(inputs).first().click().type("-renamed") + //this is idx of "Max events", which should be int cy.wrap(inputs).eq(6).click().type("wrong data") }) cy.contains(/^apply/i).should("be.enabled").click() diff --git a/ui/client/package.json b/ui/client/package.json index 094af7fb671..ca4cc9ff5e7 100644 --- a/ui/client/package.json +++ b/ui/client/package.json @@ -18,7 +18,7 @@ "start:backend-docker": "npm run clean-translations && start-server-and-test start-backend:docker http-get://localhost:8080/static/main.html start", "start:backend-staging": "npm run clean-translations && BACKEND_DOMAIN=https://staging.nussknacker.io webpack serve", "start:backend-demo": "npm run clean-translations && BACKEND_DOMAIN=https://demo.nussknacker.io webpack serve", - "start-backend:docker": "docker run -i -p 8080:8080 -e CONFIG_FORCE_scenarioTypes_streaming_engineConfig_type=stub -e NUSSKNACKER_CONFIG_FILE=/opt/nussknacker/conf/dev-application.conf --pull always -P touk/nussknacker:staging-latest", + "start-backend:docker": "docker run -i -p 8080:8080 -e CONFIG_FORCE_scenarioTypes_streaming_deploymentConfig_type=stub -e NUSSKNACKER_CONFIG_FILE=/opt/nussknacker/conf/dev-application.conf --pull always -P touk/nussknacker:staging-latest", "pretest": "npm run check", "test:unit": "jest", "test:e2e": "cypress run", diff --git a/ui/server/src/test/resources/ui.conf b/ui/server/src/test/resources/ui.conf index 6dcd2d337f8..93f2a695516 100644 --- a/ui/server/src/test/resources/ui.conf +++ b/ui/server/src/test/resources/ui.conf @@ -39,7 +39,7 @@ customProcesses { scenarioTypes { "streaming" { - engineConfig { + deploymentConfig { restUrl: "http://localhost:8081" queryableStateProxyUrl: "localhost:9123" jobManagerTimeout: 1m diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala index f45b450a975..b5aaa13147e 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala @@ -167,7 +167,7 @@ class BaseFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSu Some("1"), FixedValuesParameterEditor(fixedPossibleValues), List(FixedValuesValidator(fixedPossibleValues)), - Some("Number of theards") + Some("Number of threads") ) ) @@ -186,7 +186,7 @@ class BaseFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSu entity should include("Configured property environment (Environment) is missing") entity should include("This field value has to be an integer number") entity should include("Unknown property unknown") - entity should include("Property numberOfThreads (Number of theards) has invalid value") + entity should include("Property numberOfThreads (Number of threads) has invalid value") } } diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtypedata/ProcessingTypeDataConfigurationReaderSpec.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtypedata/ProcessingTypeDataConfigurationReaderSpec.scala index 70f32421019..7b9c1009889 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtypedata/ProcessingTypeDataConfigurationReaderSpec.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtypedata/ProcessingTypeDataConfigurationReaderSpec.scala @@ -11,7 +11,7 @@ class ProcessingTypeDataConfigurationReaderSpec extends FunSuite { """ |processTypes { | "streaming" { - | engineConfig { + | deploymentConfig { | jobManagerTimeout: 1m | restUrl: "http://localhost:8081" | queryableStateProxyUrlMissing: "localhost:9123" @@ -53,7 +53,7 @@ class ProcessingTypeDataConfigurationReaderSpec extends FunSuite { """ |scenarioTypes { | "streaming" { - | engineConfig { + | deploymentConfig { | restUrl: "http://localhost:8081" | typeMissing: "flinkStreaming" | } @@ -70,7 +70,7 @@ class ProcessingTypeDataConfigurationReaderSpec extends FunSuite { intercept[typesafe.config.ConfigException] { ProcessingTypeDataConfigurationReader.readProcessingTypeConfig(config) - }.getMessage should include("No configuration setting found for key 'engineConfig.type'") + }.getMessage should include("No configuration setting found for key 'deploymentConfig.type'") } test("should throw when no configuration is provided") {