Merge remote-tracking branch 'refs/remotes/upstream/staging' into NU-…

…1560-pan-zoom

# Conflicts:
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Fragment should add documentation url in fragment properties and show it in modal within scenario #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Fragment should allow adding input parameters and display used fragment graph in modal #4.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Fragment should allow adding input parameters and display used fragment graph in modal #7.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Fragment should display dead-ended fragment correct #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process initially clean should import JSON and save #1.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should allow pan view #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should select only fully covered (to right) #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should select only fully covered (to right) #1.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should select partially covered (to left) #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should select partially covered (to left) #1.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should switch modes, append and inverse select with shift #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should switch modes, append and inverse select with shift #1.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should switch modes, append and inverse select with shift #2.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process mouse drag should switch modes, append and inverse select with shift #3.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process should preserve condition on link move (switch) #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process should validate process on nodes paste #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process view should have node search toolbar #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process with data should allow drag component and drop on edge #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process with data should allow drag component and drop on edge #1.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process with data should allow drag node #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/Process with data should display question mark when renaming a node and updating the count #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/UndoRedo should work for copypaste #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/UndoRedo should work for drop on edge #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/UndoRedo should work for drop on edge #2.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/UndoRedo should work for move #0.png
#	designer/client/cypress/e2e/__image_snapshots__/electron/Linux/UndoRedo should work with counts #1.png
#	designer/client/src/components/graph/Graph.tsx
#	designer/client/src/components/graph/RangeSelectPlugin.ts
#	designer/client/src/components/graph/focusableStyled.ts
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - aggregates - Single Side Join #0.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - basic components - choice #0.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - basic components - filter #0.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - basic components - filter #1.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - basic components - split #0.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - basic components - union #0.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - basic components - variable #0.png
#	docs/autoScreenshotChangeDocs/Auto Screenshot Change Docs - fragments - Outputs #0.png

.run/NussknackerApp.run.xml

@@ -1,36 +1,37 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="NussknackerApp" type="Application" factoryName="Application">
-    <option name="ALTERNATIVE_JRE_PATH" value="1.8" />
+    <option name="ALTERNATIVE_JRE_PATH" value="1.8"/>
     <envs>
-      <env name="AUTHENTICATION_USERS_FILE" value="../../../nussknacker-dist/src/universal/conf/users.conf" />
-      <env name="CONSOLE_THRESHOLD_LEVEL" value="DEBUG" />
-      <env name="FLINK_REST_URL" value="http://localhost:3031" />
-      <env name="FLINK_SHOULD_VERIFY_BEFORE_DEPLOY" value="false" />
-      <env name="GRAFANA_URL" value="http://localhost:8081/grafana" />
-      <env name="INFLUXDB_URL" value="http://localhost:3086" />
-      <env name="KAFKA_ADDRESS" value="localhost:3032" />
-      <env name="NUSSKNACKER_LOG_LEVEL" value="DEBUG" />
-      <env name="OPENAPI_SERVICE_URL" value="http://localhost:5000" />
-      <env name="SCHEMA_REGISTRY_URL" value="http://localhost:3082" />
-      <env name="SQL_ENRICHER_URL" value="localhost:5432" />
-      <env name="USAGE_REPORTS_FINGERPRINT" value="development" />
-      <env name="USAGE_REPORTS_SOURCE" value="sources" />
-      <env name="INPUT_CONFIG_RESOLVE_ENV_VARIABLES" value="false" />
-      <env name="TABLES_DEFINITION_FILE" value="../../../nussknacker-dist/src/universal/conf/tables-definition.sql" />
+      <env name="AUTHENTICATION_USERS_FILE" value="../../../nussknacker-dist/src/universal/conf/users.conf"/>
+      <env name="CONSOLE_THRESHOLD_LEVEL" value="DEBUG"/>
+      <env name="FLINK_REST_URL" value="http://localhost:3031"/>
+      <env name="FLINK_SHOULD_VERIFY_BEFORE_DEPLOY" value="false"/>
+      <env name="GRAFANA_URL" value="http://localhost:8081/grafana"/>
+      <env name="INFLUXDB_URL" value="http://localhost:3086"/>
+      <env name="KAFKA_ADDRESS" value="localhost:3032"/>
+      <env name="NUSSKNACKER_LOG_LEVEL" value="DEBUG"/>
+      <env name="OPENAPI_SERVICE_URL" value="http://localhost:5000"/>
+      <env name="SCHEMA_REGISTRY_URL" value="http://localhost:3082"/>
+      <env name="SQL_ENRICHER_URL" value="localhost:5432"/>
+      <env name="USAGE_REPORTS_FINGERPRINT" value="development"/>
+      <env name="USAGE_REPORTS_SOURCE" value="sources"/>
+      <env name="INPUT_CONFIG_RESOLVE_ENV_VARIABLES" value="false"/>
+      <env name="TABLES_DEFINITION_FILE" value="../../../nussknacker-dist/src/universal/conf/tables-definition.sql"/>
     </envs>
-    <option name="INCLUDE_PROVIDED_SCOPE" value="true" />
-    <option name="MAIN_CLASS_NAME" value="pl.touk.nussknacker.ui.NussknackerApp" />
-    <module name="nussknacker-designer" />
-    <option name="VM_PARAMETERS" value="-Dconfig.override_with_env_vars=true -Dnussknacker.config.locations=../../../nussknacker-dist/src/universal/conf/dev-application.conf -Dlogback.configurationFile=../../../nussknacker-dist/src/universal/conf/logback.xml" />
-    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/designer/server/work" />
+    <option name="INCLUDE_PROVIDED_SCOPE" value="true"/>
+    <option name="MAIN_CLASS_NAME" value="pl.touk.nussknacker.ui.NussknackerApp"/>
+    <module name="nussknacker-designer"/>
+    <option name="VM_PARAMETERS"
+            value="-Dconfig.override_with_env_vars=true -Dnussknacker.config.locations=../../../nussknacker-dist/src/universal/conf/dev-application.conf -Dlogback.configurationFile=../../../nussknacker-dist/src/universal/conf/logback.xml"/>
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/designer/server/work"/>
     <extension name="coverage">
       <pattern>
-        <option name="PATTERN" value="pl.touk.nussknacker.ui.*" />
-        <option name="ENABLED" value="true" />
+        <option name="PATTERN" value="pl.touk.nussknacker.ui.*"/>
+        <option name="ENABLED" value="true"/>
       </pattern>
     </extension>
     <method v="2">
-      <option name="Make" enabled="true" />
+      <option name="Make" enabled="true"/>
     </method>
   </configuration>
 </component>

build.sbt

@@ -290,64 +290,65 @@ val flinkCommonsTextV    = "1.10.0"
 val flinkCommonsIOV      = "2.11.0"
 val avroV                = "1.11.3"
 //we should use max(version used by confluent, version acceptable by flink), https://docs.confluent.io/platform/current/installation/versions-interoperability.html - confluent version reference
-val kafkaV               = "3.6.1"
+val kafkaV               = "3.6.2"
 //TODO: Spring 5.3 has some problem with handling our PrimitiveOrWrappersPropertyAccessor
 val springV              = "5.2.23.RELEASE"
-val scalaTestV           = "3.2.17"
+val scalaTestV           = "3.2.18"
 val scalaCheckV          = "1.17.0"
 val scalaCheckVshort     = scalaCheckV.take(4).replace(".", "-")
 val scalaTestPlusV       =
-  "3.2.17.0" // has to match scalatest and scalacheck versions, see https://github.com/scalatest/scalatestplus-scalacheck/releases
+  "3.2.18.0" // has to match scalatest and scalacheck versions, see https://github.com/scalatest/scalatestplus-scalacheck/releases
 // note: Logback 1.3 requires Slf4j 2.x, but Flink has Slf4j 1.7 on its classpath
-val logbackV                = "1.2.12"
+val logbackV                = "1.2.13"
 // this is used in cloud, official JsonEncoder uses different field layout
 val logbackJsonV            = "0.1.5"
-val circeV                  = "0.14.5"
+val circeV                  = "0.14.6"
 val circeGenericExtrasV     = "0.14.3"
-val jwtCirceV               = "9.3.0"
-val jacksonV                = "2.14.3"
-val catsV                   = "2.9.0"
-val catsEffectV             = "3.5.2"
-val everitSchemaV           = "1.14.2"
+val jwtCirceV               = "10.0.0"
+val jacksonV                = "2.15.4"
+val catsV                   = "2.10.0"
+val catsEffectV             = "3.5.4"
+val everitSchemaV           = "1.14.4"
 val slf4jV                  = "1.7.36"
 val scalaLoggingV           = "3.9.5"
 val scalaCompatV            = "1.0.2"
 val ficusV                  = "1.4.7"
-val configV                 = "1.4.2"
+val configV                 = "1.4.3"
 //we want to use 5.x for lite metrics to have tags, however dropwizard development kind of freezed. Maybe we should consider micrometer?
 //In Flink metrics we use bundled dropwizard metrics v. 3.x
 // rc16+ depend on slf4j 2.x
 val dropWizardV             = "5.0.0-rc15"
-val scalaCollectionsCompatV = "2.10.0"
-val testContainersScalaV    = "0.41.0"
-val testContainersJavaV     = "1.19.3"
-val nettyV                  = "4.1.93.Final"
-val nettyReactiveStreamsV   = "2.0.8"
+val scalaCollectionsCompatV = "2.12.0"
+val testContainersScalaV    = "0.41.3"
+val testContainersJavaV     = "1.19.7"
+val nettyV                  = "4.1.109.Final"
+val nettyReactiveStreamsV   = "2.0.12"
 
 val akkaV                     = "2.6.20"
 val akkaHttpV                 = "10.2.10"
 val akkaManagementV           = "1.1.4"
 val akkaHttpCirceV            = "1.39.2"
 val slickV                    = "3.4.1"
 val slickPgV                  = "0.21.1"
-val hikariCpV                 = "5.0.1"
+val hikariCpV                 = "5.1.0"
 val hsqldbV                   = "2.7.2"
-val postgresV                 = "42.6.0"
-val flywayV                   = "9.19.1"
+val postgresV                 = "42.7.3"
+// Flway 10 requires Java 17
+val flywayV                   = "9.22.3"
 val confluentV                = "7.5.1"
-val azureKafkaSchemaRegistryV = "1.1.0-beta.1"
-val azureSchemaRegistryV      = "1.3.6"
-val azureIdentityV            = "1.9.0"
+val azureKafkaSchemaRegistryV = "1.1.1"
+val azureSchemaRegistryV      = "1.4.4"
+val azureIdentityV            = "1.12.0"
 val bcryptV                   = "0.10.2"
 val cronParserV               = "9.1.6" // 9.1.7+ requires JDK 16+
 val javaxValidationApiV       = "2.0.1.Final"
-val caffeineCacheV            = "3.1.6"
-val sttpV                     = "3.8.15"
+val caffeineCacheV            = "3.1.8"
+val sttpV                     = "3.9.5"
 val tapirV                    = "1.9.11"
 //we use legacy version because this one supports Scala 2.12
 val monocleV                  = "2.1.0"
-val jmxPrometheusJavaagentV   = "0.18.0"
-val wireMockV                 = "2.35.0"
+val jmxPrometheusJavaagentV   = "0.20.0"
+val wireMockV                 = "2.35.2"
 val findBugsV                 = "3.0.2"
 val enumeratumV               = "1.7.3"
 
@@ -931,10 +932,10 @@ lazy val flinkSchemedKafkaComponentsUtils = (project in flink("schemed-kafka-com
     name := "nussknacker-flink-schemed-kafka-components-utils",
     libraryDependencies ++= {
       Seq(
-        "org.apache.flink" % "flink-streaming-java"   % flinkV               % Provided,
-        "org.apache.flink" % "flink-avro"             % flinkV,
-        "org.apache.flink" % s"flink-connector-kafka" % flinkConnectorKafkaV % Test,
-        "org.scalatest"   %% "scalatest"              % scalaTestV           % Test
+        "org.apache.flink" % "flink-streaming-java"  % flinkV               % Provided,
+        "org.apache.flink" % "flink-avro"            % flinkV,
+        "org.apache.flink" % "flink-connector-kafka" % flinkConnectorKafkaV % Test,
+        "org.scalatest"   %% "scalatest"             % scalaTestV           % Test
       )
     }
   )
@@ -1430,7 +1431,8 @@ lazy val developmentTestsDeploymentManager = (project in development("deployment
   )
   .dependsOn(
     deploymentManagerApi % Provided,
-    flinkDeploymentManager, // for accessing flink property config
+    flinkDeploymentManager, // for accessing Flink property config
+    scenarioCompiler,       // for run tests on the Flink
     testUtils % Test
   )
 

designer/client/README.md

@@ -108,7 +108,11 @@ Just like in options above, you should run backend and frontend before and the s
 #### Using unified linux environment
 
 This is the correct option if you want to add/modify image snapshots and make sure that it was done in deterministic way.
-It runs backend in Docker container, frontend connected to this backend and after that it runs cypress tests also in Docker container.
+It runs:
+
+-   NU backend and frontend in a Docker container using docker compose
+-   [Redpanada](https://redpanda.com/) for Kafka and SchemaRegistry support, via docker compose
+-   [Cypress](https://www.cypress.io/) tests in Docker container
 
 You should run once:
 
@@ -122,6 +126,15 @@ and after that you can run multiple times:
 npm run test:e2e:linux
 ```
 
+#### backend:docker
+
+This npm task is mainly aliases for docker compose commad to prepare the envs for cypress testing:
+
+-   runs NU backend and frontend
+-   runs [Redpanada](https://redpanda.com/) for Kafka and SchemaRegistry support
+
+Behind the scenes, we clean up and run docker compose env, which is located in the `client/docker-compose.yml` file.
+
 #### Using unified linux environment with update image snapshots mode enabled
 
 To run cypress test in mode that would update image snapshots, use the same commands as in the option above but with `:update` suffix:

designer/client/cypress/e2e/compare.cy.ts

@@ -18,8 +18,8 @@ describe("Compare", () => {
             .should("be.visible")
             .should("be.enabled")
             .click();
-        cy.contains("Version to compare").siblings("select").select(1);
-        cy.contains("Difference to pick").siblings("select").select(1);
+        cy.contains("Version to compare").get("#otherVersion input").select(1);
+        cy.contains("Difference to pick").get("#differentVersion input").select(1);
         cy.contains(/^cancel$/i).should("be.visible");
         cy.get("[data-testid=window]").matchImage();
         cy.get("button[name=zoom]").click();

designer/client/cypress/e2e/compareVersions.cy.ts

@@ -64,8 +64,8 @@ describe("Compare versions", () => {
 
         // Open the compare dialog and select values to compare
         cy.contains(/^compare$/i).click();
-        cy.get("[data-testid=window]").get("#otherVersion").select(1);
-        cy.get("[data-testid=window]").get("#differentVersion").select(1);
+        cy.get("[data-testid=window]").get("#otherVersion input").select(1);
+        cy.get("[data-testid=window]").get("#differentVersion input").select(1);
 
         // Check current value for changed fragment parameter
         cy.get("[data-testid=window]")

designer/client/cypress/e2e/components.cy.ts

@@ -232,7 +232,11 @@ describe("Components list", () => {
         cy.matchQuery("?TEXT=xxx");
         cy.viewport(1600, 500);
         cy.wait(500); //ensure "loading" mask is hidden
-        cy.get("#app-container>main").matchImage();
+        cy.get("#app-container>main").matchImage({
+            screenshotConfig: {
+                blackout: ["[role='row']:not(:first-of-type) > [role='cell'][data-field='modificationDate'] span"],
+            },
+        });
     });
 
     it("should allow filtering by processing mode", () => {

designer/client/cypress/e2e/fragment.cy.ts

@@ -37,8 +37,7 @@ describe("Fragment", () => {
         cy.get("@window").contains("+").click();
         cy.get("[data-testid='fieldsRow:4']").find("[placeholder='Field name']").type("name_string_any_with_suggestion");
         toggleSettings(4);
-        cy.get("[data-testid='settings:4']").contains("Any value").click();
-        cy.get("[id$='option-1']").click({ force: true });
+        cy.get("[data-testid='settings:4']").contains("Any value").select(1);
 
         // Display Add list item errors when blank value
         cy.get("[data-testid='settings:4']").contains("User defined list").click();
@@ -80,15 +79,13 @@ describe("Fragment", () => {
         cy.get("@window").contains("+").click();
         cy.get("[data-testid='fieldsRow:5']").find("[placeholder='Field name']").type("name_string_fixed");
         toggleSettings(5);
-        cy.get("[data-testid='settings:5']").contains("Any value").click();
-        cy.get("[id$='option-0']").click({ force: true });
+        cy.get("[data-testid='settings:5']").contains("Any value").select(1);
         cy.get("[data-testid='settings:5']").contains("User defined list").click();
         cy.get("[data-testid='settings:5']").find("[id='ace-editor']").type("#meta.processName");
         cy.get("[data-testid='settings:5']").contains("Typing...").should("not.exist");
         cy.get("[data-testid='settings:5']").find("[id='ace-editor']").type("{enter}");
         cy.get("[data-testid='settings:5']").find("[role='button']").contains("#meta.processName");
-        cy.get("[data-testid='settings:5']").find("[aria-label='type-select']").eq(1).click();
-        cy.get("[id$='option-1']").click({ force: true });
+        cy.get("[data-testid='settings:5']").find("[aria-label='type-select']").eq(1).select(1);
         cy.get("[data-testid='settings:5']").find("textarea").eq(1).type("Hint text test");
 
         cy.get("[data-testid='settings:5']")
@@ -130,8 +127,7 @@ describe("Fragment", () => {
         toggleSettings(7);
 
         // Select any value with suggestions Input mode
-        cy.get("[data-testid='settings:7']").contains("Any value").click();
-        cy.get("[id$='option-1']").click({ force: true });
+        cy.get("[data-testid='settings:7']").contains("Any value").select(1);
 
         // Activate preset mode
         cy.get("[data-testid='settings:7']").contains("Preset").click();
@@ -150,8 +146,8 @@ describe("Fragment", () => {
             .contains(/preset selection/i)
             .siblings()
             .eq(0)
-            .click();
-        cy.get("[id$='option-1']").click({ force: true });
+            .find("input")
+            .select(1);
 
         // Select Initial value
         cy.get("[data-testid='settings:7']")

designer/client/cypress/e2e/processes.cy.ts

@@ -9,7 +9,7 @@ describe("Processes list", () => {
         cy.createTestProcess(`${PROCESSING_MODE}-Request-Response`, undefined, "RequestResponse", "Request-Response");
 
         // Sometimes it's a small difference with the creation date time of scenarios, that's why we need to wait to keep elements in the same order
-        cy.wait(500);
+        cy.wait(1000);
         cy.createTestProcess(`${PROCESSING_MODE}-Streaming`, undefined, undefined, "Unbounded-Stream");
     });
 
@@ -38,7 +38,7 @@ describe("Processes list", () => {
         cy.get('[data-testid="window-frame"]')
             .contains(/request-response/i)
             .click();
-        cy.get("#processCategory").select(2);
+        cy.get("#processCategory input").select(2);
         cy.contains(/^create$/i)
             .should("be.enabled")
             .click();

designer/client/cypress/e2e/testsFromFile.cy.ts

@@ -0,0 +1,34 @@
+describe("Process tests from file", () => {
+    const transactionsTopic = "transactions";
+    const scenarioName = "test-from-file-scenario";
+
+    before(() => {
+        clear();
+    });
+
+    after(() => {
+        clear();
+    });
+
+    it("should properly display results of tests from file", () => {
+        cy.viewport(1440, 1200);
+        cy.createSchema(`${transactionsTopic}-value`, "transactionsAvroSchema.json");
+        cy.createKafkaTopic(transactionsTopic);
+        cy.visitNewProcess(scenarioName, "dumbStreamKafkaSourceScenario", "DevelopmentTests");
+        cy.get("[title='run test on data from file']")
+            .first()
+            .next("[type=file]")
+            .should("exist")
+            .selectFile("cypress/fixtures/transactionsTestData.txt", { force: true });
+        cy.get("text[joint-selector='testResultsSummary']").eq(0).contains("1");
+        cy.get("[model-id='kafka']").should("be.visible").trigger("dblclick");
+        cy.get("[data-testid='window-frame']").matchImage();
+    });
+
+    function clear() {
+        cy.log("Clearing data...");
+        cy.deleteAllTestProcesses({ filter: scenarioName, force: true });
+        cy.removeSchema(`${transactionsTopic}-value`);
+        cy.removeKafkaTopic(transactionsTopic);
+    }
+});