Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[NU-1904] Add possiblity to configure deployment parameters for Eager… #7586

Open
wants to merge 9 commits into
base: staging
Choose a base branch
from
+786 −437
Open

[NU-1904] Add possiblity to configure deployment parameters for Eager… #7586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ea4a6eb
[NU-1904] Add possiblity to configure deployment parameters for Eager…
Feb 21, 2025
5e2cf31
Changes in FE
Feb 25, 2025
917c0dd
update openapi docs
Feb 25, 2025
2f476b9
some refactor
Feb 25, 2025
893f358
fixes
Feb 25, 2025
706f76d
more fixes
Feb 25, 2025
68934bb
almost
Feb 25, 2025
6a458ea
more fixes and adjustments
Feb 26, 2025
97d4e46
add readme entries and rollback changes for tests purposes
Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions ...s/src/test/scala/pl/touk/nussknacker/engine/benchmarks/interpreter/InterpreterSetup.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,26 @@ import cats.data.Validated.{Invalid, Valid}
import cats.data.ValidatedNel
import pl.touk.nussknacker.engine.Interpreter.InterpreterShape
import pl.touk.nussknacker.engine.api._
import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, DesignerWideComponentId, UnboundedStreamComponent}
import pl.touk.nussknacker.engine.api.component.{
ComponentDefinition,
DesignerWideComponentId,
NodesDeploymentData,
UnboundedStreamComponent
}
import pl.touk.nussknacker.engine.api.context.ProcessCompilationError
import pl.touk.nussknacker.engine.api.exception.NuExceptionInfo
import pl.touk.nussknacker.engine.api.process._
import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess
import pl.touk.nussknacker.engine.compile.ProcessCompilerData
import pl.touk.nussknacker.engine.compiledgraph.part.ProcessPart
import pl.touk.nussknacker.engine.definition.component.Components
import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode
import pl.touk.nussknacker.engine.definition.component.{ComponentDefinitionWithImplementation, Components}
import pl.touk.nussknacker.engine.definition.model.{ModelDefinition, ModelDefinitionWithClasses}
import pl.touk.nussknacker.engine.dict.SimpleDictRegistry
import pl.touk.nussknacker.engine.modelconfig.ComponentsUiConfig
import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector
import pl.touk.nussknacker.engine.testing.ModelDefinitionBuilder
import pl.touk.nussknacker.engine.util.Implicits._
import pl.touk.nussknacker.engine.{CustomProcessValidatorLoader, InterpretationResult, api}
import pl.touk.nussknacker.engine.{ComponentUseCase, CustomProcessValidatorLoader, InterpretationResult, api}

import scala.language.higherKinds
import scala.reflect.ClassTag
Expand Down Expand Up @@ -74,7 +78,8 @@ class InterpreterSetup[T: ClassTag] {
getClass.getClassLoader,
ProductionServiceInvocationCollector,
ComponentUseCase.EngineRuntime,
CustomProcessValidatorLoader.emptyCustomProcessValidator
CustomProcessValidatorLoader.emptyCustomProcessValidator,
NodesDeploymentData.empty,
)
}

Expand Down
4 changes: 2 additions & 2 deletions components-api/src/main/scala/pl/touk/nussknacker/engine/api/Service.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package pl.touk.nussknacker.engine.api

import pl.touk.nussknacker.engine.api.component.{AllProcessingModesComponent, Component}
import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector

import scala.concurrent.{ExecutionContext, Future}
Expand Down Expand Up @@ -37,7 +37,7 @@ trait ServiceInvoker {
def invoke(context: Context)(
implicit ec: ExecutionContext,
collector: ServiceInvocationCollector,
componentUseCase: ComponentUseCase
componentUseContext: ComponentUseContext,
): Future[Any]

}
4 changes: 3 additions & 1 deletion ...nts-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import io.circe.{Decoder, Encoder}
import pl.touk.nussknacker.engine.api.NodeId
import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData

final case class NodesDeploymentData(dataByNodeId: Map[NodeId, NodeDeploymentData])
final case class NodesDeploymentData(private val dataByNodeId: Map[NodeId, NodeDeploymentData]) {
def get(nodeId: NodeId): Option[NodeDeploymentData] = dataByNodeId.get(nodeId)
}

object NodesDeploymentData {

Expand Down
22 changes: 0 additions & 22 deletions components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/ComponentUseCase.scala
View file
Open in desktop

This file was deleted.

32 changes: 32 additions & 0 deletions ...nents-api/src/main/scala/pl/touk/nussknacker/engine/api/process/ComponentUseContext.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package pl.touk.nussknacker.engine.api.process

import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData
import pl.touk.nussknacker.engine.api.process.ComponentUseContext.EngineRuntime

/**
* Specifies the mode a node is used/invoked. It can be one of the following values:
* <ul>
* <li>EngineRuntime - component is invoked in real engine, eg. Flink.</li>
* <li>TestRuntime - component is invoked in test mode to collect results for test data. Can be used to stub real implementation.</li>
* <li>Validation - used when compiling and validating nodes by Designer. Components should not be invoked in this mode.</li>
* <li>ServiceQuery - used when component (Service) is invoked by Designer in ServiceQuery.</li>
* <li>TestDataGeneration - used when compiling, but only for purpose of generating test data. Components should not be invoked in this mode.</li>
* </ul>
*/
sealed trait ComponentUseContext {

def deploymentData(): Option[NodeDeploymentData] = this match {
case EngineRuntime(nodeData) => nodeData
case _ => None
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's use explicit cases matching. Thanks to that, we avoid some mistakes during changes around these contexts. Also, why not a normal polymorphism? I mean def deploymentData

}

}

object ComponentUseContext {
case class EngineRuntime(nodeData: Option[NodeDeploymentData]) extends ComponentUseContext
case object TestRuntime extends ComponentUseContext
case object Validation extends ComponentUseContext
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you remove contexts:

  • ServiceQuery - It was for purpose of external project, I checked their repo and found that they can replace it with EngineRuntime(None) without any damage
  • Validation - Our validation mechanism compiles components, and after that regrets result of compilation so, these components never will be used - we can pass EngineRuntime(None) there with some additional note that it is not important
  • TestDataGeneration - During test data generation things are done very similar to validation. We compile components only to get test data generator, so component is not used for anything else and it doesn't need to know the context.
    ?

Also, I would rename two other contexts:

  • EngineRuntime - scenario testing mechanism also run component on some engine so it is confusing, I would rename it to LiveDeployment/RealDeployment
  • TestRuntime - ScenarioTesting. Meantime we clarified this name for naming this thing. We have scenarioTesting in endpoints and in other places in the code

case object ServiceQuery extends ComponentUseContext
case object TestDataGeneration extends ComponentUseContext

}
10 changes: 5 additions & 5 deletions ...ents/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import org.scalatest.matchers.should.Matchers
import org.scalatest.{BeforeAndAfterAll, Outcome}
import pl.touk.nussknacker.engine.api._
import pl.touk.nussknacker.engine.api.parameter.ParameterName
import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.api.test.EmptyInvocationCollector.Instance
import pl.touk.nussknacker.engine.api.typed.TypedMap
import pl.touk.nussknacker.engine.util.ResourceLoader
Expand All @@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersA
import pl.touk.nussknacker.http.backend.FixedAsyncHttpClientBackendProvider
import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers}
import pl.touk.nussknacker.openapi.parser.SwaggerParser
import pl.touk.nussknacker.openapi.{ApiKeySecret, OpenAPIServicesConfig, SecurityConfig, SecuritySchemeName}
import pl.touk.nussknacker.openapi.{ApiKeySecret, OpenAPIServicesConfig, SecuritySchemeName}
import pl.touk.nussknacker.test.PatientScalaFutures

import java.net.URL
Expand All @@ -31,9 +31,9 @@ class OpenAPIServiceSpec
with LazyLogging
with PatientScalaFutures {

implicit val componentUseCase: ComponentUseCase = ComponentUseCase.EngineRuntime
implicit val metaData: MetaData = MetaData("testProc", StreamMetaData())
implicit val context: Context = Context("testContextId", Map.empty)
implicit val componentUseCase: ComponentUseContext = ComponentUseContext.EngineRuntime(None)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rename componentUseCase to componentUseContext

implicit val metaData: MetaData = MetaData("testProc", StreamMetaData())
implicit val context: Context = Context("testContextId", Map.empty)
val jobData = JobData(metaData, ProcessVersion.empty.copy(processName = metaData.name))

type FixtureParam = EagerServiceWithStaticParametersAndReturnType
Expand Down
4 changes: 2 additions & 2 deletions ...onents/openapi/src/main/scala/pl/touk/nussknacker/openapi/enrichers/SwaggerEnricher.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package pl.touk.nussknacker.openapi.enrichers
import org.asynchttpclient.DefaultAsyncHttpClient
import pl.touk.nussknacker.engine.api.definition.Parameter
import pl.touk.nussknacker.engine.api.parameter.ParameterName
import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.api.runtimecontext.EngineRuntimeContext
import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
import pl.touk.nussknacker.engine.api.typed.typing
Expand Down Expand Up @@ -63,7 +63,7 @@ class SwaggerEnricher(
collector: ServiceInvocationCollector,
contextId: ContextId,
metaData: MetaData,
componentUseCase: ComponentUseCase
componentUseContext: ComponentUseContext
): Future[AnyRef] =
measuring {
swaggerHttpService.invoke(
Expand Down
8 changes: 4 additions & 4 deletions components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package pl.touk.nussknacker.openapi
import cats.data.Validated
import cats.data.Validated.{Invalid, Valid}
import org.apache.commons.io.IOUtils
import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api._
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.util.runtimecontext.TestEngineRuntimeContext
import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersAndReturnType
import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers}
Expand All @@ -19,9 +19,9 @@ trait BaseOpenAPITest {

protected val baseConfig: OpenAPIServicesConfig = OpenAPIServicesConfig(new URL("http://foo"))

implicit val componentUseCase: ComponentUseCase = ComponentUseCase.EngineRuntime
implicit val metaData: MetaData = MetaData("testProc", StreamMetaData())
implicit val context: Context = Context("testContextId", Map.empty)
implicit val componentUseContext: ComponentUseContext = ComponentUseContext.EngineRuntime(None)
implicit val metaData: MetaData = MetaData("testProc", StreamMetaData())
implicit val context: Context = Context("testContextId", Map.empty)
private val jobData = JobData(metaData, ProcessVersion.empty.copy(processName = metaData.name))
private val runtimeContext = TestEngineRuntimeContext(jobData)

Expand Down
5 changes: 2 additions & 3 deletions components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseEnricherInvoker.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package pl.touk.nussknacker.sql.service

import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
import pl.touk.nussknacker.engine.api.typed.typing
import pl.touk.nussknacker.engine.api.{Context, Params, ServiceInvoker}
import pl.touk.nussknacker.engine.util.service.AsyncExecutionTimeMeasurement
import pl.touk.nussknacker.sql.db.WithDBConnectionPool
Expand Down Expand Up @@ -34,7 +33,7 @@ class DatabaseEnricherInvoker(
override def invoke(context: Context)(
implicit ec: ExecutionContext,
collector: ServiceInvocationCollector,
componentUseCase: ComponentUseCase
componentUseContext: ComponentUseContext,
): Future[queryExecutor.QueryResult] = {
getTimeMeasurement().measuring {
queryDatabase(queryArgumentsExtractor(argsCount, params, context))
Expand Down
7 changes: 3 additions & 4 deletions ...sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseEnricherInvokerWithCache.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
package pl.touk.nussknacker.sql.service

import com.github.benmanes.caffeine.cache.{AsyncCache, Caffeine}
import pl.touk.nussknacker.engine.api.{Context, Params}
import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
import pl.touk.nussknacker.engine.api.typed.typing
import pl.touk.nussknacker.engine.api.{Context, Params}
import pl.touk.nussknacker.engine.util.service.AsyncExecutionTimeMeasurement
import pl.touk.nussknacker.sql.db.query.{QueryArguments, QueryResultStrategy}
import pl.touk.nussknacker.sql.db.schema.TableDefinition
Expand Down Expand Up @@ -53,7 +52,7 @@ class DatabaseEnricherInvokerWithCache(
override def invoke(context: Context)(
implicit ec: ExecutionContext,
collector: ServiceInvocationCollector,
componentUseCase: ComponentUseCase
componentUseContext: ComponentUseContext,
): Future[queryExecutor.QueryResult] = {
getTimeMeasurement().measuring {
val queryArguments = queryArgumentsExtractor(argsCount, params, context)
Expand Down
12 changes: 6 additions & 6 deletions ...ents/sql/src/test/scala/pl/touk/nussknacker/sql/utils/BaseDatabaseQueryEnricherTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import pl.touk.nussknacker.engine.api.context.ProcessCompilationError.EmptyProce
import pl.touk.nussknacker.engine.api.context.ValidationContext
import pl.touk.nussknacker.engine.api.context.transformation.{FailedToDefineParameter, OutputVariableNameValue}
import pl.touk.nussknacker.engine.api.parameter.ParameterName
import pl.touk.nussknacker.engine.api.process.ComponentUseCase
import pl.touk.nussknacker.engine.api.process.ComponentUseContext
import pl.touk.nussknacker.engine.api.test.EmptyInvocationCollector
import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
import pl.touk.nussknacker.engine.api.typed.typing
Expand All @@ -19,11 +19,11 @@ import scala.concurrent.ExecutionContext

trait BaseDatabaseQueryEnricherTest extends AnyFunSuite with Matchers with BeforeAndAfterAll {

implicit val ec: ExecutionContext = ExecutionContext.Implicits.global
implicit val context: Context = Context("", Map.empty)
implicit val metaData: MetaData = MetaData("", StreamMetaData())
implicit val collector: ServiceInvocationCollector = EmptyInvocationCollector.Instance
implicit val componentUseCase: ComponentUseCase = ComponentUseCase.TestRuntime
implicit val ec: ExecutionContext = ExecutionContext.Implicits.global
implicit val context: Context = Context("", Map.empty)
implicit val metaData: MetaData = MetaData("", StreamMetaData())
implicit val collector: ServiceInvocationCollector = EmptyInvocationCollector.Instance
implicit val componentUseContext: ComponentUseContext = ComponentUseContext.TestRuntime

val jobData: JobData = JobData(metaData, ProcessVersion.empty.copy(processName = metaData.name))

Expand Down
10 changes: 5 additions & 5 deletions designer/client/src/components/modals/AdvancedParametersSection.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,19 @@ import { Typography } from "@mui/material";
import AccordionDetails from "@mui/material/AccordionDetails";

interface Props {
nodeId: string;
componentId: string;
}

export function AdvancedParametersSection({ children, nodeId }: PropsWithChildren<Props>): JSX.Element {
export function AdvancedParametersSection({ children, componentId }: PropsWithChildren<Props>): JSX.Element {
return (
<Accordion disableGutters elevation={0} sx={{ border: 0, "&::before": { display: "none" } }}>
<AccordionSummary
expandIcon={<ExpandMoreIcon sx={{ color: "inherit" }} />}
aria-controls={`${nodeId}-content`}
id={`${nodeId}-header`}
aria-controls={`${componentId}-content`}
id={`${componentId}-header`}
sx={{ flexDirection: "row-reverse", border: 0 }}
>
<Typography>{nodeId}</Typography>
<Typography>{componentId}</Typography>
</AccordionSummary>
<AccordionDetails>{children}</AccordionDetails>
</Accordion>
Expand Down
Loading