Update Nussknacker version to 1.0.0 (refactor to component provider) (#…

…184) * update nussknacker version * added component providers * naming fix * fix config accessing * fix configs names in tests * fix configs names in tests * add categories, version update * print version in proper way * simplify signature * simplify signature * fix clash by rebase * Update to Nussknacker 1.0.0 and its dependencies Define explicit minio image version * Fix compatibility checking * Update prinz version
prinz-nussknacker · Oct 20, 2021 · cad48b5 · cad48b5
1 parent 1c6bb7c
commit cad48b5
Show file tree

Hide file tree

Showing 43 changed files with 338 additions and 368 deletions.
diff --git a/README.md b/README.md
@@ -25,11 +25,11 @@ If you are already signed to GitHub in your project, just add any of these lines
 to add Prinz dependencies to your `.sbt` project
 
 ```sbt
-"pl.touk.nussknacker.prinz" %% "prinz" % "0.1.0-SNAPSHOT"
-"pl.touk.nussknacker.prinz" %% "prinz-mlflow" % "0.1.0-SNAPSHOT"
-"pl.touk.nussknacker.prinz" %% "prinz-pmml" % "0.1.0-SNAPSHOT"
-"pl.touk.nussknacker.prinz" %% "prinz-h2o" % "0.1.0-SNAPSHOT"
-"pl.touk.nussknacker.prinz" %% "prinz-proxy" % "0.1.0-SNAPSHOT"
+"pl.touk.nussknacker.prinz" %% "prinz" % "1.0.0-SNAPSHOT"
+"pl.touk.nussknacker.prinz" %% "prinz-mlflow" % "1.0.0-SNAPSHOT"
+"pl.touk.nussknacker.prinz" %% "prinz-pmml" % "1.0.0-SNAPSHOT"
+"pl.touk.nussknacker.prinz" %% "prinz-h2o" % "1.0.0-SNAPSHOT"
+"pl.touk.nussknacker.prinz" %% "prinz-proxy" % "1.0.0-SNAPSHOT"
 ```
 
 ## Authors

diff --git a/build.sbt b/build.sbt
@@ -1,13 +1,13 @@
 import sbtassembly.MergeStrategy
 
-val prinzV = "0.1.0-SNAPSHOT"
+val prinzV = "1.0.0-SNAPSHOT"
 val prinzOrg = "pl.touk.nussknacker.prinz"
 val repositoryOwner = "prinz-nussknacker"
 val repositoryName = "prinz"
 
 // Dependency versions
 val scalaV = "2.12.10"
-val nussknackerV = "0.3.0"
+val nussknackerV = "1.0.0"
 val sttpV = "3.0.0-RC7"
 val scalatestV = "3.2.2"
 val minioS3V = "8.0.0"

diff --git a/dev-environment/README.md b/dev-environment/README.md
@@ -7,9 +7,9 @@ To use it on CI the `secrets.GITHUB_TOKEN` is used to authenticate to repository
 To use it on your local machine you have to generate Personal Access Token
 on [this website](https://github.com/settings/tokens) and save it to `~/token.txt` file. Then use
 ```
-cat ~/token.txt | docker login https://docker.pkg.github.com -u YOUR_GITHUB_USERNAME --password-stdin
+cat ~/token.txt | docker login https://ghcr.io -u YOUR_GITHUB_USERNAME --password-stdin
 ```
-to login to Github Docker Registry and use the prepared version.
+to login to GitHub Docker Registry and use the prepared version.
 
 ## Publish updated image
 To publish the updated version of any image you should build a new version locally with command

diff --git a/dev-environment/create_environment.sh b/dev-environment/create_environment.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 scalaV="2.12"
-prinzV="0.1.0-SNAPSHOT"
+prinzV="1.0.0-SNAPSHOT"
 
 COMP_FILES=""
 ENV_FILE="-f docker-compose-env.yaml"

diff --git a/dev-environment/docker-compose-env.yaml b/dev-environment/docker-compose-env.yaml
@@ -2,19 +2,29 @@ version: '3.8'
 
 services:
 
-  nussknacker:
-    container_name: nussknacker
-    build: ./nussknacker
+  designer:
+    container_name: nussknacker_designer
+    image: touk/nussknacker:1.0.0
     ports:
-      - "3000:3000"
-      - "8080:8080"
+      - "3081:8080"
     environment:
-      NUSSKNACKER_CONFIG_FILE: >-
-        ${NUSSKNACKER_CONFIG_FILE:-/opt/prinz-sample/prinz-application.conf}
+      # multiple, comma separated, config files can be used. They will be merged in order, via HOCON fallback mechanism
+      # https://github.com/lightbend/config/blob/master/HOCON.md#config-object-merging-and-file-merging
+      CONFIG_FILE: ${NUSSKNACKER_CONFIG_FILE-/opt/nussknacker/conf/application.conf,/opt/nussknacker/conf/prinz.conf}
+      JDK_JAVA_OPTIONS: -Xmx256M
+      FLINK_REST_URL: http://jobmanager:8081
+      FLINK_QUERYABLE_STATE_PROXY_URL: taskmanager:9069
+      KAFKA_ADDRESS: kafka:9092
+      SCHEMA_REGISTRY_URL: http://schemaregistry:8081
+      COUNTS_URL: http://influxdb:8086/query
+    volumes:
+      - storage_designer:/opt/nussknacker/storage
+      # this is needed to be able to verify savepoints during deployments
+      - storage_flink:/opt/flink/data
+      - ./nussknacker/opt/prinz-sample/prinz.conf:/opt/nussknacker/conf/prinz.conf
+      - ./nussknacker/opt/prinz-sample/prinz-sample-assembly-1.0.0-SNAPSHOT.jar:/opt/prinz-sample/prinz-sample.jar
     networks:
       - dev-bridge-net
-    depends_on:
-      - jobmanager
 
   zookeeper:
     container_name: nussknacker_zookeeper
@@ -23,8 +33,8 @@ services:
       ZOO_MY_ID: 1
       JVMFLAGS: "-Xms64m -Xmx128m"
     volumes:
-      - nussknacker_storage_zookeeper_datalog:/datalog
-      - nussknacker_storage_zookeeper_data:/data
+      - storage_zookeeper_datalog:/datalog
+      - storage_zookeeper_data:/data
     networks:
       - dev-bridge-net
 
@@ -35,7 +45,7 @@ services:
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
       SCHEMA_REGISTRY_HOST_NAME: schemaregistry
     ports:
-      - 3082:8081
+      - "3082:8081"
     depends_on:
       - zookeeper
       - kafka
@@ -49,32 +59,36 @@ services:
     container_name: nussknacker_kafka
     image: wurstmeister/kafka:2.12-2.3.0
     ports:
-      - "3032:9092"
+      - "3032:3032"
     environment:
       HOSTNAME_COMMAND: "docker info | grep ^Name: | cut -d' ' -f 2"
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+      KAFKA_LISTENERS: INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:3032
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://localhost:3032
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CREATE_TOPICS: "processedEvents:1:1"
       KAFKA_BROKER_ID: 1
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_HEAP_OPTS: -Xms128m -Xmx512m
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - nussknacker_storage_kafka_data:/kafka
+      - storage_kafka_data:/kafka
     networks:
       - dev-bridge-net
     depends_on:
       - zookeeper
 
   influxdb:
     container_name: nussknacker_influxdb
-    image: influxdb:1.7.10-alpine
+    image: influxdb:1.8.4-alpine
+    ports:
+      - "3086:8086"
     environment:
       INFLUXDB_DB: esp
     volumes:
       - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf
-      - nussknacker_storage_influxdb:/var/lib/influxdb
+      - storage_influxdb:/var/lib/influxdb
     networks:
       - dev-bridge-net
 
@@ -88,31 +102,37 @@ services:
 
   jobmanager:
     container_name: nussknacker_jobmanager
-    image: flink:1.11.2-scala_2.12-java11
+    image: flink:1.13.1-scala_2.12-java11
     ports:
       - "3031:8081"
     entrypoint: /flink-entrypoint.sh
     command: jobmanager
+    environment:
+      # those environment variables are duplicated here and in designer service,
+      # in case of setup with designer run in other network than flink
+      KAFKA_ADDRESS: kafka:9092
+      SCHEMA_REGISTRY_URL: http://schemaregistry:8081
     depends_on:
       - zookeeper
       - kafka
       - influxdb
     volumes:
       - ./flink/flink-conf.yaml:/tmp/flink-conf.yaml
       - ./flink/flink-entrypoint.sh:/flink-entrypoint.sh
-      - nussknacker_storage_jobmanager:/opt/flink/data
     networks:
       - dev-bridge-net
 
   taskmanager:
     container_name: nussknacker_taskmanager
-    image: flink:1.11.2-scala_2.12-java11
+    image: flink:1.13.1-scala_2.12-java11
     ports:
       - "3063:9069"
       - "9009:9009"
       - "9008:9008"
     entrypoint: /flink-entrypoint.sh
     command: taskmanager
+    environment:
+      TASK_MANAGER_NUMBER_OF_TASK_SLOTS: 40
     depends_on:
       - zookeeper
       - kafka
@@ -121,7 +141,7 @@ services:
     volumes:
       - ./flink/flink-conf.yaml:/tmp/flink-conf.yaml
       - ./flink/flink-entrypoint.sh:/flink-entrypoint.sh
-      - nussknacker_storage_taskmanager:/opt/flink/data
+      - storage_flink:/opt/flink/data
     ulimits:
       nproc: 70000
       nofile:
@@ -136,9 +156,15 @@ networks:
     external: true
 
 volumes:
-  nussknacker_storage_zookeeper_datalog:
-  nussknacker_storage_zookeeper_data:
-  nussknacker_storage_kafka_data:
-  nussknacker_storage_jobmanager:
-  nussknacker_storage_taskmanager:
-  nussknacker_storage_influxdb:
+  storage_zookeeper_datalog:
+    name: nussknacker_storage_zookeeper_datalog
+  storage_zookeeper_data:
+    name: nussknacker_storage_zookeeper_data
+  storage_kafka_data:
+    name: nussknacker_storage_kafka_data
+  storage_flink:
+    name: nussknacker_storage_flink
+  storage_influxdb:
+    name: nussknacker_storage_influxdb
+  storage_designer:
+    name: nussknacker_storage_designer
diff --git a/dev-environment/flink/flink-conf.yaml b/dev-environment/flink/flink-conf.yaml
@@ -17,14 +17,13 @@
 ################################################################################
 
 
-################################################################################
+#==============================================================================
 # Common
-################################################################################
+#==============================================================================
 
-# The host on which the JobManager runs. Only used in
-# non-high-availability mode. The JobManager process will use
-# this hostname to bind the listening servers to. The TaskManagers
-# will try to connect to the JobManager on that host.
+# The host on which the JobManager runs. Only used in non-high-availability mode.
+# The JobManager process will use this hostname to bind the listening servers to.
+# The TaskManagers will try to connect to the JobManager on that host.
 
 taskmanager.hostname: taskmanager
 
@@ -41,10 +40,9 @@ env.java.opts.taskmanager: "
   -Dcom.sun.management.jmxremote.authenticate=false
   -Dcom.sun.management.jmxremote.ssl=false"
 
-# The number of task slots that each TaskManager offers.
-# Each slot runs one parallel pipeline.
+# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
 
-taskmanager.numberOfTaskSlots: 40
+taskmanager.numberOfTaskSlots: <configured_by_flink_entrypoint>
 
 jobmanager.rpc.address: jobmanager
 
@@ -60,9 +58,9 @@ jobmanager.heap.size: 256m
 
 parallelism.default: 8
 
-################################################################################
+#==============================================================================
 # Web Frontend
-################################################################################
+#==============================================================================
 
 # The port under which the web-based runtime monitor listens.
 # A value of -1 deactivates the web server.
@@ -73,33 +71,46 @@ web.log.path: /opt/flink/data/logs
 
 high-availability: zookeeper
 high-availability.zookeeper.quorum: zookeeper:2181
-high-availability.zookeeper.path.root: /demo
+high-availability.zookeeper.path.root: /flink
 high-availability.cluster-id: nussknacker
 high-availability.storageDir: file:///opt/flink/data/storage
 
 state.backend: filesystem
 state.checkpoints.dir: file:///opt/flink/data/checkpoints
 state.savepoints.dir: file:///opt/flink/data/savepoints
 
+#Below are base settings for rocksdb metrics, that can be used for grafana dashboards
+state.backend.rocksdb.metrics.estimate-num-keys: true
+state.backend.rocksdb.metrics.estimate-live-data-size: true
+state.backend.rocksdb.metrics.cur-size-all-mem-tables: true
+state.backend.rocksdb.metrics.size-all-mem-tables: true
+
+#We can have many jobs per cluster, in such setting managed memory is not easy to tune
+state.backend.rocksdb.memory.managed: false
+
+#For frequent writes increase the value as needed. Currently RocksDB settings can only be changed per Flink cluster
+state.backend.rocksdb.writebuffer.size: 256m
+
+
 metrics.reporters: influxdb_reporter
 
-metrics.reporter.influxdb_reporter.class:
-  org.apache.flink.metrics.influxdb.InfluxdbReporter
+metrics.reporter.influxdb_reporter.class: org.apache.flink.metrics.influxdb.InfluxdbReporter
 metrics.reporter.influxdb_reporter.host: telegraf
 metrics.reporter.influxdb_reporter.port: 8087
 metrics.reporter.influxdb_reporter.db: esp
 
-metrics.scope.jm: demo.<host>.jobmanagerGlobal
-metrics.scope.jm.job: demo.<host>.jobmanagerJob.<job_name>
-metrics.scope.tm: demo.<host>.taskmanagerGlobal.<tm_id>
-metrics.scope.tm.job: demo.<host>.taskmanagerJob.<tm_id>.<job_name>
-metrics.scope.task:
-  demo.<host>.taskmanagerTask.<tm_id>.<job_name>.<task_name>.<subtask_index>
-metrics.scope.operator:
-  demo.<host>.taskmanagerTask.<tm_id>.<job_name>.<operator_name>.<subtask_index>
+metrics.scope.jm: local.<host>.jobmanagerGlobal
+metrics.scope.jm.job: local.<host>.jobmanagerJob.<job_name>
+metrics.scope.tm: local.<host>.taskmanagerGlobal.<tm_id>
+metrics.scope.tm.job: local.<host>.taskmanagerJob.<tm_id>.<job_name>
+metrics.scope.task: local.<host>.taskmanagerTask.<tm_id>.<job_name>.<task_name>.<subtask_index>
+metrics.scope.operator: local.<host>.taskmanagerTask.<tm_id>.<job_name>.<operator_name>.<subtask_index>
 
-classloader.resolve-order: parent-first
+#Uncomment if you encounter problems with classloading with complex custom components
+#classloader.resolve-order: parent-first
 akka.framesize: 209715200b
 
 query.server.port: 6125
 blob.server.port: 6124
+
+queryable-state.enable: true
diff --git a/dev-environment/minio/Dockerfile b/dev-environment/minio/Dockerfile
@@ -1,4 +1,4 @@
-FROM minio/minio:latest
+FROM minio/minio:RELEASE.2021-06-17T00-10-46Z-28-gac7697426
 
 ENV MINIO_HOME /opt/minio
 ENV BUCKET_NAME mlflow

diff --git a/dev-environment/nussknacker/Dockerfile b/dev-environment/nussknacker/Dockerfile