Merge pull request #121 from learningOrchestra/feature-tensorflow-ste…

…ps-and-architecture-refactoring

Feature tensorflow steps and architecture refactoring

docker-compose.yml

@@ -32,9 +32,9 @@ services:
     depends_on:
       - databaseprimary
       - images
-    deploy: *default-deploy-worker
+    deploy: *default-deploy-manager
     volumes:
-      - "database_api:/database_api"
+      - "database_api:/datasets"
     networks:
       - database
     environment: *default-service-database-env
@@ -178,14 +178,14 @@ services:
       - spark
     environment: *default-service-database-env
 
-  modelbuilder:
-    build: microservices/model_builder_image
-    image: 127.0.0.1:5050/model_builder
+  builder:
+    build: microservices/builder_image
+    image: 127.0.0.1:5050/builder
     ports:
       - "5002:5002"
       - "41100:41100"
     extra_hosts:
-      - "modelbuilder:0.0.0.0"
+      - "builder:0.0.0.0"
     depends_on:
       - databaseprimary
       - images
@@ -235,8 +235,9 @@ services:
     volumes:
       - "database_executor:/explore"
       - "database_executor:/transform"
-      - "default_model:/models"
+      - "model:/models"
       - "binary_executor:/binaries"
+      - "code_executor:/code_executions"
     networks:
       - database
     environment: *default-service-database-env
@@ -253,9 +254,9 @@ services:
       - database
       - spark
 
-  defaultmodel:
-    build: microservices/default_model_image
-    image: 127.0.0.1:5050/default_model
+  model:
+    build: microservices/model_image
+    image: 127.0.0.1:5050/model
     ports:
       - "5007:5007"
     depends_on:
@@ -266,7 +267,11 @@ services:
       - database
     environment: *default-service-database-env
     volumes:
-      - "default_model:/models"
+      - "code_executor:/code_executions"
+      - "database_executor:/explore"
+      - "database_executor:/transform"
+      - "model:/models"
+      - "binary_executor:/binaries"
 
   binaryexecutor:
     build: microservices/binary_executor_image
@@ -281,9 +286,10 @@ services:
       - database
     environment: *default-service-database-env
     volumes:
-      - "default_model:/models"
+      - "model:/models"
       - "binary_executor:/binaries"
       - "database_executor:/transform"
+      - "code_executor:/code_executions"
 
   codexecutor:
     build: microservices/code_executor_image
@@ -295,10 +301,12 @@ services:
       - images
     deploy: *default-deploy-manager
     volumes:
+      - "code_executor:/code_executions"
       - "database_executor:/explore"
       - "database_executor:/transform"
-      - "default_model:/models"
+      - "model:/models"
       - "binary_executor:/binaries"
+      - "database_api:/datasets"
     networks:
       - database
     environment: *default-service-database-env
@@ -313,6 +321,7 @@ volumes:
   database:
   database_api:
   database_executor:
-  default_model:
+  code_executor:
+  model:
   binary_executor:
   portainer:

microservices/binary_executor_image/Dockerfile

@@ -9,5 +9,6 @@ ENV MICROSERVICE_PORT 5008
 ENV BINARY_VOLUME_PATH "/binaries"
 ENV MODELS_VOLUME_PATH "/models"
 ENV TRANSFORM_VOLUME_PATH "/transform"
+ENV CODE_EXECUTOR_VOLUME_PATH "/code_executions"
 
 CMD ["python", "server.py"]

microservices/binary_executor_image/binary_execution.py

@@ -2,10 +2,12 @@
 from concurrent.futures import ThreadPoolExecutor
 from utils import Database, Data, Metadata, ObjectStorage
 from constants import Constants
+import traceback
 
 
 class Parameters:
     __DATASET_KEY_CHARACTER = "$"
+    __CLASS_INSTANCE_CHARACTER = "#"
     __DATASET_WITH_OBJECT_KEY_CHARACTER = "."
     __REMOVE_KEY_CHARACTER = ""
 
@@ -17,23 +19,61 @@ def treat(self, method_parameters: dict) -> dict:
         parameters = method_parameters.copy()
 
         for name, value in parameters.items():
-            if self.__is_dataset(value):
-                dataset_name = self.__get_dataset_name_from_value(
-                    value)
-                if self.__has_dot_in_dataset_name(value):
-                    object_name = self.__get_name_after_dot_from_value(value)
+            if type(value) is list:
+                new_value = []
+                for item in value:
+                    new_value.append(self.__treat_value(item))
+                parameters[name] = new_value
+            else:
+                parameters[name] = self.__treat_value(value)
 
-                    parameters[name] = self.__data.get_object_from_dataset(
-                        dataset_name, object_name)
+        return parameters
 
-                else:
-                    parameters[name] = self.__data.get_dataset_content(
-                        dataset_name)
+    def __treat_value(self, value: object) -> object:
+        if self.__is_dataset(value):
+            dataset_name = self.__get_dataset_name_from_value(
+                value)
 
-        return parameters
+            if self.__has_dot_in_dataset_name(value):
+                object_name = self.__get_name_after_dot_from_value(value)
+                return self.__data.get_object_from_dataset(
+                    dataset_name, object_name)
+
+            else:
+                return self.__data.get_dataset_content(
+                    dataset_name)
+
+        elif self.__is_a_class_instance(value):
+            return self.__get_a_class_instance(value)
+
+        else:
+            return value
+
+    def __get_a_class_instance(self, class_code: str) -> object:
+        class_instance_name = "class_instance"
+        class_instance = None
+        context_variables = {}
 
-    def __is_dataset(self, value: str) -> bool:
-        return self.__DATASET_KEY_CHARACTER in value
+        class_code = class_code.replace(
+            self.__CLASS_INSTANCE_CHARACTER,
+            f'{class_instance_name}=')
+
+        import tensorflow
+        exec(class_code, locals(), context_variables)
+
+        return context_variables[class_instance_name]
+
+    def __is_a_class_instance(self, value: object) -> bool:
+        if type(value) != str:
+            return False
+        else:
+            return self.__CLASS_INSTANCE_CHARACTER in value
+
+    def __is_dataset(self, value: object) -> bool:
+        if type(value) != str:
+            return False
+        else:
+            return self.__DATASET_KEY_CHARACTER in value
 
     def __get_dataset_name_from_value(self, value: str) -> str:
         dataset_name = value.replace(self.__DATASET_KEY_CHARACTER,
@@ -121,6 +161,7 @@ def __pipeline(self,
                                                          flag=True)
 
         except Exception as exception:
+            traceback.print_exc()
             self.__metadata_creator.create_execution_document(
                 self.executor_name,
                 description,
@@ -135,7 +176,14 @@ def __pipeline(self,
 
     def __execute_a_object_method(self, class_instance: object, method: str,
                                   parameters: dict) -> object:
-        model_method = getattr(class_instance, method)
+        class_method = getattr(class_instance, method)
 
         treated_parameters = self.__parameters_handler.treat(parameters)
-        return model_method(**treated_parameters)
+        method_result = class_method(**treated_parameters)
+
+        if self.executor_service_type == Constants.TRAIN_TENSORFLOW_TYPE or \
+                self.executor_service_type == Constants.TRAIN_SCIKITLEARN_TYPE or \
+                method_result is None:
+            return class_instance
+
+        return method_result

microservices/binary_executor_image/constants.py

@@ -1,6 +1,7 @@
 class Constants:
     MODULE_PATH_FIELD_NAME = "modulePath"
     CLASS_FIELD_NAME = "class"
+    MODEL_NAME_FIELD_NAME = "modelName"
     PARENT_NAME_FIELD_NAME = "parentName"
     NAME_FIELD_NAME = "name"
     FINISHED_FIELD_NAME = "finished"
@@ -13,6 +14,7 @@ class Constants:
     MODELS_VOLUME_PATH = "MODELS_VOLUME_PATH"
     BINARY_VOLUME_PATH = "BINARY_VOLUME_PATH"
     TRANSFORM_VOLUME_PATH = "TRANSFORM_VOLUME_PATH"
+    CODE_EXECUTOR_VOLUME_PATH = "CODE_EXECUTOR_VOLUME_PATH"
 
     DELETED_MESSAGE = "deleted file"
 
@@ -33,21 +35,43 @@ class Constants:
 
     MESSAGE_RESULT = "result"
 
+    MODEL_SCIKITLEARN_TYPE = "model/scikitlearn"
+    MODEL_TENSORFLOW_TYPE = "model/tensorflow"
+
+    TUNE_SCIKITLEARN_TYPE = "tune/scikitlearn"
+    TUNE_TENSORFLOW_TYPE = "tune/tensorflow"
+
+    TRAIN_SCIKITLEARN_TYPE = "train/scikitlearn"
+    TRAIN_TENSORFLOW_TYPE = "train/tensorflow"
+
+    EVALUATE_SCIKITLEARN_TYPE = "evaluate/scikitlearn"
+    EVALUATE_TENSORFLOW_TYPE = "evaluate/tensorflow"
+
+    PREDICT_SCIKITLEARN_TYPE = "predict/scikitlearn"
+    PREDICT_TENSORFLOW_TYPE = "predict/tensorflow"
+
+    PYTHON_FUNCTION_TYPE = "function/python"
+    DATASET_TENSORFLOW_TYPE = "dataset/tensorflow"
+
+    TRANSFORM_SCIKITLEARN_TYPE = "transform/scikitlearn"
+    TRANSFORM_TENSORFLOW_TYPE = "transform/tensorflow"
+
+    EXPLORE_SCIKITLEARN_TYPE = "explore/scikitlearn"
+    EXPLORE_TENSORFLOW_TYPE = "explore/tensorflow"
+
+    API_PATH = "/api/learningOrchestra/v1/"
+
     MICROSERVICE_URI_SWITCHER = {
-        "tune": "/api/learningOrchestra/v1/tune/",
-        "train": "/api/learningOrchestra/v1/train/",
-        "evaluate": "/api/learningOrchestra/v1/evaluate/",
-        "predict": "/api/learningOrchestra/v1/predict/"
+        TUNE_SCIKITLEARN_TYPE: f'{API_PATH}{TUNE_SCIKITLEARN_TYPE}',
+        TUNE_TENSORFLOW_TYPE: f'{API_PATH}{TUNE_TENSORFLOW_TYPE}',
+        TRAIN_SCIKITLEARN_TYPE: f'{API_PATH}{TRAIN_SCIKITLEARN_TYPE}',
+        TRAIN_TENSORFLOW_TYPE: f'{API_PATH}{TRAIN_TENSORFLOW_TYPE}',
+        EVALUATE_SCIKITLEARN_TYPE: f'{API_PATH}{EVALUATE_SCIKITLEARN_TYPE}',
+        EVALUATE_TENSORFLOW_TYPE: f'{API_PATH}{EVALUATE_TENSORFLOW_TYPE}',
+        PREDICT_SCIKITLEARN_TYPE: f'{API_PATH}{PREDICT_SCIKITLEARN_TYPE}',
+        PREDICT_TENSORFLOW_TYPE: f'{API_PATH}{PREDICT_TENSORFLOW_TYPE}',
     }
 
-    DEFAULT_MODEL_TYPE = "defaultModel"
-    TUNE_TYPE = "tune"
-    TRAIN_TYPE = "train"
-    EVALUATE_TYPE = "evaluate"
-    PREDICT_TYPE = "predict"
-    TRANSFORM_TYPE = "transform"
-    PYTHON_TRANSFORM_TYPE = "pythonTransform"
-
     MICROSERVICE_URI_PATH = "/binaryExecutor"
     MICROSERVICE_URI_GET_PARAMS = "?query={}&limit=20&skip=0"
 

microservices/binary_executor_image/requirements.txt

@@ -2,5 +2,9 @@ pymongo==3.10.1
 flask==1.1.2
 datetime==4.3
 pytz==2020.1
-scikit-learn
+dill==0.3.3
 pandas==1.2.0
+# Don't set a specific version in dependencies in bellow,
+# they are used directly by user on API request.
+scikit-learn
+tensorflow