Merge remote-tracking branch 'upstream/master' into issue-40

.argo/argo-saas-unit-tests.yaml

@@ -15,6 +15,10 @@ steps:
    template: saas-unit-test-base
    arguments:
      parameters.COMMAND: /src/saas/axops/run-test-axops.sh
+  TEMPLATE-TEST:
+   template: saas-unit-test-base
+   arguments:
+     parameters.COMMAND: /src/saas/common/run-test.sh
   #AXAMM-TEST:
   #  template: saas-unit-test-base
   #  arguments:
@@ -35,8 +39,6 @@ steps:
     template: saas-unit-test-base
     arguments:
       parameters.COMMAND: /src/saas/axops/run-test-modules-2.sh
-  # yaml-checker:
-  #   template: axchecker-container
 outputs:
   artifacts:
     coverage-report-axops:

.argo/test-yamls/fixtures-dynamic.yaml

@@ -81,7 +81,7 @@ fixtures:
 - DYN_FIX_WITH_OUTPUTS:
     template: test-dynamic-fixture-container-with-outputs
 steps:
-- WEB-CLIENT-INLINED:
+- SLEEPER:
    image: alpine:latest
    command: [sh, -c]
    args: ["sleep 60"]
@@ -106,4 +106,43 @@ resources:
 outputs:
   artifacts:
     BIN-DIR:
-      path: /bin
+      path: /bin
+
+---
+type: workflow
+version: 1
+name: test-fixtures-dynamic-inputs
+description: Workflow which supplies input parameters to dynamic fixtures
+inputs:
+  parameters:
+    COMMIT:
+      default: "%%session.commit%%"
+    REPO:
+      default: "%%session.repo%%"
+    ECHO:
+      default: hello world
+fixtures:
+- DYN_FIX_WITH_INPUTS:
+    template: test-dynamic-fixture-container-with-inputs
+steps:
+- SLEEPER:
+   image: alpine:latest
+   command: [sh, -c]
+   args: ["sleep 60"]
+   resources:
+     cpu_cores: 0.05
+     mem_mib: 64
+
+---
+type: container
+version: 1
+name: test-dynamic-fixture-container-with-inputs
+image: alpine:latest
+command: [sh, -c]
+args: ["echo %%inputs.parameters.ECHO%%; sleep 999999"]
+resources:
+  cpu_cores: 0.05
+  mem_mib: 64
+inputs:
+  parameters:
+    ECHO:

common/python/ax/cluster_management/app/cluster_upgrader.py

@@ -15,6 +15,7 @@
 import yaml
 from pprint import pformat
 
+from ax.cloud import Cloud
 from ax.cloud.aws import AWS_DEFAULT_PROFILE
 from ax.meta import AXCustomerId
 from ax.platform.ax_cluster_info import AXClusterInfo
@@ -71,21 +72,22 @@ def run(self):
         upgrade_service = True
 
         if self._cfg.target_software_info.kube_installer_version == self._current_software_info.kube_installer_version \
-            and self._cfg.target_software_info.kube_version == self._current_software_info.kube_version:
+            and self._cfg.target_software_info.kube_version == self._current_software_info.kube_version \
+                and not self._cfg.force_upgrade:
             upgrade_kube = False
 
         if self._cfg.target_software_info.image_namespace == self._current_software_info.image_namespace \
             and self._cfg.target_software_info.image_version == self._current_software_info.image_version \
             and self._cfg.target_software_info.image_version != "latest" \
-            and not upgrade_kube:
+            and not upgrade_kube \
+                and not self._cfg.force_upgrade:
             upgrade_service = False
 
-        if not upgrade_service and not upgrade_kube and not self._cfg.force_upgrade:
+        if not upgrade_service and not upgrade_kube:
             logger.info("%sCluster's software versions is not changed, not performing upgrade.%s", COLOR_GREEN, COLOR_NORM)
             logger.info("%sIf you want to force upgrade cluster, please specify --force-upgrade flag.%s", COLOR_YELLOW, COLOR_NORM)
             return
 
-
         if self._cfg.dry_run:
             logger.info("DRY RUN: upgrading cluster %s", self._name_id)
             return
@@ -186,7 +188,8 @@ def _upgrade_kube(self):
             "OLD_KUBE_VERSION": self._current_software_info.kube_version,
             "NEW_KUBE_VERSION": self._cfg.target_software_info.kube_version,
             "NEW_CLUSTER_INSTALL_VERSION": self._cfg.target_software_info.kube_installer_version,
-            "ARGO_AWS_REGION": self._cluster_config.get_region()
+            "ARGO_AWS_REGION": self._cluster_config.get_region(),
+            "AX_TARGET_CLOUD": Cloud().target_cloud()
         }
 
         if self._cfg.cloud_profile:
@@ -198,7 +201,6 @@ def _upgrade_kube(self):
         env.update(os.environ)
         subprocess.check_call(["upgrade-kubernetes"], env=env)
 
-
     def _start_platform(self):
         """
         This step brings up Argo platform services

saas/argocli/src/applatix.io/argo/cmd/cluster.go

@@ -65,10 +65,10 @@ func clusterShell(cmd *cobra.Command, args []string) {
 	homePath := usr.HomeDir
 
 	clusterManagerImage := fmt.Sprintf("%s/%s/axclustermanager:%s", registry, namespace, version)
-	log.Printf("Getting the most up to date docker image (%s) for cluster management...\n", clusterManagerImage)
+	fmt.Printf("Getting the most up to date docker image (%s) for cluster management...\n", clusterManagerImage)
 	runCmdTTY(dockerPath, "pull", clusterManagerImage)
 
-	log.Println("Entering cluster management shell...")
+	fmt.Println("Entering cluster management shell...")
 	volAWS := fmt.Sprintf("%s/.aws:/root/.aws", homePath)
 	volKube := fmt.Sprintf("%s/.kube:/tmp/ax_kube", homePath)
 	volSSH := fmt.Sprintf("%s/.ssh:/root/.ssh", homePath)

saas/argocli/src/applatix.io/argo/cmd/job.go

@@ -370,7 +370,7 @@ func jobLogs(cmd *cobra.Command, args []string) {
 			if err != nil {
 				log.Fatalf("Failed to decode line '%s': %v", string(line), err)
 			}
-			log.Print(jl.Log)
+			fmt.Print(jl.Log)
 		}
 	default:
 

saas/argocli/src/applatix.io/argo/cmd/login.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/user"
 	"path"
+	"regexp"
 	"strings"
 
 	"applatix.io/api"
@@ -16,17 +17,26 @@ import (
 )
 
 var (
-	loginConfig   string // --config
+	loginArgs     loginFlags
 	clusterConfig api.ClusterConfig
+
+	// regex to match SSL certificate related failures from net/http
+	x509Error = regexp.MustCompile(".*(x509.*)")
 )
 
+type loginFlags struct {
+	config   string // --config
+	insecure bool   // --insecure
+}
+
 func init() {
 	RootCmd.AddCommand(loginCmd)
 
-	loginCmd.Flags().StringVar(&loginConfig, "config", "", "Configuration name")
+	loginCmd.Flags().StringVar(&loginArgs.config, "config", "", "Configuration name")
 	loginCmd.Flags().StringVar(&clusterConfig.URL, "url", "", "Cluster URL")
 	loginCmd.Flags().StringVar(&clusterConfig.Username, "username", "", "Username")
 	loginCmd.Flags().StringVar(&clusterConfig.Password, "password", "", "Password")
+	loginCmd.Flags().BoolVar(&loginArgs.insecure, "insecure", false, "Allow config to use insecure option for invalid certificates")
 }
 
 var loginCmd = &cobra.Command{
@@ -38,12 +48,12 @@ var loginCmd = &cobra.Command{
 		}
 		reader := bufio.NewReader(os.Stdin)
 
-		if loginConfig == "" {
+		if loginArgs.config == "" {
 			fmt.Printf("Enter a configuration name (%s): ", api.DefaultConfigName)
-			loginConfig, _ = reader.ReadString('\n')
-			loginConfig = strings.TrimSpace(loginConfig)
-			if loginConfig == "" {
-				loginConfig = api.DefaultConfigName
+			loginArgs.config, _ = reader.ReadString('\n')
+			loginArgs.config = strings.TrimSpace(loginArgs.config)
+			if loginArgs.config == "" {
+				loginArgs.config = api.DefaultConfigName
 			}
 		}
 
@@ -80,12 +90,16 @@ var loginCmd = &cobra.Command{
 		if axErr != nil {
 			// TODO: need to completely rework axerror to preserve original,
 			// underlying error and not rely on this clunky string search
-			if axErr.Code == axerror.ERR_AX_HTTP_CONNECTION.Code && (strings.Contains(axErr.Message, "certificate is not valid") || strings.Contains(axErr.Message, "certificate is valid for")) {
-				fmt.Printf("Cluster is using an invalid or self-signed certificate. Proceed insecurely (y/n)? ")
-				insecure, _ := reader.ReadString('\n')
-				insecure = strings.TrimSpace(strings.ToLower(insecure))
-				if insecure != "y" && insecure != "yes" {
-					os.Exit(1)
+			if axErr.Code == axerror.ERR_AX_HTTP_CONNECTION.Code && x509Error.MatchString(axErr.Message) {
+				errorMsg := x509Error.ReplaceAllString(axErr.Message, "${1}")
+				fmt.Printf("%s: Cluster is using an invalid certificate: %s\n", ansiFormat("WARNING", FgRed), errorMsg)
+				if !loginArgs.insecure {
+					fmt.Printf("Proceed insecurely (y/n)? ")
+					insecure, _ := reader.ReadString('\n')
+					insecure = strings.TrimSpace(strings.ToLower(insecure))
+					if insecure != "y" && insecure != "yes" {
+						os.Exit(1)
+					}
 				}
 				// Try again insecurely
 				newTrue := true
@@ -104,11 +118,11 @@ var loginCmd = &cobra.Command{
 		if err != nil {
 			log.Fatalln(err)
 		}
-		configPath := path.Join(usr.HomeDir, api.ArgoDir, loginConfig)
+		configPath := path.Join(usr.HomeDir, api.ArgoDir, loginArgs.config)
 		err = clusterConfig.WriteConfigFile(configPath)
 		if err != nil {
 			log.Fatalf("Failed to write config file: %v\n", err)
 		}
-		log.Printf("Config written to: %s\n", configPath)
+		fmt.Printf("Config written to: %s\n", configPath)
 	},
 }

saas/axops/src/applatix.io/axops/service/embed.go

@@ -229,7 +229,15 @@ func EmbedWorkflowTemplate(tmpl *template.WorkflowTemplate, ctx *template.Templa
 						Template: eCtrTmpl.(*EmbeddedContainerTemplate),
 						Service:  &Service{},
 					}
-					cRef.Arguments = fixTmplRef.Arguments
+					if fixTmplRef.Arguments != nil {
+						cRef.Arguments = fixTmplRef.Arguments
+					} else {
+						cRef.Arguments = make(template.Arguments)
+					}
+					axErr = inferArgumentsToChild(tmpl, ctx.Templates[fixTmplRef.Template], cRef.Arguments)
+					if axErr != nil {
+						return nil, axErr
+					}
 					eTmpl.Fixtures[i][fixRefName] = &EmbeddedFixtureTemplateRef{
 						EmbeddedContainerTemplateRef: &cRef,
 					}

saas/common/run-test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+SRCROOT=`dirname $0`/../../
+SRCROOT=`cd $SRCROOT;pwd`
+source $SRCROOT/saas/build_env.sh
+source $SRCROOT/saas/test-helper.sh
+
+set -e
+
+go test applatix.io/template
+

saas/common/src/applatix.io/template/bad/container.yaml

@@ -83,3 +83,27 @@ command: ["sh", "-c"]
 args: ["echo '%%echo%%'"]
 resources:
   cpu_cores: abc
+
+---
+type: container
+version: 1
+name: input-artifact-missing-path
+description: input artifacts to containers should specify path
+image: alpine:latest
+command: ["sh", "-c"]
+args: ["echo '%%echo%%'"]
+inputs:
+  artifacts:
+    CODE:
+
+---
+type: container
+version: 1
+name: input-volume-missing-mount_path
+description: input volume to containers should specify mount_path
+image: alpine:latest
+command: ["sh", "-c"]
+args: ["echo '%%echo%%'"]
+inputs:
+  volumes:
+    CODE:

saas/common/src/applatix.io/template/bad/deployment.yaml

@@ -65,3 +65,43 @@ containers:
       parameters.sleep: "%%volumes.mylogvol%%"
       volumes.data-vol: "%%inputs.volumes.mydatavol%%"
       volumes.log-vol: "%%volumes.mylogvol%%"
+
+---
+type: deployment
+version: 1
+name: deployment-with-input-artifact-path
+description: deployments should not set 'path' for input artifacts
+application_name: sleep-service
+deployment_name: sleep-service
+inputs:
+  artifacts:
+    DATA:
+      path: /data
+containers:
+  userdb:
+    image: postgres:9.6.1
+    inputs:
+      artifacts:
+        DATA:
+          from: "%%inputs.artifacts.DATA%%"
+          path: /data
+
+---
+type: deployment
+version: 1
+name: deployment-with-input-volume-mount_path
+description: deployments should not set 'path' for input volumes
+application_name: sleep-service
+deployment_name: sleep-service
+inputs:
+  volumes:
+    DATA:
+      mount_path: /data
+containers:
+  userdb:
+    image: postgres:9.6.1
+    inputs:
+      volumes:
+        DATA:
+          from: "%%inputs.artifacts.DATA%%"
+          mount_path: /data

saas/common/src/applatix.io/template/bad/workflow.yaml

@@ -301,4 +301,44 @@ args: ["find /src && && sleep 5"]
 inputs:
   artifacts:
     ARTIFACT_TO_CONSUME:
-      path: /src
+      path: /src
+
+---
+type: workflow
+version: 1
+name: workflow-with-input-artifact-path
+description: workflows should not set 'path' for input artifacts
+inputs:
+  artifacts:
+    ARTIFACT_TO_CONSUME:
+      path: /src
+steps:
+- CONSUME_INLINE:
+    image: alpine:latest
+    command: ["sh", "-c"]
+    args: ["find /src && && sleep 5"]
+    inputs:
+      artifacts:
+        ARTIFACT_TO_CONSUME:
+          from: "%%inputs.artifacts.ARTIFACT_TO_CONSUME%%"
+          path: /src
+
+---
+type: workflow
+version: 1
+name: workflow-with-input-volume-mount_path
+description: workflows should not set 'mount_path' for input volumes
+inputs:
+  volumes:
+    DATA:
+      mount_path: /src
+steps:
+- CONSUME_INLINE:
+    image: alpine:latest
+    command: ["sh", "-c"]
+    args: ["find /src && && sleep 5"]
+    inputs:
+      volumes:
+        DATA:
+          from: "%%inputs.volumes.DATA%%"
+          mount_path: /src