updated cloud function deployment handling

docker/alpine/build.yaml

@@ -38,7 +38,7 @@ pipeline:
     action: docker:build
     path: ${buildPath}/build/
     arguments:
-#      '--no-cache': ''
+      '--no-cache': ''
     tag:
       image: go_endly
       version: $version
@@ -69,7 +69,7 @@ pipeline:
       force: true
       path: ${buildPath}/compact/
       arguments:
-#        '--no-cache': ''
+        '--no-cache': ''
       tag:
         image: endly
         version: ${version}-alpine3.8

docker/ubuntu/build.yaml

@@ -39,7 +39,7 @@ pipeline:
       action: docker:build
       path: ${buildPath}/build/
       arguments:
-#        '--no-cache': ''
+        '--no-cache': ''
       tag:
         image: goendly
         version: ${version}-ubuntu16.04
@@ -67,7 +67,7 @@ pipeline:
       action: docker:build
       path: ${buildPath}/compact/
       arguments:
-#        '--no-cache': ''
+        '--no-cache': ''
       tag:
         image: endly
         version: ${version}-ubuntu16.04

go.mod

@@ -55,8 +55,8 @@ require (
 	github.com/viant/asc v0.2.0
 	github.com/viant/assertly v0.4.1
 	github.com/viant/bgc v0.5.0
-	github.com/viant/dsc v0.4.0
-	github.com/viant/dsunit v0.6.0
+	github.com/viant/dsc v0.4.1
+	github.com/viant/dsunit v0.6.1
 	github.com/viant/neatly v0.7.0
 	github.com/viant/toolbox v0.12.0
 	github.com/yuin/gopher-lua v0.0.0-20181109042959-a0dfe84f6227 // indirect

go.sum

@@ -157,6 +157,8 @@ github.com/viant/dsc v0.3.1 h1:nacuk40qVGLAJWbHEoMzE9FrSMVlQvTcRElamRE4PyI=
 github.com/viant/dsc v0.3.1/go.mod h1:vkBPh3XSXUBB/ePbEO0VsKPaiN4JLRuB3QVftbS6KI4=
 github.com/viant/dsc v0.4.0 h1:b4v2ukl/okzAGqX+SdIi8mNN7G+w39VPcVqWpnfyGiQ=
 github.com/viant/dsc v0.4.0/go.mod h1:vkBPh3XSXUBB/ePbEO0VsKPaiN4JLRuB3QVftbS6KI4=
+github.com/viant/dsc v0.4.1 h1:DQFOfEY8f7LjACqqOaHE2wCgkpiQHoDKeMlDU8bqpLE=
+github.com/viant/dsc v0.4.1/go.mod h1:vkBPh3XSXUBB/ePbEO0VsKPaiN4JLRuB3QVftbS6KI4=
 github.com/viant/dsunit v0.2.1 h1:6dQrsMLNQIgd3lQJBZ6vGXJOrV/r7E9JMZZ1rmxkWhI=
 github.com/viant/dsunit v0.2.1/go.mod h1:QL5nCpnROplJ6lNbuh4aHlov+1/y3vyPgdVg2BUOkrw=
 github.com/viant/dsunit v0.3.0 h1:tyuaYQQcpHAmFEBpwuqjuZ5hVhXM2zSQK38UsxklT4E=
@@ -173,6 +175,8 @@ github.com/viant/dsunit v0.5.0 h1:rBbJcNGO1Uu/LOLF11/Ej4KhAfYpKWVJCJB8hBF9rYU=
 github.com/viant/dsunit v0.5.0/go.mod h1:QL5nCpnROplJ6lNbuh4aHlov+1/y3vyPgdVg2BUOkrw=
 github.com/viant/dsunit v0.6.0 h1:1FrI65GRfGsdlYF82SNf933HhyJpX9HqthbDYR4x/0Y=
 github.com/viant/dsunit v0.6.0/go.mod h1:QL5nCpnROplJ6lNbuh4aHlov+1/y3vyPgdVg2BUOkrw=
+github.com/viant/dsunit v0.6.1 h1:0KzTqPJF3O/OpohLPNhGpemly/rF7HSkwWJaM8CtEQE=
+github.com/viant/dsunit v0.6.1/go.mod h1:QL5nCpnROplJ6lNbuh4aHlov+1/y3vyPgdVg2BUOkrw=
 github.com/viant/neatly v0.4.3 h1:ysOUO3uHxOa94HSSIFTcbjnVbtSXVOGAEfmQmZcYQu8=
 github.com/viant/neatly v0.4.3/go.mod h1:4G3EAdqY6PW7e0m825UD/imWZ04Kd6pMXkkWvGK5q78=
 github.com/viant/neatly v0.4.4 h1:KCp/MeVpBhBVGpMfsP03/PeaCekWQZglwMwdDHC1NyY=

model/inline_workflow.go

@@ -245,12 +245,15 @@ func (p *InlineWorkflow) AsWorkflow(name string, baseURL string) (*Workflow, err
 	}
 	root := p.buildTask("", map[string]interface{}{})
 	tagID := name
-	for _, entry := range p.Pipeline {
-		if err = p.buildWorkflowNodes(entry.Key, entry.Value, root, tagID, p.State); err != nil {
-			return nil, err
+
+	if len( p.Pipeline) > 0 {
+		for _, entry := range p.Pipeline {
+			if err = p.buildWorkflowNodes(entry.Key, entry.Value, root, tagID, p.State); err != nil {
+				return nil, err
+			}
 		}
 	}
-	p.normalize(root.TasksNode)
+
 	if len(root.Tasks) > 0 {
 		workflow.TasksNode = root.TasksNode
 	} else {
@@ -260,9 +263,10 @@ func (p *InlineWorkflow) AsWorkflow(name string, baseURL string) (*Workflow, err
 	}
 	p.workflow = workflow
 	return workflow, nil
-
 }
 
+
+
 func (p *InlineWorkflow) normalize(node *TasksNode) {
 	for _, task := range node.Tasks {
 		if task.Name == CatchTask {

system/cloud/gc/cloudfunctions/README.md

@@ -87,6 +87,23 @@ pipeline:
         source:
           URL: test/
     ```
+3. Deploying with eventTrigger
+    @deploy_with_trigger
+    ```yaml
+    pipeline:
+      deploy:
+        action: gc/cloudfunctions:deploy
+        '@name': MyFunction
+        entryPoint: MyFunctionFN
+        runtime: go111
+        eventTrigger:
+          eventType: google.storage.object.finalize
+          resource: projects/_/buckets/myBucket
+        source:
+          URL: test/
+    
+    ```
+
 
 ###### Calling function
 

system/cloud/gc/cloudfunctions/deploy.yaml

@@ -2,6 +2,7 @@ defaults:
   credentials: am
 pipeline:
   deploy:
+    when: false
     action: gc/cloudfunctions:deploy
     '@name': HelloWorld
     entryPoint: HelloWorld

system/cloud/gc/cloudfunctions/helper.go

@@ -37,5 +37,11 @@ func getIgnoreList(service storage.Service, URL string) []string {
 	if err != nil {
 		return list
 	}
-	return strings.Split(toolbox.AsString(content),",")
+	for _, item := range  strings.Split(toolbox.AsString(content),",") {
+		if strings.HasPrefix(item, "#") {
+			continue
+		}
+		list = append(list, item)
+	}
+	return list
 }

system/cloud/gc/cloudfunctions/service.go

@@ -26,8 +26,6 @@ const (
 	ServiceID = "gc/cloudfunctions"
 )
 
-
-
 //no operation service
 type service struct {
 	*endly.AbstractService
@@ -64,6 +62,11 @@ func (s *service) Deploy(context *endly.Context, request *DeployRequest) (*Deplo
 	response.Function, err = s.get(context, &GetRequest{
 		Name: request.Name,
 	})
+
+	if response.Function.Status == "OFFLINE" {
+		context.Publish(gc.NewOutputEvent(request.Name, "deploy", response))
+		return nil, fmt.Errorf("failed to deploy funciton")
+	}
 	return response, err
 }
 
@@ -100,27 +103,24 @@ func (s *service) get(context *endly.Context, request *GetRequest) (*cloudfuncti
 
 }
 
-
-
-
 func (s *service) getFunctionPackageReader(resource *url.Resource) (io.ReadCloser, error) {
 	storageService, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
 	if err != nil {
 		return nil, err
 	}
-	object, err  := storageService.StorageObject(resource.URL)
+	object, err := storageService.StorageObject(resource.URL)
 	if err != nil {
 		return nil, err
 	}
 	if object.IsContent() {
-		return  storageService.DownloadWithURL(resource.URL)
+		return storageService.DownloadWithURL(resource.URL)
 	}
 	ignoreList := getIgnoreList(storageService, toolbox.URLPathJoin(resource.URL, ".gcloudignore"))
 	writer := new(bytes.Buffer)
 	archive := zip.NewWriter(writer)
 	err = storage.ArchiveWithFilter(storageService, resource.URL, archive, func(candidate storage.Object) bool {
 		candidateName := candidate.FileInfo().Name()
-		if strings.HasSuffix(candidateName, ".zip")  || strings.HasPrefix(candidateName, "gcloudignore"){
+		if strings.HasSuffix(candidateName, ".zip") || strings.HasPrefix(candidateName, "gcloudignore") {
 			return false
 		}
 		if len(ignoreList) == 0 {
@@ -139,14 +139,20 @@ func (s *service) getFunctionPackageReader(resource *url.Resource) (io.ReadClose
 	return ioutil.NopCloser(bytes.NewReader(payload)), err
 }
 
-
 func (s *service) deploy(context *endly.Context, request *DeployRequest) (*cloudfunctions.Operation, error) {
 	ctxClient, err := GetClient(context)
 	if err != nil {
 		return nil, err
 	}
 	parent := s.expandWithContext(context, ctxClient.CredConfig, request.Region, parentLocationTemplate)
 	request.Name = s.expandWithContext(context, ctxClient.CredConfig, request.Region, request.Name)
+
+
+	//TODO add support for simple name based on trigger type
+	if request.EventTrigger != nil {
+		request.EventTrigger.Resource = s.expandWithContext(context, ctxClient.CredConfig, request.Region, request.EventTrigger.Resource)
+	}
+
 	projectService := cloudfunctions.NewProjectsLocationsFunctionsService(ctxClient.service)
 	cloudFunction, err := projectService.Get(request.Name).Do()
 	if err != nil {
@@ -186,7 +192,6 @@ func (s *service) deploy(context *endly.Context, request *DeployRequest) (*cloud
 	return updateCall.Do()
 }
 
-
 func (s *service) list(context *endly.Context, request *ListRequest) (*ListResponse, error) {
 	ctxClient, err := GetClient(context)
 	if err != nil {
@@ -196,16 +201,15 @@ func (s *service) list(context *endly.Context, request *ListRequest) (*ListRespo
 	parent := s.expandWithContext(context, ctxClient.CredConfig, request.Region, parentLocationTemplate)
 	listCall := projectService.List(parent)
 	listCall.Context(ctxClient.Context())
-	list , err := listCall.Do()
+	list, err := listCall.Do()
 	if err != nil {
 		return nil, err
 	}
 	return &ListResponse{
-		Function:list.Functions,
+		Function: list.Functions,
 	}, nil
 }
 
-
 func (s *service) delete(context *endly.Context, request *DeleteRequest) (*DeleteResponse, error) {
 	ctxClient, err := GetClient(context)
 	if err != nil {
@@ -241,7 +245,7 @@ func (s *service) call(context *endly.Context, request *CallRequest) (*cloudfunc
 	projectService := cloudfunctions.NewProjectsLocationsFunctionsService(ctxClient.service)
 	request.Name = s.expandWithContext(context, ctxClient.CredConfig, request.Region, request.Name)
 	callFunctionRequest := &cloudfunctions.CallFunctionRequest{}
-	if request.Data != nil {//TODO check for binary to encode with base64
+	if request.Data != nil { //TODO check for binary to encode with base64
 		dataText := ""
 		if toolbox.IsMap(request.Data) || toolbox.IsSlice(request.Data) {
 			JSON, err := toolbox.AsJSONText(request.Data)
@@ -259,7 +263,6 @@ func (s *service) call(context *endly.Context, request *CallRequest) (*cloudfunc
 	return call.Do()
 }
 
-
 func (s *service) registerRoutes() {
 	client := &cloudfunctions.Service{}
 	routes, err := gc.BuildRoutes(client, func(name string) string {
@@ -409,15 +412,14 @@ func (s *service) registerRoutes() {
 		},
 	})
 	for _, route := range routes {
-		 if _, err := s.Route(route.Action);err == nil {
-		 	continue
-		 }
+		if _, err := s.Route(route.Action); err == nil {
+			continue
+		}
 		route.OnRawRequest = InitRequest
 		s.Register(route)
 	}
 }
 
-
 //New creates a new Dataflow service
 func New() endly.Service {
 	var result = &service{

system/cloud/gc/cloudfunctions/service_test.go

@@ -42,7 +42,7 @@ func TestNew(t *testing.T) {
 	response := make(map[string]interface{})
 	err = endly.Run(context, request, &response)
 	assert.Nil(t, err)
-	toolbox.DumpIndent(response, true)
+
 }
 
 
@@ -66,7 +66,6 @@ func TestService_Deploy(t *testing.T) {
 			EntryPoint:   "HelloWorld",
 			Runtime:      "go111",
 			HttpsTrigger: &cloudfunctions.HttpsTrigger{},
-
 		},
 		Source:   url.NewResource(path.Join(parent, "test/")),
 	}, &response)