Increase length limit for ID and Name for pipeline (128) and connecto…

…r (256) (#1331) * increase connector and pipeline id and length limit to 128 * add validation for id, name and desc to pkg/provisioning config remove pipeline health check cmd, obsolated by `-pipelines.exit-on-error` flag --------- Co-authored-by: Lyubo Kamenov <lyubo@meroxa.io>
ConduitIO · Jan 8, 2024 · afc939c · afc939c
1 parent 2f3205f
commit afc939c
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 161 deletions.
diff --git a/Makefile b/Makefile
@@ -9,7 +9,7 @@ VERSION=`git describe --tags --dirty`
 GO_VERSION_CHECK=`./scripts/check-go-version.sh`
 
 # The build target should stay at the top since we want it to be the default target.
-build: check-go-version pkg/web/ui/dist build-pipeline-check
+build: check-go-version pkg/web/ui/dist
 	go build -ldflags "-X 'github.com/conduitio/conduit/pkg/conduit.version=${VERSION}'" -o conduit -tags ui ./cmd/conduit/main.go
 	@echo "\nBuild complete. Enjoy using Conduit!"
 	@echo "Get started by running:"
@@ -39,9 +39,6 @@ build-server: check-go-version
 	go build -ldflags "-X 'github.com/conduitio/conduit/pkg/conduit.version=${VERSION}'" -o conduit ./cmd/conduit/main.go
 	@echo "build version: ${VERSION}"
 
-build-pipeline-check: check-go-version
-	go build -o conduit-pipeline-check ./cmd/conduit-pipeline-check/main.go
-
 run:
 	go run ./cmd/conduit/main.go
 
@@ -56,7 +53,6 @@ proto-lint:
 
 clean:
 	@rm -f conduit
-	@rm -f conduit-pipeline-check
 	@rm -rf pkg/web/ui/dist
 
 download:

diff --git a/cmd/conduit-pipeline-check/main.go b/cmd/conduit-pipeline-check/main.go
diff --git a/pkg/connector/service.go b/pkg/connector/service.go
@@ -29,6 +29,11 @@ import (
 
 var idRegex = regexp.MustCompile(`^[A-Za-z0-9-_:.]*$`)
 
+const (
+	IDLengthLimit   = 256
+	NameLengthLimit = 256
+)
+
 // Service manages connectors.
 type Service struct {
 	logger log.CtxLogger
@@ -291,7 +296,7 @@ func (s *Service) validateConnector(cfg Config, id string) error {
 	if cfg.Name == "" {
 		multierr = multierror.Append(multierr, ErrNameMissing)
 	}
-	if len(cfg.Name) > 64 {
+	if len(cfg.Name) > NameLengthLimit {
 		multierr = multierror.Append(multierr, ErrNameOverLimit)
 	}
 	if id == "" {
@@ -301,7 +306,7 @@ func (s *Service) validateConnector(cfg Config, id string) error {
 	if !matched {
 		multierr = multierror.Append(multierr, ErrInvalidCharacters)
 	}
-	if len(id) > 64 {
+	if len(id) > IDLengthLimit {
 		multierr = multierror.Append(multierr, ErrIDOverLimit)
 	}
 

diff --git a/pkg/connector/service_test.go b/pkg/connector/service_test.go
@@ -17,6 +17,7 @@ package connector
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -308,16 +309,16 @@ func TestService_Create_ValidateSuccess(t *testing.T) {
 		data   Config
 	}{{
 		name:   "valid config name",
-		connID: uuid.NewString(),
+		connID: strings.Repeat("a", 128),
 		data: Config{
-			Name:     "Name#@-/_0%$",
+			Name:     strings.Repeat("a", 128),
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}, {
 		name:   "valid connector ID",
 		connID: "Aa0-_.",
 		data: Config{
-			Name:     "test-connector",
+			Name:     "Name#@-/_0%$",
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}}
@@ -361,16 +362,16 @@ func TestService_Create_ValidateError(t *testing.T) {
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}, {
-		name:    "connector name over 64 characters",
+		name:    "connector name over 256 characters",
 		connID:  uuid.NewString(),
 		errType: ErrNameOverLimit,
 		data: Config{
-			Name:     "aaaaaaaaa1bbbbbbbbb2ccccccccc3ddddddddd4eeeeeeeee5fffffffff6ggggg",
+			Name:     strings.Repeat("a", 257),
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}, {
-		name:    "connector ID over 64 characters",
-		connID:  "aaaaaaaaa1bbbbbbbbb2ccccccccc3ddddddddd4eeeeeeeee5fffffffff6ggggg",
+		name:    "connector ID over 256 characters",
+		connID:  strings.Repeat("a", 257),
 		errType: ErrIDOverLimit,
 		data: Config{
 			Name:     "test-connector",

diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go
@@ -29,6 +29,12 @@ import (
 
 var idRegex = regexp.MustCompile(`^[A-Za-z0-9-_:.]*$`)
 
+const (
+	IDLengthLimit          = 128
+	NameLengthLimit        = 128
+	DescriptionLengthLimit = 8192
+)
+
 type FailureEvent struct {
 	// ID is the ID of the pipeline which failed.
 	ID    string
@@ -338,10 +344,10 @@ func (s *Service) validatePipeline(cfg Config, id string) error {
 	if s.instanceNames[cfg.Name] {
 		multierr = multierror.Append(multierr, ErrNameAlreadyExists)
 	}
-	if len(cfg.Name) > 64 {
+	if len(cfg.Name) > NameLengthLimit {
 		multierr = multierror.Append(multierr, ErrNameOverLimit)
 	}
-	if len(cfg.Description) > 8192 {
+	if len(cfg.Description) > DescriptionLengthLimit {
 		multierr = multierror.Append(multierr, ErrDescriptionOverLimit)
 	}
 	if id == "" {
@@ -351,7 +357,7 @@ func (s *Service) validatePipeline(cfg Config, id string) error {
 	if !matched {
 		multierr = multierror.Append(multierr, ErrInvalidCharacters)
 	}
-	if len(id) > 64 {
+	if len(id) > IDLengthLimit {
 		multierr = multierror.Append(multierr, ErrIDOverLimit)
 	}
 

diff --git a/pkg/pipeline/service_test.go b/pkg/pipeline/service_test.go
@@ -17,6 +17,7 @@ package pipeline
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/conduitio/conduit/pkg/foundation/cerrors"
@@ -155,21 +156,21 @@ func TestService_Create_ValidateSuccess(t *testing.T) {
 	service := NewService(logger, db)
 
 	testCases := []struct {
-		name   string
-		connID string
-		data   Config
+		name string
+		id   string
+		data Config
 	}{{
-		name:   "valid config name",
-		connID: uuid.NewString(),
+		name: "valid config name",
+		id:   strings.Repeat("a", 128),
 		data: Config{
-			Name:        "Name#@-/_0%$",
-			Description: "",
+			Name:        strings.Repeat("a", 128),
+			Description: strings.Repeat("a", 8192),
 		},
 	}, {
-		name:   "valid pipeline ID",
-		connID: "Aa0-_:.",
+		name: "valid pipeline ID",
+		id:   "Aa0-_:.",
 		data: Config{
-			Name:        "test-pipeline",
+			Name:        "Name#@-/_0%$",
 			Description: "",
 		},
 	}}
@@ -178,7 +179,7 @@ func TestService_Create_ValidateSuccess(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := service.Create(
 				ctx,
-				tt.connID,
+				tt.id,
 				tt.data,
 				ProvisionTypeAPI,
 			)
@@ -198,39 +199,47 @@ func TestService_Create_ValidateError(t *testing.T) {
 
 	testCases := []struct {
 		name    string
-		connID  string
+		id      string
 		errType error
 		data    Config
 	}{{
 		name:    "empty config name",
-		connID:  uuid.NewString(),
+		id:      uuid.NewString(),
 		errType: ErrNameMissing,
 		data: Config{
 			Name:        "",
 			Description: "",
 		},
 	}, {
-		name:    "pipeline name over 64 characters",
-		connID:  uuid.NewString(),
+		name:    "pipeline name over 128 characters",
+		id:      uuid.NewString(),
 		errType: ErrNameOverLimit,
 		data: Config{
-			Name:        "aaaaaaaaa1bbbbbbbbb2ccccccccc3ddddddddd4eeeeeeeee5fffffffff6ggggg",
+			Name:        strings.Repeat("a", 129),
 			Description: "",
 		},
 	}, {
-		name:    "pipeline ID over 64 characters",
-		connID:  "aaaaaaaaa1bbbbbbbbb2ccccccccc3ddddddddd4eeeeeeeee5fffffffff6ggggg",
+		name:    "pipeline ID over 128 characters",
+		id:      strings.Repeat("a", 129),
 		errType: ErrIDOverLimit,
 		data: Config{
-			Name:        "test-connector",
+			Name:        "test-pipeline",
 			Description: "",
 		},
 	}, {
-		name:    "invalid characters in connector ID",
-		connID:  "a%bc",
+		name:    "pipeline description over 8192 characters",
+		id:      uuid.NewString(),
+		errType: ErrDescriptionOverLimit,
+		data: Config{
+			Name:        "test-pipeline",
+			Description: strings.Repeat("a", 8193),
+		},
+	}, {
+		name:    "invalid characters in pipeline ID",
+		id:      "a%bc",
 		errType: ErrInvalidCharacters,
 		data: Config{
-			Name:        "test-connector",
+			Name:        "test-pipeline",
 			Description: "",
 		},
 	}}
@@ -239,7 +248,7 @@ func TestService_Create_ValidateError(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := service.Create(
 				ctx,
-				tt.connID,
+				tt.id,
 				tt.data,
 				ProvisionTypeAPI,
 			)