diff --git a/assets/openapi.json b/assets/openapi.json index 4e1a872..a4ecb3d 100755 --- a/assets/openapi.json +++ b/assets/openapi.json @@ -1091,6 +1091,16 @@ "schema": { "$ref": "#/components/schemas/ID" } + }, + { + "name": "remote-source", + "in": "query", + "description": "Remote source git URL from which get the source files.", + "required": false, + "example": "git+https://github.com/konveyor/move2kube", + "schema": { + "$ref": "#/components/schemas/RemoteSource" + } } ], "responses": { @@ -1313,6 +1323,16 @@ "schema": { "$ref": "#/components/schemas/ID" } + }, + { + "name": "skip-qa", + "in": "query", + "description": "Boolean to skip interactive QA.", + "required": false, + "example": "true", + "schema": { + "type": "boolean" + } } ], "requestBody": { @@ -2295,6 +2315,12 @@ "description": "A unique ID.", "example": "id-1234" }, + "RemoteSource": { + "pattern": "^git[+](https|ssh)://[a-zA-Z0-9]+([-.]{1}[a-zA-Z0-9]+)*[.][a-zA-Z]{2,5}(:[0-9]{1,5})?(\/.*)?$", + "type": "string", + "description": "A git URL.", + "example": "git+https://github.com/konveyor/move2kube" + }, "Error": { "required": [ "error" diff --git a/internal/common/constants.go b/internal/common/constants.go index bc626ba..1ec0473 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -65,6 +65,8 @@ var ( AuthServerClient gocloak.GoCloak // ID_REGEXP is the regexp used to check if a Id is valid ID_REGEXP = regexp.MustCompile("^[a-zA-Z0-9-_]+$") + // REMOTE_SOURCE_REGEXP is the regexp used to check if a remote source is valid + REMOTE_SOURCE_REGEXP = regexp.MustCompile(`^git\+(https|ssh)://[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,5}(:[0-9]{1,5})?(\/.*)?$`) // INVALID_NAME_CHARS_REGEXP is the regexp used to replace invalid name characters with hyphen INVALID_NAME_CHARS_REGEXP = regexp.MustCompile("[^a-z0-9-]") // AUTHZ_HEADER is the authorization header diff --git a/internal/common/utils.go b/internal/common/utils.go index 18bc875..8e4bccd 100644 --- a/internal/common/utils.go +++ b/internal/common/utils.go @@ -123,6 +123,11 @@ func IsValidId(id string) bool { return ID_REGEXP.MatchString(id) } +// IsRemoteSource returns true if the provided remoteSource is valid +func IsRemoteSource(remoteSource string) bool { + return REMOTE_SOURCE_REGEXP.MatchString(remoteSource) +} + // IsStringPresent checks if a value is present in a slice func IsStringPresent(list []string, value string) bool { for _, val := range list { diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go index 827418d..f0a49b9 100644 --- a/internal/filesystem/filesystem.go +++ b/internal/filesystem/filesystem.go @@ -1172,7 +1172,7 @@ func (fs *FileSystem) deleteProjectInput(t *bolt.Tx, workspaceId, projectId, pro // StartPlanning starts the generation of a plan for a project. // If plan generation is ongoing it will return an error. -func (fs *FileSystem) StartPlanning(workspaceId, projectId string, debugMode bool) error { +func (fs *FileSystem) StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error { logrus.Trace("FileSystem.StartPlanning start") defer logrus.Trace("FileSystem.StartPlanning end") db, err := fs.GetDatabase(false) @@ -1182,11 +1182,11 @@ func (fs *FileSystem) StartPlanning(workspaceId, projectId string, debugMode boo } defer db.Close() return db.Update(func(t *bolt.Tx) error { - return fs.startPlanning(t, workspaceId, projectId, debugMode) + return fs.startPlanning(t, workspaceId, projectId, remoteSource, debugMode) }) } -func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, debugMode bool) error { +func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId, remoteSource string, debugMode bool) error { logrus.Trace("FileSystem.startPlanning start") defer logrus.Trace("FileSystem.startPlanning end") // check conditions @@ -1198,7 +1198,7 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, d return types.ErrorOngoing{Id: projectId} } - if !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] { + if remoteSource == "" && !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] { if !project.Status[types.ProjectStatusInputReference] { return types.ErrorValidation{Reason: "the project has no source folders or customization folders as input"} } @@ -1261,8 +1261,8 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, d return fmt.Errorf("failed to resolve the temporary directory '%s' as a symbolic link. Error: %w", currentRunDir, err) } // default is empty string, if the input source is given, value is updated. - currentRunSrcDir := "" - if project.Status[types.ProjectStatusInputSources] { + currentRunSrcDir := remoteSource + if currentRunSrcDir == "" && project.Status[types.ProjectStatusInputSources] { currentRunSrcDir = filepath.Join(currentRunDir, SOURCES_DIR) currentRunSrcDirSrc := filepath.Join(projInputsDir, EXPANDED_DIR, SOURCES_DIR) if err := copyDir(currentRunSrcDirSrc, currentRunSrcDir); err != nil { @@ -1305,7 +1305,7 @@ func (fs *FileSystem) startPlanning(t *bolt.Tx, workspaceId, projectId string, d inpPathSrc := "" inpPathDst := "" workInp := work.Inputs[inp.Id] - if workInp.Type == types.ProjectInputSources { + if workInp.Type == types.ProjectInputSources && remoteSource != "" { if currentRunSrcDir == "" { currentRunSrcDir = filepath.Join(currentRunDir, SOURCES_DIR) } @@ -1515,18 +1515,18 @@ func (fs *FileSystem) deletePlan(t *bolt.Tx, workspaceId, projectId string) erro } // ResumeTransformation resumes a transformation that did not finish -func (fs *FileSystem) ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode bool) error { +func (fs *FileSystem) ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error { db, err := fs.GetDatabase(false) if err != nil { return err } defer db.Close() return db.Update(func(t *bolt.Tx) error { - return fs.resumeTransformation(t, workspaceId, projectId, projOutputId, debugMode) + return fs.resumeTransformation(t, workspaceId, projectId, projOutputId, debugMode, skipQA) }) } -func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, projOutputId string, debugMode bool) error { +func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error { // check conditions project, err := fs.readProject(t, workspaceId, projectId) if err != nil { @@ -1617,23 +1617,23 @@ func (fs *FileSystem) resumeTransformation(t *bolt.Tx, workspaceId, projectId, p currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...) } // resume the transformation - go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, true) + go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, true) return nil } // StartTransformation starts the transformation for a project. -func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode bool) error { +func (fs *FileSystem) StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error { db, err := fs.GetDatabase(false) if err != nil { return err } defer db.Close() return db.Update(func(t *bolt.Tx) error { - return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode) + return fs.startTransformation(t, workspaceId, projectId, projOutput, plan, debugMode, skipQA) }) } -func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode bool) error { +func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error { // check conditions project, err := fs.readProject(t, workspaceId, projectId) if err != nil { @@ -1642,7 +1642,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str if _, ok := project.Outputs[projOutput.Id]; ok { return types.ErrorIdAlreadyInUse{Id: projOutput.Id} } - if !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] { + if !project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusInputCustomizations] && !project.Status[types.ProjectStatusRemoteInputSources] { if !project.Status[types.ProjectStatusInputReference] { return types.ErrorValidation{Reason: "the project has no source or customization folders as input"} } @@ -1727,7 +1727,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str // default is empty string, if the input source is given, the value is updated currentRunSrcDir := "" // copy the source and customizations directories into the run directory - if project.Status[types.ProjectStatusInputSources] { + if project.Status[types.ProjectStatusInputSources] && !project.Status[types.ProjectStatusRemoteInputSources] { currentRunSrcDir = filepath.Join(currentRunDir, SOURCES_DIR) srcPath := filepath.Join(projInputsDir, EXPANDED_DIR, SOURCES_DIR) if err := copyDir(srcPath, currentRunSrcDir); err != nil { @@ -1811,7 +1811,7 @@ func (fs *FileSystem) startTransformation(t *bolt.Tx, workspaceId, projectId str currentRunConfigPaths = append(commonConfigPaths, currentRunConfigPaths...) } // start the transformation - go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, false) + go fs.runTransform(currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message, qaServerMeta.Port, transformCh, workspaceId, projectId, projOutput, debugMode, skipQA, false) logrus.Infof("Waiting for QA engine to start for the output '%s' of the project '%s'", projOutput.Id, projectId) if err := <-transformCh; err != nil { return fmt.Errorf("failed to start the transformation and qa engine. Error: %w", err) @@ -2156,7 +2156,7 @@ func NewFileSystem() (*FileSystem, error) { } for _, project := range projects { for _, projOutput := range project.Outputs { - if err := fileSystem.ResumeTransformation(workspace.Id, project.Id, projOutput.Id, false); err != nil { + if err := fileSystem.ResumeTransformation(workspace.Id, project.Id, projOutput.Id, false, false); err != nil { logrus.Debugf("failed to resume the transformation for output with id: %s of project id: %s . Error: %q", projOutput.Id, project.Id, err) } } @@ -2206,8 +2206,8 @@ func validateAndProcessPlan(plan string, shouldProcess bool) (string, error) { return "", fmt.Errorf("'spec.sourceDir' is missing from the plan") } else if pSpecSourceDir, ok := pSpecSourceDirI.(string); !ok { return "", fmt.Errorf("'spec.sourceDir' is not a string. Actual value is %+v of type %T", pSpecSourceDirI, pSpecSourceDirI) - } else if pSpecSourceDir != SOURCES_DIR && pSpecSourceDir != "" { - return "", fmt.Errorf("'spec.sourceDir' is invalid. Expected 'source' . Actual: %s", pSpecSourceDir) + } else if pSpecSourceDir != SOURCES_DIR && pSpecSourceDir != "" && !strings.HasPrefix(pSpecSourceDir, "git+https://") { + return "", fmt.Errorf("'spec.sourceDir' is invalid. Expected 'source' or 'git+https:// . Actual: %s", pSpecSourceDir) } else { // TODO: better processing of the plan pMeta["name"], _ = common.NormalizeName(pMetaName) @@ -2333,6 +2333,9 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri } // update state logrus.Debug("planning finished. inside Update. just before update start") + if strings.HasPrefix(currentRunSrcDir, "git+https://") { + project.Status[types.ProjectStatusRemoteInputSources] = true + } project.Status[types.ProjectStatusPlanning] = false project.Status[types.ProjectStatusPlan] = true project.Status[types.ProjectStatusStalePlan] = false @@ -2372,7 +2375,7 @@ func (fs *FileSystem) runPlan(currentRunDir string, currentRunConfigPaths []stri return err } -func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, overwriteOutDir bool) error { +func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths []string, currentRunSrcDir, currentRunCustDir, currentRunOutDir, message string, port int, transformCh chan error, workspaceId, projectId string, projOutput types.ProjectOutput, debugMode bool, skipQA bool, overwriteOutDir bool) error { logrus.Infof("Starting transformation in %s with configs from %+v and source from %s , customizations from %s and output to %s", currentRunDir, currentRunConfigPaths, currentRunSrcDir, currentRunCustDir, currentRunOutDir) portStr, err := cast.ToStringE(port) if err != nil { @@ -2389,6 +2392,9 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [ if verbose { cmdArgs = append(cmdArgs, "--log-level", "trace") } + if skipQA { + cmdArgs = append(cmdArgs, "--qa-skip") + } if !common.Config.EnableLocalExecution { cmdArgs = append(cmdArgs, "--disable-local-execution") } @@ -2421,6 +2427,19 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [ logrus.Errorf("failed to start the transform command. Error: %q", err) return err } + flag := true + go func() { + if err := cmd.Wait(); err != nil { + logrus.Errorf("failed to wait for end of the transform command. Error: %q", err) + } + logrus.Debugf("Closing transformCh: %t", flag) + if flag { + flag = false + transformCh <- nil + close(transformCh) + } + + }() wg := sync.WaitGroup{} outCh := make(chan string, 10) stdoutReader := bufio.NewReader(stdout) @@ -2448,17 +2467,17 @@ func (fs *FileSystem) runTransform(currentRunDir string, currentRunConfigPaths [ } text, err = stderrReader.ReadString('\n') } - logrus.Debugf("failed to fetch the stderr of move2kube transform. Error: %q", err) + logrus.Infof("failed to fetch the stderr of move2kube transform. Error: %q", err) wg.Done() }() go func() { wg.Wait() close(outCh) }() - flag := true for outputLine := range outCh { if flag && strings.Contains(outputLine, portStr) { flag = false + logrus.Debug("Closing transformCh") transformCh <- nil close(transformCh) } diff --git a/internal/filesystem/interface.go b/internal/filesystem/interface.go index a4cca47..59a4661 100644 --- a/internal/filesystem/interface.go +++ b/internal/filesystem/interface.go @@ -40,12 +40,12 @@ type IFileSystem interface { CreateProjectInput(workspaceId, projectId string, projInput types.ProjectInput, file io.Reader, isCommon bool) error ReadProjectInput(workspaceId, projectId, projInputId string, isCommon bool) (projInput types.ProjectInput, file io.Reader, err error) DeleteProjectInput(workspaceId, projectId, projInputId string, isCommon bool) error - StartPlanning(workspaceId, projectId string, debugMode bool) error + StartPlanning(workspaceId, projectId, remoteSource string, debugMode bool) error ReadPlan(workspaceId, projectId string) (plan io.Reader, err error) UpdatePlan(workspaceId, projectId string, plan io.Reader) error DeletePlan(workspaceId, projectId string) error - StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode bool) error - ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode bool) error + StartTransformation(workspaceId, projectId string, projOutput types.ProjectOutput, plan io.Reader, debugMode, skipQA bool) error + ResumeTransformation(workspaceId, projectId, projOutputId string, debugMode, skipQA bool) error ReadProjectOutput(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error) ReadProjectOutputGraph(workspaceId, projectId, projOutputId string) (projOutput types.ProjectOutput, file io.Reader, err error) DeleteProjectOutput(workspaceId, projectId, projOutputId string) error diff --git a/internal/move2kubeapi/handlers/handlers.go b/internal/move2kubeapi/handlers/handlers.go index daf0564..e5f4bbe 100644 --- a/internal/move2kubeapi/handlers/handlers.go +++ b/internal/move2kubeapi/handlers/handlers.go @@ -30,6 +30,10 @@ import ( ) const ( + // SKIP_QA_QUERY_PARAM is the name of the query parameter used for skipping QA + SKIP_QA_QUERY_PARAM = "skip-qa" + // REMOTE_SOURCE_QUERY_PARAM is the URL of the git remote to be used as source + REMOTE_SOURCE_QUERY_PARAM = "remote-source" // DEBUG_QUERY_PARAM is the name of the query parameter used for debug mode DEBUG_QUERY_PARAM = "debug" // WORKSPACE_ID_ROUTE_VAR is the route variable that contains the workspace Id diff --git a/internal/move2kubeapi/handlers/outputs.go b/internal/move2kubeapi/handlers/outputs.go index ca885b5..29ee4ab 100644 --- a/internal/move2kubeapi/handlers/outputs.go +++ b/internal/move2kubeapi/handlers/outputs.go @@ -54,6 +54,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) { planReader = nil } debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true" + skipQA := r.URL.Query().Get(SKIP_QA_QUERY_PARAM) == "true" timestamp, _, err := common.GetTimestamp() if err != nil { logrus.Errorf("failed to get the timestamp. Error: %q", err) @@ -65,7 +66,7 @@ func HandleStartTransformation(w http.ResponseWriter, r *http.Request) { projOutput.Timestamp = timestamp projOutput.Name = projOutput.Id // This isn't really used anywhere projOutput.Status = types.ProjectOutputStatusInProgress - if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode); err != nil { + if err := m2kFS.StartTransformation(workspaceId, projectId, projOutput, planReader, debugMode, skipQA); err != nil { logrus.Errorf("failed to start the transformation. Error: %q", err) if notExErr, ok := err.(types.ErrorDoesNotExist); ok { if notExErr.Id == "plan" { diff --git a/internal/move2kubeapi/handlers/plan.go b/internal/move2kubeapi/handlers/plan.go index 3fdeb97..9fbc004 100644 --- a/internal/move2kubeapi/handlers/plan.go +++ b/internal/move2kubeapi/handlers/plan.go @@ -39,8 +39,14 @@ func HandleStartPlanning(w http.ResponseWriter, r *http.Request) { sendErrorJSON(w, "invalid id", http.StatusBadRequest) return } + remoteSource := r.URL.Query().Get(REMOTE_SOURCE_QUERY_PARAM) + if remoteSource != "" && !common.IsRemoteSource(remoteSource) { + logrus.Errorf("invalid remote source format; not matching regexp %s. Actual: %s", common.REMOTE_SOURCE_REGEXP, remoteSource) + sendErrorJSON(w, "invalid remote source format", http.StatusBadRequest) + return + } debugMode := r.URL.Query().Get(DEBUG_QUERY_PARAM) == "true" - if err := m2kFS.StartPlanning(workspaceId, projectId, debugMode); err != nil { + if err := m2kFS.StartPlanning(workspaceId, projectId, remoteSource, debugMode); err != nil { logrus.Errorf("failed to start plan generation. Error: %q", err) if _, ok := err.(types.ErrorDoesNotExist); ok { w.WriteHeader(http.StatusNotFound) diff --git a/internal/types/types.go b/internal/types/types.go index 256357f..3d5dce1 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -346,6 +346,8 @@ func (e ErrorTokenUnverifiable) Error() string { type ProjectStatus string const ( + // ProjectStatusRemoteInputSources indicates the project has a remlote git source + ProjectStatusRemoteInputSources ProjectStatus = "remote" // ProjectStatusInputSources indicates the project has source folder uploaded ProjectStatusInputSources ProjectStatus = "sources" // ProjectStatusInputCustomizations indicates the project has customizations folder uploaded