This repository has been archived by the owner on Jul 19, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Add validate flow on local directory if path was given
* validate without fetch * add comment * validate without fetch * add comment * template to read local data as github data * fetch local data as github data and validate on it * add hierarchy for github vs gitlab * fetch gitlab files from local * fetch gitlab files from local * iterate only on cicd relevant to the scm * change validate flags and running options * refactor path handling * refactor before PR * Cr fixes * Cr fixes
- Loading branch information
Showing
13 changed files
with
424 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| package localConnector | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "regexp" | ||
| "strings" | ||
|
|
||
| githubConnector "github.com/allero-io/allero/pkg/connectors/github" | ||
| gitlabConnector "github.com/allero-io/allero/pkg/connectors/gitlab" | ||
| "github.com/allero-io/allero/pkg/fileManager" | ||
| ) | ||
|
|
||
| type LocalConnector struct { | ||
| absoluteRootPath string | ||
| } | ||
|
|
||
| func New() *LocalConnector { | ||
| return &LocalConnector{ | ||
| absoluteRootPath: "", | ||
| } | ||
| } | ||
|
|
||
| func (lc *LocalConnector) Get(path string) error { | ||
| abs, err := filepath.Abs(path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| lc.absoluteRootPath = abs | ||
| var localJsonObject LocalRoot | ||
| githubJsonObject := make(map[string]*githubConnector.GithubOwner) | ||
| err = lc.getLocalGithub(githubJsonObject) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| localJsonObject.GithubData = githubJsonObject | ||
|
|
||
| gitlabJsonObject := make(map[string]*gitlabConnector.GitlabGroup) | ||
| err = lc.getLocalGitlab(gitlabJsonObject) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| localJsonObject.GitlabData = gitlabJsonObject | ||
|
|
||
| localJson, err := json.MarshalIndent(localJsonObject, "", " ") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| alleroHomedir := fileManager.GetAlleroHomedir() | ||
| return fileManager.WriteToFile(fmt.Sprintf("%s/repo_files/local.json", alleroHomedir), localJson) | ||
| } | ||
|
|
||
| func (lc *LocalConnector) walkAndMatchedFiles(dir string, regex string) ([]string, error) { | ||
|
|
||
| var allFiles []string | ||
| err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { | ||
| if matched, _ := regexp.MatchString(regex, path); matched { | ||
| relativePath := strings.TrimPrefix(path, lc.absoluteRootPath+"/") | ||
| allFiles = append(allFiles, relativePath) | ||
| } | ||
|
|
||
| return nil | ||
| }) | ||
|
|
||
| return allFiles, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| package localConnector | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "path" | ||
| "path/filepath" | ||
|
|
||
| "github.com/allero-io/allero/pkg/connectors" | ||
| githubConnector "github.com/allero-io/allero/pkg/connectors/github" | ||
| "github.com/allero-io/allero/pkg/fileManager" | ||
| ) | ||
|
|
||
| func (lc *LocalConnector) getLocalGithub(githubJsonObject map[string]*githubConnector.GithubOwner) error { | ||
| err := lc.addRootPathAsNewRepo(githubJsonObject) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| escapedRepoName := connectors.EscapeJsonKey(lc.absoluteRootPath) | ||
| err = lc.processGithubWorkflowFiles(githubJsonObject, escapedRepoName) | ||
| if err != nil { | ||
| fmt.Println(err) | ||
| return err | ||
| } | ||
| return nil | ||
|
|
||
| } | ||
|
|
||
| func (lc *LocalConnector) addRootPathAsNewRepo(githubJsonObject map[string]*githubConnector.GithubOwner) error { | ||
| githubJsonObject["local_owner"] = &githubConnector.GithubOwner{ | ||
| Name: "sudo", | ||
| Type: "local_github", | ||
| ID: 0, | ||
| Repositories: make(map[string]*githubConnector.GithubRepository), | ||
| } | ||
|
|
||
| escapedRepoName := connectors.EscapeJsonKey(lc.absoluteRootPath) | ||
|
|
||
| githubJsonObject["local_owner"].Repositories[escapedRepoName] = &githubConnector.GithubRepository{ | ||
| Name: escapedRepoName, | ||
| FullName: escapedRepoName, | ||
| ID: 0, | ||
| ProgrammingLanguages: nil, | ||
| GithubActionsWorkflows: make(map[string]*githubConnector.PipelineFile), | ||
| JfrogPipelines: make(map[string]*githubConnector.PipelineFile), | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (lc *LocalConnector) processGithubWorkflowFiles(githubJsonObject map[string]*githubConnector.GithubOwner, repoName string) error { | ||
| workflowFilesChan, _ := lc.getWorkflowFilesEntities(repoName) | ||
| var processingError error | ||
|
|
||
| for workflowFile := range workflowFilesChan { | ||
| fullPath := filepath.Join(lc.absoluteRootPath, workflowFile.RelativePath) | ||
| content, err := fileManager.ReadFile(fullPath) | ||
| if err != nil { | ||
| processingError = fmt.Errorf("failed to get content for file %s", fullPath) | ||
| continue | ||
| } | ||
|
|
||
| jsonContentBytes, err := connectors.YamlToJson(content) | ||
| if err != nil { | ||
| processingError = err | ||
| continue | ||
| } | ||
|
|
||
| jsonContent := make(map[string]interface{}) | ||
| err = json.Unmarshal(jsonContentBytes, &jsonContent) | ||
| if err != nil { | ||
| processingError = err | ||
| continue | ||
| } | ||
|
|
||
| workflowFile.Content = jsonContent | ||
| escapedFilename := connectors.EscapeJsonKey(workflowFile.Filename) | ||
|
|
||
| if workflowFile.Origin == "github_actions" { | ||
| githubJsonObject["local_owner"].Repositories[repoName].GithubActionsWorkflows[escapedFilename] = workflowFile | ||
| } else if workflowFile.Origin == "jfrog_pipelines" { | ||
| githubJsonObject["local_owner"].Repositories[repoName].JfrogPipelines[escapedFilename] = workflowFile | ||
| } else { | ||
| processingError = fmt.Errorf("unsupported CICD platform %s for file %s from repository %s", workflowFile.Origin, workflowFile.RelativePath, repoName) | ||
| continue | ||
| } | ||
| } | ||
|
|
||
| return processingError | ||
| } | ||
|
|
||
| func (lc *LocalConnector) getWorkflowFilesEntities(repoName string) (chan *githubConnector.PipelineFile, error) { | ||
| workflowFilesEntitiesChan := make(chan *githubConnector.PipelineFile) | ||
|
|
||
| var getEntitiesErr error | ||
| go func() { | ||
| defer close(workflowFilesEntitiesChan) | ||
|
|
||
| for _, cicdPlatform := range connectors.SUPPORTED_CICD_PLATFORMS { | ||
| if !cicdPlatform.GithubValid { | ||
| continue | ||
| } | ||
| relevantFilesPaths, err := lc.walkAndMatchedFiles(lc.absoluteRootPath, cicdPlatform.RelevantFilesRegex) | ||
| if err != nil { | ||
| return | ||
| } | ||
| for _, filePath := range relevantFilesPaths { | ||
| workflowFilesEntitiesChan <- &githubConnector.PipelineFile{ | ||
| RelativePath: filePath, | ||
| Filename: path.Base(filePath), | ||
| Origin: cicdPlatform.Name, | ||
| } | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| return workflowFilesEntitiesChan, getEntitiesErr | ||
| } |
Oops, something went wrong.