diff --git a/backend/plugins/sonarqube/e2e/hotspot_test.go b/backend/plugins/sonarqube/e2e/hotspot_test.go new file mode 100644 index 00000000000..5b4339334f6 --- /dev/null +++ b/backend/plugins/sonarqube/e2e/hotspot_test.go @@ -0,0 +1,70 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/securitytesting" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/sonarqube/impl" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" + "github.com/apache/incubator-devlake/plugins/sonarqube/tasks" +) + +func TestSonarqubeHotspotDataFlow(t *testing.T) { + + var sonarqube impl.Sonarqube + dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube) + + // import raw data table + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_sonarqube_hotspots.csv", + "_raw_sonarqube_hotspots") + + // Standard data + taskData := &tasks.SonarqubeTaskData{ + Options: &tasks.SonarqubeOptions{ + ConnectionId: 1, + ProjectKey: "f5a50c63-2e8f-4107-9014-853f6f467757", + }, + } + // Interfered data + taskData2 := &tasks.SonarqubeTaskData{ + Options: &tasks.SonarqubeOptions{ + ConnectionId: 2, + ProjectKey: "testWarrenEtcd", + }, + } + + // verify extraction + dataflowTester.FlushTabler(&models.SonarqubeHotspot{}) + dataflowTester.Subtask(tasks.ExtractHotspotsMeta, taskData) + + dataflowTester.Subtask(tasks.ExtractHotspotsMeta, taskData2) + dataflowTester.VerifyTableWithOptions(&models.SonarqubeHotspot{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_sonarqube_hotspots.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify convertor + dataflowTester.FlushTabler(&securitytesting.StIssue{}) + dataflowTester.Subtask(tasks.ConvertHotspotsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&securitytesting.StIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_hotspots.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv new file mode 100644 index 00000000000..12278fdf20d --- /dev/null +++ b/backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv @@ -0,0 +1,6 @@ +"id","params","data","url","input","created_at" +5517,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBamj46XwcL-YZOPh"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""REVIEWED"",""resolution"":""SAFE"",""line"":295,""message"":""Make sure that using this pseudorandom number generator is safe here."",""author"":""julian.qian@airbnb.com"",""creationDate"":""2016-04-25T19:02:16+0000"",""updateDate"":""2022-12-21T08:30:43+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=f5a50c63-2e8f-4107-9014-853f6f467757&ps=100,null,2023-02-09 02:30:20.274 +5518,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBajH46XwcL-YZONY"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""TO_REVIEW"",""line"":143,""message"":""Make sure that using this pseudorandom number generator is safe here."",""author"":""peng.ye@airbnb.com"",""creationDate"":""2016-02-26T22:41:48+0000"",""updateDate"":""2022-12-20T14:50:30+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=f5a50c63-2e8f-4107-9014-853f6f467757&ps=100,null,2023-02-09 02:30:20.274 +5519,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBajH46XwcL-YZONZ"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""TO_REVIEW"",""line"":172,""message"":""Make sure that using this pseudorandom number generator is safe here."",""author"":""peng.ye@airbnb.com"",""creationDate"":""2016-02-26T22:41:48+0000"",""updateDate"":""2022-12-20T14:50:30+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=f5a50c63-2e8f-4107-9014-853f6f467757&ps=100,null,2023-02-09 02:30:20.274 +5727,"{""connectionId"":2,""ProjectKey"":""testWarrenEtcd""}","{""key"":""AYYrCCi2CVdTZLqEeJeQ"",""component"":""testWarrenEtcd:tests/common/user_test.go"",""project"":""testWarrenEtcd"",""securityCategory"":""auth"",""vulnerabilityProbability"":""HIGH"",""status"":""TO_REVIEW"",""line"":110,""message"":""\""password\"" detected here, make sure this is not a hard-coded credential."",""author"":"""",""creationDate"":""2023-02-07T08:39:18+0000"",""updateDate"":""2023-02-07T08:39:18+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=testWarrenEtcd&ps=100,null,2023-02-10 04:18:55.028 +5728,"{""connectionId"":2,""ProjectKey"":""testWarrenEtcd""}","{""key"":""AYYrCCi2CVdTZLqEeJeS"",""component"":""testWarrenEtcd:tests/common/user_test.go"",""project"":""testWarrenEtcd"",""securityCategory"":""auth"",""vulnerabilityProbability"":""HIGH"",""status"":""TO_REVIEW"",""line"":148,""message"":""\""password\"" detected here, make sure this is not a hard-coded credential."",""author"":"""",""creationDate"":""2023-02-07T08:39:18+0000"",""updateDate"":""2023-02-07T08:39:18+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=testWarrenEtcd&ps=100,null,2023-02-10 04:18:55.028 diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv b/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv new file mode 100644 index 00000000000..b700d465cce --- /dev/null +++ b/backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv @@ -0,0 +1,6 @@ +connection_id,hotspot_key,rule_key,component,project_key,line,status,message,author,assignee,security_category,vulnerability_probability,creation_date,update_date +1,AYUwBajH46XwcL-YZONY,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,peng.ye@airbnb.com,,weak-cryptography,MEDIUM,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00 +1,AYUwBajH46XwcL-YZONZ,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,peng.ye@airbnb.com,,weak-cryptography,MEDIUM,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00 +1,AYUwBamj46XwcL-YZOPh,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make sure that using this pseudorandom number generator is safe here.,julian.qian@airbnb.com,,weak-cryptography,MEDIUM,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00 +2,AYYrCCi2CVdTZLqEeJeQ,,testWarrenEtcd:tests/common/user_test.go,testWarrenEtcd,110,TO_REVIEW,"""password"" detected here, make sure this is not a hard-coded credential.",,,auth,HIGH,2023-02-07T08:39:18.000+00:00,2023-02-07T08:39:18.000+00:00 +2,AYYrCCi2CVdTZLqEeJeS,,testWarrenEtcd:tests/common/user_test.go,testWarrenEtcd,148,TO_REVIEW,"""password"" detected here, make sure this is not a hard-coded credential.",,,auth,HIGH,2023-02-07T08:39:18.000+00:00,2023-02-07T08:39:18.000+00:00 diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv new file mode 100644 index 00000000000..5f7a7de8546 --- /dev/null +++ b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv @@ -0,0 +1,4 @@ +id,rule,severity,component,project_key,line,status,message,debt,effort,commit_author_email,assignee,hash,tags,type,scope,start_line,end_line,start_offset,end_offset,vulnerability_probability,security_category,creation_date,update_date +sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,,,peng.ye@airbnb.com,,,,HOTSPOTS,,143,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00 +sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,,,peng.ye@airbnb.com,,,,HOTSPOTS,,172,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00 +sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make sure that using this pseudorandom number generator is safe here.,,,julian.qian@airbnb.com,,,,HOTSPOTS,,295,0,0,0,MEDIUM,,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00 diff --git a/backend/plugins/sonarqube/tasks/hotspots_extractor.go b/backend/plugins/sonarqube/tasks/hotspots_extractor.go index b6daae80b65..d231fa5a592 100644 --- a/backend/plugins/sonarqube/tasks/hotspots_extractor.go +++ b/backend/plugins/sonarqube/tasks/hotspots_extractor.go @@ -33,12 +33,41 @@ func ExtractHotspots(taskCtx plugin.SubTaskContext) errors.Error { extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) { - body := &models.SonarqubeHotspot{} - err := errors.Convert(json.Unmarshal(resData.Data, body)) + var res struct { + Key string `json:"key" gorm:"primaryKey"` + RuleKey string `json:"ruleKey"` + Component string `json:"component" gorm:"index"` + ProjectKey string `json:"project" gorm:"index"` + Line int `json:"line"` + Status string `json:"status"` + Message string `json:"message"` + Author string `json:"author"` + Assignee string `json:"assignee"` + SecurityCategory string `json:"securityCategory"` + VulnerabilityProbability string `json:"vulnerabilityProbability"` + CreationDate *helper.Iso8601Time `json:"creationDate"` + UpdateDate *helper.Iso8601Time `json:"updateDate"` + } + err := errors.Convert(json.Unmarshal(resData.Data, &res)) if err != nil { return nil, err } - body.ConnectionId = data.Options.ConnectionId + body := &models.SonarqubeHotspot{ + ConnectionId: data.Options.ConnectionId, + HotspotKey: res.Key, + RuleKey: res.RuleKey, + Component: res.Component, + ProjectKey: res.ProjectKey, + Line: res.Line, + Status: res.Status, + Message: res.Message, + Author: res.Author, + Assignee: res.Assignee, + SecurityCategory: res.SecurityCategory, + VulnerabilityProbability: res.VulnerabilityProbability, + CreationDate: res.CreationDate, + UpdateDate: res.UpdateDate, + } return []interface{}{body}, nil }, })