Skip to content

Commit

Permalink
test(sonarqube): add hotspots e2e
Browse files Browse the repository at this point in the history
  • Loading branch information
@warren830
warren830 committed Feb 10, 2023
1 parent c6dd289 commit 6ce44d6
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 3 deletions.
70 changes: 70 additions & 0 deletions backend/plugins/sonarqube/e2e/hotspot_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"testing"

"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/securitytesting"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/sonarqube/impl"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
"github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
)

func TestSonarqubeHotspotDataFlow(t *testing.T) {

var sonarqube impl.Sonarqube
dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube)

// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_sonarqube_hotspots.csv",
"_raw_sonarqube_hotspots")

// Standard data
taskData := &tasks.SonarqubeTaskData{
Options: &tasks.SonarqubeOptions{
ConnectionId: 1,
ProjectKey: "f5a50c63-2e8f-4107-9014-853f6f467757",
},
}
// Interfered data
taskData2 := &tasks.SonarqubeTaskData{
Options: &tasks.SonarqubeOptions{
ConnectionId: 2,
ProjectKey: "testWarrenEtcd",
},
}

// verify extraction
dataflowTester.FlushTabler(&models.SonarqubeHotspot{})
dataflowTester.Subtask(tasks.ExtractHotspotsMeta, taskData)

dataflowTester.Subtask(tasks.ExtractHotspotsMeta, taskData2)
dataflowTester.VerifyTableWithOptions(&models.SonarqubeHotspot{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_sonarqube_hotspots.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})

// verify convertor
dataflowTester.FlushTabler(&securitytesting.StIssue{})
dataflowTester.Subtask(tasks.ConvertHotspotsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&securitytesting.StIssue{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/issue_hotspots.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}
6 changes: 6 additions & 0 deletions backend/plugins/sonarqube/e2e/raw_tables/_raw_sonarqube_hotspots.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
"id","params","data","url","input","created_at"
5517,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBamj46XwcL-YZOPh"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""REVIEWED"",""resolution"":""SAFE"",""line"":295,""message"":""Make sure that using this pseudorandom number generator is safe here."",""author"":""julian.qian@airbnb.com"",""creationDate"":""2016-04-25T19:02:16+0000"",""updateDate"":""2022-12-21T08:30:43+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=f5a50c63-2e8f-4107-9014-853f6f467757&ps=100,null,2023-02-09 02:30:20.274
5518,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBajH46XwcL-YZONY"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""TO_REVIEW"",""line"":143,""message"":""Make sure that using this pseudorandom number generator is safe here."",""author"":""peng.ye@airbnb.com"",""creationDate"":""2016-02-26T22:41:48+0000"",""updateDate"":""2022-12-20T14:50:30+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=f5a50c63-2e8f-4107-9014-853f6f467757&ps=100,null,2023-02-09 02:30:20.274
5519,"{""connectionId"":1,""ProjectKey"":""f5a50c63-2e8f-4107-9014-853f6f467757""}","{""key"":""AYUwBajH46XwcL-YZONZ"",""component"":""f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java"",""project"":""f5a50c63-2e8f-4107-9014-853f6f467757"",""securityCategory"":""weak-cryptography"",""vulnerabilityProbability"":""MEDIUM"",""status"":""TO_REVIEW"",""line"":172,""message"":""Make sure that using this pseudorandom number generator is safe here."",""author"":""peng.ye@airbnb.com"",""creationDate"":""2016-02-26T22:41:48+0000"",""updateDate"":""2022-12-20T14:50:30+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=f5a50c63-2e8f-4107-9014-853f6f467757&ps=100,null,2023-02-09 02:30:20.274
5727,"{""connectionId"":2,""ProjectKey"":""testWarrenEtcd""}","{""key"":""AYYrCCi2CVdTZLqEeJeQ"",""component"":""testWarrenEtcd:tests/common/user_test.go"",""project"":""testWarrenEtcd"",""securityCategory"":""auth"",""vulnerabilityProbability"":""HIGH"",""status"":""TO_REVIEW"",""line"":110,""message"":""\""password\"" detected here, make sure this is not a hard-coded credential."",""author"":"""",""creationDate"":""2023-02-07T08:39:18+0000"",""updateDate"":""2023-02-07T08:39:18+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=testWarrenEtcd&ps=100,null,2023-02-10 04:18:55.028
5728,"{""connectionId"":2,""ProjectKey"":""testWarrenEtcd""}","{""key"":""AYYrCCi2CVdTZLqEeJeS"",""component"":""testWarrenEtcd:tests/common/user_test.go"",""project"":""testWarrenEtcd"",""securityCategory"":""auth"",""vulnerabilityProbability"":""HIGH"",""status"":""TO_REVIEW"",""line"":148,""message"":""\""password\"" detected here, make sure this is not a hard-coded credential."",""author"":"""",""creationDate"":""2023-02-07T08:39:18+0000"",""updateDate"":""2023-02-07T08:39:18+0000""}",http://35.89.154.221:30008/api/hotspots/search?p=1&projectKey=testWarrenEtcd&ps=100,null,2023-02-10 04:18:55.028
6 changes: 6 additions & 0 deletions backend/plugins/sonarqube/e2e/snapshot_tables/_tool_sonarqube_hotspots.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
connection_id,hotspot_key,rule_key,component,project_key,line,status,message,author,assignee,security_category,vulnerability_probability,creation_date,update_date
1,AYUwBajH46XwcL-YZONY,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,peng.ye@airbnb.com,,weak-cryptography,MEDIUM,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
1,AYUwBajH46XwcL-YZONZ,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,peng.ye@airbnb.com,,weak-cryptography,MEDIUM,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
1,AYUwBamj46XwcL-YZOPh,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make sure that using this pseudorandom number generator is safe here.,julian.qian@airbnb.com,,weak-cryptography,MEDIUM,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
2,AYYrCCi2CVdTZLqEeJeQ,,testWarrenEtcd:tests/common/user_test.go,testWarrenEtcd,110,TO_REVIEW,"""password"" detected here, make sure this is not a hard-coded credential.",,,auth,HIGH,2023-02-07T08:39:18.000+00:00,2023-02-07T08:39:18.000+00:00
2,AYYrCCi2CVdTZLqEeJeS,,testWarrenEtcd:tests/common/user_test.go,testWarrenEtcd,148,TO_REVIEW,"""password"" detected here, make sure this is not a hard-coded credential.",,,auth,HIGH,2023-02-07T08:39:18.000+00:00,2023-02-07T08:39:18.000+00:00
4 changes: 4 additions & 0 deletions backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
id,rule,severity,component,project_key,line,status,message,debt,effort,commit_author_email,assignee,hash,tags,type,scope,start_line,end_line,start_offset,end_offset,vulnerability_probability,security_category,creation_date,update_date
sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,,,peng.ye@airbnb.com,,,,HOTSPOTS,,143,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make sure that using this pseudorandom number generator is safe here.,,,peng.ye@airbnb.com,,,,HOTSPOTS,,172,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make sure that using this pseudorandom number generator is safe here.,,,julian.qian@airbnb.com,,,,HOTSPOTS,,295,0,0,0,MEDIUM,,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
35 changes: 32 additions & 3 deletions backend/plugins/sonarqube/tasks/hotspots_extractor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,41 @@ func ExtractHotspots(taskCtx plugin.SubTaskContext) errors.Error {
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) {
body := &models.SonarqubeHotspot{}
err := errors.Convert(json.Unmarshal(resData.Data, body))
var res struct {
Key string `json:"key" gorm:"primaryKey"`
RuleKey string `json:"ruleKey"`
Component string `json:"component" gorm:"index"`
ProjectKey string `json:"project" gorm:"index"`
Line int `json:"line"`
Status string `json:"status"`
Message string `json:"message"`
Author string `json:"author"`
Assignee string `json:"assignee"`
SecurityCategory string `json:"securityCategory"`
VulnerabilityProbability string `json:"vulnerabilityProbability"`
CreationDate *helper.Iso8601Time `json:"creationDate"`
UpdateDate *helper.Iso8601Time `json:"updateDate"`
}
err := errors.Convert(json.Unmarshal(resData.Data, &res))
if err != nil {
return nil, err
}
body.ConnectionId = data.Options.ConnectionId
body := &models.SonarqubeHotspot{
ConnectionId: data.Options.ConnectionId,
HotspotKey: res.Key,
RuleKey: res.RuleKey,
Component: res.Component,
ProjectKey: res.ProjectKey,
Line: res.Line,
Status: res.Status,
Message: res.Message,
Author: res.Author,
Assignee: res.Assignee,
SecurityCategory: res.SecurityCategory,
VulnerabilityProbability: res.VulnerabilityProbability,
CreationDate: res.CreationDate,
UpdateDate: res.UpdateDate,
}
return []interface{}{body}, nil
},
})
Expand Down

0 comments on commit 6ce44d6

Please sign in to comment.