From 4f55581d766a3d388fa1bc605436b46607453186 Mon Sep 17 00:00:00 2001
From: Yingchu Chen <yingchu.chen@merico.dev>
Date: Tue, 21 Feb 2023 21:28:12 +0800
Subject: [PATCH] fix(sonarqube): fix data loss

---
 backend/plugins/sonarqube/impl/impl.go        | 38 ++++++++-----------
 .../sonarqube/tasks/issues_collector.go       |  1 -
 .../sonarqube/tasks/issues_extractor.go       |  7 +---
 .../releases/lake-v0.14.0/docker-compose.yml  |  3 ++
 .../releases/lake-v0.15.0/docker-compose.yml  |  4 ++
 docker-compose.yml                            |  4 ++
 6 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go
index e4655274d03..c3be0ba08de 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -123,30 +123,22 @@ func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
 		Options:   op,
 		ApiClient: apiClient,
 	}
-	if op.ProjectKey != "" {
-		var scope *models.SonarqubeProject
-		var apiProject *tasks.SonarqubeApiProject
-		// support v100 & advance mode
-		// If we still cannot find the record in db, we have to request from remote server and save it to db
-		db := taskCtx.GetDal()
-		err = db.First(&scope, dal.Where("connection_id = ? AND project_key = ?", op.ConnectionId, op.ProjectKey))
-		if err != nil && db.IsErrorNotFound(err) {
-			apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
-			if err != nil {
-				return nil, err
-			}
-			logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
-			scope = tasks.ConvertProject(apiProject)
-			scope.ConnectionId = op.ConnectionId
-			err = taskCtx.GetDal().CreateIfNotExist(&scope)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if err != nil {
-			return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project: %s", op.ProjectKey))
-		}
+	// even we have project in _tool_sonaqube_projects, we still need to collect project to update LastAnalysisDate
+	var scope *models.SonarqubeProject
+	var apiProject *tasks.SonarqubeApiProject
+	apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
+	if err != nil {
+		return nil, err
 	}
+	logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
+	scope = tasks.ConvertProject(apiProject)
+	scope.ConnectionId = op.ConnectionId
+	err = taskCtx.GetDal().CreateOrUpdate(&scope)
+	if err != nil {
+		return nil, err
+	}
+	taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime()
+
 	var createdDateAfter time.Time
 	if op.CreatedDateAfter != "" {
 		createdDateAfter, err = errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go b/backend/plugins/sonarqube/tasks/issues_collector.go
index 4c12a4fb3fd..8e937e7d3a5 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -35,7 +35,6 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
 	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUES_TABLE)
 	logger := taskCtx.GetLogger()
 	logger.Info("collect issues")
-
 	collectorWithState, err := helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
 	if err != nil {
 		return err
diff --git a/backend/plugins/sonarqube/tasks/issues_extractor.go b/backend/plugins/sonarqube/tasks/issues_extractor.go
index 2670910adaa..3facf5ead44 100644
--- a/backend/plugins/sonarqube/tasks/issues_extractor.go
+++ b/backend/plugins/sonarqube/tasks/issues_extractor.go
@@ -22,13 +22,12 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"hash"
-	"strings"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/sonarqube/models"
+	"hash"
+	"strings"
 )
 
 var _ plugin.SubTaskEntryPoint = ExtractIssues
@@ -45,7 +44,6 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
 			if err != nil {
 				return nil, err
 			}
-
 			sonarqubeIssue := &models.SonarqubeIssue{
 				ConnectionId: data.Options.ConnectionId,
 				IssueKey:     body.Key,
@@ -91,7 +89,6 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error {
 					results = append(results, codeBlock)
 				}
 			}
-
 			return results, nil
 		},
 	})
diff --git a/devops/releases/lake-v0.14.0/docker-compose.yml b/devops/releases/lake-v0.14.0/docker-compose.yml
index f912a430722..e9b4d79b94a 100644
--- a/devops/releases/lake-v0.14.0/docker-compose.yml
+++ b/devops/releases/lake-v0.14.0/docker-compose.yml
@@ -27,6 +27,9 @@ services:
       MYSQL_DATABASE: lake
       MYSQL_USER: merico
       MYSQL_PASSWORD: merico
+    command:
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_bin
 
   grafana:
     image: apache/devlake-dashboard:v0.14.0
diff --git a/devops/releases/lake-v0.15.0/docker-compose.yml b/devops/releases/lake-v0.15.0/docker-compose.yml
index 956602f46c6..32dac09f23b 100644
--- a/devops/releases/lake-v0.15.0/docker-compose.yml
+++ b/devops/releases/lake-v0.15.0/docker-compose.yml
@@ -27,6 +27,10 @@ services:
       MYSQL_DATABASE: lake
       MYSQL_USER: merico
       MYSQL_PASSWORD: merico
+    command:
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_bin
+
 
   grafana:
     image: apache/devlake-dashboard:v0.15.0
diff --git a/docker-compose.yml b/docker-compose.yml
index 1183b17c535..5fe61e7a0b1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,6 +27,10 @@ services:
       MYSQL_DATABASE: lake
       MYSQL_USER: merico
       MYSQL_PASSWORD: merico
+    command:
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_bin
+
 
   # postgres:
   #   image: postgres:14.2