feat: support Zentao task worklogs

backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_worklogs.csv

@@ -0,0 +1,3 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ProjectId"":48}","{""id"":106,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":48,""execution"":49,""account"":""devlake"",""work"":""sample worklog"",""vision"":""rnd"",""date":""2025-02-20"",""left"":5,""consumed"":11.3,""begin"":0,""end"":0,""extra"":null,""order"":0,""deleted"":""0""}",http://iwater.red:8000/api.php/v1/tasks/135/estimate,{"id":135},2025-02-21 06:28:36.902
+2,"{""ConnectionId"":1,""ProjectId"":48}","{""id"":107,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":48,""execution"":49,""account"":""devlake"",""work"":"""",""vision"":""rnd"",""date":""2025-02-20"",""left"":1,""consumed"":4,""begin"":0,""end"":0,""extra"":null,""order"":0,""deleted"":""0""}",http://iwater.red:8000/api.php/v1/tasks/135/estimate,{"id":135},2025-02-21 06:28:37.001

backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_worklogs.csv

@@ -0,0 +1,3 @@
+connection_id,id,object_id,object_type,project,execution,product,account,work,vision,date,left,consumed,begin,end,extra,order,deleted
+1,106,135,task,48,49,,devlake,sample worklog,rnd,2025-02-20,5,11.3,0,0,NULL,0,0
+1,107,135,task,48,49,,devlake,,rnd,2025-02-20,1,4,0,0,NULL,0,0

backend/plugins/zentao/e2e/snapshot_tables/issue_worklogs.csv

@@ -0,0 +1,3 @@
+id,author_id,comment,time_spent_minutes,logged_date,started_date,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+zentao:ZentaoWorklog:1:106,zentao:ZentaoAccount:1:1,sample worklog,678,2025-02-20T08:00:00.000+00:00,2025-02-20T08:00:00.000+00:00,zentao:ZentaoTask:1:135,"{""ConnectionId"":1,""ProjectId"":48}",_raw_zentao_api_task_worklogs,1,
+zentao:ZentaoWorklog:1:107,zentao:ZentaoAccount:1:1,,240,2025-02-20T08:00:00.000+00:00,2025-02-20T08:00:00.000+00:00,zentao:ZentaoTask:1:135,"{""ConnectionId"":1,""ProjectId"":48}",_raw_zentao_api_task_worklogs,2,

backend/plugins/zentao/e2e/task_worklogs_test.go

@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/apache/incubator-devlake/core/models/common"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/plugins/zentao/impl"
+	"github.com/apache/incubator-devlake/plugins/zentao/models"
+	"github.com/apache/incubator-devlake/plugins/zentao/tasks"
+)
+
+func TestZentaoTaskWorklogDataFlow(t *testing.T) {
+
+	var zentao impl.Zentao
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao)
+
+	taskData := &tasks.ZentaoTaskData{
+		Options: &tasks.ZentaoOptions{
+			ConnectionId: 1,
+			ProjectId:    48,
+		},
+		ApiClient:   getFakeAPIClient(),
+		HomePageURL: getFakeHomepage(),
+	}
+
+	// import _raw_zentao_api_task_worklogs raw data table
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_task_worklogs.csv",
+		"_raw_zentao_api_task_worklogs")
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_zentao_accounts.csv", &models.ZentaoAccount{})
+
+	// verify worklogs extraction
+	dataflowTester.FlushTabler(&models.ZentaoWorklog{})
+	dataflowTester.Subtask(tasks.ExtractTaskWorklogsMeta, taskData)
+	dataflowTester.VerifyTableWithOptions(&models.ZentaoWorklog{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/_tool_zentao_worklogs.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+
+	// verify task repo commit conversion
+	dataflowTester.FlushTabler(&crossdomain.IssueWorklog{})
+	dataflowTester.Subtask(tasks.ConvertTaskWorklogsMeta, taskData)
+	dataflowTester.VerifyTableWithOptions(&crossdomain.IssueWorklog{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/issue_worklogs.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+}

backend/plugins/zentao/impl/impl.go

@@ -93,6 +93,7 @@ func (p Zentao) GetTablesInfo() []dal.Tabler {
 		&models.ZentaoExecutionSummary{},
 		&models.ZentaoProductSummary{},
 		&models.ZentaoProjectStory{},
+		&models.ZentaoWorklog{},
 	}
 }
 
@@ -162,6 +163,10 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
 
 		tasks.DBGetChangelogMeta,
 		tasks.ConvertChangelogMeta,
+
+		tasks.CollectTaskWorklogsMeta,
+		tasks.ExtractTaskWorklogsMeta,
+		tasks.ConvertTaskWorklogsMeta,
 	}
 }
 

backend/plugins/zentao/models/migrationscripts/20250219_add_worklogs.go

@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+	"github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts/archived"
+)
+
+type addWorklogs struct{}
+
+func (*addWorklogs) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(
+		basicRes,
+		&archived.ZentaoWorklog{},
+	)
+}
+
+func (*addWorklogs) Version() uint64 {
+	return 20250219153329
+}
+
+func (*addWorklogs) Name() string {
+	return "add table _tool_zentao_worklogs"
+}

backend/plugins/zentao/models/migrationscripts/archived/worklog.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type ZentaoWorklog struct {
+	archived.NoPKModel
+	ConnectionId uint64  `gorm:"primaryKey;type:BIGINT NOT NULL"`
+	Id           uint64  `gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false" json:"id"`
+	ObjectId     uint64  `json:"objectID"`
+	ObjectType   string  `json:"objectType"`
+	Project      uint64  `json:"project"`
+	Execution    uint64  `json:"execution"`
+	Product      string  `json:"product"`
+	Account      string  `json:"account"`
+	Work         string  `json:"work"`
+	Vision       string  `json:"vision"`
+	Date         string  `json:"date"`
+	Left         float32 `json:"left"`
+	Consumed     float32 `json:"consumed"`
+	Begin        uint64  `json:"begin"`
+	End          uint64  `json:"end"`
+	Extra        *string `json:"extra"`
+	Order        uint64  `json:"order"`
+	Deleted      string  `json:"deleted"`
+}
+
+func (ZentaoWorklog) TableName() string {
+	return "_tool_zentao_worklogs"
+}

backend/plugins/zentao/models/migrationscripts/register.go

@@ -32,5 +32,6 @@ func All() []plugin.MigrationScript {
 		new(addExecutionStoryAndExecutionSummary),
 		new(addRawParamTableForScope),
 		new(dropTotalReal),
+		new(addWorklogs),
 	}
 }

backend/plugins/zentao/models/worklog.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+	"github.com/apache/incubator-devlake/core/models/common"
+)
+
+type ZentaoWorklog struct {
+	ConnectionId uint64  `gorm:"primaryKey;type:BIGINT NOT NULL"`
+	Id           int64   `gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false" json:"id"`
+	ObjectId     int64   `json:"objectID"`
+	ObjectType   string  `json:"objectType"`
+	Project      int64   `json:"project"`
+	Execution    int64   `json:"execution"`
+	Product      string  `json:"product"`
+	Account      string  `json:"account"`
+	Work         string  `json:"work"`
+	Vision       string  `json:"vision"`
+	Date         string  `json:"date"`
+	Left         float32 `json:"left"`
+	Consumed     float32 `json:"consumed"`
+	Begin        int64   `json:"begin"`
+	End          int64   `json:"end"`
+	Extra        *string `json:"extra"`
+	Order        int64   `json:"order"`
+	Deleted      string  `json:"deleted"`
+	common.NoPKModel
+}
+
+func (ZentaoWorklog) TableName() string {
+	return "_tool_zentao_worklogs"
+}

backend/plugins/zentao/tasks/task_worklog_collector.go

@@ -0,0 +1,121 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"reflect"
+
+	"github.com/apache/incubator-devlake/core/dal"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/zentao/models"
+)
+
+const RAW_TASK_WORKLOGS_TABLE = "zentao_api_task_worklogs"
+
+var CollectTaskWorklogsMeta = plugin.SubTaskMeta{
+	Name:             "collectTaskWorklogs",
+	EntryPoint:       CollectTaskWorklogs,
+	EnabledByDefault: true,
+	Description:      "collect Zentao task work logs, supports both timeFilter and diffSync.",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_TICKET},
+}
+
+type Input struct {
+	Id uint64 `json:"id"`
+}
+
+func CollectTaskWorklogs(taskCtx plugin.SubTaskContext) errors.Error {
+	db := taskCtx.GetDal()
+	data := taskCtx.GetData().(*ZentaoTaskData)
+
+	logger := taskCtx.GetLogger()
+
+	apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
+		Ctx:     taskCtx,
+		Options: data.Options,
+		Table:   RAW_TASK_WORKLOGS_TABLE,
+	})
+	if err != nil {
+		return err
+	}
+
+	// load task IDs from db
+	clauses := []dal.Clause{
+		dal.Select("id"),
+		dal.From(&models.ZentaoTask{}),
+		dal.Where(
+			"project = ? AND connection_id = ?",
+			data.Options.ProjectId, data.Options.ConnectionId,
+		),
+	}
+	if apiCollector.IsIncremental() && apiCollector.GetSince() != nil {
+		clauses = append(clauses, dal.Where("last_edited_date IS NOT NULL AND last_edited_date > ?", apiCollector.GetSince()))
+	}
+
+	// construct the input iterator
+	cursor, err := db.Cursor(clauses...)
+	if err != nil {
+		return err
+	}
+	iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(Input{}))
+	if err != nil {
+		return err
+	}
+
+	// collect task worklogs
+	err = apiCollector.InitCollector(api.ApiCollectorArgs{
+		Input:         iterator,
+		ApiClient:     data.ApiClient,
+		UrlTemplate:   "tasks/{{ .Input.Id }}/estimate",
+		Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
+			return nil, nil
+		},
+		ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
+			var data struct {
+				Effort json.RawMessage `json:"effort"`
+			}
+			err := api.UnmarshalResponse(res, &data)
+			if err != nil {
+				return nil, err
+			}
+
+			if string(data.Effort) == "{}" || string(data.Effort) == "null" {
+				return nil, nil
+			}
+
+			var efforts []json.RawMessage
+			jsonErr := json.Unmarshal(data.Effort, &efforts)
+			if jsonErr != nil {
+				return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal efforts")
+			}
+			return efforts, nil
+		},
+		AfterResponse: ignoreHTTPStatus404,
+	})
+	if err != nil {
+		logger.Error(err, "collect Zentao task worklogs error")
+		return err
+	}
+
+	return apiCollector.Execute()
+}