feat: add teambition plugin

backend/go.mod

@@ -46,6 +46,7 @@ require (
 )
 
 require (
+	github.com/golang-jwt/jwt/v5 v5.0.0-rc.1 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/Microsoft/go-winio v0.5.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect

backend/go.sum

@@ -244,7 +244,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA=
 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v5 v5.0.0-rc.1 h1:tDQ1LjKga657layZ4JLsRdxgvupebc0xuPwRNuTfUgs=
+github.com/golang-jwt/jwt/v5 v5.0.0-rc.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

backend/plugins/teambition/api/blueprint200.go

@@ -0,0 +1,107 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"fmt"
+	"github.com/apache/incubator-devlake/plugins/teambition/models"
+	"time"
+
+	"github.com/apache/incubator-devlake/core/dal"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
+	plan := make(plugin.PipelinePlan, len(bpScopes))
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId, syncPolicy)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(bpScopes, connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	plan plugin.PipelinePlan,
+	bpScopes []*plugin.BlueprintScopeV200,
+	connectionId uint64,
+	syncPolicy *plugin.BlueprintSyncPolicy,
+) (plugin.PipelinePlan, errors.Error) {
+	for i, bpScope := range bpScopes {
+		stage := plan[i]
+		if stage == nil {
+			stage = plugin.PipelineStage{}
+		}
+		// construct task options for teambition
+		options := make(map[string]interface{})
+		options["projectId"] = bpScope.Id
+		options["connectionId"] = connectionId
+		if syncPolicy.TimeAfter != nil {
+			options["timeAfter"] = syncPolicy.TimeAfter.Format(time.RFC3339)
+		}
+
+		subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, bpScope.Entities)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, &plugin.PipelineTask{
+			Plugin:   "teambition",
+			Subtasks: subtasks,
+			Options:  options,
+		})
+		plan[i] = stage
+	}
+
+	return plan, nil
+}
+
+func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connectionId uint64) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0)
+	for _, bpScope := range bpScopes {
+		project := &models.TeambitionProject{}
+		// get project from db
+		err := basicRes.GetDal().First(project,
+			dal.Where(`connection_id = ? and id = ?`,
+				connectionId, bpScope.Id))
+		if err != nil {
+			return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project %s", bpScope.Id))
+		}
+		// add board to scopes
+		if utils.StringsContains(bpScope.Entities, plugin.DOMAIN_TYPE_TICKET) {
+			domainBoard := &ticket.Board{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.TeambitionProject{}).Generate(project.ConnectionId, project.Id),
+				},
+				Name: project.Name,
+			}
+			scopes = append(scopes, domainBoard)
+		}
+	}
+	return scopes, nil
+}

backend/plugins/teambition/api/connection.go

@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"context"
+	"fmt"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/teambition/models"
+	"github.com/apache/incubator-devlake/plugins/teambition/tasks"
+	"github.com/apache/incubator-devlake/server/api/shared"
+	"net/http"
+)
+
+type TeambitionTestConnResponse struct {
+	shared.ApiBody
+	Connection *models.TeambitionConn
+}
+
+// TestConnection @Summary test teambition connection
+// @Description Test teambition Connection
+// @Tags plugins/teambition
+// @Param body body models.TeambitionConn true "json body"
+// @Success 200  {object} TeambitionTestConnResponse "Success"
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/teambition/test [POST]
+func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	// process input
+	var connection models.TeambitionConn
+	err := api.Decode(input.Body, &connection, vld)
+	if err != nil {
+		return nil, err
+	}
+
+	// test connection
+	apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := apiClient.Get("/org/info?orgId="+connection.TenantId, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code: %d", res.StatusCode))
+	}
+	resBody := tasks.TeambitionComRes[any]{}
+	err = api.UnmarshalResponse(res, &resBody)
+	if err != nil {
+		return nil, err
+	}
+	if resBody.Code != http.StatusOK {
+		return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code: %d", res.StatusCode))
+	}
+
+	body := TeambitionTestConnResponse{}
+	body.Success = true
+	body.Message = "success"
+	body.Connection = &connection
+	// output
+	return &plugin.ApiResourceOutput{Body: body, Status: 200}, nil
+}
+
+// PostConnections @Summary create teambition connection
+// @Description Create teambition connection
+// @Tags plugins/teambition
+// @Param body body models.TeambitionConnection true "json body"
+// @Success 200  {object} models.TeambitionConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/teambition/connections [POST]
+func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	// update from request and save to database
+	connection := &models.TeambitionConnection{}
+	err := connectionHelper.Create(connection, input)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
+}
+
+// PatchConnection @Summary patch teambition connection
+// @Description Patch teambition connection
+// @Tags plugins/teambition
+// @Param body body models.TeambitionConnection true "json body"
+// @Success 200  {object} models.TeambitionConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/teambition/connections/{connectionId} [PATCH]
+func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.TeambitionConnection{}
+	err := connectionHelper.Patch(connection, input)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection}, nil
+}
+
+// DeleteConnection @Summary delete a teambition connection
+// @Description Delete a teambition connection
+// @Tags plugins/teambition
+// @Success 200  {object} models.TeambitionConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/teambition/connections/{connectionId} [DELETE]
+func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.TeambitionConnection{}
+	err := connectionHelper.First(connection, input.Params)
+	if err != nil {
+		return nil, err
+	}
+	err = connectionHelper.Delete(connection)
+	return &plugin.ApiResourceOutput{Body: connection}, err
+}
+
+// ListConnections @Summary get all teambition connections
+// @Description Get all teambition connections
+// @Tags plugins/teambition
+// @Success 200  {object} []models.TeambitionConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/teambition/connections [GET]
+func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	var connections []models.TeambitionConnection
+	err := connectionHelper.List(&connections)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
+}
+
+// GetConnection @Summary get teambition connection detail
+// @Description Get teambition connection detail
+// @Tags plugins/teambition
+// @Success 200  {object} models.TeambitionConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/teambition/connections/{connectionId} [GET]
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.TeambitionConnection{}
+	err := connectionHelper.First(connection, input.Params)
+	return &plugin.ApiResourceOutput{Body: connection}, err
+}

backend/plugins/teambition/api/init.go

@@ -0,0 +1,37 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/go-playground/validator/v10"
+)
+
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes context.BasicRes
+
+func Init(br context.BasicRes) {
+	basicRes = br
+	vld = validator.New()
+	connectionHelper = helper.NewConnectionHelper(
+		basicRes,
+		vld,
+	)
+}

backend/plugins/teambition/e2e/changelogs_test.go

@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"github.com/apache/incubator-devlake/core/models/common"
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/plugins/teambition/impl"
+	"github.com/apache/incubator-devlake/plugins/teambition/models"
+	"github.com/apache/incubator-devlake/plugins/teambition/tasks"
+	"testing"
+)
+
+func TestTeambitionChangelogs(t *testing.T) {
+
+	var teambition impl.Teambition
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "teambition", teambition)
+
+	taskData := &tasks.TeambitionTaskData{
+		Options: &tasks.TeambitionOptions{
+			ConnectionId: 1,
+			ProjectId:    "64132c94f0d59df1c9825ab8",
+		},
+	}
+
+	// import raw data table
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_teambition_api_task_activities.csv",
+		"_raw_teambition_api_task_activities")
+	dataflowTester.FlushTabler(&models.TeambitionTaskActivity{})
+
+	// verify extraction
+	dataflowTester.Subtask(tasks.ExtractTaskActivitiesMeta, taskData)
+	dataflowTester.VerifyTableWithOptions(
+		models.TeambitionTaskActivity{},
+		e2ehelper.TableOptions{
+			CSVRelPath:   "./snapshot_tables/_tool_teambition_task_activities.csv",
+			IgnoreTypes:  []interface{}{common.NoPKModel{}},
+			IgnoreFields: []string{"created", "updated", "start_date", "end_date", "create_time", "update_time"},
+		},
+	)
+
+	dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
+	dataflowTester.Subtask(tasks.ConvertTaskChangelogMeta, taskData)
+	dataflowTester.VerifyTableWithOptions(
+		ticket.IssueChangelogs{},
+		e2ehelper.TableOptions{
+			CSVRelPath:   "./snapshot_tables/issue_changelogs.csv",
+			IgnoreFields: []string{"created_date"},
+			IgnoreTypes:  []interface{}{domainlayer.DomainEntity{}},
+		},
+	)
+}