cherry pick all #7708 related commits to release-v1.0 (#7759)

* feat(jira): collector account from issue changelog items as much as possible

* fix(jira): fix migration scripts

* feat(jira): add `_tool_jira_issue_fields`, collect account field from the new table

* fix(jira): fix testes

* fix(jira): fix `reporter` field in issue change log items

* fix(jira): fix account id when there is no from/to values

* fix(jira): fix OriginalToValue in issue_changelogs

* fix(jira): don't use `tmpFromAccountId` and `tmpToAccountId` in issue changelogs

* fix(jira): remove all usage of `tmpFromAccountId` and `tmpToAccountId`

* refactor(jira): remove some codes

* fix(jira): fix e2e test

* fix(jira): fix wrong key in `issueFieldMap`

backend/helpers/e2ehelper/data_flow_tester.go

@@ -460,7 +460,7 @@ func formatDbValue(value interface{}, nullable bool) string {
 	return ``
 }
 
-// ColumnWithRawData create an Column string with _raw_data_* appending
+// ColumnWithRawData create a Column string with _raw_data_* appending
 func ColumnWithRawData(column ...string) []string {
 	return append(
 		column,

backend/plugins/jira/e2e/changelog_test.go

@@ -38,6 +38,7 @@ func TestIssueChangelogDataFlow(t *testing.T) {
 	}
 	// import raw data table
 	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_changelogs.csv", "_raw_jira_api_issue_changelogs")
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv", &models.JiraIssueField{})
 	dataflowTester.FlushTabler(&models.JiraIssueChangelogs{})
 	dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{})
 	dataflowTester.FlushTabler(&models.JiraAccount{})
@@ -68,6 +69,8 @@ func TestIssueChangelogDataFlow(t *testing.T) {
 			"from_string",
 			"to_value",
 			"to_string",
+			"tmp_from_account_id",
+			"tmp_to_account_id",
 		),
 	)
 	dataflowTester.VerifyTable(
@@ -83,11 +86,36 @@ func TestIssueChangelogDataFlow(t *testing.T) {
 			"timezone",
 		),
 	)
+
+	// import raw data: _raw_jira_api_issue_fields
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_fields.csv", "_raw_jira_api_issue_fields")
+	dataflowTester.FlushTabler(&models.JiraIssueField{})
+	dataflowTester.Subtask(tasks.ExtractIssueFieldsMeta, taskData)
+	dataflowTester.VerifyTable(
+		models.JiraIssueField{},
+		"./snapshot_tables/_tool_jira_issue_fields.csv",
+		e2ehelper.ColumnWithRawData(
+			"connection_id",
+			"id",
+			"board_id",
+			"name",
+			"custom",
+			"orderable",
+			"navigable",
+			"searchable",
+			"schema_type",
+			"schema_items",
+			"schema_custom",
+			"schema_custom_id",
+			"sche_custom_system"),
+	)
+
 	// verify changelog conversion
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_changelogs.csv", &models.JiraIssueChangelogs{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_changelog_items.csv", &models.JiraIssueChangelogItems{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_statuses_for_changelog.csv", &models.JiraStatus{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_board_issues_for_changelog.csv", &models.JiraBoardIssue{})
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv", &models.JiraIssueField{})
 	dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
 	dataflowTester.Subtask(tasks.ConvertIssueChangelogsMeta, taskData)
 	dataflowTester.VerifyTable(

backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issue_fields.csv

@@ -0,0 +1,4 @@
+
+"id","params","data","url","input","created_at"
+70,"{""ConnectionId"":2,""BoardId"":8}","{""id"":""customfield_10315"",""name"":""cu用户单选"",""custom"":true,""orderable"":true,""navigable"":true,""searchable"":true,""clauseNames"":[""cf[10315]"",""user_single""],""schema"":{""type"":""user"",""custom"":""com.atlassian.jira.plugin.system.customfieldtypes:userpicker"",""customId"":10315}}","http://172.26.129.92:8092/rest/api/2/field","null","2024-07-11 06:49:14.517"
+66,"{""ConnectionId"":2,""BoardId"":8}","{""id"":""customfield_10400"",""name"":""gitBranch"",""custom"":true,""orderable"":true,""navigable"":true,""searchable"":true,""clauseNames"":[""cf[10400]"",""gitBranch""],""schema"":{""type"":""any"",""custom"":""com.xiplink.jira.git.jira_git_plugin:gitbranches"",""customId"":10400}}","http://172.26.129.92:8092/rest/api/2/field","null","2024-07-11 06:49:14.517"

backend/plugins/jira/e2e/snapshot_tables/_tool_jira_issue_fields.csv

@@ -0,0 +1,3 @@
+connection_id,board_id,id,name,custom,orderable,navigable,searchable,schema_type,schema_items,schema_custom,schema_custom_id,sche_custom_system,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+2,8,customfield_10315,cu用户单选,1,1,1,1,user,,com.atlassian.jira.plugin.system.customfieldtypes:userpicker,10315,,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issue_fields,70,
+2,8,customfield_10400,gitBranch,1,1,1,1,any,,com.xiplink.jira.git.jira_git_plugin:gitbranches,10400,,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issue_fields,66,

backend/plugins/jira/impl/impl.go

@@ -90,6 +90,7 @@ func (p Jira) GetTablesInfo() []dal.Tabler {
 		&models.JiraIssueComment{},
 		&models.JiraIssueRelationship{},
 		&models.JiraScopeConfig{},
+		&models.JiraIssueField{},
 	}
 }
 
@@ -103,6 +104,9 @@ func (p Jira) Name() string {
 
 func (p Jira) SubTaskMetas() []plugin.SubTaskMeta {
 	return []plugin.SubTaskMeta{
+		tasks.CollectIssueFieldsMeta,
+		tasks.ExtractIssueFieldsMeta,
+
 		tasks.CollectBoardFilterBeginMeta,
 
 		tasks.CollectStatusMeta,

backend/plugins/jira/models/issue_changelog.go

@@ -40,15 +40,17 @@ type JiraIssueChangelogItems struct {
 	common.NoPKModel
 
 	// collected fields
-	ConnectionId uint64 `gorm:"primaryKey"`
-	ChangelogId  uint64 `gorm:"primaryKey"`
-	Field        string `gorm:"primaryKey"`
-	FieldType    string
-	FieldId      string
-	FromValue    string
-	FromString   string
-	ToValue      string
-	ToString     string
+	ConnectionId     uint64 `gorm:"primaryKey"`
+	ChangelogId      uint64 `gorm:"primaryKey"`
+	Field            string `gorm:"primaryKey"`
+	FieldType        string
+	FieldId          string
+	FromValue        string
+	FromString       string
+	ToValue          string
+	ToString         string
+	TmpFromAccountId string
+	TmpToAccountId   string
 }
 
 func (JiraIssueChangelogs) TableName() string {

backend/plugins/jira/models/issue_field.go

@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/core/models/common"
+
+type JiraIssueField struct {
+	common.NoPKModel
+	ConnectionId uint64 `gorm:"primaryKey"`
+	BoardId      uint64 `gorm:"primaryKey"`
+
+	ID         string `json:"id" gorm:"primaryKey"`
+	Name       string `json:"name"`
+	Custom     bool   `json:"custom"`
+	Orderable  bool   `json:"orderable"`
+	Navigable  bool   `json:"navigable"`
+	Searchable bool   `json:"searchable"`
+	//ClauseNames      []string `json:"clauseNames"`
+	SchemaType       string `json:"schema_type"`
+	SchemaItems      string `json:"schema_items"`
+	SchemaCustom     string `json:"schema_custom"`
+	SchemaCustomID   int    `json:"schema_custom_id"`
+	ScheCustomSystem string `json:"sche_custom_system"`
+}
+
+func (JiraIssueField) TableName() string {
+	return "_tool_jira_issue_fields"
+}

backend/plugins/jira/models/migrationscripts/20240709_add_tmp_account_id_to_issue_changelog_item.go

@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addTmpAccountIdToJiraIssueChangelogItem)(nil)
+
+type jiraIssueChangelogItems20240709 struct {
+	TmpFromAccountId string
+	TmpToAccountId   string
+}
+
+func (jiraIssueChangelogItems20240709) TableName() string {
+	return "_tool_jira_issue_changelog_items"
+}
+
+type addTmpAccountIdToJiraIssueChangelogItem struct{}
+
+func (script *addTmpAccountIdToJiraIssueChangelogItem) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&jiraIssueChangelogItems20240709{})
+}
+
+func (*addTmpAccountIdToJiraIssueChangelogItem) Version() uint64 {
+	return 20240709134200
+}
+
+func (*addTmpAccountIdToJiraIssueChangelogItem) Name() string {
+	return "add TmpFromAccountId and TmpToAccountId to _tool_jira_issue_changelog_items"
+}

backend/plugins/jira/models/migrationscripts/20240710_add_issue_field_table.go

@@ -0,0 +1,41 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived"
+)
+
+var _ plugin.MigrationScript = (*addIssueFieldTable)(nil)
+
+type addIssueFieldTable struct{}
+
+func (script *addIssueFieldTable) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&archived.JiraIssueField{})
+}
+
+func (*addIssueFieldTable) Version() uint64 {
+	return 20240710100000
+}
+
+func (*addIssueFieldTable) Name() string {
+	return "init table _tool_jira_issue_fields"
+}

backend/plugins/jira/models/migrationscripts/archived/issue_field.go

@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+
+type JiraIssueField struct {
+	archived.NoPKModel
+	ConnectionId uint64 `gorm:"primaryKey"`
+	BoardId      uint64 `gorm:"primaryKey"`
+
+	ID               string `json:"id" gorm:"primaryKey"`
+	Name             string `json:"name"`
+	Custom           bool   `json:"custom"`
+	Orderable        bool   `json:"orderable"`
+	Navigable        bool   `json:"navigable"`
+	Searchable       bool   `json:"searchable"`
+	SchemaType       string `json:"schema_type"`
+	SchemaItems      string `json:"schema_items"`
+	SchemaCustom     string `json:"schema_custom"`
+	SchemaCustomID   int    `json:"schema_custom_id"`
+	ScheCustomSystem string `json:"sche_custom_system"`
+}
+
+func (JiraIssueField) TableName() string {
+	return "_tool_jira_issue_fields"
+}

backend/plugins/jira/models/migrationscripts/register.go

@@ -49,5 +49,7 @@ func All() []plugin.MigrationScript {
 		new(addFilterJQL),
 		new(addWorklogToIssue),
 		new(addSubtaskToIssue),
+		new(addTmpAccountIdToJiraIssueChangelogItem),
+		new(addIssueFieldTable),
 	}
 }

backend/plugins/jira/tasks/apiv2models/changelog.go

@@ -46,38 +46,46 @@ func (c Changelog) ToToolLayer(connectionId, issueId uint64, issueUpdated *time.
 }
 
 type ChangelogItem struct {
-	Field      string `json:"field"`
-	Fieldtype  string `json:"fieldtype"`
+	Field     string `json:"field"`
+	Fieldtype string `json:"fieldtype"`
+	FieldId   string `json:"fieldId"`
+
 	FromValue  string `json:"from"`
 	FromString string `json:"fromString"`
-	ToValue    string `json:"to"`
-	ToString   string `json:"toString"`
+
+	ToValue  string `json:"to"`
+	ToString string `json:"toString"`
+
+	TmpFromAccountId string `json:"tmpFromAccountId,omitempty"`
+	TmpToAccountId   string `json:"tmpToAccountId,omitempty"`
 }
 
 func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.JiraIssueChangelogItems {
 	item := &models.JiraIssueChangelogItems{
-		ConnectionId: connectionId,
-		ChangelogId:  changelogId,
-		Field:        c.Field,
-		FieldType:    c.Fieldtype,
-		FromValue:    c.FromValue,
-		FromString:   c.FromString,
-		ToValue:      c.ToValue,
-		ToString:     c.ToString,
+		ConnectionId:     connectionId,
+		ChangelogId:      changelogId,
+		Field:            c.Field,
+		FieldType:        c.Fieldtype,
+		FromValue:        c.FromValue,
+		FromString:       c.FromString,
+		ToValue:          c.ToValue,
+		ToString:         c.ToString,
+		TmpFromAccountId: c.TmpFromAccountId,
+		TmpToAccountId:   c.TmpToAccountId,
 	}
 	return item
 }
 
-func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount {
-	if c.Field != "assignee" {
-		return nil
-	}
+func (c ChangelogItem) ExtractUser(connectionId uint64, userFieldMaps map[string]struct{}) []*models.JiraAccount {
 	var result []*models.JiraAccount
-	if c.FromValue != "" {
-		result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue})
-	}
-	if c.ToValue != "" {
-		result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue})
+	_, ok := userFieldMaps[c.Field]
+	if c.Field == "assignee" || c.Field == "reporter" || ok {
+		if c.FromValue != "" {
+			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue})
+		}
+		if c.ToValue != "" {
+			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue})
+		}
 	}
 	return result
 }

backend/plugins/jira/tasks/apiv2models/issue.go

@@ -300,7 +300,7 @@ func (i *Issue) SetAllFields(raw json.RawMessage) errors.Error {
 	return nil
 }
 
-func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) {
+func (i Issue) ExtractEntities(connectionId uint64, userFieldMaps map[string]struct{}) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) {
 	issue := i.toToolLayer(connectionId)
 	var comments []*models.JiraIssueComment
 	var worklogs []*models.JiraWorklog
@@ -341,7 +341,7 @@ func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue
 			}
 			for _, item := range changelog.Items {
 				changelogItems = append(changelogItems, item.ToToolLayer(connectionId, changelog.ID))
-				users = append(users, item.ExtractUser(connectionId)...)
+				users = append(users, item.ExtractUser(connectionId, userFieldMaps)...)
 			}
 		}
 	}