Skip to content

Commit

Permalink
8216 featurecustomize add support for incremental csv upload in the c…
Browse files Browse the repository at this point in the history
…ustomize plugin (apache#8218)

* feat: support incremental import for 

- /plugins/customize/csvfiles/issues.csv
- /plugins/customize/csvfiles/issue_repo_commits.csv

[Feature][Customize] Add Support for Incremental CSV Upload in the Customize Plugin apache#8216
  • Loading branch information
narrowizard authored Nov 26, 2024
1 parent 8c7f3c2 commit 1dfb12d
Show file tree
Hide file tree
Showing 19 changed files with 224 additions and 126 deletions.
14 changes: 12 additions & 2 deletions backend/plugins/customize/api/csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const maxMemory = 32 << 20 // 32 MB
// @Accept multipart/form-data
// @Param boardId formData string true "the ID of the board"
// @Param boardName formData string true "the name of the board"
// @Param incremental formData bool false "whether to import incrementally"
// @Param file formData file true "select file to upload"
// @Produce json
// @Success 200
Expand All @@ -47,6 +48,10 @@ func (h *Handlers) ImportIssue(input *plugin.ApiResourceInput) (*plugin.ApiResou
}
// nolint
defer file.Close()
incremental := false
if input.Request.FormValue("incremental") == "true" {
incremental = true
}
boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
if boardId == "" {
return nil, errors.BadInput.New("empty boardId")
Expand All @@ -59,7 +64,7 @@ func (h *Handlers) ImportIssue(input *plugin.ApiResourceInput) (*plugin.ApiResou
if err != nil {
return nil, err
}
return nil, h.svc.ImportIssue(boardId, file)
return nil, h.svc.ImportIssue(boardId, file, incremental)
}

// ImportIssueCommit accepts a CSV file, parses and saves it to the database
Expand Down Expand Up @@ -94,6 +99,7 @@ func (h *Handlers) ImportIssueCommit(input *plugin.ApiResourceInput) (*plugin.Ap
// @Tags plugins/customize
// @Accept multipart/form-data
// @Param boardId formData string true "the ID of the board"
// @Param incremental formData bool false "whether to import incrementally"
// @Param file formData file true "select file to upload"
// @Produce json
// @Success 200
Expand All @@ -111,7 +117,11 @@ func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput) (*plugi
if boardId == "" {
return nil, errors.Default.New("empty boardId")
}
return nil, h.svc.ImportIssueRepoCommit(boardId, file)
incremental := false
if input.Request.FormValue("incremental") == "true" {
incremental = true
}
return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental)
}

func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser, errors.Error) {
Expand Down
4 changes: 2 additions & 2 deletions backend/plugins/customize/e2e/import_issue_commits_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ func TestImportIssueCommitDataFlow(t *testing.T) {
}
defer f.Close()
// import data
err := svc.ImportIssueCommit(`{"ConnectionId":1,"BoardId":8}`, f)
err := svc.ImportIssueCommit(`csv-board`, f)
if err != nil {
t.Fatal(err)
}
dataflowTester.VerifyTableWithRawData(
crossdomain.IssueCommit{},
"snapshot_tables/issue_commits.csv",
"snapshot_tables/issue_commits_from_import_issue_commit.csv",
[]string{
"issue_id",
"commit_sha",
Expand Down
31 changes: 25 additions & 6 deletions backend/plugins/customize/e2e/import_issue_repo_commits_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ limitations under the License.
package e2e

import (
"os"
"testing"

"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
"github.com/apache/incubator-devlake/plugins/customize/service"
"os"
"testing"
)

func TestImportIssueRepoCommitDataFlow(t *testing.T) {
Expand All @@ -33,18 +35,35 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) {
// create tables `issue_repo_commits` and `issue_commits`
dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
dataflowTester.FlushTabler(&crossdomain.IssueCommit{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})

dataflowTester.ImportCsvIntoTabler("raw_tables/issue_repo_commits_original.csv", &crossdomain.IssueRepoCommit{})
dataflowTester.ImportCsvIntoTabler("raw_tables/board_issues.csv", &ticket.BoardIssue{})

svc := service.NewService(dataflowTester.Dal)

f, err1 := os.Open("raw_tables/issue_repo_commits.csv")
issueRepoCommitsFile, err1 := os.Open("raw_tables/issue_repo_commits.csv")
if err1 != nil {
t.Fatal(err1)
}
defer f.Close()
defer issueRepoCommitsFile.Close()
// import data
err := svc.ImportIssueRepoCommit(`{"ConnectionId":1,"BoardId":8}`, f)
err := svc.ImportIssueRepoCommit("csv-board", issueRepoCommitsFile, false)
if err != nil {
t.Fatal(err)
}

// import data incrementally
issueRepoCommitsIncrementalFile, err2 := os.Open("raw_tables/issue_repo_commits_incremental.csv")
if err2 != nil {
t.Fatal(err2)
}
defer issueRepoCommitsIncrementalFile.Close()
err = svc.ImportIssueRepoCommit("csv-board", issueRepoCommitsIncrementalFile, true)
if err != nil {
t.Fatal(err)
}

dataflowTester.VerifyTableWithRawData(
crossdomain.IssueRepoCommit{},
"snapshot_tables/issue_repo_commits.csv",
Expand All @@ -58,7 +77,7 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) {
})
dataflowTester.VerifyTableWithRawData(
crossdomain.IssueCommit{},
"snapshot_tables/issue_commits.csv",
"snapshot_tables/issue_commits_from_import_issue_repo_commit.csv",
[]string{
"issue_id",
"commit_sha",
Expand Down
26 changes: 21 additions & 5 deletions backend/plugins/customize/e2e/import_issues_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,21 +89,37 @@ func TestImportIssueDataFlow(t *testing.T) {
t.Fatal(err1)
}
defer issueFile.Close()
err = svc.ImportIssue("csv-board", issueFile)
err = svc.ImportIssue("csv-board", issueFile, false)
if err != nil {
t.Fatal(err)
}

issueFile2, err2 := os.Open("raw_tables/issues_input2.csv")
issueAppendToFile1, err2 := os.Open("raw_tables/issues_input_incremental.csv")
if err2 != nil {
t.Fatal(err2)
}
defer issueAppendToFile1.Close()
err = svc.ImportIssue("csv-board", issueAppendToFile1, true)
if err != nil {
t.Fatal(err)
}
issueFile2, err3 := os.Open("raw_tables/issues_input2.csv")
if err3 != nil {
t.Fatal(err3)
}
defer issueFile2.Close()
err = svc.ImportIssue("csv-board2", issueFile2)
err = svc.ImportIssue("csv-board2", issueFile2, false)
if err != nil {
t.Fatal(err)
}
issueToOverwriteFile2, err4 := os.Open("raw_tables/issues_input2_overwrite.csv")
if err4 != nil {
t.Fatal(err4)
}
defer issueToOverwriteFile2.Close()
err = svc.ImportIssue("csv-board2", issueToOverwriteFile2, false)
if err != nil {
t.Fatal(err)
}

dataflowTester.VerifyTableWithRawData(
ticket.Issue{},
"snapshot_tables/issues_output.csv",
Expand Down
16 changes: 16 additions & 0 deletions backend/plugins/customize/e2e/raw_tables/board_issues.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv-board,csv:1,,,0,
csv-board,csv:10,,,0,
csv-board,csv:11,,,0,
csv-board,csv:12,,,0,
csv-board,csv:13,,,0,
csv-board,csv:14,,,0,
csv-board2,jira:JiraIssue:1:10063,,,0,
csv-board2,jira:JiraIssue:1:10064,,,0,
csv-board2,jira:JiraIssue:1:10065,,,0,
csv-board2,jira:JiraIssue:1:10066,,,0,
csv-board2,jira:JiraIssue:1:10139,,,0,
csv-board2,jira:JiraIssue:1:10145,,,0,
csv-board2,jira:JiraIssue:1:10159,,,0,
csv-board2,jira:JiraIssue:1:10202,,,0,
csv-board2,jira:JiraIssue:1:10203,,,0,
12 changes: 0 additions & 12 deletions backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,3 @@ csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_removed,commit_id_should_be_removed,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,,
3 changes: 0 additions & 3 deletions backend/plugins/customize/e2e/raw_tables/issues_input.csv
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,3 @@ id,url,issue_key,title,original_type,original_status,created_date,resolution_dat
csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"
4 changes: 2 additions & 2 deletions backend/plugins/customize/e2e/raw_tables/issues_input2.csv
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue title edited,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"
25 changes: 0 additions & 25 deletions backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv

This file was deleted.

Loading

0 comments on commit 1dfb12d

Please sign in to comment.