diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_extractor.go index d2d24b48c9e..0024c3ead6e 100644 --- a/backend/plugins/gitlab/tasks/account_extractor.go +++ b/backend/plugins/gitlab/tasks/account_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "strings" "github.com/apache/incubator-devlake/core/errors" @@ -41,52 +40,23 @@ var ExtractAccountsMeta = plugin.SubTaskMeta{ } func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE) + subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_USER_TABLE) // Do not extract createdUserAt if we are not using /users API - var skipCreatedUserAt = strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") - - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - var userRes models.GitlabAccount - err := errors.Convert(json.Unmarshal(row.Data, &userRes)) - if err != nil { - return nil, err - } + skipCreatedUserAt := strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") + subtaskCommonArgs.SubtaskConfig = map[string]any{ + "skipCreatedUserAt": skipCreatedUserAt, + } - results := make([]interface{}, 0) - var GitlabAccount *models.GitlabAccount + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[models.GitlabAccount]{ + SubtaskCommonArgs: subtaskCommonArgs, + Extract: func(userRes *models.GitlabAccount, _ *api.RawData) ([]interface{}, errors.Error) { + account := *userRes + account.ConnectionId = data.Options.ConnectionId if skipCreatedUserAt { - GitlabAccount = &models.GitlabAccount{ - ConnectionId: data.Options.ConnectionId, - GitlabId: userRes.GitlabId, - Username: userRes.Username, - Name: userRes.Name, - State: userRes.State, - MembershipState: userRes.MembershipState, - AvatarUrl: userRes.AvatarUrl, - WebUrl: userRes.WebUrl, - Email: userRes.Email, - } - } else { - GitlabAccount = &models.GitlabAccount{ - ConnectionId: data.Options.ConnectionId, - GitlabId: userRes.GitlabId, - Username: userRes.Username, - Name: userRes.Name, - State: userRes.State, - MembershipState: userRes.MembershipState, - AvatarUrl: userRes.AvatarUrl, - WebUrl: userRes.WebUrl, - Email: userRes.Email, - CreatedUserAt: userRes.CreatedUserAt, - } + account.CreatedUserAt = nil } - - results = append(results, GitlabAccount) - - return results, nil + return []interface{}{&account}, nil }, }) @@ -94,10 +64,5 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { return err } - err = extractor.Execute() - if err != nil { - return err - } - - return nil + return extractor.Execute() } diff --git a/backend/plugins/gitlab/tasks/tag_extractor.go b/backend/plugins/gitlab/tasks/tag_extractor.go index 60dac78a449..4f0a511dea2 100644 --- a/backend/plugins/gitlab/tasks/tag_extractor.go +++ b/backend/plugins/gitlab/tasks/tag_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -51,27 +49,17 @@ var ExtractTagMeta = plugin.SubTaskMeta{ } func ExtractApiTag(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_TAG_TABLE) - - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - // need to extract 1 kind of entities here - results := make([]interface{}, 0, 1) + subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_TAG_TABLE) - gitlabApiTag := &GitlabApiTag{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiTag)) - if err != nil { - return nil, err - } + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabApiTag]{ + SubtaskCommonArgs: subtaskCommonArgs, + Extract: func(gitlabApiTag *GitlabApiTag, _ *api.RawData) ([]interface{}, errors.Error) { gitlabTag, err := convertTag(gitlabApiTag) if err != nil { return nil, err } gitlabTag.ConnectionId = data.Options.ConnectionId - results = append(results, gitlabTag) - - return results, nil + return []interface{}{gitlabTag}, nil }, })