fix(gitlab): use stateful extractors for users and tags

backend/plugins/gitlab/tasks/account_extractor.go

@@ -18,7 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
 	"strings"
 
 	"github.com/apache/incubator-devlake/core/errors"
@@ -41,63 +40,29 @@ var ExtractAccountsMeta = plugin.SubTaskMeta{
 }
 
 func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE)
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_USER_TABLE)
 
 	// Do not extract createdUserAt if we are not using /users API
-	var skipCreatedUserAt = strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com")
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			var userRes models.GitlabAccount
-			err := errors.Convert(json.Unmarshal(row.Data, &userRes))
-			if err != nil {
-				return nil, err
-			}
+	skipCreatedUserAt := strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com")
+	subtaskCommonArgs.SubtaskConfig = map[string]any{
+		"skipCreatedUserAt": skipCreatedUserAt,
+	}
 
-			results := make([]interface{}, 0)
-			var GitlabAccount *models.GitlabAccount
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[models.GitlabAccount]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(userRes *models.GitlabAccount, _ *api.RawData) ([]interface{}, errors.Error) {
+			account := *userRes
+			account.ConnectionId = data.Options.ConnectionId
 			if skipCreatedUserAt {
-				GitlabAccount = &models.GitlabAccount{
-					ConnectionId:    data.Options.ConnectionId,
-					GitlabId:        userRes.GitlabId,
-					Username:        userRes.Username,
-					Name:            userRes.Name,
-					State:           userRes.State,
-					MembershipState: userRes.MembershipState,
-					AvatarUrl:       userRes.AvatarUrl,
-					WebUrl:          userRes.WebUrl,
-					Email:           userRes.Email,
-				}
-			} else {
-				GitlabAccount = &models.GitlabAccount{
-					ConnectionId:    data.Options.ConnectionId,
-					GitlabId:        userRes.GitlabId,
-					Username:        userRes.Username,
-					Name:            userRes.Name,
-					State:           userRes.State,
-					MembershipState: userRes.MembershipState,
-					AvatarUrl:       userRes.AvatarUrl,
-					WebUrl:          userRes.WebUrl,
-					Email:           userRes.Email,
-					CreatedUserAt:   userRes.CreatedUserAt,
-				}
+				account.CreatedUserAt = nil
 			}
-
-			results = append(results, GitlabAccount)
-
-			return results, nil
+			return []interface{}{&account}, nil
 		},
 	})
 
 	if err != nil {
 		return err
 	}
 
-	err = extractor.Execute()
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return extractor.Execute()
 }

backend/plugins/gitlab/tasks/tag_extractor.go

@@ -18,8 +18,6 @@ limitations under the License.
 package tasks
 
 import (
-	"encoding/json"
-
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -51,27 +49,17 @@ var ExtractTagMeta = plugin.SubTaskMeta{
 }
 
 func ExtractApiTag(taskCtx plugin.SubTaskContext) errors.Error {
-	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_TAG_TABLE)
-
-	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
-		RawDataSubTaskArgs: *rawDataSubTaskArgs,
-		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			// need to extract 1 kind of entities here
-			results := make([]interface{}, 0, 1)
+	subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_TAG_TABLE)
 
-			gitlabApiTag := &GitlabApiTag{}
-			err := errors.Convert(json.Unmarshal(row.Data, gitlabApiTag))
-			if err != nil {
-				return nil, err
-			}
+	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabApiTag]{
+		SubtaskCommonArgs: subtaskCommonArgs,
+		Extract: func(gitlabApiTag *GitlabApiTag, _ *api.RawData) ([]interface{}, errors.Error) {
 			gitlabTag, err := convertTag(gitlabApiTag)
 			if err != nil {
 				return nil, err
 			}
 			gitlabTag.ConnectionId = data.Options.ConnectionId
-			results = append(results, gitlabTag)
-
-			return results, nil
+			return []interface{}{gitlabTag}, nil
 		},
 	})