fix(gitlab): fetch resoruces nested under groups

goto · Feb 28, 2024 · e8bdf03 · e8bdf03
1 parent f676c96
commit e8bdf03
Show file tree

Hide file tree

Showing 3 changed files with 90 additions and 62 deletions.
diff --git a/plugins/providers/gitlab/provider.go b/plugins/providers/gitlab/provider.go
@@ -71,58 +71,71 @@ func (p *provider) GetResources(ctx context.Context, pc *domain.ProviderConfig)
 		return nil, err
 	}
 
-	var resources []*domain.Resource
-	resourceTypes := pc.GetResourceTypes()
-
-	var mu sync.Mutex
 	eg, ctx := errgroup.WithContext(ctx)
+	eg.SetLimit(20)
 
-	if utils.ContainsString(resourceTypes, resourceTypeGroup) {
-		eg.Go(func() error {
-			groups, err := fetchResources(ctx,
-				func(listOpt gitlab.ListOptions, reqOpts ...gitlab.RequestOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) {
-					return client.Groups.ListGroups(&gitlab.ListGroupsOptions{ListOptions: listOpt}, reqOpts...)
-				},
-				func(g *gitlab.Group) *domain.Resource {
-					r := group{*g, pc.Type, pc.URN}.toResource()
-					return &r
-				},
-			)
-			if err != nil {
-				return fmt.Errorf("unable to fetch groups: %w", err)
-			}
-
-			mu.Lock()
-			resources = append(resources, groups...)
-			mu.Unlock()
+	var mu sync.Mutex
+	var resources []*domain.Resource
+	resourceTypes := pc.GetResourceTypes()
 
-			return nil
-		})
+	groups, err := fetchResources(ctx,
+		func(listOpt gitlab.ListOptions, reqOpts ...gitlab.RequestOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) {
+			return client.Groups.ListGroups(&gitlab.ListGroupsOptions{ListOptions: listOpt}, reqOpts...)
+		},
+		func(g *gitlab.Group) *domain.Resource {
+			r := group{*g, pc.Type, pc.URN}.toResource()
+			return &r
+		},
+	)
+	if err != nil {
+		p.logger.Error(ctx, "unable to fetch groups", "provider_urn", pc.URN, "error", err)
+		return nil, fmt.Errorf("unable to fetch groups: %w", err)
 	}
 
-	if utils.ContainsString(resourceTypes, resourceTypeProject) {
+	for _, group := range groups {
+		group := group
 		eg.Go(func() error {
-			projects, err := fetchResources(ctx,
-				func(listOpt gitlab.ListOptions, reqOpts ...gitlab.RequestOptionFunc) ([]*gitlab.Project, *gitlab.Response, error) {
-					return client.Projects.ListProjects(&gitlab.ListProjectsOptions{ListOptions: listOpt}, reqOpts...)
-				},
-				func(p *gitlab.Project) *domain.Resource {
-					r := project{*p, pc.Type, pc.URN}.toResource()
-					return &r
-				},
-			)
-			if err != nil {
-				return fmt.Errorf("unable to fetch projects: %w", err)
+			if utils.ContainsString(resourceTypes, resourceTypeProject) {
+				projects, err := fetchResources(ctx,
+					func(listOpt gitlab.ListOptions, reqOpts ...gitlab.RequestOptionFunc) ([]*gitlab.Project, *gitlab.Response, error) {
+						falseBool := false
+						return client.Groups.ListGroupProjects(
+							group.URN,
+							&gitlab.ListGroupProjectsOptions{
+								ListOptions: listOpt,
+								WithShared:  &falseBool,
+							},
+							reqOpts...,
+						)
+					},
+					func(p *gitlab.Project) *domain.Resource {
+						r := project{*p, pc.Type, pc.URN}.toResource()
+						return &r
+					},
+				)
+				if err != nil {
+					p.logger.Error(ctx, "unable to fetch projects under a group", "provider_urn", pc.URN, "group_id", group.URN, "error", err)
+					return fmt.Errorf("unable to fetch projects under group %q: %w", group.URN, err)
+				}
+
+				if utils.ContainsString(resourceTypes, resourceTypeGroup) {
+					group.Children = projects
+				} else {
+					mu.Lock()
+					resources = append(resources, projects...)
+					mu.Unlock()
+				}
 			}
 
-			mu.Lock()
-			resources = append(resources, projects...)
-			mu.Unlock()
-
+			// TODO: handle group <> sub-groups hierarchy
+			if utils.ContainsString(resourceTypes, resourceTypeGroup) {
+				mu.Lock()
+				resources = append(resources, group)
+				mu.Unlock()
+			}
 			return nil
 		})
 	}
-
 	if err := eg.Wait(); err != nil {
 		return nil, err
 	}

diff --git a/plugins/providers/gitlab/provider_test.go b/plugins/providers/gitlab/provider_test.go
@@ -2,6 +2,7 @@ package gitlab_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -20,10 +21,10 @@ import (
 
 var (
 	groupsEndpoint             = "/api/v4/groups"
+	groupProjectsEndpoint      = func(gID string) string { return fmt.Sprintf("/api/v4/groups/%s/projects", gID) }
 	groupMembersEndpoint       = func(gID string) string { return fmt.Sprintf("/api/v4/groups/%s/members", gID) }
 	groupMemberDetailsEndpoint = func(gID, uID string) string { return fmt.Sprintf("/api/v4/groups/%s/members/%s", gID, uID) }
 
-	projectsEndpoint             = "/api/v4/projects"
 	projectMembersEndpoint       = func(pID string) string { return fmt.Sprintf("/api/v4/projects/%s/members", pID) }
 	projectMemberDetailsEndpoint = func(pID, uID string) string { return fmt.Sprintf("/api/v4/projects/%s/members/%s", pID, uID) }
 )
@@ -78,34 +79,48 @@ func TestCreateConfig(t *testing.T) {
 
 func TestGetResources(t *testing.T) {
 	t.Run("should return gitlab resources on success", func(t *testing.T) {
-		mux := http.NewServeMux()
-		mux.HandleFunc(groupsEndpoint, func(w http.ResponseWriter, r *http.Request) {
-			switch r.Method {
-			case http.MethodGet:
-				groups, err := readFixtures("testdata/groups/page_1.json")
-				require.NoError(t, err)
-				w.WriteHeader(http.StatusOK)
-				w.Write(groups)
-				return
-			default:
-				w.WriteHeader(http.StatusMethodNotAllowed)
-				w.Write(nil)
-			}
-		})
-		mux.HandleFunc(projectsEndpoint, func(w http.ResponseWriter, r *http.Request) {
+		dummyGroupsBytes, err := readFixtures("testdata/groups/page_1.json")
+		require.NoError(t, err)
+		var dummyGroups []map[string]interface{}
+		err = json.Unmarshal(dummyGroupsBytes, &dummyGroups)
+		require.NoError(t, err)
+
+		server := http.NewServeMux()
+		server.HandleFunc(groupsEndpoint, func(w http.ResponseWriter, r *http.Request) {
 			switch r.Method {
 			case http.MethodGet:
-				groups, err := readFixtures("testdata/projects/page_1.json")
-				require.NoError(t, err)
 				w.WriteHeader(http.StatusOK)
-				w.Write(groups)
+				w.Write(dummyGroupsBytes)
 				return
 			default:
 				w.WriteHeader(http.StatusMethodNotAllowed)
 				w.Write(nil)
 			}
 		})
-		ts := httptest.NewServer(mux)
+		for _, g := range dummyGroups {
+			gID := fmt.Sprintf("%v", g["id"])
+			server.HandleFunc(groupProjectsEndpoint(gID), func(w http.ResponseWriter, r *http.Request) {
+				withShared := r.URL.Query().Get("with_shared")
+				switch r.Method {
+				case http.MethodGet:
+					assert.Equal(t, "false", withShared)
+					if gID == "1" {
+						projects, err := readFixtures("testdata/projects/page_1.json")
+						require.NoError(t, err)
+						w.WriteHeader(http.StatusOK)
+						w.Write(projects)
+						return
+					}
+					w.WriteHeader(http.StatusOK)
+					w.Write([]byte("[]"))
+					return
+				default:
+					w.WriteHeader(http.StatusMethodNotAllowed)
+					w.Write(nil)
+				}
+			})
+		}
+		ts := httptest.NewServer(server)
 		defer ts.Close()
 
 		encryptor := new(mocks.Encryptor)

diff --git a/plugins/providers/gitlab/resource.go b/plugins/providers/gitlab/resource.go
@@ -53,7 +53,7 @@ func (p project) toResource() domain.Resource {
 		Type:         resourceTypeProject,
 		URN:          strID,
 		GlobalURN:    utils.GetGlobalURN("gitlab", p.providerURN, resourceTypeProject, strID),
-		Name:         p.Name,
+		Name:         p.NameWithNamespace,
 		Details: map[string]interface{}{
 			resource.ReservedDetailsKeyMetadata: map[string]interface{}{
 				"description":         p.Description,