[Integration][Gitlab] Added support for gitlab member ingestion (#767)

integrations/gitlab/CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+0.1.141 (2024-11-13)
+===================
+
+### Features
+
+- Added support for gitlab member ingestion (PORT-7708)
+
+
 0.1.140 (2024-11-12)
 ====================
 

integrations/gitlab/gitlab_integration/core/async_fetcher.py

@@ -12,6 +12,8 @@
     ProjectPipeline,
     Issue,
     Group,
+    User,
+    GroupMember,
     ProjectFile,
 )
 from loguru import logger
@@ -36,6 +38,8 @@ async def fetch_single(
                 Issue,
                 Project,
                 Group,
+                User,
+                GroupMember,
             ],
         ],
         *args,

integrations/gitlab/gitlab_integration/events/hooks/base.py

@@ -1,9 +1,16 @@
 from abc import ABC, abstractmethod
-from typing import List, Any
+from typing import List, Any, Dict, Optional
+import typing
 from loguru import logger
-from gitlab.v4.objects import Project
-
+from gitlab.v4.objects import Project, Group
+from gitlab.base import RESTObject
 from gitlab_integration.gitlab_service import GitlabService
+from port_ocean.context.ocean import ocean
+from port_ocean.context.event import event
+from gitlab_integration.git_integration import (
+    GitlabPortAppConfig,
+    GitlabMemberSelector,
+)
 
 
 class HookHandler(ABC):
@@ -20,6 +27,41 @@ def __init__(
     async def on_hook(self, event: str, body: dict[str, Any]) -> None:
         pass
 
+    async def _register_object_with_members(self, kind: str, gitlab_object: RESTObject):
+        resource_configs = typing.cast(
+            GitlabPortAppConfig, event.port_app_config
+        ).resources
+
+        matching_resource_configs = [
+            resource_config
+            for resource_config in resource_configs
+            if (
+                resource_config.kind == kind
+                and isinstance(resource_config.selector, GitlabMemberSelector)
+            )
+        ]
+
+        if not matching_resource_configs:
+            logger.info(
+                "Resource not found in port app config, update port app config to include the resource type"
+            )
+            return
+
+        for resource_config in matching_resource_configs:
+            include_bot_members = resource_config.selector.include_bot_members
+            include_inherited_members = (
+                resource_config.selector.include_inherited_members
+            )
+
+            object_result: RESTObject = (
+                await self.gitlab_service.enrich_object_with_members(
+                    gitlab_object,
+                    include_bot_members,
+                    include_inherited_members,
+                )
+            )
+            await ocean.register_raw(resource_config.kind, [object_result.asdict()])
+
 
 class ProjectHandler(HookHandler):
     async def on_hook(self, event: str, body: dict[str, Any]) -> None:
@@ -49,3 +91,23 @@ async def on_hook(self, event: str, body: dict[str, Any]) -> None:
     @abstractmethod
     async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         pass
+
+
+class GroupHandler(HookHandler):
+    async def on_hook(self, event: str, body: dict[str, Any]) -> None:
+        logger.info(f"Handling {event}")
+
+        group_id = body.get("group_id", body.get("group", {}).get("id"))
+        group = await self.gitlab_service.get_group(group_id)
+        await self._on_hook(body, group)
+        group_path = body.get("full_path", body.get("group_path"))
+        logger.info(f"Finished handling {event} for group {group_path}")
+
+    @abstractmethod
+    async def _on_hook(
+        self, body: dict[str, Any], gitlab_group: Optional[Group]
+    ) -> None:
+        pass
+
+    async def _register_group(self, kind: str, gitlab_group: Dict[str, Any]) -> None:
+        await ocean.register_raw(kind, [gitlab_group])

integrations/gitlab/gitlab_integration/events/hooks/group.py

@@ -1,39 +1,48 @@
-from typing import Any
+from typing import Any, Optional
 
 from loguru import logger
 
-from gitlab_integration.events.hooks.base import HookHandler
 from gitlab_integration.utils import ObjectKind
+from gitlab_integration.events.hooks.base import GroupHandler
 from port_ocean.context.ocean import ocean
+from gitlab.v4.objects import Group
 
 
-class GroupHook(HookHandler):
+class Groups(GroupHandler):
     events = ["Subgroup Hook"]
-    system_events = [
-        "group_destroy",
-        "group_create",
-        "group_rename",
-    ]
+    system_events = ["group_destroy", "group_create", "group_rename"]
 
-    async def on_hook(self, event: str, body: dict[str, Any]) -> None:
-        event_name = body["event_name"]
-
-        logger.info(f"Handling {event_name} for {event}")
-
-        group_id = body["group_id"] if "group_id" in body else body["group"]["id"]
-
-        logger.info(f"Handling hook {event} for group {group_id}")
-
-        group = await self.gitlab_service.get_group(group_id)
+    async def _on_hook(
+        self, body: dict[str, Any], gitlab_group: Optional[Group]
+    ) -> None:
 
+        group_id = body.get("group_id")
         group_full_path = body.get("full_path")
-        if group:
-            await ocean.register_raw(ObjectKind.GROUP, [group.asdict()])
+        event_name = body["event_name"]
+
+        logger.info(
+            f"Handling event '{event_name}' for group with ID '{group_id}' and full path '{group_full_path}'"
+        )
+        if gitlab_group:
+            await self._register_group(
+                ObjectKind.GROUP,
+                gitlab_group.asdict(),
+            )
+            await self._register_object_with_members(
+                ObjectKind.GROUPWITHMEMBERS, gitlab_group
+            )
+            logger.info(f"Registered group {group_id}")
         elif (
             group_full_path
             and self.gitlab_service.should_run_for_path(group_full_path)
             and event_name in ("subgroup_destroy", "group_destroy")
         ):
             await ocean.unregister_raw(ObjectKind.GROUP, [body])
+            await ocean.unregister_raw(ObjectKind.GROUPWITHMEMBERS, [body])
+            logger.info(f"Unregistered group {group_id}")
+            return
+
         else:
-            logger.info(f"Group {group_id} was filtered for event {event}. Skipping...")
+            logger.info(
+                f"Group {group_id} was filtered for event {event_name}. Skipping..."
+            )

integrations/gitlab/gitlab_integration/events/hooks/members.py

@@ -0,0 +1,29 @@
+from typing import Any, Optional
+from loguru import logger
+
+from gitlab_integration.utils import ObjectKind
+from gitlab_integration.events.hooks.base import GroupHandler
+from gitlab.v4.objects import Group
+
+
+class Members(GroupHandler):
+    events = ["Member Hook"]
+    system_events = [
+        "user_remove_from_group",
+        "user_update_for_group",
+        "user_add_to_group",
+    ]
+
+    async def _on_hook(
+        self, body: dict[str, Any], gitlab_group: Optional[Group]
+    ) -> None:
+        if gitlab_group:
+            event_name, user_username = (body["event_name"], body["user_username"])
+            logger.info(f"Handling {event_name} for group member {user_username}")
+            await self._register_object_with_members(
+                ObjectKind.GROUPWITHMEMBERS, gitlab_group
+            )
+        else:
+            logger.info(
+                f"Group member's group {body['group_id']} was filtered for event {body['event_name']}. Skipping..."
+            )

integrations/gitlab/gitlab_integration/events/hooks/push.py

@@ -103,7 +103,10 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
             enriched_project = await self.gitlab_service.enrich_project_with_extras(
                 gitlab_project
             )
-            await ocean.register_raw(ObjectKind.PROJECT, [enriched_project])
+            await ocean.register_raw(ObjectKind.PROJECT, [enriched_project.asdict()])
+            await self._register_object_with_members(
+                ObjectKind.PROJECTWITHMEMBERS, enriched_project
+            )
 
         else:
             logger.debug(

integrations/gitlab/gitlab_integration/events/setup.py

@@ -11,7 +11,8 @@
 from gitlab_integration.events.hooks.merge_request import MergeRequest
 from gitlab_integration.events.hooks.pipelines import Pipelines
 from gitlab_integration.events.hooks.push import PushHook
-from gitlab_integration.events.hooks.group import GroupHook
+from gitlab_integration.events.hooks.members import Members
+from gitlab_integration.events.hooks.group import Groups
 from gitlab_integration.events.hooks.project_files import ProjectFiles
 from gitlab_integration.gitlab_service import GitlabService
 from gitlab_integration.models.webhook_groups_override_config import (
@@ -122,7 +123,8 @@ def setup_listeners(gitlab_service: GitlabService, group_id: str) -> None:
         Job(gitlab_service),
         Issues(gitlab_service),
         Pipelines(gitlab_service),
-        GroupHook(gitlab_service),
+        Groups(gitlab_service),
+        Members(gitlab_service),
         ProjectFiles(gitlab_service),
     ]
     for handler in handlers:
@@ -140,7 +142,8 @@ def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None:
         Job,
         Issues,
         Pipelines,
-        GroupHook,
+        Groups,
+        Members,
         ProjectFiles,
     ]
     for handler in handlers:

integrations/gitlab/gitlab_integration/git_integration.py

@@ -121,6 +121,25 @@ class GitlabResourceConfig(ResourceConfig):
     selector: GitlabSelector
 
 
+class GitlabMemberSelector(Selector):
+
+    include_inherited_members: bool = Field(
+        alias="includeInheritedMembers",
+        default=False,
+        description="If set to true, the integration will include inherited members in the group members list. Default value is false",
+    )
+    include_bot_members: bool = Field(
+        alias="includeBotMembers",
+        default=True,
+        description="If set to false, bots will be filtered out from the members list. Default value is true",
+    )
+
+
+class GitlabObjectWithMembersResourceConfig(ResourceConfig):
+    kind: Literal["project-with-members", "group-with-members"]
+    selector: GitlabMemberSelector
+
+
 class FilesSelector(BaseModel):
     path: str = Field(description="The path to get the files from")
     repos: List[str] = Field(
@@ -146,7 +165,7 @@ class GitlabPortAppConfig(PortAppConfig):
     project_visibility_filter: str | None = Field(
         alias="projectVisibilityFilter", default=None
     )
-    resources: list[GitLabFilesResourceConfig | GitlabResourceConfig] = Field(default_factory=list)  # type: ignore
+    resources: list[GitlabObjectWithMembersResourceConfig | GitLabFilesResourceConfig | GitlabResourceConfig] = Field(default_factory=list)  # type: ignore
 
 
 def _get_project_from_cache(project_id: int) -> Project | None: