diff --git a/gitlab/client.py b/gitlab/client.py index ed5803b5c..89c3806cb 100644 --- a/gitlab/client.py +++ b/gitlab/client.py @@ -1167,6 +1167,7 @@ def __init__( url: str, query_data: Dict[str, Any], get_next: bool = True, + dedupe: bool = True, **kwargs: Any, ) -> None: self._gl = gl @@ -1174,6 +1175,9 @@ def __init__( # Preserve kwargs for subsequent queries self._kwargs = kwargs.copy() + self._dedupe = dedupe + self._retrieved_object_ids: set[int] = set() + self._query(url, query_data, **self._kwargs) self._get_next = get_next @@ -1205,6 +1209,21 @@ def _query( error_message="Failed to parse the server message" ) from e + if self._dedupe: + duplicate_ids = ( + set(o["id"] for o in self._data) & self._retrieved_object_ids + ) + if duplicate_ids: + utils.warn( + message=( + f"During pagination duplicate object(s) with id(s) " + f"{duplicate_ids} returned from Gitlab and filtered" + ), + category=UserWarning, + ) + self._data = [o for o in self._data if o["id"] not in duplicate_ids] + self._retrieved_object_ids.update(o["id"] for o in self._data) + self._current = 0 @property