fix: filter duplicates from previous page during pagination

ericfrederich · ericfrederich · commit cd22af55ddf6 · 2024-09-17T09:43:33.000-04:00
fixes #2979
diff --git a/gitlab/client.py b/gitlab/client.py
@@ -1174,6 +1174,8 @@ def __init__(
         # Preserve kwargs for subsequent queries
         self._kwargs = kwargs.copy()
 
+        self._prev_page_objects = []
+
         self._query(url, query_data, **self._kwargs)
         self._get_next = get_next
 
@@ -1199,12 +1201,26 @@ def _query(
         self._total: Optional[str] = result.headers.get("X-Total")
 
         try:
-            self._data: List[Dict[str, Any]] = result.json()
+            data: List[Dict[str, Any]] = result.json()
         except Exception as e:
             raise gitlab.exceptions.GitlabParsingError(
                 error_message="Failed to parse the server message"
             ) from e
 
+        self._data = []
+        for item in data:
+            if item in self._prev_page_objects:
+                utils.warn(
+                    message=(
+                        f"During pagination duplicate object with id "
+                        f"{item['id']} returned from Gitlab and filtered"
+                    ),
+                    category=UserWarning,
+                )
+                continue
+            self._data.append(item)
+
+        self._prev_page_objects = list(self._data)
         self._current = 0
 
     @property
