feat: make deduplication optional

ericfrederich · ericfrederich · commit 99cee2a3af8c · 2024-10-15T10:51:50.000-04:00
diff --git a/gitlab/client.py b/gitlab/client.py
@@ -1167,13 +1167,15 @@ def __init__(
         url: str,
         query_data: Dict[str, Any],
         get_next: bool = True,
+        dedupe: bool = True,
         **kwargs: Any,
     ) -> None:
         self._gl = gl
 
         # Preserve kwargs for subsequent queries
         self._kwargs = kwargs.copy()
 
+        self._dedupe = dedupe
         self._retrieved_object_ids: set[int] = set()
 
         self._query(url, query_data, **self._kwargs)
@@ -1207,17 +1209,20 @@ def _query(
                 error_message="Failed to parse the server message"
             ) from e
 
-        duplicate_ids = set(o["id"] for o in self._data) & self._retrieved_object_ids
-        if duplicate_ids:
-            utils.warn(
-                message=(
-                    f"During pagination duplicate object(s) with id(s) "
-                    f"{duplicate_ids} returned from Gitlab and filtered"
-                ),
-                category=UserWarning,
+        if self._dedupe:
+            duplicate_ids = (
+                set(o["id"] for o in self._data) & self._retrieved_object_ids
             )
-        self._data = [o for o in self._data if o["id"] not in duplicate_ids]
-        self._retrieved_object_ids.update(o["id"] for o in self._data)
+            if duplicate_ids:
+                utils.warn(
+                    message=(
+                        f"During pagination duplicate object(s) with id(s) "
+                        f"{duplicate_ids} returned from Gitlab and filtered"
+                    ),
+                    category=UserWarning,
+                )
+            self._data = [o for o in self._data if o["id"] not in duplicate_ids]
+            self._retrieved_object_ids.update(o["id"] for o in self._data)
 
         self._current = 0
 
