Enable debugging for urllib

juliogonzalez · juliogonzalez · commit d1a1b4e5feaa · 2026-04-13T11:28:42.000+02:00
As of today, using -d or --debug does not offer debugging for the requests made using urllib. This is needed for debugging connection issues, in particular when there are redirects involved. We need this monkey patching because between Python 3.5.2 [1] and 3.11.X, Python stopped respecting http.client.HTTPConnection.debuglevel, and the fix [2] seems got only available starting with Python 3.12 When we start using only 3.12 and later, we can rollback this and just change the code to set http.client.HTTPConnection.debuglevel to 1, when debugging is enabled [1] https://stackoverflow.com/a/74416575 [2] python/cpython#99353 BEFORE: $ rm -rf test/; python3 -m obs_maven.core -d ../spacewalk/java/buildconf/ivy/obs-maven-config.yaml test -a antlr4-runtime DEBUG:root:Reading configuration INFO:root:Processing artifact antlr4-runtime DEBUG:root:Parsing https://download.opensuse.org/repositories/systemsmanagement:/Uyuni:/Master/openSUSE_Leap_15.6/repodata/repomd.xml DEBUG:root:Loading RPMs from cache file: .obs-to-maven-cache/Uyuni/63cc3f113c0efce8106bf832c89795ff5111983a9e4bd5cea7742462e48c6221-primary.xml.gz.data DEBUG:root:package mtime: 1775727845, [] INFO:root:Downloading /tmp/obsmvn-rwdr_yt4/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm DEBUG:root:Getting binary from: https://download.opensuse.org/repositories/systemsmanagement:/Uyuni:/Master/openSUSE_Leap_15.6/noarch/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm DEBUG:root:not linked: /usr/share/doc/packages/antlr4-java /usr/share/doc/packages/antlr4-java/CHANGES.txt /usr/share/doc/packages/antlr4-java/README.md /usr/share/java/antlr4 /usr/share/java/antlr4/antlr4-runtime.jar /usr/share/licenses/antlr4-java /usr/share/licenses/antlr4-java/LICENSE.txt /usr/share/maven-metadata/antlr4-antlr4-runtime.xml /usr/share/maven-poms/antlr4 /usr/share/maven-poms/antlr4/antlr4-master.pom /usr/share/maven-poms/antlr4/antlr4-runtime.pom DEBUG:root:full pattern: ^/usr/.*/antlr4-runtime[^/]*\.jar$ INFO:root:extracting /usr/share/java/antlr4/antlr4-runtime.jar to /tmp/obsmvn-rwdr_yt4/antlr4-runtime.jar INFO:root:deploying /tmp/obsmvn-rwdr_yt4/antlr4-runtime.jar to test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.jar DEBUG:root:Setting mtime 1775727845 on test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.jar DEBUG:root:Computing test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.jar.sha1 DEBUG:root:Computing test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.pom.sha1 AFTER: $ rm -rf test/; python3 -m obs_maven.core -d ../spacewalk/java/buildconf/ivy/obs-maven-config.yaml test -a antlr4-runtime DEBUG:root:Reading configuration INFO:root:Processing artifact antlr4-runtime DEBUG:root:Parsing https://download.opensuse.org/repositories/systemsmanagement:/Uyuni:/Master/openSUSE_Leap_15.6/repodata/repomd.xml DEBUG:root:HTTPS Request: GET /repositories/systemsmanagement:/Uyuni:/Master/openSUSE_Leap_15.6/repodata/repomd.xml HTTP/1.1 DEBUG:root:HTTPS Response: 200 OK DEBUG:root:Loading RPMs from cache file: .obs-to-maven-cache/Uyuni/63cc3f113c0efce8106bf832c89795ff5111983a9e4bd5cea7742462e48c6221-primary.xml.gz.data DEBUG:root:package mtime: 1775727845, [] INFO:root:Downloading /tmp/obsmvn-f2s6lyau/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm DEBUG:root:Getting binary from: https://download.opensuse.org/repositories/systemsmanagement:/Uyuni:/Master/openSUSE_Leap_15.6/noarch/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm DEBUG:root:HTTPS Request: GET /repositories/systemsmanagement:/Uyuni:/Master/openSUSE_Leap_15.6/noarch/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm HTTP/1.1 DEBUG:root:HTTPS Response: 302 Found DEBUG:root:HTTPS Redirect to: https://ftp.gwdg.de/pub/opensuse/repositories/systemsmanagement%3A/Uyuni%3A/Master/openSUSE_Leap_15.6/noarch/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm DEBUG:root:HTTPS Request: GET /pub/opensuse/repositories/systemsmanagement%3A/Uyuni%3A/Master/openSUSE_Leap_15.6/noarch/antlr4-java-4.13.0-241000.1.12.uyuni5.noarch.rpm HTTP/1.1 DEBUG:root:HTTPS Response: 200 OK DEBUG:root:not linked: /usr/share/doc/packages/antlr4-java /usr/share/doc/packages/antlr4-java/CHANGES.txt /usr/share/doc/packages/antlr4-java/README.md /usr/share/java/antlr4 /usr/share/java/antlr4/antlr4-runtime.jar /usr/share/licenses/antlr4-java /usr/share/licenses/antlr4-java/LICENSE.txt /usr/share/maven-metadata/antlr4-antlr4-runtime.xml /usr/share/maven-poms/antlr4 /usr/share/maven-poms/antlr4/antlr4-master.pom /usr/share/maven-poms/antlr4/antlr4-runtime.pom DEBUG:root:full pattern: ^/usr/.*/antlr4-runtime[^/]*\.jar$ INFO:root:extracting /usr/share/java/antlr4/antlr4-runtime.jar to /tmp/obsmvn-f2s6lyau/antlr4-runtime.jar INFO:root:deploying /tmp/obsmvn-f2s6lyau/antlr4-runtime.jar to test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.jar DEBUG:root:Setting mtime 1775727845 on test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.jar DEBUG:root:Computing test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.jar.sha1 DEBUG:root:Computing test/suse/antlr4-runtime/4.13.0/antlr4-runtime-4.13.0.pom.sha1
diff --git a/obs_maven/core.py b/obs_maven/core.py
@@ -16,12 +16,14 @@
 # You should have received a copy of the GNU General Public License
 
 import argparse
+import http.client
 import logging
 import os
 import os.path
 import shutil
 import sys
 import tempfile
+import urllib.request
 import yaml
 import xml.etree.ElementTree as ET
 
@@ -32,6 +34,56 @@
 logging.basicConfig(level=logging.INFO)
 
 
+class _DebugConnectionMixin:
+    """Mixin class that adds debug logging to HTTP connections"""
+    _protocol_name = "HTTP"
+
+    def send(self, data):
+        if isinstance(data, bytes):
+            # Log the first line (request line) for readability
+            first_line = data.split(b'\r\n')[0].decode('ascii', errors='replace')
+            logging.debug(f"{self._protocol_name} Request: {first_line}")
+        return super().send(data)
+
+    def getresponse(self, *args, **kwargs):
+        response = super().getresponse(*args, **kwargs)
+        logging.debug(f"{self._protocol_name} Response: {response.status} {response.reason}")
+        if 300 <= response.status < 400:
+            location = response.getheader('Location', 'N/A')
+            logging.debug(f"{self._protocol_name} Redirect to: {location}")
+        return response
+
+
+class _DebugHTTPConnection(_DebugConnectionMixin, http.client.HTTPConnection):
+    """HTTPConnection subclass that logs debug info through Python logging"""
+    _protocol_name = "HTTP"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.set_debuglevel(1)
+
+
+class _DebugHTTPSConnection(_DebugConnectionMixin, http.client.HTTPSConnection):
+    """HTTPSConnection subclass that logs debug info through Python logging"""
+    _protocol_name = "HTTPS"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.set_debuglevel(1)
+
+
+class HTTPDebugHandler(urllib.request.HTTPHandler):
+    """Custom HTTP handler that enables debug output for urllib requests"""
+    def http_open(self, req):
+        return self.do_open(_DebugHTTPConnection, req)
+
+
+class HTTPSDebugHandler(urllib.request.HTTPSHandler):
+    """Custom HTTPS handler that enables debug output for urllib requests"""
+    def https_open(self, req):
+        return self.do_open(_DebugHTTPSConnection, req)
+
+
 class Configuration:
     def __init__(self, config_path, repo, cache_path, allowed_artifacts):
         data = {}
@@ -107,6 +159,10 @@ def main():
     args = parser.parse_args()
 
     logging.getLogger().setLevel(args.loglevel)
+    if args.loglevel == logging.DEBUG:
+        # Enable HTTP debugging by installing custom handlers
+        opener = urllib.request.build_opener(HTTPDebugHandler(), HTTPSDebugHandler())
+        urllib.request.install_opener(opener)
     logging.debug("Reading configuration")
     config = Configuration(args.config, args.out, args.cache, args.allowed_artifacts)
     tmp = tempfile.mkdtemp(prefix="obsmvn-")
