Fix unicode issues in HttpError and BatchHttpRequest

methane · methane · commit 0915761b342e · 2015-03-26T13:55:52.000+09:00
1) HttpError parses http response by json.loads. It accepts only
unicode on PY3 while response is bytes. So decode response first.

2) BatchHttpRequest uses email.parser.FeedParser. It accepts only
unicode on PY3, too. So encode parsed response to emulate normal
http response.
diff --git a/googleapiclient/errors.py b/googleapiclient/errors.py
@@ -37,14 +37,16 @@ class HttpError(Error):
   @util.positional(3)
   def __init__(self, resp, content, uri=None):
     self.resp = resp
+    if not isinstance(content, bytes):
+        raise TypeError("HTTP content should be bytes")
     self.content = content
     self.uri = uri
 
   def _get_reason(self):
     """Calculate the reason for the error from the response content."""
     reason = self.resp.reason
     try:
-      data = json.loads(self.content)
+      data = json.loads(self.content.decode('utf-8'))
       reason = data['error']['message']
     except (ValueError, KeyError):
       pass
diff --git a/googleapiclient/http.py b/googleapiclient/http.py
@@ -1257,6 +1257,10 @@ def _execute(self, http, order, requests):
 
     # Prepend with a content-type header so FeedParser can handle it.
     header = 'content-type: %s\r\n\r\n' % resp['content-type']
+    # PY3's FeedParser only accepts unicode. So we should decode content
+    # here, and encode each payload again.
+    if six.PY3:
+      content = content.decode('utf-8')
     for_parser = header + content
 
     parser = FeedParser()
@@ -1270,6 +1274,9 @@ def _execute(self, http, order, requests):
     for part in mime_response.get_payload():
       request_id = self._header_to_id(part['Content-ID'])
       response, content = self._deserialize_response(part.get_payload())
+      # We encode content here to emulate normal http response.
+      if isinstance(content, six.text_type):
+        content = content.encode('utf-8')
       self._responses[request_id] = (response, content)
 
   @util.positional(1)
@@ -1536,6 +1543,8 @@ def request(self, uri,
         content = body
     elif content == 'echo_request_uri':
       content = uri
+    if isinstance(content, six.text_type):
+      content = content.encode('utf-8')
     return httplib2.Response(resp), content
 
 
diff --git a/tests/test_discovery.py b/tests/test_discovery.py
@@ -1048,7 +1048,7 @@ def test_resumable_media_handle_resume_of_upload_of_unknown_size(self):
           'Content-Range': 'bytes */14',
           'content-length': '0'
           }
-      self.assertEqual(expected, json.loads(e.content),
+      self.assertEqual(expected, json.loads(e.content.decode('utf-8')),
         'Should send an empty body when requesting the current upload status.')
 
   def test_pickle(self):
@@ -1186,7 +1186,7 @@ def test_get_media(self):
       ({'status': '200'}, 'standing in for media'),
       ])
     response = request.execute(http=http)
-    self.assertEqual('standing in for media', response)
+    self.assertEqual(b'standing in for media', response)
 
 
 if __name__ == '__main__':
diff --git a/tests/test_errors.py b/tests/test_errors.py
@@ -28,7 +28,7 @@
 from googleapiclient.errors import HttpError
 
 
-JSON_ERROR_CONTENT = """
+JSON_ERROR_CONTENT = b"""
 {
  "error": {
   "errors": [
@@ -65,36 +65,36 @@ def test_json_body(self):
 
   def test_bad_json_body(self):
     """Test handling of bodies with invalid json."""
-    resp, content = fake_response('{',
+    resp, content = fake_response(b'{',
         { 'status':'400', 'content-type': 'application/json'},
         reason='Failed')
     error = HttpError(resp, content)
     self.assertEqual(str(error), '<HttpError 400 "Failed">')
 
   def test_with_uri(self):
     """Test handling of passing in the request uri."""
-    resp, content = fake_response('{',
+    resp, content = fake_response(b'{',
         {'status':'400', 'content-type': 'application/json'},
         reason='Failure')
     error = HttpError(resp, content, uri='http://example.org')
     self.assertEqual(str(error), '<HttpError 400 when requesting http://example.org returned "Failure">')
 
   def test_missing_message_json_body(self):
     """Test handling of bodies with missing expected 'message' element."""
-    resp, content = fake_response('{}',
+    resp, content = fake_response(b'{}',
         {'status':'400', 'content-type': 'application/json'},
         reason='Failed')
     error = HttpError(resp, content)
     self.assertEqual(str(error), '<HttpError 400 "Failed">')
 
   def test_non_json(self):
     """Test handling of non-JSON bodies"""
-    resp, content = fake_response('}NOT OK', {'status':'400'})
+    resp, content = fake_response(b'}NOT OK', {'status':'400'})
     error = HttpError(resp, content)
     self.assertEqual(str(error), '<HttpError 400 "Ok">')
 
   def test_missing_reason(self):
     """Test an empty dict with a missing resp.reason."""
-    resp, content = fake_response('}NOT OK', {'status': '400'}, reason=None)
+    resp, content = fake_response(b'}NOT OK', {'status': '400'}, reason=None)
     error = HttpError(resp, content)
     self.assertEqual(str(error), '<HttpError 400 "">')
diff --git a/tests/test_http.py b/tests/test_http.py
@@ -460,7 +460,7 @@ def test_media_io_base_download_retries_5xx(self):
 ETag: "etag/pony"\r\n\r\n{"answer": 42}"""
 
 
-BATCH_RESPONSE = """--batch_foobarbaz
+BATCH_RESPONSE = b"""--batch_foobarbaz
 Content-Type: application/http
 Content-Transfer-Encoding: binary
 Content-ID: <randomness+1>
@@ -482,7 +482,7 @@ def test_media_io_base_download_retries_5xx(self):
 --batch_foobarbaz--"""
 
 
-BATCH_ERROR_RESPONSE = """--batch_foobarbaz
+BATCH_ERROR_RESPONSE = b"""--batch_foobarbaz
 Content-Type: application/http
 Content-Transfer-Encoding: binary
 Content-ID: <randomness+1>
@@ -518,7 +518,7 @@ def test_media_io_base_download_retries_5xx(self):
 --batch_foobarbaz--"""
 
 
-BATCH_RESPONSE_WITH_401 = """--batch_foobarbaz
+BATCH_RESPONSE_WITH_401 = b"""--batch_foobarbaz
 Content-Type: application/http
 Content-Transfer-Encoding: binary
 Content-ID: <randomness+1>
@@ -541,7 +541,7 @@ def test_media_io_base_download_retries_5xx(self):
 --batch_foobarbaz--"""
 
 
-BATCH_SINGLE_RESPONSE = """--batch_foobarbaz
+BATCH_SINGLE_RESPONSE = b"""--batch_foobarbaz
 Content-Type: application/http
 Content-Transfer-Encoding: binary
 Content-ID: <randomness+1>
@@ -928,7 +928,7 @@ def _postproc(resp, content):
 
     # Query parameters should be sent in the body.
     response = req.execute()
-    self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)
+    self.assertEqual(b'q=' + b'a' * MAX_URI_LENGTH + b'%3F%26', response)
 
     # Extra headers should be set.
     response = req.execute()
diff --git a/tests/test_json_model.py b/tests/test_json_model.py
@@ -150,7 +150,7 @@ def test_bad_response(self):
     model = JsonModel(data_wrapper=False)
     resp = httplib2.Response({'status': '401'})
     resp.reason = 'Unauthorized'
-    content = '{"error": {"message": "not authorized"}}'
+    content = b'{"error": {"message": "not authorized"}}'
 
     try:
       content = model.response(resp, content)
diff --git a/tests/test_mocks.py b/tests/test_mocks.py
@@ -134,15 +134,15 @@ def test_simple_matching_dict_body(self):
   def test_errors(self):
     errorResponse = httplib2.Response({'status': 500, 'reason': 'Server Error'})
     requestBuilder = RequestMockBuilder({
-        'plus.activities.list': (errorResponse, '{}')
+        'plus.activities.list': (errorResponse, b'{}')
         })
     plus = build('plus', 'v1', http=self.http, requestBuilder=requestBuilder)
 
     try:
       activity = plus.activities().list(collection='public', userId='me').execute()
       self.fail('An exception should have been thrown')
     except HttpError as e:
-      self.assertEqual('{}', e.content)
+      self.assertEqual(b'{}', e.content)
       self.assertEqual(500, e.resp.status)
       self.assertEqual('Server Error', e.resp.reason)
 
