From 3189185450269d8f62c1521efa398a099f6ede60 Mon Sep 17 00:00:00 2001 From: Gopal Kalidindi Date: Thu, 3 May 2018 18:31:16 -0700 Subject: [PATCH] Fixed/Implemented Chunked Read. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The document mentions that InfluxDBClient.Query method returns one ResultSet per data chunk if chunking is enabled by setting the chunked parameter to True. But only one ResultSet containing all data rows was being returned. This in effect is like having no chunking and retrieving all data in one shot. Also, for chunked data read to work, streaming should be enabled on the underlying requests.Session.request method by setting the stream parameter to True. This was not being done. Made the following changes to make the chunked read work as documented. * Updated the method _read_chunked_response to make it a generator that yields one ResultSet per data chunk. * Enabling streaming on requests.Session.request by passing the parameter “stream=True” whenever chunking is enabled on InfluxDBClient.Query. * Updated the entire call chain from InfluxDBClient.Query to requests.Session.request method to support streaming by adding the parameter “stream”. --- influxdb/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/influxdb/client.py b/influxdb/client.py index f5c0b55b..20cabb89 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -200,7 +200,7 @@ def switch_user(self, username, password): self._username = username self._password = password - def request(self, url, method='GET', params=None, data=None, + def request(self, url, method='GET', params=None, data=None, stream=False, expected_response_code=200, headers=None): """Make a HTTP request to the InfluxDB API. @@ -246,6 +246,7 @@ def request(self, url, method='GET', params=None, data=None, auth=(self._username, self._password), params=params, data=data, + stream=stream, headers=headers, proxies=self._proxies, verify=self._verify_ssl, @@ -315,8 +316,8 @@ def write(self, data, params=None, expected_response_code=204, @staticmethod def _read_chunked_response(response, raise_errors=True): - result_set = {} for line in response.iter_lines(): + result_set = {} if isinstance(line, bytes): line = line.decode('utf-8') data = json.loads(line) @@ -325,7 +326,7 @@ def _read_chunked_response(response, raise_errors=True): if isinstance(result[_key], list): result_set.setdefault( _key, []).extend(result[_key]) - return ResultSet(result_set, raise_errors=raise_errors) + yield ResultSet(result_set, raise_errors=raise_errors) def query(self, query, @@ -391,6 +392,7 @@ def query(self, method='GET', params=params, data=None, + stream=True if chunked else False, expected_response_code=expected_response_code )