diff --git a/.travis.yml b/.travis.yml index 1dd2c78d..2259dc06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ addons: - wget matrix: + allow_failures: + - python: 3.4 + env: TOX_ENV=docs include: - python: 2.7 env: TOX_ENV=py27 diff --git a/dev-requirements.txt b/dev-requirements.txt index 78d40a24..7d75102e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ requests nose mock pandas -Sphinx==1.2.3 +Sphinx==1.5.5 sphinx_rtd_theme wheel twine diff --git a/examples/tutorial.py b/examples/tutorial.py index e790ece5..92ae7e3b 100644 --- a/examples/tutorial.py +++ b/examples/tutorial.py @@ -41,7 +41,7 @@ def main(host='localhost', port=8086): print("Write points: {0}".format(json_body)) client.write_points(json_body) - print("Queying data: " + query) + print("Querying data: " + query) result = client.query(query) print("Result: {0}".format(result)) diff --git a/examples/tutorial_pandas.py b/examples/tutorial_pandas.py index c9a09fde..855f5740 100644 --- a/examples/tutorial_pandas.py +++ b/examples/tutorial_pandas.py @@ -7,7 +7,9 @@ def main(host='localhost', port=8086): user = 'root' password = 'root' - dbname = 'example' + dbname = 'demo' + # Temporarily used to avoid line protocol time conversion issues #412, #426, #431. + protocol = 'json' client = DataFrameClient(host, port, user, password, dbname) @@ -20,16 +22,16 @@ def main(host='localhost', port=8086): client.create_database(dbname) print("Write DataFrame") - client.write_points(df, 'demo') + client.write_points(df, 'demo', protocol=protocol) print("Write DataFrame with Tags") - client.write_points(df, 'demo', {'k1': 'v1', 'k2': 'v2'}) + client.write_points(df, 'demo', {'k1': 'v1', 'k2': 'v2'}, protocol=protocol) print("Read DataFrame") client.query("select * from demo") print("Delete database: " + dbname) - client.delete_database(dbname) + client.drop_database(dbname) def parse_args(): diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 00739c49..90c049c9 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -17,4 +17,4 @@ ] -__version__ = '4.0.0' +__version__ = '4.1.0' diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index ef1fa78f..7683aeef 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -140,7 +140,7 @@ def query(self, query, chunked=False, database=None): """ results = super(DataFrameClient, self).query(query, database=database) - if query.upper().startswith("SELECT"): + if query.strip().upper().startswith("SELECT"): if len(results) > 0: return self._to_dataframe(results) else: @@ -227,7 +227,7 @@ def _convert_dataframe_to_lines(self, measurement, field_columns=None, tag_columns=None, - global_tags={}, + global_tags=None, time_precision=None, numeric_precision=None): @@ -366,7 +366,7 @@ def _stringify_dataframe(self, if datatype == 'field': # If dealing with fields, format ints and strings correctly - dataframe[int_columns] = dataframe[int_columns] + 'i' + dataframe[int_columns] += 'i' dataframe[string_columns] = '"' + dataframe[string_columns] + '"' elif datatype == 'tag': dataframe = dataframe.apply(_escape_pandas_series) diff --git a/influxdb/client.py b/influxdb/client.py index ab9aa409..5f415abc 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -53,6 +53,9 @@ class InfluxDBClient(object): :param timeout: number of seconds Requests will wait for your client to establish a connection, defaults to None :type timeout: int + :param retries: number of retries your client will try before aborting, + defaults to 3. 0 indicates try until success + :type retries: int :param use_udp: use UDP to connect to InfluxDB, defaults to False :type use_udp: bool :param udp_port: UDP port to connect to InfluxDB, defaults to 4444 @@ -70,6 +73,7 @@ def __init__(self, ssl=False, verify_ssl=False, timeout=None, + retries=3, use_udp=False, udp_port=4444, proxies=None, @@ -81,6 +85,7 @@ def __init__(self, self._password = password self._database = database self._timeout = timeout + self._retries = retries self._verify_ssl = verify_ssl @@ -205,6 +210,8 @@ def request(self, url, method='GET', params=None, data=None, :param expected_response_code: the expected response code of the request, defaults to 200 :type expected_response_code: int + :param headers: headers to add to the request + :type headers: dict :returns: the response from the request :rtype: :class:`requests.Response` :raises InfluxDBServerError: if the response code is any server error @@ -223,9 +230,10 @@ def request(self, url, method='GET', params=None, data=None, if isinstance(data, (dict, list)): data = json.dumps(data) - # Try to send the request a maximum of three times. (see #103) - # TODO (aviau): Make this configurable. - for i in range(0, 3): + # Try to send the request more than once by default (see #103) + retry = True + _try = 0 + while retry: try: response = self._session.request( method=method, @@ -240,12 +248,14 @@ def request(self, url, method='GET', params=None, data=None, ) break except requests.exceptions.ConnectionError as e: - if i < 2: - continue - else: - raise e + _try += 1 + if self._retries != 0: + retry = _try < self._retries - if response.status_code >= 500 and response.status_code < 600: + else: + raise requests.exceptions.ConnectionError + + if 500 <= response.status_code < 600: raise InfluxDBServerError(response.content) elif response.status_code == expected_response_code: return response @@ -293,13 +303,27 @@ def write(self, data, params=None, expected_response_code=204, ) return True + def _read_chunked_response(self, response, raise_errors=True): + result_set = {} + for line in response.iter_lines(): + if isinstance(line, bytes): + line = line.decode('utf-8') + data = json.loads(line) + for result in data.get('results', []): + for _key in result: + if type(result[_key]) == list: + result_set.setdefault(_key, []).extend(result[_key]) + return ResultSet(result_set, raise_errors=raise_errors) + def query(self, query, params=None, epoch=None, expected_response_code=200, database=None, - raise_errors=True): + raise_errors=True, + chunked=False, + chunk_size=0): """Send a query to InfluxDB. :param query: the actual query string @@ -308,6 +332,11 @@ def query(self, :param params: additional parameters for the request, defaults to {} :type params: dict + :param epoch: response timestamps to be in epoch format either 'h', + 'm', 's', 'ms', 'u', or 'ns',defaults to `None` which is + RFC3339 UTC format with nanosecond precision + :type epoch: str + :param expected_response_code: the expected status code of response, defaults to 200 :type expected_response_code: int @@ -319,6 +348,14 @@ def query(self, returns errors, defaults to True :type raise_errors: bool + :param chunked: Enable to use chunked responses from InfluxDB. + With ``chunked`` enabled, one ResultSet is returned per chunk + containing all results within that chunk + :type chunked: bool + + :param chunk_size: Size of each chunk to tell InfluxDB to use. + :type chunk_size: int + :returns: the queried data :rtype: :class:`~.ResultSet` """ @@ -331,6 +368,11 @@ def query(self, if epoch is not None: params['epoch'] = epoch + if chunked: + params['chunked'] = 'true' + if chunk_size > 0: + params['chunk_size'] = chunk_size + response = self.request( url="query", method='GET', @@ -339,6 +381,9 @@ def query(self, expected_response_code=expected_response_code ) + if chunked: + return self._read_chunked_response(response) + data = response.json() results = [ @@ -366,7 +411,7 @@ def write_points(self, :param points: the list of points to be written in the database :type points: list of dictionaries, each dictionary represents a point - :type data: (if protocol is 'json') list of dicts, where each dict + :type points: (if protocol is 'json') list of dicts, where each dict represents a point. (if protocol is 'line') sequence of line protocol strings. :param time_precision: Either 's', 'm', 'ms' or 'u', defaults to None @@ -545,7 +590,7 @@ def alter_retention_policy(self, name, database=None, :type duration: str :param replication: the new replication of the existing retention policy - :type replication: str + :type replication: int :param default: whether or not to set the modified policy as default :type default: bool @@ -674,9 +719,9 @@ def delete_series(self, database=None, measurement=None, tags=None): deleted, defaults to client's current database :type database: str :param measurement: Delete all series from a measurement - :type id: str + :type measurement: str :param tags: Delete all series that match given tags - :type id: dict + :type tags: dict """ database = database or self._database query_str = 'DROP SERIES' @@ -779,7 +824,7 @@ def send_packet(self, packet, protocol='json'): if protocol == 'json': data = make_lines(packet).encode('utf-8') elif protocol == 'line': - data = ('\n'.join(data) + '\n').encode('utf-8') + data = ('\n'.join(packet) + '\n').encode('utf-8') self.udp_socket.sendto(data, (self._host, self.udp_port)) diff --git a/influxdb/influxdb08/client.py b/influxdb/influxdb08/client.py index a3b31639..8955ab76 100644 --- a/influxdb/influxdb08/client.py +++ b/influxdb/influxdb08/client.py @@ -55,6 +55,9 @@ class InfluxDBClient(object): :param verify_ssl: verify SSL certificates for HTTPS requests, defaults is False :type verify_ssl: boolean + :param retries: number of retries your client will try before aborting, + defaults to 3. 0 indicates try until success + :type retries: int :param timeout: number of seconds Requests will wait for your client to establish a connection, defaults to None :type timeout: int @@ -73,6 +76,7 @@ def __init__(self, ssl=False, verify_ssl=False, timeout=None, + retries=3, use_udp=False, udp_port=4444): """ @@ -84,6 +88,7 @@ def __init__(self, self._password = password self._database = database self._timeout = timeout + self._retries = retries self._verify_ssl = verify_ssl @@ -228,9 +233,10 @@ def request(self, url, method='GET', params=None, data=None, if data is not None and not isinstance(data, str): data = json.dumps(data) - # Try to send the request a maximum of three times. (see #103) - # TODO (aviau): Make this configurable. - for i in range(0, 3): + retry = True + _try = 0 + # Try to send the request more than once by default (see #103) + while retry: try: response = session.request( method=method, @@ -244,10 +250,11 @@ def request(self, url, method='GET', params=None, data=None, break except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - if i < 2: - continue - else: - raise e + _try += 1 + if self._retries != 0: + retry = _try < self._retries + else: + raise requests.exceptions.ConnectionError if response.status_code == expected_response_code: return response diff --git a/influxdb/line_protocol.py b/influxdb/line_protocol.py index 7c8c8f24..59d93bff 100644 --- a/influxdb/line_protocol.py +++ b/influxdb/line_protocol.py @@ -5,13 +5,12 @@ from __future__ import print_function from __future__ import unicode_literals -from copy import copy from datetime import datetime from numbers import Integral from pytz import UTC from dateutil.parser import parse -from six import binary_type, text_type, integer_types, PY2 +from six import iteritems, binary_type, text_type, integer_types, PY2 EPOCH = UTC.localize(datetime.utcfromtimestamp(0)) @@ -75,12 +74,22 @@ def quote_literal(value): ) +def _is_float(value): + try: + float(value) + except ValueError: + return False + return True + + def _escape_value(value): value = _get_unicode(value) if isinstance(value, text_type) and value != '': return quote_ident(value) elif isinstance(value, integer_types) and not isinstance(value, bool): return str(value) + 'i' + elif _is_float(value): + return repr(value) else: return str(value) @@ -108,7 +117,7 @@ def make_lines(data, precision=None): matching the line protocol introduced in InfluxDB 0.9.0. """ lines = [] - static_tags = data.get('tags', None) + static_tags = data.get('tags') for point in data['points']: elements = [] @@ -119,32 +128,29 @@ def make_lines(data, precision=None): key_values = [measurement] # add tags - if static_tags is None: - tags = point.get('tags', {}) + if static_tags: + tags = dict(static_tags) # make a copy, since we'll modify + tags.update(point.get('tags') or {}) else: - tags = copy(static_tags) - tags.update(point.get('tags', {})) + tags = point.get('tags') or {} # tags should be sorted client-side to take load off server - for tag_key in sorted(tags.keys()): + for tag_key, tag_value in sorted(iteritems(tags)): key = _escape_tag(tag_key) - value = _escape_tag(tags[tag_key]) + value = _escape_tag(tag_value) if key != '' and value != '': - key_values.append("{key}={value}".format(key=key, value=value)) + key_values.append(key + "=" + value) key_values = ','.join(key_values) elements.append(key_values) # add fields field_values = [] - for field_key in sorted(point['fields'].keys()): + for field_key, field_value in sorted(iteritems(point['fields'])): key = _escape_tag(field_key) - value = _escape_value(point['fields'][field_key]) + value = _escape_value(field_value) if key != '' and value != '': - field_values.append("{key}={value}".format( - key=key, - value=value - )) + field_values.append(key + "=" + value) field_values = ','.join(field_values) elements.append(field_values) diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index f586df3f..e319c0e5 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -32,6 +32,7 @@ import unittest from influxdb import InfluxDBClient +from influxdb.resultset import ResultSet def _build_response_object(status_code=200, content=""): @@ -644,6 +645,61 @@ def connection_error(self, *args, **kwargs): with self.assertRaises(requests.exceptions.ConnectionError): cli.write_points(self.dummy_points) + @mock.patch('requests.Session.request') + def test_random_request_retry(self, mock_request): + """Tests that a random number of connection errors will be handled""" + + class CustomMock(object): + def __init__(self, retries): + self.i = 0 + self.retries = retries + + def connection_error(self, *args, **kwargs): + self.i += 1 + + if self.i < self.retries: + raise requests.exceptions.ConnectionError + else: + r = requests.Response() + r.status_code = 204 + return r + + retries = random.randint(1, 100) + mock_request.side_effect = CustomMock(retries).connection_error + + cli = InfluxDBClient(database='db', retries=retries) + cli.write_points( + self.dummy_points + ) + + @mock.patch('requests.Session.request') + def test_random_request_retry_raises(self, mock_request): + """Tests that a random number of connection errors plus one \ + will not be handled""" + + class CustomMock(object): + def __init__(self, retries): + self.i = 0 + self.retries = retries + + def connection_error(self, *args, **kwargs): + self.i += 1 + + if self.i < self.retries + 1: + raise requests.exceptions.ConnectionError + else: + r = requests.Response() + r.status_code = 200 + return r + + retries = random.randint(1, 100) + mock_request.side_effect = CustomMock(retries).connection_error + + cli = InfluxDBClient(database='db', retries=retries) + + with self.assertRaises(requests.exceptions.ConnectionError): + cli.write_points(self.dummy_points) + def test_get_list_users(self): example_response = ( '{"results":[{"series":[{"columns":["user","admin"],' @@ -792,6 +848,43 @@ def test_invalid_port_fails(self): with self.assertRaises(ValueError): InfluxDBClient('host', '80/redir', 'username', 'password') + def test_chunked_response(self): + example_response = \ + u'{"results":[{"statement_id":0,"series":' \ + '[{"name":"cpu","columns":["fieldKey","fieldType"],"values":' \ + '[["value","integer"]]}],"partial":true}]}\n{"results":' \ + '[{"statement_id":0,"series":[{"name":"iops","columns":' \ + '["fieldKey","fieldType"],"values":[["value","integer"]]}],' \ + '"partial":true}]}\n{"results":[{"statement_id":0,"series":' \ + '[{"name":"load","columns":["fieldKey","fieldType"],"values":' \ + '[["value","integer"]]}],"partial":true}]}\n{"results":' \ + '[{"statement_id":0,"series":[{"name":"memory","columns":' \ + '["fieldKey","fieldType"],"values":[["value","integer"]]}]}]}\n' + + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + text=example_response + ) + response = self.cli.query('show series limit 4 offset 0', + chunked=True, chunk_size=4) + self.assertTrue(len(response) == 4) + self.assertEqual(response.__repr__(), ResultSet( + {'series': [{'values': [['value', 'integer']], + 'name': 'cpu', + 'columns': ['fieldKey', 'fieldType']}, + {'values': [['value', 'integer']], + 'name': 'iops', + 'columns': ['fieldKey', 'fieldType']}, + {'values': [['value', 'integer']], + 'name': 'load', + 'columns': ['fieldKey', 'fieldType']}, + {'values': [['value', 'integer']], + 'name': 'memory', + 'columns': ['fieldKey', 'fieldType']}]} + ).__repr__()) + class FakeClient(InfluxDBClient): @@ -800,7 +893,7 @@ def __init__(self, *args, **kwargs): def query(self, query, - params={}, + params=None, expected_response_code=200, database=None): if query == 'Fail': diff --git a/influxdb/tests/helper_test.py b/influxdb/tests/helper_test.py index 10546286..44392f80 100644 --- a/influxdb/tests/helper_test.py +++ b/influxdb/tests/helper_test.py @@ -261,9 +261,9 @@ class Meta: self.assertEqual(point2['time'], yesterday) def testInvalidHelpers(self): - ''' + """ Tests errors in invalid helpers. - ''' + """ class MissingMeta(SeriesHelper): pass diff --git a/influxdb/tests/influxdb08/helper_test.py b/influxdb/tests/influxdb08/helper_test.py index e744d1e4..c9ce311f 100644 --- a/influxdb/tests/influxdb08/helper_test.py +++ b/influxdb/tests/influxdb08/helper_test.py @@ -83,9 +83,9 @@ def testSingleSeriesName(self): 'Resetting helper did not empty datapoints.') def testSeveralSeriesNames(self): - ''' + """ Tests JSON conversion when there is only one series name. - ''' + """ TestSeriesHelper.MySeriesHelper(server_name='us.east-1', time=159) TestSeriesHelper.MySeriesHelper(server_name='fr.paris-10', time=158) TestSeriesHelper.MySeriesHelper(server_name='lu.lux', time=157) @@ -116,9 +116,9 @@ def testSeveralSeriesNames(self): 'Resetting helper did not empty datapoints.') def testInvalidHelpers(self): - ''' + """ Tests errors in invalid helpers. - ''' + """ class MissingMeta(SeriesHelper): pass diff --git a/influxdb/tests/server_tests/base.py b/influxdb/tests/server_tests/base.py index 3566d7ba..f217fce1 100644 --- a/influxdb/tests/server_tests/base.py +++ b/influxdb/tests/server_tests/base.py @@ -41,9 +41,9 @@ def _teardown_influxdb_server(inst): class SingleTestCaseWithServerMixin(object): - ''' A mixin for unittest.TestCase to start an influxdb server instance + """ A mixin for unittest.TestCase to start an influxdb server instance in a temporary directory **for each test function/case** - ''' + """ # 'influxdb_template_conf' attribute must be set # on the TestCase class or instance. @@ -53,10 +53,10 @@ class SingleTestCaseWithServerMixin(object): class ManyTestCasesWithServerMixin(object): - ''' Same than SingleTestCaseWithServerMixin + """ Same than SingleTestCaseWithServerMixin but creates a single instance for the whole class. Also pre-creates a fresh database: 'db'. - ''' + """ # 'influxdb_template_conf' attribute must be set on the class itself ! diff --git a/influxdb/tests/test_line_protocol.py b/influxdb/tests/test_line_protocol.py index 726f8705..61f54557 100644 --- a/influxdb/tests/test_line_protocol.py +++ b/influxdb/tests/test_line_protocol.py @@ -119,3 +119,20 @@ def test_quote_literal(self): line_protocol.quote_literal(r"""\foo ' bar " Örf"""), r"""'\\foo \' bar " Örf'""" ) + + def test_float_with_long_decimal_fraction(self): + """Ensure precision is preserved when casting floats into strings.""" + data = { + "points": [ + { + "measurement": "test", + "fields": { + "float_val": 1.0000000000000009, + } + } + ] + } + self.assertEqual( + line_protocol.make_lines(data), + 'test float_val=1.0000000000000009\n' + ) diff --git a/tox.ini b/tox.ini index 856a4717..15060e37 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt pandas - Sphinx==1.2.3 + Sphinx==1.5.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build