From 75d7fe8e58add673832c41f9fd34086f001ca22d Mon Sep 17 00:00:00 2001 From: tux Date: Thu, 17 May 2018 18:34:48 +0200 Subject: [PATCH 01/26] Add _to_dict_drop_na function and use it in _convert_dataframe_to_json --- influxdb/_dataframe_client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 2444a77f..ec489983 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -211,8 +211,8 @@ def _to_dataframe(self, rs, dropna=True): return result - @staticmethod - def _convert_dataframe_to_json(dataframe, + def _convert_dataframe_to_json(self, + dataframe, measurement, tags=None, tag_columns=None, @@ -262,7 +262,7 @@ def _convert_dataframe_to_json(dataframe, 'time': np.int64(ts.value / precision_factor)} for ts, tag, rec in zip(dataframe.index, dataframe[tag_columns].to_dict('record'), - dataframe[field_columns].to_dict('record')) + self._to_dict_drop_na(dataframe[field_columns])) ] return points @@ -382,6 +382,10 @@ def format_line(line): points = (measurement + tags + ' ' + fields + ' ' + time).tolist() return points + @staticmethod + def _to_dict_drop_na(dframe): + return [{k:v for k,v in m.items() if pd.notnull(v)} for m in dframe.to_dict(orient='rows')] + @staticmethod def _stringify_dataframe(dframe, numeric_precision, datatype='field'): From 8c20ea3d4a5769aab2c45166be95484098229c3f Mon Sep 17 00:00:00 2001 From: tux Date: Fri, 18 May 2018 10:03:07 +0200 Subject: [PATCH 02/26] Satisfy flake8 --- influxdb/_dataframe_client.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index ec489983..373d1fff 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -255,14 +255,16 @@ def _convert_dataframe_to_json(self, "h": 1e9 * 3600, }.get(time_precision, 1) + dataframe_zip = zip(dataframe.index, + dataframe[tag_columns].to_dict('record'), + self._todict_dropna(dataframe[field_columns])) + points = [ {'measurement': measurement, 'tags': dict(list(tag.items()) + list(tags.items())), 'fields': rec, 'time': np.int64(ts.value / precision_factor)} - for ts, tag, rec in zip(dataframe.index, - dataframe[tag_columns].to_dict('record'), - self._to_dict_drop_na(dataframe[field_columns])) + for ts, tag, rec in dataframe_zip ] return points @@ -383,8 +385,10 @@ def format_line(line): return points @staticmethod - def _to_dict_drop_na(dframe): - return [{k:v for k,v in m.items() if pd.notnull(v)} for m in dframe.to_dict(orient='rows')] + def _todict_dropna(dataframe): + return [{k: v for k, v in m.items() + if pd.notnull(v)} + for m in dataframe.to_dict(orient='rows')] @staticmethod def _stringify_dataframe(dframe, numeric_precision, datatype='field'): From 119844ad71b980a41cfb9d8eda8b0977d78c961b Mon Sep 17 00:00:00 2001 From: tux Date: Fri, 18 May 2018 10:22:19 +0200 Subject: [PATCH 03/26] Add dropna on tag columns --- influxdb/_dataframe_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 373d1fff..d71283cf 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -256,7 +256,7 @@ def _convert_dataframe_to_json(self, }.get(time_precision, 1) dataframe_zip = zip(dataframe.index, - dataframe[tag_columns].to_dict('record'), + self._todict_dropna(dataframe[tag_columns]), self._todict_dropna(dataframe[field_columns])) points = [ From 2c155879daf99e966c23b8fe1f9b3c907d1b80a4 Mon Sep 17 00:00:00 2001 From: tux Date: Fri, 18 May 2018 10:22:56 +0200 Subject: [PATCH 04/26] Change to_dict orient param to original value --- influxdb/_dataframe_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index d71283cf..34f2ded1 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -388,7 +388,7 @@ def format_line(line): def _todict_dropna(dataframe): return [{k: v for k, v in m.items() if pd.notnull(v)} - for m in dataframe.to_dict(orient='rows')] + for m in dataframe.to_dict('record')] @staticmethod def _stringify_dataframe(dframe, numeric_precision, datatype='field'): From fe9203b3c9e035378c6bc2d888a50aff67f4b9ef Mon Sep 17 00:00:00 2001 From: tux Date: Sun, 29 Jul 2018 11:23:43 +0200 Subject: [PATCH 05/26] Add nan values insertion test --- influxdb/tests/dataframe_client_test.py | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 78f5437f..f489b99d 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -12,6 +12,7 @@ import unittest import warnings import requests_mock +import numpy as np from influxdb.tests import skipIfPYpy, using_pypy from nose.tools import raises @@ -33,6 +34,35 @@ def setUp(self): # By default, raise exceptions on warnings warnings.simplefilter('error', FutureWarning) + def test_write_points_with_nan_values(self): + """Test write points with NaN values.""" + now = pd.Timestamp('1970-01-01 00:00+00:00') + dataframe = pd.DataFrame(data=[["1", 1, 1.0], ["2", 2, 2.0]], + index=[now, now + timedelta(hours=1)], + columns=["column_one", "column_two", + "column_three"]) + + dataframe.replace(2, np.nan) + + expected = ( + b"foo column_one=\"1\",column_two=1i,column_three=1.0 0\n" + b"foo column_one=\"2\",column_two=2i,column_three=2.0 " + b"3600000000000\n" + ) + + with requests_mock.Mocker() as m: + m.register_uri(requests_mock.POST, + "http://localhost:8086/write", + status_code=204) + + cli = DataFrameClient(database='db') + + cli.write_points(dataframe, 'foo') + self.assertEqual(m.last_request.body, expected) + + cli.write_points(dataframe, 'foo', tags=None) + self.assertEqual(m.last_request.body, expected) + def test_write_points_from_dataframe(self): """Test write points from df in TestDataFrameClient object.""" now = pd.Timestamp('1970-01-01 00:00+00:00') From 25272aa321b2c993a0e737832f75402f819ab226 Mon Sep 17 00:00:00 2001 From: tux Date: Sun, 29 Jul 2018 11:36:25 +0200 Subject: [PATCH 06/26] Add numpy to test requirements --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index 9b31f5f1..e7bfd966 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,3 +2,4 @@ nose nose-cov mock requests-mock +numpy From 188269c083d3ffdd5f55f6a0597942e3ac9a878b Mon Sep 17 00:00:00 2001 From: tux Date: Sun, 29 Jul 2018 12:01:41 +0200 Subject: [PATCH 07/26] Revert last commit --- test-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e7bfd966..9b31f5f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,4 +2,3 @@ nose nose-cov mock requests-mock -numpy From 32072162f1b7d479139a3891328945a60c0f6c68 Mon Sep 17 00:00:00 2001 From: tux Date: Wed, 22 Aug 2018 18:50:21 +0200 Subject: [PATCH 08/26] Remove numpy import --- influxdb/tests/dataframe_client_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index c713d443..d975aa8e 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -12,7 +12,6 @@ import unittest import warnings import requests_mock -import numpy as np from influxdb.tests import skipIfPYpy, using_pypy from nose.tools import raises From 112a9abd7a4d122a9bf73846dd1a36db717433a9 Mon Sep 17 00:00:00 2001 From: tux Date: Wed, 22 Aug 2018 19:53:03 +0200 Subject: [PATCH 09/26] Add trick to remove numpy dependency --- influxdb/tests/dataframe_client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index d975aa8e..cb6162f7 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -41,7 +41,7 @@ def test_write_points_with_nan_values(self): columns=["column_one", "column_two", "column_three"]) - dataframe.replace(2, np.nan) + dataframe.replace('2', float("NaN")) expected = ( b"foo column_one=\"1\",column_two=1i,column_three=1.0 0\n" From eaf818de0f3d297f1a4fa28c6c97212d4eaa3c11 Mon Sep 17 00:00:00 2001 From: aviau Date: Fri, 7 Dec 2018 15:56:57 -0500 Subject: [PATCH 10/26] 5.2.1 release --- influxdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/__init__.py b/influxdb/__init__.py index 03f74581..a1eb3789 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.0' +__version__ = '5.2.1' From 5ad21886ed965eec7ec28f5ad60b8d5e1e323184 Mon Sep 17 00:00:00 2001 From: aviau Date: Fri, 7 Dec 2018 15:59:02 -0500 Subject: [PATCH 11/26] release 5.2.1 in changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c156b678..14a9abf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added + +### Changed + +### Removed + +## [v5.2.1] - 2018-12-07 ### Added ### Changed From ec86d397b9cb2c00b5d32ee00bdca03e564da5fd Mon Sep 17 00:00:00 2001 From: xginn8 Date: Tue, 12 Feb 2019 12:48:49 -0300 Subject: [PATCH 12/26] Unpin setuptools to fix travis (#674) * Unpin setuptools to fix travis Signed-off-by: Matthew McGinn * Add some ignores for new flake8 tests Signed-off-by: Matthew McGinn --- .travis.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f3d4a5d..22626f40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: install: - pip install tox-travis - - pip install setuptools==20.6.6 + - pip install setuptools - pip install coveralls - mkdir -p "influxdb_install/${INFLUXDB_VER}" - if [ -n "${INFLUXDB_VER}" ] ; then wget "https://dl.influxdata.com/influxdb/releases/influxdb_${INFLUXDB_VER}_amd64.deb" ; fi diff --git a/tox.ini b/tox.ini index d0d87fec..2f9c212c 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ commands = nosetests -v --with-doctest {posargs} deps = flake8 pep8-naming -commands = flake8 influxdb +commands = flake8 --ignore=W503,W504,W605,N802,F821 influxdb [testenv:pep257] deps = pydocstyle From fbfca02c2692406905af9066cd718a35aff2bab4 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Thu, 14 Mar 2019 11:25:58 +0000 Subject: [PATCH 13/26] unpin pypy (#682) cryptography-2.5 is not compatible with PyPy < 5.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 22626f40..a1cf7b55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "2.7" - "3.5" - "3.6" - - "pypy-5.3.1" + - "pypy" - "pypy3" env: From f3f890c1eef38a3388cd38fe30acb3552f35a534 Mon Sep 17 00:00:00 2001 From: xginn8 Date: Thu, 14 Mar 2019 09:11:52 -0400 Subject: [PATCH 14/26] Rename all mixedCase globals to snake case to appease N816 (#689) Signed-off-by: Matthew McGinn --- influxdb/tests/__init__.py | 6 ++-- influxdb/tests/dataframe_client_test.py | 4 +-- .../tests/influxdb08/dataframe_client_test.py | 4 +-- .../server_tests/client_test_with_server.py | 34 +++++++++---------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/influxdb/tests/__init__.py b/influxdb/tests/__init__.py index adf2f20c..f7c5dfb9 100644 --- a/influxdb/tests/__init__.py +++ b/influxdb/tests/__init__.py @@ -12,10 +12,10 @@ import unittest using_pypy = hasattr(sys, "pypy_version_info") -skipIfPYpy = unittest.skipIf(using_pypy, "Skipping this test on pypy.") +skip_if_pypy = unittest.skipIf(using_pypy, "Skipping this test on pypy.") _skip_server_tests = os.environ.get( 'INFLUXDB_PYTHON_SKIP_SERVER_TESTS', None) == 'True' -skipServerTests = unittest.skipIf(_skip_server_tests, - "Skipping server tests...") +skip_server_tests = unittest.skipIf(_skip_server_tests, + "Skipping server tests...") diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index cb6162f7..c6ca5a7d 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -13,7 +13,7 @@ import warnings import requests_mock -from influxdb.tests import skipIfPYpy, using_pypy +from influxdb.tests import skip_if_pypy, using_pypy from nose.tools import raises from .client_test import _mocked_session @@ -24,7 +24,7 @@ from influxdb import DataFrameClient -@skipIfPYpy +@skip_if_pypy class TestDataFrameClient(unittest.TestCase): """Set up a test DataFrameClient object.""" diff --git a/influxdb/tests/influxdb08/dataframe_client_test.py b/influxdb/tests/influxdb08/dataframe_client_test.py index 6e6fa2cc..0a766af0 100644 --- a/influxdb/tests/influxdb08/dataframe_client_test.py +++ b/influxdb/tests/influxdb08/dataframe_client_test.py @@ -12,7 +12,7 @@ from nose.tools import raises -from influxdb.tests import skipIfPYpy, using_pypy +from influxdb.tests import skip_if_pypy, using_pypy from .client_test import _mocked_session @@ -22,7 +22,7 @@ from influxdb.influxdb08 import DataFrameClient -@skipIfPYpy +@skip_if_pypy class TestDataFrameClient(unittest.TestCase): """Define the DataFramClient test object.""" diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 2f8a2097..4dbc1b75 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -23,7 +23,7 @@ from influxdb import InfluxDBClient from influxdb.exceptions import InfluxDBClientError -from influxdb.tests import skipIfPYpy, using_pypy, skipServerTests +from influxdb.tests import skip_if_pypy, using_pypy, skip_server_tests from influxdb.tests.server_tests.base import ManyTestCasesWithServerMixin from influxdb.tests.server_tests.base import SingleTestCaseWithServerMixin @@ -82,7 +82,7 @@ def point(series_name, timestamp=None, tags=None, **fields): ] if not using_pypy: - dummy_pointDF = { + dummy_point_df = { "measurement": "cpu_load_short", "tags": {"host": "server01", "region": "us-west"}, @@ -90,7 +90,7 @@ def point(series_name, timestamp=None, tags=None, **fields): [[0.64]], columns=['value'], index=pd.to_datetime(["2009-11-10T23:00:00Z"])) } - dummy_pointsDF = [{ + dummy_points_df = [{ "measurement": "cpu_load_short", "tags": {"host": "server01", "region": "us-west"}, "dataframe": pd.DataFrame( @@ -120,7 +120,7 @@ def point(series_name, timestamp=None, tags=None, **fields): ] -@skipServerTests +@skip_server_tests class SimpleTests(SingleTestCaseWithServerMixin, unittest.TestCase): """Define the class of simple tests.""" @@ -267,7 +267,7 @@ def test_invalid_port_fails(self): InfluxDBClient('host', '80/redir', 'username', 'password') -@skipServerTests +@skip_server_tests class CommonTests(ManyTestCasesWithServerMixin, unittest.TestCase): """Define a class to handle common tests for the server.""" @@ -293,15 +293,15 @@ def test_write_points(self): """Test writing points to the server.""" self.assertIs(True, self.cli.write_points(dummy_point)) - @skipIfPYpy + @skip_if_pypy def test_write_points_DF(self): """Test writing points with dataframe.""" self.assertIs( True, self.cliDF.write_points( - dummy_pointDF['dataframe'], - dummy_pointDF['measurement'], - dummy_pointDF['tags'] + dummy_point_df['dataframe'], + dummy_point_df['measurement'], + dummy_point_df['tags'] ) ) @@ -342,7 +342,7 @@ def test_write_points_check_read_DF(self): rsp = self.cliDF.query('SELECT * FROM cpu_load_short') assert_frame_equal( rsp['cpu_load_short'], - dummy_pointDF['dataframe'] + dummy_point_df['dataframe'] ) # Query with Tags @@ -351,7 +351,7 @@ def test_write_points_check_read_DF(self): assert_frame_equal( rsp[('cpu_load_short', (('host', 'server01'), ('region', 'us-west')))], - dummy_pointDF['dataframe'] + dummy_point_df['dataframe'] ) def test_write_multiple_points_different_series(self): @@ -407,21 +407,21 @@ def test_write_multiple_points_different_series_DF(self): for i in range(2): self.assertIs( True, self.cliDF.write_points( - dummy_pointsDF[i]['dataframe'], - dummy_pointsDF[i]['measurement'], - dummy_pointsDF[i]['tags'])) + dummy_points_df[i]['dataframe'], + dummy_points_df[i]['measurement'], + dummy_points_df[i]['tags'])) time.sleep(1) rsp = self.cliDF.query('SELECT * FROM cpu_load_short') assert_frame_equal( rsp['cpu_load_short'], - dummy_pointsDF[0]['dataframe'] + dummy_points_df[0]['dataframe'] ) rsp = self.cliDF.query('SELECT * FROM memory') assert_frame_equal( rsp['memory'], - dummy_pointsDF[1]['dataframe'] + dummy_points_df[1]['dataframe'] ) def test_write_points_batch(self): @@ -786,7 +786,7 @@ def test_query_multiple_series(self): self.cli.write_points(pts) -@skipServerTests +@skip_server_tests class UdpTests(ManyTestCasesWithServerMixin, unittest.TestCase): """Define a class to test UDP series.""" From 2d17360e7f2c1f00588b362d77ae762142207f33 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Thu, 14 Mar 2019 10:33:50 -0300 Subject: [PATCH 15/26] Fixup small test docstring typo Signed-off-by: Matthew McGinn --- influxdb/tests/dataframe_client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index c6ca5a7d..80145933 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -866,7 +866,7 @@ def test_query_into_dataframe(self): assert_frame_equal(expected[k], result[k]) def test_multiquery_into_dataframe(self): - """Test multiquyer into df for TestDataFrameClient object.""" + """Test multiquery into df for TestDataFrameClient object.""" data = { "results": [ { From 5147aed455ef99e0f01bb6909ee40a693107852a Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Thu, 14 Mar 2019 14:04:32 +0000 Subject: [PATCH 16/26] Fix tz localize (#684) * fix already tz-aware error * fix tests tz_localize * update CHANGELOG.md --- CHANGELOG.md | 1 + influxdb/_dataframe_client.py | 3 ++- influxdb/tests/dataframe_client_test.py | 15 ++++++++++----- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14a9abf4..590bd4f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed +- Fix 'TypeError: Already tz-aware' introduced with recent versions of Panda (#671, #676, thx @f4bsch @clslgrnc) ### Removed diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 763a845c..2de2023d 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -202,7 +202,8 @@ def _to_dataframe(self, rs, dropna=True): df = pd.DataFrame(data) df.time = pd.to_datetime(df.time) df.set_index('time', inplace=True) - df.index = df.index.tz_localize('UTC') + if df.index.tzinfo is None: + df.index = df.index.tz_localize('UTC') df.index.name = None result[key].append(df) for key, data in result.items(): diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 80145933..3c460077 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -847,13 +847,15 @@ def test_query_into_dataframe(self): pd1 = pd.DataFrame( [[23422]], columns=['value'], index=pd.to_datetime(["2009-11-10T23:00:00Z"])) - pd1.index = pd1.index.tz_localize('UTC') + if pd1.index.tzinfo is None: + pd1.index = pd1.index.tz_localize('UTC') pd2 = pd.DataFrame( [[23422], [23422], [23422]], columns=['value'], index=pd.to_datetime(["2009-11-10T23:00:00Z", "2009-11-10T23:00:00Z", "2009-11-10T23:00:00Z"])) - pd2.index = pd2.index.tz_localize('UTC') + if pd2.index.tzinfo is None: + pd2.index = pd2.index.tz_localize('UTC') expected = { ('network', (('direction', ''),)): pd1, ('network', (('direction', 'in'),)): pd2 @@ -900,11 +902,14 @@ def test_multiquery_into_dataframe(self): index=pd.to_datetime([ "2015-01-29 21:55:43.702900257+0000", "2015-01-29 21:55:43.702900257+0000", - "2015-06-11 20:46:02+0000"])).tz_localize('UTC') + "2015-06-11 20:46:02+0000"])) + if pd1.index.tzinfo is None: + pd1.index = pd1.index.tz_localize('UTC') pd2 = pd.DataFrame( [[3]], columns=['count'], - index=pd.to_datetime(["1970-01-01 00:00:00+00:00"]))\ - .tz_localize('UTC') + index=pd.to_datetime(["1970-01-01 00:00:00+00:00"])) + if pd2.index.tzinfo is None: + pd2.index = pd2.index.tz_localize('UTC') expected = [{'cpu_load_short': pd1}, {'cpu_load_short': pd2}] cli = DataFrameClient('host', 8086, 'username', 'password', 'db') From f93010bcd7e39632bd6c6fca7bbf34a67cd4c299 Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Thu, 14 Mar 2019 11:19:55 -0300 Subject: [PATCH 17/26] Bump version to 5.2.2 Signed-off-by: Matthew McGinn --- CHANGELOG.md | 7 ++++++- influxdb/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 590bd4f3..035476ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed -- Fix 'TypeError: Already tz-aware' introduced with recent versions of Panda (#671, #676, thx @f4bsch @clslgrnc) ### Removed +## [v5.2.2] - 2019-03-14 +### Added + +### Changed +- Fix 'TypeError: Already tz-aware' introduced with recent versions of Panda (#671, #676, thx @f4bsch @clslgrnc) + ## [v5.2.1] - 2018-12-07 ### Added diff --git a/influxdb/__init__.py b/influxdb/__init__.py index a1eb3789..288880b1 100644 --- a/influxdb/__init__.py +++ b/influxdb/__init__.py @@ -18,4 +18,4 @@ ] -__version__ = '5.2.1' +__version__ = '5.2.2' From 9fd9d6e282b89c400094d0087d727ccfbb56dda3 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Fri, 15 Mar 2019 17:13:43 +0000 Subject: [PATCH 18/26] [WIP] add py37 and recent influxdb (#692) * add py37 and recent influxdb * remove useless py34 dep * use py36 for pydocstyle (py27 soon deprecated) * ugly fix to numpy inconsistencies * py37 is not in ubuntu 14.04 * move import numpy and add noqa * get 3.7 into travis matrix * get 3.7 into travis matrix --- .travis.yml | 36 +++++++++++++++++++++---- influxdb/tests/dataframe_client_test.py | 18 ++++++++++--- tox.ini | 30 ++++++++++++++------- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index a1cf7b55..8c660b67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,12 @@ python: - "pypy3" env: - - INFLUXDB_VER=1.2.4 - - INFLUXDB_VER=1.3.9 - - INFLUXDB_VER=1.4.2 - - INFLUXDB_VER=1.5.4 + - INFLUXDB_VER=1.2.4 # 2017-05-08 + - INFLUXDB_VER=1.3.9 # 2018-01-19 + - INFLUXDB_VER=1.4.3 # 2018-01-30 + - INFLUXDB_VER=1.5.4 # 2018-06-22 + - INFLUXDB_VER=1.6.4 # 2018-10-24 + - INFLUXDB_VER=1.7.4 # 2019-02-14 addons: apt: @@ -20,7 +22,31 @@ addons: matrix: include: - - python: 2.7 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.2.4 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.3.9 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.4.3 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.5.4 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.6.4 + - python: 3.7 + dist: xenial + sudo: true + env: INFLUXDB_VER=1.7.4 + - python: 3.6 env: TOX_ENV=pep257 - python: 3.6 env: TOX_ENV=docs diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 3c460077..68bce912 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -22,6 +22,7 @@ import pandas as pd from pandas.util.testing import assert_frame_equal from influxdb import DataFrameClient + import numpy @skip_if_pypy @@ -425,10 +426,16 @@ def test_write_points_from_dataframe_with_numeric_precision(self): ["2", 2, 2.2222222222222]], index=[now, now + timedelta(hours=1)]) - expected_default_precision = ( - b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' - b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' - ) + if tuple(map(int, numpy.version.version.split('.'))) <= (1, 13, 3): + expected_default_precision = ( + b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' + b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' + ) + else: + expected_default_precision = ( + b'foo,hello=there 0=\"1\",1=1i,2=1.1111111111111 0\n' + b'foo,hello=there 0=\"2\",1=2i,2=2.2222222222222 3600000000000\n' # noqa E501 line too long + ) expected_specified_precision = ( b'foo,hello=there 0=\"1\",1=1i,2=1.1111 0\n' @@ -448,6 +455,9 @@ def test_write_points_from_dataframe_with_numeric_precision(self): cli = DataFrameClient(database='db') cli.write_points(dataframe, "foo", {"hello": "there"}) + print(expected_default_precision) + print(m.last_request.body) + self.assertEqual(m.last_request.body, expected_default_precision) cli = DataFrameClient(database='db') diff --git a/tox.ini b/tox.ini index 2f9c212c..4a1921e2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,28 @@ [tox] -envlist = py27, py35, py36, pypy, pypy3, flake8, pep257, coverage, docs +envlist = py27, py35, py36, py37, pypy, pypy3, flake8, pep257, coverage, docs [testenv] passenv = INFLUXDB_PYTHON_INFLUXD_PATH setenv = INFLUXDB_PYTHON_SKIP_SERVER_TESTS=False deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt - py27,py34,py35,py36: pandas==0.20.1 - py27,py34,py35,py36: numpy==1.13.3 + py27: pandas==0.21.1 + py27: numpy==1.13.3 + py35: pandas==0.22.0 + py35: numpy==1.14.6 + py36: pandas==0.23.4 + py36: numpy==1.15.4 + py37: pandas==0.24.2 + py37: numpy==1.16.2 # Only install pandas with non-pypy interpreters +# Testing all combinations would be too expensive commands = nosetests -v --with-doctest {posargs} [testenv:flake8] deps = flake8 pep8-naming -commands = flake8 --ignore=W503,W504,W605,N802,F821 influxdb +commands = flake8 influxdb [testenv:pep257] deps = pydocstyle @@ -26,19 +33,22 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt pandas coverage - numpy==1.13.3 + numpy commands = nosetests -v --with-coverage --cover-html --cover-package=influxdb [testenv:docs] deps = -r{toxinidir}/requirements.txt - pandas==0.20.1 - numpy==1.13.3 - Sphinx==1.5.5 + pandas==0.24.2 + numpy==1.16.2 + Sphinx==1.8.5 sphinx_rtd_theme commands = sphinx-build -b html docs/source docs/build [flake8] -ignore = N802,F821,E402 -# E402: module level import not at top of file +ignore = W503,W504,W605,N802,F821,E402 +# W503: Line break occurred before a binary operator +# W504: Line break occurred after a binary operator +# W605: invalid escape sequence # N802: nosetests's setUp function # F821: False positive in intluxdb/dataframe_client.py +# E402: module level import not at top of file From e5d2589147559f7a997466456444e95dcb357ec3 Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Sat, 16 Mar 2019 17:47:41 +0000 Subject: [PATCH 19/26] Python and influxdb supported versions (#693) * numpy might use non-numerical version * update * update CHANGELOG.md --- CHANGELOG.md | 1 + README.rst | 4 ++-- influxdb/tests/dataframe_client_test.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 035476ab..d18d5bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added ### Changed +- Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) ### Removed diff --git a/README.rst b/README.rst index d4f9611c..026171b2 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ InfluxDB is an open-source distributed time series database, find more about Inf InfluxDB pre v1.1.0 users ------------------------- -This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.2, and v1.5.4. +This module is tested with InfluxDB versions: v1.2.4, v1.3.9, v1.4.3, v1.5.4, v1.6.4, and 1.7.4. Those users still on InfluxDB v0.8.x users may still use the legacy client by importing ``from influxdb.influxdb08 import InfluxDBClient``. @@ -59,7 +59,7 @@ On Debian/Ubuntu, you can install it with this command:: Dependencies ------------ -The influxdb-python distribution is supported and tested on Python 2.7, 3.5, 3.6, PyPy and PyPy3. +The influxdb-python distribution is supported and tested on Python 2.7, 3.5, 3.6, 3.7, PyPy and PyPy3. **Note:** Python <3.5 are currently untested. See ``.travis.yml``. diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 68bce912..99685844 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -426,7 +426,7 @@ def test_write_points_from_dataframe_with_numeric_precision(self): ["2", 2, 2.2222222222222]], index=[now, now + timedelta(hours=1)]) - if tuple(map(int, numpy.version.version.split('.'))) <= (1, 13, 3): + if numpy.lib.NumpyVersion(numpy.__version__) <= '1.13.3': expected_default_precision = ( b'foo,hello=there 0=\"1\",1=1i,2=1.11111111111 0\n' b'foo,hello=there 0=\"2\",1=2i,2=2.22222222222 3600000000000\n' From 170d47e2ac8c64991339e66595501171bfac89cb Mon Sep 17 00:00:00 2001 From: Matthew McGinn Date: Tue, 19 Mar 2019 23:27:47 -0500 Subject: [PATCH 20/26] Add CODEOWNERS file for automatic reviewers on GitHub Signed-off-by: Matthew McGinn --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..0acbd7c8 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @aviau @xginn8 @sebito91 From a3d28f251bd8debee736fdb3e074b85a849fb31c Mon Sep 17 00:00:00 2001 From: Colas Le Guernic Date: Wed, 20 Mar 2019 04:30:26 +0000 Subject: [PATCH 21/26] Parameter binding for client's `query()` method (#678) * add bind_params to query * tutorial for bind_params --- CHANGELOG.md | 1 + examples/tutorial.py | 9 ++++++++- influxdb/_dataframe_client.py | 12 ++++++++++++ influxdb/client.py | 18 ++++++++++++++++++ influxdb/tests/dataframe_client_test.py | 7 ++++--- .../server_tests/client_test_with_server.py | 4 +++- 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d18d5bc4..a5bc07fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) ### Changed - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) diff --git a/examples/tutorial.py b/examples/tutorial.py index 4083bfc5..12cd49c1 100644 --- a/examples/tutorial.py +++ b/examples/tutorial.py @@ -13,7 +13,9 @@ def main(host='localhost', port=8086): dbname = 'example' dbuser = 'smly' dbuser_password = 'my_secret_password' - query = 'select value from cpu_load_short;' + query = 'select Float_value from cpu_load_short;' + query_where = 'select Int_value from cpu_load_short where host=$host;' + bind_params = {'host': 'server01'} json_body = [ { "measurement": "cpu_load_short", @@ -50,6 +52,11 @@ def main(host='localhost', port=8086): print("Result: {0}".format(result)) + print("Querying data: " + query_where) + result = client.query(query_where, bind_params=bind_params) + + print("Result: {0}".format(result)) + print("Switch user: " + user) client.switch_user(user, password) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 2de2023d..33a8b562 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -142,6 +142,7 @@ def write_points(self, def query(self, query, params=None, + bind_params=None, epoch=None, expected_response_code=200, database=None, @@ -153,8 +154,18 @@ def query(self, """ Query data into a DataFrame. + .. danger:: + In order to avoid injection vulnerabilities (similar to `SQL + injection `_ + vulnerabilities), do not directly include untrusted data into the + ``query`` parameter, use ``bind_params`` instead. + :param query: the actual query string :param params: additional parameters for the request, defaults to {} + :param bind_params: bind parameters for the query: + any variable in the query written as ``'$var_name'`` will be + replaced with ``bind_params['var_name']``. Only works in the + ``WHERE`` clause and takes precedence over ``params['params']`` :param epoch: response timestamps to be in epoch format either 'h', 'm', 's', 'ms', 'u', or 'ns',defaults to `None` which is RFC3339 UTC format with nanosecond precision @@ -172,6 +183,7 @@ def query(self, :rtype: :class:`~.ResultSet` """ query_args = dict(params=params, + bind_params=bind_params, epoch=epoch, expected_response_code=expected_response_code, raise_errors=raise_errors, diff --git a/influxdb/client.py b/influxdb/client.py index 8f8b14ae..e94ae25d 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -345,6 +345,7 @@ def _read_chunked_response(response, raise_errors=True): def query(self, query, params=None, + bind_params=None, epoch=None, expected_response_code=200, database=None, @@ -354,6 +355,12 @@ def query(self, method="GET"): """Send a query to InfluxDB. + .. danger:: + In order to avoid injection vulnerabilities (similar to `SQL + injection `_ + vulnerabilities), do not directly include untrusted data into the + ``query`` parameter, use ``bind_params`` instead. + :param query: the actual query string :type query: str @@ -361,6 +368,12 @@ def query(self, defaults to {} :type params: dict + :param bind_params: bind parameters for the query: + any variable in the query written as ``'$var_name'`` will be + replaced with ``bind_params['var_name']``. Only works in the + ``WHERE`` clause and takes precedence over ``params['params']`` + :type bind_params: dict + :param epoch: response timestamps to be in epoch format either 'h', 'm', 's', 'ms', 'u', or 'ns',defaults to `None` which is RFC3339 UTC format with nanosecond precision @@ -394,6 +407,11 @@ def query(self, if params is None: params = {} + if bind_params is not None: + params_dict = json.loads(params.get('params', '{}')) + params_dict.update(bind_params) + params['params'] = json.dumps(params_dict) + params['q'] = query params['db'] = database or self._database diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 99685844..6cb47083 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -923,10 +923,11 @@ def test_multiquery_into_dataframe(self): expected = [{'cpu_load_short': pd1}, {'cpu_load_short': pd2}] cli = DataFrameClient('host', 8086, 'username', 'password', 'db') - iql = "SELECT value FROM cpu_load_short WHERE region='us-west';"\ - "SELECT count(value) FROM cpu_load_short WHERE region='us-west'" + iql = "SELECT value FROM cpu_load_short WHERE region=$region;"\ + "SELECT count(value) FROM cpu_load_short WHERE region=$region" + bind_params = {'region': 'us-west'} with _mocked_session(cli, 'GET', 200, data): - result = cli.query(iql) + result = cli.query(iql, bind_params=bind_params) for r, e in zip(result, expected): for k in e: assert_frame_equal(e[k], r[k]) diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 4dbc1b75..121d2c82 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -440,7 +440,9 @@ def test_write_points_batch(self): batch_size=2) time.sleep(5) net_in = self.cli.query("SELECT value FROM network " - "WHERE direction='in'").raw + "WHERE direction=$dir", + bind_params={'dir': 'in'} + ).raw net_out = self.cli.query("SELECT value FROM network " "WHERE direction='out'").raw cpu = self.cli.query("SELECT value FROM cpu_usage").raw From 9e876dc433352bef7df507a955e3ecaf5d2c571b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Dudek?= <45991310+lukaszdudek-silvair@users.noreply.github.com> Date: Mon, 1 Apr 2019 19:05:02 +0200 Subject: [PATCH 22/26] Add CQs management methods to the client (#681) * Add CQs management methods to the client --- CHANGELOG.md | 2 + influxdb/client.py | 92 +++++++++++++++ influxdb/tests/client_test.py | 108 ++++++++++++++++++ .../server_tests/client_test_with_server.py | 30 +++++ 4 files changed, 232 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5bc07fb..9834a5ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Add `get_list_continuous_queries`, `drop_continuous_query`, and `create_continuous_query` management methods for + continuous queries (#681 thx @lukaszdudek-silvair) - query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) ### Changed diff --git a/influxdb/client.py b/influxdb/client.py index e94ae25d..d365643d 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -926,6 +926,98 @@ def get_list_privileges(self, username): text = "SHOW GRANTS FOR {0}".format(quote_ident(username)) return list(self.query(text).get_points()) + def get_list_continuous_queries(self): + """Get the list of continuous queries in InfluxDB. + + :return: all CQs in InfluxDB + :rtype: list of dictionaries + + :Example: + + :: + + >> cqs = client.get_list_cqs() + >> cqs + [ + { + u'db1': [] + }, + { + u'db2': [ + { + u'name': u'vampire', + u'query': u'CREATE CONTINUOUS QUERY vampire ON ' + 'mydb BEGIN SELECT count(dracula) INTO ' + 'mydb.autogen.all_of_them FROM ' + 'mydb.autogen.one GROUP BY time(5m) END' + } + ] + } + ] + """ + query_string = "SHOW CONTINUOUS QUERIES" + return [{sk[0]: list(p)} for sk, p in self.query(query_string).items()] + + def create_continuous_query(self, name, select, database=None, + resample_opts=None): + r"""Create a continuous query for a database. + + :param name: the name of continuous query to create + :type name: str + :param select: select statement for the continuous query + :type select: str + :param database: the database for which the continuous query is + created. Defaults to current client's database + :type database: str + :param resample_opts: resample options + :type resample_opts: str + + :Example: + + :: + + >> select_clause = 'SELECT mean("value") INTO "cpu_mean" ' \ + ... 'FROM "cpu" GROUP BY time(1m)' + >> client.create_continuous_query( + ... 'cpu_mean', select_clause, 'db_name', 'EVERY 10s FOR 2m' + ... ) + >> client.get_list_continuous_queries() + [ + { + 'db_name': [ + { + 'name': 'cpu_mean', + 'query': 'CREATE CONTINUOUS QUERY "cpu_mean" ' + 'ON "db_name" ' + 'RESAMPLE EVERY 10s FOR 2m ' + 'BEGIN SELECT mean("value") ' + 'INTO "cpu_mean" FROM "cpu" ' + 'GROUP BY time(1m) END' + } + ] + } + ] + """ + query_string = ( + "CREATE CONTINUOUS QUERY {0} ON {1}{2} BEGIN {3} END" + ).format(quote_ident(name), quote_ident(database or self._database), + ' RESAMPLE ' + resample_opts if resample_opts else '', select) + self.query(query_string) + + def drop_continuous_query(self, name, database=None): + """Drop an existing continuous query for a database. + + :param name: the name of continuous query to drop + :type name: str + :param database: the database for which the continuous query is + dropped. Defaults to current client's database + :type database: str + """ + query_string = ( + "DROP CONTINUOUS QUERY {0} ON {1}" + ).format(quote_ident(name), quote_ident(database or self._database)) + self.query(query_string) + def send_packet(self, packet, protocol='json', time_precision=None): """Send an UDP packet. diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e27eef17..e1a30b81 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -1027,6 +1027,114 @@ def test_get_list_privileges_fails(self): with _mocked_session(cli, 'get', 401): cli.get_list_privileges('test') + def test_get_list_continuous_queries(self): + """Test getting a list of continuous queries.""" + data = { + "results": [ + { + "statement_id": 0, + "series": [ + { + "name": "testdb01", + "columns": ["name", "query"], + "values": [["testname01", "testquery01"], + ["testname02", "testquery02"]] + }, + { + "name": "testdb02", + "columns": ["name", "query"], + "values": [["testname03", "testquery03"]] + }, + { + "name": "testdb03", + "columns": ["name", "query"] + } + ] + } + ] + } + + with _mocked_session(self.cli, 'get', 200, json.dumps(data)): + self.assertListEqual( + self.cli.get_list_continuous_queries(), + [ + { + 'testdb01': [ + {'name': 'testname01', 'query': 'testquery01'}, + {'name': 'testname02', 'query': 'testquery02'} + ] + }, + { + 'testdb02': [ + {'name': 'testname03', 'query': 'testquery03'} + ] + }, + { + 'testdb03': [] + } + ] + ) + + @raises(Exception) + def test_get_list_continuous_queries_fails(self): + """Test failing to get a list of continuous queries.""" + with _mocked_session(self.cli, 'get', 400): + self.cli.get_list_continuous_queries() + + def test_create_continuous_query(self): + """Test continuous query creation.""" + data = {"results": [{}]} + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + text=json.dumps(data) + ) + query = 'SELECT count("value") INTO "6_months"."events" FROM ' \ + '"events" GROUP BY time(10m)' + self.cli.create_continuous_query('cq_name', query, 'db_name') + self.assertEqual( + m.last_request.qs['q'][0], + 'create continuous query "cq_name" on "db_name" begin select ' + 'count("value") into "6_months"."events" from "events" group ' + 'by time(10m) end' + ) + self.cli.create_continuous_query('cq_name', query, 'db_name', + 'EVERY 10s FOR 2m') + self.assertEqual( + m.last_request.qs['q'][0], + 'create continuous query "cq_name" on "db_name" resample ' + 'every 10s for 2m begin select count("value") into ' + '"6_months"."events" from "events" group by time(10m) end' + ) + + @raises(Exception) + def test_create_continuous_query_fails(self): + """Test failing to create a continuous query.""" + with _mocked_session(self.cli, 'get', 400): + self.cli.create_continuous_query('cq_name', 'select', 'db_name') + + def test_drop_continuous_query(self): + """Test dropping a continuous query.""" + data = {"results": [{}]} + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.GET, + "http://localhost:8086/query", + text=json.dumps(data) + ) + self.cli.drop_continuous_query('cq_name', 'db_name') + self.assertEqual( + m.last_request.qs['q'][0], + 'drop continuous query "cq_name" on "db_name"' + ) + + @raises(Exception) + def test_drop_continuous_query_fails(self): + """Test failing to drop a continuous query.""" + with _mocked_session(self.cli, 'get', 400): + self.cli.drop_continuous_query('cq_name', 'db_name') + def test_invalid_port_fails(self): """Test invalid port fail for TestInfluxDBClient object.""" with self.assertRaises(ValueError): diff --git a/influxdb/tests/server_tests/client_test_with_server.py b/influxdb/tests/server_tests/client_test_with_server.py index 121d2c82..fda3f720 100644 --- a/influxdb/tests/server_tests/client_test_with_server.py +++ b/influxdb/tests/server_tests/client_test_with_server.py @@ -722,6 +722,36 @@ def test_drop_retention_policy(self): rsp ) + def test_create_continuous_query(self): + """Test continuous query creation.""" + self.cli.create_retention_policy('some_rp', '1d', 1) + query = 'select count("value") into "some_rp"."events" from ' \ + '"events" group by time(10m)' + self.cli.create_continuous_query('test_cq', query, 'db') + cqs = self.cli.get_list_continuous_queries() + expected_cqs = [ + { + 'db': [ + { + 'name': 'test_cq', + 'query': 'CREATE CONTINUOUS QUERY test_cq ON db ' + 'BEGIN SELECT count(value) INTO ' + 'db.some_rp.events FROM db.autogen.events ' + 'GROUP BY time(10m) END' + } + ] + } + ] + self.assertEqual(cqs, expected_cqs) + + def test_drop_continuous_query(self): + """Test continuous query drop.""" + self.test_create_continuous_query() + self.cli.drop_continuous_query('test_cq', 'db') + cqs = self.cli.get_list_continuous_queries() + expected_cqs = [{'db': []}] + self.assertEqual(cqs, expected_cqs) + def test_issue_143(self): """Test for PR#143 from repo.""" pt = partial(point, 'a_series_name', timestamp='2015-03-30T16:16:37Z') From 6f356f0702a0c297346073ec365655289148cb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A7=8B=E8=91=89?= Date: Sun, 7 Apr 2019 09:32:09 -0500 Subject: [PATCH 23/26] Fix a warning under Python 3.7 (#697) * Fix a warning under Python 3.7 Signed-off-by: Matthew McGinn --- CHANGELOG.md | 1 + setup.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9834a5ef..33302f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) +- Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) ### Removed diff --git a/setup.py b/setup.py index cd6e4e9b..d44875f6 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ tests_require=test_requires, install_requires=requires, extras_require={'test': test_requires}, - classifiers=( + classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', @@ -55,5 +55,5 @@ 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', - ), + ], ) From 15d82319ca653930a50a502a3b4174bf80741a4a Mon Sep 17 00:00:00 2001 From: xginn8 Date: Sun, 7 Apr 2019 20:09:36 -0500 Subject: [PATCH 24/26] Update delete_series docstring to differentiate between drop_database (#699) Closes #666 Signed-off-by: Matthew McGinn --- CHANGELOG.md | 1 + influxdb/client.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33302f16..a7630a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) - Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) +- Update documentation for empty `delete_series` confusion ### Removed diff --git a/influxdb/client.py b/influxdb/client.py index d365643d..dc77bfc1 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -827,7 +827,9 @@ def set_user_password(self, username, password): def delete_series(self, database=None, measurement=None, tags=None): """Delete series from a database. - Series can be filtered by measurement and tags. + Series must be filtered by either measurement and tags. + This method cannot be used to delete all series, use + `drop_database` instead. :param database: the database from which the series should be deleted, defaults to client's current database From c24d8286af118a1ac1e2b72d72355ba1402292d5 Mon Sep 17 00:00:00 2001 From: Ron Rothman Date: Mon, 8 Apr 2019 13:13:13 -0400 Subject: [PATCH 25/26] add consistency parameter to write_points (#664) * add consistency parameter to write_points [https://github.com/influxdata/influxdb-python/issues/643] --- CHANGELOG.md | 1 + influxdb/client.py | 21 +++++++++++++++++---- influxdb/tests/client_test.py | 26 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7630a9d..7f1503b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - query() now accepts a bind_params argument for parameter binding (#678 thx @clslgrnc) ### Changed +- Add consistency param to InfluxDBClient.write_points (#643 thx @RonRothman) - Update test suite to add support for Python 3.7 and InfluxDB v1.6.4 and 1.7.4 (#692 thx @clslgrnc) - Update classifiers tuple to list in setup.py (#697 thx @Hanaasagi) - Update documentation for empty `delete_series` confusion diff --git a/influxdb/client.py b/influxdb/client.py index dc77bfc1..8ac557d3 100644 --- a/influxdb/client.py +++ b/influxdb/client.py @@ -458,7 +458,8 @@ def write_points(self, retention_policy=None, tags=None, batch_size=None, - protocol='json' + protocol='json', + consistency=None ): """Write to multiple time series names. @@ -486,6 +487,9 @@ def write_points(self, :type batch_size: int :param protocol: Protocol for writing data. Either 'line' or 'json'. :type protocol: str + :param consistency: Consistency for the points. + One of {'any','one','quorum','all'}. + :type consistency: str :returns: True, if the operation is successful :rtype: bool @@ -498,14 +502,16 @@ def write_points(self, time_precision=time_precision, database=database, retention_policy=retention_policy, - tags=tags, protocol=protocol) + tags=tags, protocol=protocol, + consistency=consistency) return True return self._write_points(points=points, time_precision=time_precision, database=database, retention_policy=retention_policy, - tags=tags, protocol=protocol) + tags=tags, protocol=protocol, + consistency=consistency) def ping(self): """Check connectivity to InfluxDB. @@ -531,12 +537,16 @@ def _write_points(self, database, retention_policy, tags, - protocol='json'): + protocol='json', + consistency=None): if time_precision not in ['n', 'u', 'ms', 's', 'm', 'h', None]: raise ValueError( "Invalid time precision is given. " "(use 'n', 'u', 'ms', 's', 'm' or 'h')") + if consistency not in ['any', 'one', 'quorum', 'all', None]: + raise ValueError('Invalid consistency: {}'.format(consistency)) + if protocol == 'json': data = { 'points': points @@ -551,6 +561,9 @@ def _write_points(self, 'db': database or self._database } + if consistency is not None: + params['consistency'] = consistency + if time_precision is not None: params['precision'] = time_precision diff --git a/influxdb/tests/client_test.py b/influxdb/tests/client_test.py index e1a30b81..e4cc7e11 100644 --- a/influxdb/tests/client_test.py +++ b/influxdb/tests/client_test.py @@ -337,6 +337,23 @@ def test_write_points_with_precision(self): m.last_request.body, ) + def test_write_points_with_consistency(self): + """Test write points with consistency for TestInfluxDBClient object.""" + with requests_mock.Mocker() as m: + m.register_uri( + requests_mock.POST, + 'http://localhost:8086/write', + status_code=204 + ) + + cli = InfluxDBClient(database='db') + + cli.write_points(self.dummy_points, consistency='any') + self.assertEqual( + m.last_request.qs, + {'db': ['db'], 'consistency': ['any']} + ) + def test_write_points_with_precision_udp(self): """Test write points with precision for TestInfluxDBClient object.""" s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -409,6 +426,15 @@ def test_write_points_bad_precision(self): time_precision='g' ) + def test_write_points_bad_consistency(self): + """Test write points w/bad consistency value.""" + cli = InfluxDBClient() + with self.assertRaises(ValueError): + cli.write_points( + self.dummy_points, + consistency='boo' + ) + @raises(Exception) def test_write_points_with_precision_fails(self): """Test write points w/precision fail for TestInfluxDBClient object.""" From a0cd98731e486187d79ea451d2d9b8648a793177 Mon Sep 17 00:00:00 2001 From: Shan Desai Date: Wed, 10 Apr 2019 15:30:45 +0200 Subject: [PATCH 26/26] Add Example for sending information to DB via UDP (#648) Due to lack of documentation for UDP, this example provides basic usage of sending information points via UDP. The code structure followed is similar, if not same as other examples in the `examples` directory. Signed-off-by: Shantanoo --- docs/source/examples.rst | 6 ++++ examples/tutorial_udp.py | 66 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 examples/tutorial_udp.py diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 2c85fbda..fdda62a9 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -25,3 +25,9 @@ Tutorials - SeriesHelper .. literalinclude:: ../../examples/tutorial_serieshelper.py :language: python + +Tutorials - UDP +=============== + +.. literalinclude:: ../../examples/tutorial_udp.py + :language: python diff --git a/examples/tutorial_udp.py b/examples/tutorial_udp.py new file mode 100644 index 00000000..517ae858 --- /dev/null +++ b/examples/tutorial_udp.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +"""Example for sending batch information to InfluxDB via UDP.""" + +""" +INFO: In order to use UDP, one should enable the UDP service from the +`influxdb.conf` under section + [[udp]] + enabled = true + bind-address = ":8089" # port number for sending data via UDP + database = "udp1" # name of database to be stored + [[udp]] + enabled = true + bind-address = ":8090" + database = "udp2" +""" + + +import argparse + +from influxdb import InfluxDBClient + + +def main(uport): + """Instantiate connection to the InfluxDB.""" + # NOTE: structure of the UDP packet is different than that of information + # sent via HTTP + json_body = { + "tags": { + "host": "server01", + "region": "us-west" + }, + "time": "2009-11-10T23:00:00Z", + "points": [{ + "measurement": "cpu_load_short", + "fields": { + "value": 0.64 + } + }, + { + "measurement": "cpu_load_short", + "fields": { + "value": 0.67 + } + }] + } + + # make `use_udp` True and add `udp_port` number from `influxdb.conf` file + # no need to mention the database name since it is already configured + client = InfluxDBClient(use_udp=True, udp_port=uport) + + # Instead of `write_points` use `send_packet` + client.send_packet(json_body) + + +def parse_args(): + """Parse the args.""" + parser = argparse.ArgumentParser( + description='example code to play with InfluxDB along with UDP Port') + parser.add_argument('--uport', type=int, required=True, + help=' UDP port of InfluxDB') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + main(uport=args.uport)