From f4b3035ded79cd0db7a111642bbe8d95c8943c25 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Thu, 5 Oct 2017 16:22:14 -0300 Subject: [PATCH 1/4] keep threads alive on connection failures --- .gitignore | 2 ++ splitio/segments.py | 14 +++++++++++++- splitio/splits.py | 13 ++++++++++++- splitio/tests/test_segments.py | 13 +++++++------ splitio/tests/test_splits.py | 11 ++++++----- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 993b9639..52cf8f38 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,5 @@ target/ # rope autocomplete .ropeproject/ + +*.swp # vim backup files diff --git a/splitio/segments.py b/splitio/segments.py index 2a60842a..babf69bb 100644 --- a/splitio/segments.py +++ b/splitio/segments.py @@ -203,7 +203,15 @@ def refresh_segment(self): while True: response = self._segment_change_fetcher.fetch(self._name, self._change_number) - if self._change_number >= response['till']: + + # If the response fails, and doesn't return a dict, or + # returns a dict without the 'till' attribute, abort this + # execution. + if ( + not isinstance(response, dict) + or 'till' not in response + or self._change_number >= response['till'] + ): return if len(response['added']) > 0 or len(response['removed']) > 0: @@ -248,6 +256,10 @@ def _timer_start(self): def _timer_refresh(self): """Responsible for setting the periodic calls to _refresh_segment using a Timer thread.""" if self._stopped: + self._logger.error('Previous fetch failed, skipping this iteration ' + 'and rescheduling segment refresh.') + self._stopped = False + self._timer_start() return try: diff --git a/splitio/splits.py b/splitio/splits.py index 477be7a1..328dd145 100644 --- a/splitio/splits.py +++ b/splitio/splits.py @@ -438,7 +438,14 @@ def refresh_splits(self, block_until_ready=False): response = self._split_change_fetcher.fetch( self._change_number) - if self._change_number >= response['till']: + # If the response fails, and doesn't return a dict, or + # returns a dict without the 'till' attribute, abort this + # execution. + if ( + not isinstance(response, dict) + or 'till' not in response + or self._change_number >= response['till'] + ): return if 'splits' in response and len(response['splits']) > 0: @@ -476,6 +483,10 @@ def _timer_refresh(self): Timer thread """ if self._stopped: + self._logger.error('Previous fetch failed, skipping this iteration ' + 'and rescheduling segment refresh.') + self._stopped = False + self._timer_start() return try: diff --git a/splitio/tests/test_segments.py b/splitio/tests/test_segments.py index d28faa65..907807c4 100644 --- a/splitio/tests/test_segments.py +++ b/splitio/tests/test_segments.py @@ -293,12 +293,13 @@ def test_doesnt_call_executor_submit_if_stopped(self): self.some_executor.submit.assert_not_called() - def test_new_timer_not_created_if_stopped(self): - """Tests that if the segment refresh is stopped, no new Timer is created""" - self.segment._stopped = True - self.segment._timer_refresh() - - self.timer_mock.assert_not_called() +# This no longer makes sense since it will only be stopped for one iteration. +# def test_new_timer_not_created_if_stopped(self): +# """Tests that if the segment refresh is stopped, no new Timer is created""" +# self.segment._stopped = True +# self.segment._timer_refresh() +# +# self.timer_mock.assert_not_called() class SegmentChangeFetcherTests(TestCase, MockUtilsMixin): diff --git a/splitio/tests/test_splits.py b/splitio/tests/test_splits.py index 797a1bfb..3e70f235 100644 --- a/splitio/tests/test_splits.py +++ b/splitio/tests/test_splits.py @@ -304,11 +304,12 @@ def test_no_thread_created_if_stopped(self): self.fetcher._timer_refresh() self.thread_mock.assert_not_called() - def test_timer_start_not_called_if_stopped(self): - """Tests that _timer_refresh doesn't call start_tiemer if it is stopped""" - self.fetcher.stopped = True - self.fetcher._timer_refresh() - self.timer_start_mock.assert_not_called() +# This no longer makes sense since it will only be stopped for one iteration. +# def test_timer_start_not_called_if_stopped(self): +# """Tests that _timer_refresh doesn't call start_tiemer if it is stopped""" +# self.fetcher.stopped = True +# self.fetcher._timer_refresh() +# self.timer_start_mock.assert_not_called() def test_timer_start_called_if_thread_raises_exception(self): """ From 02d317c490c953a87bb1ffeba72bffb730004c45 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 6 Oct 2017 14:35:00 -0300 Subject: [PATCH 2/4] remove unnecessary logic --- splitio/redis_support.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/splitio/redis_support.py b/splitio/redis_support.py index 38c0f616..7e39e74d 100644 --- a/splitio/redis_support.py +++ b/splitio/redis_support.py @@ -59,7 +59,8 @@ def disabled_period(self, disabled_period): def disable(self): """Disables the automatic update process. This method will be called if the update fails for some reason. Use enable to re-enable the update process.""" - self._redis.setex(RedisSegmentCache._DISABLED_KEY, 1, self._disabled_period) + # self._redis.setex(RedisSegmentCache._DISABLED_KEY, 1, self._disabled_period) + pass def enable(self): """Enables the automatic update process.""" @@ -70,7 +71,8 @@ def is_enabled(self): :return: Whether the update process is enabled or not. :rtype: bool """ - return not self._redis.exists(RedisSegmentCache._DISABLED_KEY) + return True + # return not self._redis.exists(RedisSegmentCache._DISABLED_KEY) def register_segment(self, segment_name): """Register a segment for inclusion in the automatic update process. @@ -174,7 +176,8 @@ def _get_split_key(self, split_name): def disable(self): """Disables the automatic split update process for the specified disabled period. This method will be called if there's an exception while updating the splits.""" - self._redis.setex(RedisSplitCache._DISABLED_KEY, 1, self._disabled_period) + # self._redis.setex(RedisSplitCache._DISABLED_KEY, 1, self._disabled_period) + pass def enable(self): """Enables the automatic split update process.""" @@ -185,7 +188,8 @@ def is_enabled(self): :return: Whether the update process is enabled or not. :rtype: bool """ - return not self._redis.exists(RedisSplitCache._DISABLED_KEY) + return True + # return not self._redis.exists(RedisSplitCache._DISABLED_KEY) def get_change_number(self): change_number = self._redis.get(RedisSplitCache._KEY_TILL_TEMPLATE.format( @@ -274,14 +278,16 @@ def disable(self): """Disables the automatic impressions report process and the registration of any impressions for the specificed disabled period. This method will be called if there's an exception while trying to send the impressions back to Split.""" - self._redis.setex(RedisImpressionsCache._DISABLED_KEY, 1, self._disabled_period) + # self._redis.setex(RedisImpressionsCache._DISABLED_KEY, 1, self._disabled_period) + pass def is_enabled(self): """ :return: Whether the automatic report process and impressions registration are enabled. :rtype: bool """ - return not self._redis.exists(RedisImpressionsCache._DISABLED_KEY) + # return not self._redis.exists(RedisImpressionsCache._DISABLED_KEY) + return True def _build_impressions_dict(self, impressions): """Buils a dictionary of impressions that groups them based on their feature name. @@ -450,14 +456,16 @@ def disable(self): """Disables the automatic metrics report process and the registration of any metrics for the specified disabled period. This method will be called if there's an exception while trying to send the metrics back to Split.""" - self._redis.setex(RedisMetricsCache._DISABLED_KEY, 1, self._disabled_period) + # self._redis.setex(RedisMetricsCache._DISABLED_KEY, 1, self._disabled_period) + pass def is_enabled(self): """ :return: Whether the automatic report process and metrics registration are enabled. :rtype: bool """ - return not self._redis.exists(RedisMetricsCache._DISABLED_KEY) + # return not self._redis.exists(RedisMetricsCache._DISABLED_KEY) + return True def _get_count_field(self, counter): """Builds the field name for a counter on the metrics redis hash. From 193cda0b65afe23687b9e80616006777ee9c9afb Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 6 Oct 2017 14:56:45 -0300 Subject: [PATCH 3/4] remove unnecessary tests --- splitio/tests/test_redis_support.py | 182 ++++++++++++++-------------- 1 file changed, 93 insertions(+), 89 deletions(-) diff --git a/splitio/tests/test_redis_support.py b/splitio/tests/test_redis_support.py index d468bd3f..638fb94f 100644 --- a/splitio/tests/test_redis_support.py +++ b/splitio/tests/test_redis_support.py @@ -32,28 +32,29 @@ def setUp(self): self.some_redis = mock.MagicMock() self.a_segment_cache = RedisSegmentCache(self.some_redis) - def test_disable_sets_disabled_key(self): - """Test that disable sets the disabled key for segments""" - self.a_segment_cache.disable() - self.some_redis.setex.assert_called_once_with('SPLITIO.segments.__disabled__', 1, - self.a_segment_cache.disabled_period) - - def test_enable_deletes_disabled_key(self): - """Test that enable deletes the disabled key for segments""" - self.a_segment_cache.enable() - self.some_redis.delete.assert_called_once_with('SPLITIO.segments.__disabled__') - - def test_is_enabled_returns_false_if_disabled_key_exists(self): - """Test that is_enabled returns False if disabled key exists""" - self.some_redis.exists.return_value = True - self.assertFalse(self.a_segment_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.segments.__disabled__') - - def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): - """Test that is_enabled returns True if disabled key doesn't exist""" - self.some_redis.exists.return_value = False - self.assertTrue(self.a_segment_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.segments.__disabled__') +# THESE TESTS ARE NO LONGER VALID +# def test_disable_sets_disabled_key(self): +# """Test that disable sets the disabled key for segments""" +# self.a_segment_cache.disable() +# self.some_redis.setex.assert_called_once_with('SPLITIO.segments.__disabled__', 1, +# self.a_segment_cache.disabled_period) +# +# def test_enable_deletes_disabled_key(self): +# """Test that enable deletes the disabled key for segments""" +# self.a_segment_cache.enable() +# self.some_redis.delete.assert_called_once_with('SPLITIO.segments.__disabled__') +# +# def test_is_enabled_returns_false_if_disabled_key_exists(self): +# """Test that is_enabled returns False if disabled key exists""" +# self.some_redis.exists.return_value = True +# self.assertFalse(self.a_segment_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.segments.__disabled__') +# +# def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): +# """Test that is_enabled returns True if disabled key doesn't exist""" +# self.some_redis.exists.return_value = False +# self.assertTrue(self.a_segment_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.segments.__disabled__') def test_register_segment_adds_segment_name_to_register_segments_set(self): """Test that register_segment adds segment name to registered segments set""" @@ -129,28 +130,29 @@ def setUp(self): self.some_redis = mock.MagicMock() self.a_split_cache = RedisSplitCache(self.some_redis) - def test_disable_sets_disabled_key(self): - """Test that disable sets the disabled key for splits""" - self.a_split_cache.disable() - self.some_redis.setex.assert_called_once_with('SPLITIO.split.__disabled__', 1, - self.a_split_cache.disabled_period) - - def test_enable_deletes_disabled_key(self): - """Test that enable deletes the disabled key for splits""" - self.a_split_cache.enable() - self.some_redis.delete.assert_called_once_with('SPLITIO.split.__disabled__') - - def test_is_enabled_returns_false_if_disabled_key_exists(self): - """Test that is_enabled returns False if disabled key exists""" - self.some_redis.exists.return_value = True - self.assertFalse(self.a_split_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.split.__disabled__') - - def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): - """Test that is_enabled returns True if disabled key doesn't exist""" - self.some_redis.exists.return_value = False - self.assertTrue(self.a_split_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.split.__disabled__') +# THESE TESTS ARE NO LONGER VALID +# def test_disable_sets_disabled_key(self): +# """Test that disable sets the disabled key for splits""" +# self.a_split_cache.disable() +# self.some_redis.setex.assert_called_once_with('SPLITIO.split.__disabled__', 1, +# self.a_split_cache.disabled_period) +# +# def test_enable_deletes_disabled_key(self): +# """Test that enable deletes the disabled key for splits""" +# self.a_split_cache.enable() +# self.some_redis.delete.assert_called_once_with('SPLITIO.split.__disabled__') +# +# def test_is_enabled_returns_false_if_disabled_key_exists(self): +# """Test that is_enabled returns False if disabled key exists""" +# self.some_redis.exists.return_value = True +# self.assertFalse(self.a_split_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.split.__disabled__') +# +# def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): +# """Test that is_enabled returns True if disabled key doesn't exist""" +# self.some_redis.exists.return_value = False +# self.assertTrue(self.a_split_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.split.__disabled__') def test_set_change_number_sets_change_number_key(self): """Test that set_change_number sets the change number key""" @@ -201,29 +203,30 @@ def setUp(self): self.build_impressions_dict_mock = self.patch_object(self.an_impressions_cache, '_build_impressions_dict') - def test_disable_sets_disabled_key(self): - """Test that disable sets the disabled key for impressions""" - self.an_impressions_cache.disable() - self.some_redis.setex.assert_called_once_with('SPLITIO.impressions.__disabled__', 1, - self.an_impressions_cache.disabled_period) - - def test_enable_deletes_disabled_key(self): - """Test that enable deletes the disabled key for impressions""" - self.an_impressions_cache.enable() - self.some_redis.delete.assert_called_once_with('SPLITIO.impressions.__disabled__') - - def test_is_enabled_returns_false_if_disabled_key_exists(self): - """Test that is_enabled returns False if disabled key exists""" - self.some_redis.exists.return_value = True - self.assertFalse(self.an_impressions_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.impressions.__disabled__') - - def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): - """Test that is_enabled returns True if disabled key doesn't exist""" - self.some_redis.exists.return_value = False - self.assertTrue(self.an_impressions_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.impressions.__disabled__') - +# THESE TESTS ARE NO LONGER VALID +# def test_disable_sets_disabled_key(self): +# """Test that disable sets the disabled key for impressions""" +# self.an_impressions_cache.disable() +# self.some_redis.setex.assert_called_once_with('SPLITIO.impressions.__disabled__', 1, +# self.an_impressions_cache.disabled_period) +# +# def test_enable_deletes_disabled_key(self): +# """Test that enable deletes the disabled key for impressions""" +# self.an_impressions_cache.enable() +# self.some_redis.delete.assert_called_once_with('SPLITIO.impressions.__disabled__') +# +# def test_is_enabled_returns_false_if_disabled_key_exists(self): +# """Test that is_enabled returns False if disabled key exists""" +# self.some_redis.exists.return_value = True +# self.assertFalse(self.an_impressions_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.impressions.__disabled__') +# +# def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): +# """Test that is_enabled returns True if disabled key doesn't exist""" +# self.some_redis.exists.return_value = False +# self.assertTrue(self.an_impressions_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.impressions.__disabled__') +# def test_fetch_all_doesnt_call_build_impressions_dict_if_no_impressions_cached(self): """Test that fetch_all doesn't call _build_impressions_dict if no impressions are cached""" self.some_redis.lrange.return_value = None @@ -289,28 +292,29 @@ def setUp(self): self.build_metrics_from_cache_response_mock = self.patch_object( self.a_metrics_cache, '_build_metrics_from_cache_response') - def test_disable_sets_disabled_key(self): - """Test that disable sets the disabled key for metrics""" - self.a_metrics_cache.disable() - self.some_redis.setex.assert_called_once_with('SPLITIO.metrics.__disabled__', 1, - self.a_metrics_cache.disabled_period) - - def test_enable_deletes_disabled_key(self): - """Test that enable deletes the disabled key for metrics""" - self.a_metrics_cache.enable() - self.some_redis.delete.assert_called_once_with('SPLITIO.metrics.__disabled__') - - def test_is_enabled_returns_false_if_disabled_key_exists(self): - """Test that is_enabled returns False if disabled key exists""" - self.some_redis.exists.return_value = True - self.assertFalse(self.a_metrics_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.metrics.__disabled__') - - def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): - """Test that is_enabled returns True if disabled key doesn't exist""" - self.some_redis.exists.return_value = False - self.assertTrue(self.a_metrics_cache.is_enabled()) - self.some_redis.exists.assert_called_once_with('SPLITIO.metrics.__disabled__') +# THESE TESTS ARE NO LONGER VALID +# def test_disable_sets_disabled_key(self): +# """Test that disable sets the disabled key for metrics""" +# self.a_metrics_cache.disable() +# self.some_redis.setex.assert_called_once_with('SPLITIO.metrics.__disabled__', 1, +# self.a_metrics_cache.disabled_period) +# +# def test_enable_deletes_disabled_key(self): +# """Test that enable deletes the disabled key for metrics""" +# self.a_metrics_cache.enable() +# self.some_redis.delete.assert_called_once_with('SPLITIO.metrics.__disabled__') +# +# def test_is_enabled_returns_false_if_disabled_key_exists(self): +# """Test that is_enabled returns False if disabled key exists""" +# self.some_redis.exists.return_value = True +# self.assertFalse(self.a_metrics_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.metrics.__disabled__') +# +# def test_is_enabled_returns_true_if_disabled_key_doesnt_exist(self): +# """Test that is_enabled returns True if disabled key doesn't exist""" +# self.some_redis.exists.return_value = False +# self.assertTrue(self.a_metrics_cache.is_enabled()) +# self.some_redis.exists.assert_called_once_with('SPLITIO.metrics.__disabled__') def test_increment_count_calls_conditional_eval_with_increment_count_script(self): """Test that increment_count calls _conditional_eval with the increment count script""" From fc307db1224d869502ce43c7a13736a28d4e7809 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 6 Oct 2017 15:33:33 -0300 Subject: [PATCH 4/4] update changelog and version number --- CHANGES.txt | 2 ++ splitio/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index f2d51268..2379ec96 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,5 @@ +4.1.2 (Oct 6, 2017) + - Prevent fetcher & recorder threads from being killed after a connection failure. 4.1.1 (May 25, 2017) - Python 3 fixed incompatible comparison between None an integer 4.1.0 (May 16, 2017) diff --git a/splitio/version.py b/splitio/version.py index 47cbba72..96e55858 100644 --- a/splitio/version.py +++ b/splitio/version.py @@ -1 +1 @@ -__version__ = '4.1.1' +__version__ = '4.1.2'