diff --git a/splitio/api/splits.py b/splitio/api/splits.py index dcbb46f7..619306a1 100644 --- a/splitio/api/splits.py +++ b/splitio/api/splits.py @@ -37,11 +37,20 @@ def __init__(self, client, sdk_key, sdk_metadata, telemetry_runtime_producer): self._spec_version = SPEC_VERSION self._last_proxy_check_timestamp = 0 self.clear_storage = False + self._old_spec_since = None - def _check_last_proxy_check_timestamp(self): + def _check_last_proxy_check_timestamp(self, since): if self._spec_version == _SPEC_1_1 and ((utctime_ms() - self._last_proxy_check_timestamp) >= _PROXY_CHECK_INTERVAL_MILLISECONDS_SS): _LOGGER.info("Switching to new Feature flag spec (%s) and fetching.", SPEC_VERSION); self._spec_version = SPEC_VERSION + self._old_spec_since = since + + def _check_old_spec_since(self, change_number): + if self._spec_version == _SPEC_1_1 and self._old_spec_since is not None: + since = self._old_spec_since + self._old_spec_since = None + return since + return change_number class SplitsAPI(SplitsAPIBase): # pylint: disable=too-few-public-methods @@ -77,7 +86,9 @@ def fetch_splits(self, change_number, rbs_change_number, fetch_options): :rtype: dict """ try: - self._check_last_proxy_check_timestamp() + self._check_last_proxy_check_timestamp(change_number) + change_number = self._check_old_spec_since(change_number) + query, extra_headers = build_fetch(change_number, fetch_options, self._metadata, rbs_change_number) response = self._client.get( 'sdk', @@ -145,7 +156,9 @@ async def fetch_splits(self, change_number, rbs_change_number, fetch_options): :rtype: dict """ try: - self._check_last_proxy_check_timestamp() + self._check_last_proxy_check_timestamp(change_number) + change_number = self._check_old_spec_since(change_number) + query, extra_headers = build_fetch(change_number, fetch_options, self._metadata, rbs_change_number) response = await self._client.get( 'sdk', diff --git a/splitio/client/factory.py b/splitio/client/factory.py index 7c56819f..f6070243 100644 --- a/splitio/client/factory.py +++ b/splitio/client/factory.py @@ -564,7 +564,7 @@ def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pyl synchronizers = SplitSynchronizers( SplitSynchronizer(apis['splits'], storages['splits'], storages['rule_based_segments']), - SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']), + SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments'], storages['rule_based_segments']), ImpressionSynchronizer(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']), @@ -693,7 +693,7 @@ async def _build_in_memory_factory_async(api_key, cfg, sdk_url=None, events_url= synchronizers = SplitSynchronizers( SplitSynchronizerAsync(apis['splits'], storages['splits'], storages['rule_based_segments']), - SegmentSynchronizerAsync(apis['segments'], storages['splits'], storages['segments']), + SegmentSynchronizerAsync(apis['segments'], storages['splits'], storages['segments'], storages['rule_based_segments']), ImpressionSynchronizerAsync(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizerAsync(apis['events'], storages['events'], cfg['eventsBulkSize']), diff --git a/splitio/models/rule_based_segments.py b/splitio/models/rule_based_segments.py index dd964055..f7bf3f4d 100644 --- a/splitio/models/rule_based_segments.py +++ b/splitio/models/rule_based_segments.py @@ -152,6 +152,14 @@ def get_excluded_segments(self): """Return excluded segments""" return self._segments + def get_excluded_standard_segments(self): + """Return excluded segments""" + to_return = [] + for segment in self._segments: + if segment.type == SegmentType.STANDARD: + to_return.append(segment.name) + return to_return + def to_json(self): """Return a JSON representation of this object.""" return { diff --git a/splitio/storage/inmemmory.py b/splitio/storage/inmemmory.py index c7c1a7bf..e1740b72 100644 --- a/splitio/storage/inmemmory.py +++ b/splitio/storage/inmemmory.py @@ -233,7 +233,7 @@ def contains(self, segment_names): def fetch_many(self, segment_names): return {rb_segment_name: self.get(rb_segment_name) for rb_segment_name in segment_names} - + class InMemoryRuleBasedSegmentStorageAsync(RuleBasedSegmentsStorage): """InMemory implementation of a feature flag storage base.""" def __init__(self): diff --git a/splitio/sync/segment.py b/splitio/sync/segment.py index 59d9fad8..a87759e1 100644 --- a/splitio/sync/segment.py +++ b/splitio/sync/segment.py @@ -10,6 +10,7 @@ from splitio.util.backoff import Backoff from splitio.optional.loaders import asyncio, aiofiles from splitio.sync import util +from splitio.util.storage_helper import get_standard_segment_names_in_rbs_storage, get_standard_segment_names_in_rbs_storage_async from splitio.optional.loaders import asyncio _LOGGER = logging.getLogger(__name__) @@ -22,7 +23,7 @@ class SegmentSynchronizer(object): - def __init__(self, segment_api, feature_flag_storage, segment_storage): + def __init__(self, segment_api, feature_flag_storage, segment_storage, rule_based_segment_storage): """ Class constructor. @@ -39,6 +40,7 @@ def __init__(self, segment_api, feature_flag_storage, segment_storage): self._api = segment_api self._feature_flag_storage = feature_flag_storage self._segment_storage = segment_storage + self._rule_based_segment_storage = rule_based_segment_storage self._worker_pool = workerpool.WorkerPool(_MAX_WORKERS, self.synchronize_segment) self._worker_pool.start() self._backoff = Backoff( @@ -181,9 +183,12 @@ def synchronize_segments(self, segment_names = None, dont_wait = False): :rtype: bool """ if segment_names is None: - segment_names = self._feature_flag_storage.get_segment_names() + segment_names = set(self._feature_flag_storage.get_segment_names()) + segment_names.update(get_standard_segment_names_in_rbs_storage(self._rule_based_segment_storage)) for segment_name in segment_names: + _LOGGER.debug("Adding segment name to sync worker") + _LOGGER.debug(segment_name) self._worker_pool.submit_work(segment_name) if (dont_wait): return True @@ -204,7 +209,7 @@ def segment_exist_in_storage(self, segment_name): class SegmentSynchronizerAsync(object): - def __init__(self, segment_api, feature_flag_storage, segment_storage): + def __init__(self, segment_api, feature_flag_storage, segment_storage, rule_based_segment_storage): """ Class constructor. @@ -221,6 +226,7 @@ def __init__(self, segment_api, feature_flag_storage, segment_storage): self._api = segment_api self._feature_flag_storage = feature_flag_storage self._segment_storage = segment_storage + self._rule_based_segment_storage = rule_based_segment_storage self._worker_pool = workerpool.WorkerPoolAsync(_MAX_WORKERS, self.synchronize_segment) self._worker_pool.start() self._backoff = Backoff( @@ -364,7 +370,8 @@ async def synchronize_segments(self, segment_names = None, dont_wait = False): :rtype: bool """ if segment_names is None: - segment_names = await self._feature_flag_storage.get_segment_names() + segment_names = set(await self._feature_flag_storage.get_segment_names()) + segment_names.update(await get_standard_segment_names_in_rbs_storage_async(self._rule_based_segment_storage)) self._jobs = await self._worker_pool.submit_work(segment_names) if (dont_wait): diff --git a/splitio/sync/split.py b/splitio/sync/split.py index dfc58811..1d1722f6 100644 --- a/splitio/sync/split.py +++ b/splitio/sync/split.py @@ -139,7 +139,7 @@ def _fetch_until(self, fetch_options, till=None, rbs_till=None): rbs_segment_list = update_rule_based_segment_storage(self._rule_based_segment_storage, fetched_rule_based_segments, feature_flag_changes.get('rbs')['t'], self._api.clear_storage) fetched_feature_flags = [(splits.from_raw(feature_flag)) for feature_flag in feature_flag_changes.get('ff').get('d', [])] - segment_list = update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes.get('ff')['t'], self._api.clear_storage) + segment_list.update(update_feature_flag_storage(self._feature_flag_storage, fetched_feature_flags, feature_flag_changes.get('ff')['t'], self._api.clear_storage)) segment_list.update(rbs_segment_list) if feature_flag_changes.get('ff')['t'] == feature_flag_changes.get('ff')['s'] and feature_flag_changes.get('rbs')['t'] == feature_flag_changes.get('rbs')['s']: diff --git a/splitio/util/storage_helper.py b/splitio/util/storage_helper.py index ad5d93eb..81fdef65 100644 --- a/splitio/util/storage_helper.py +++ b/splitio/util/storage_helper.py @@ -1,6 +1,7 @@ """Storage Helper.""" import logging from splitio.models import splits +from splitio.models import rule_based_segments _LOGGER = logging.getLogger(__name__) @@ -58,7 +59,7 @@ def update_rule_based_segment_storage(rule_based_segment_storage, rule_based_seg for rule_based_segment in rule_based_segments: if rule_based_segment.status == splits.Status.ACTIVE: to_add.append(rule_based_segment) - segment_list.update(set(_get_segment_names(rule_based_segment.excluded.get_excluded_segments()))) + segment_list.update(set(rule_based_segment.excluded.get_excluded_standard_segments())) segment_list.update(rule_based_segment.get_condition_segment_names()) else: if rule_based_segment_storage.get(rule_based_segment.name) is not None: @@ -66,9 +67,21 @@ def update_rule_based_segment_storage(rule_based_segment_storage, rule_based_seg rule_based_segment_storage.update(to_add, to_delete, change_number) return segment_list + +def get_standard_segment_names_in_rbs_storage(rule_based_segment_storage): + """ + Retrieve a list of all standard segments names. -def _get_segment_names(excluded_segments): - return [excluded_segment.name for excluded_segment in excluded_segments] + :return: Set of segment names. + :rtype: Set(str) + """ + segment_list = set() + for rb_segment in rule_based_segment_storage.get_segment_names(): + rb_segment_obj = rule_based_segment_storage.get(rb_segment) + segment_list.update(set(rb_segment_obj.excluded.get_excluded_standard_segments())) + segment_list.update(rb_segment_obj.get_condition_segment_names()) + + return segment_list async def update_feature_flag_storage_async(feature_flag_storage, feature_flags, change_number, clear_storage=False): """ @@ -124,7 +137,7 @@ async def update_rule_based_segment_storage_async(rule_based_segment_storage, ru for rule_based_segment in rule_based_segments: if rule_based_segment.status == splits.Status.ACTIVE: to_add.append(rule_based_segment) - segment_list.update(set(_get_segment_names(rule_based_segment.excluded.get_excluded_segments()))) + segment_list.update(set(rule_based_segment.excluded.get_excluded_standard_segments())) segment_list.update(rule_based_segment.get_condition_segment_names()) else: if await rule_based_segment_storage.get(rule_based_segment.name) is not None: @@ -133,6 +146,22 @@ async def update_rule_based_segment_storage_async(rule_based_segment_storage, ru await rule_based_segment_storage.update(to_add, to_delete, change_number) return segment_list +async def get_standard_segment_names_in_rbs_storage_async(rule_based_segment_storage): + """ + Retrieve a list of all standard segments names. + + :return: Set of segment names. + :rtype: Set(str) + """ + segment_list = set() + segment_names = await rule_based_segment_storage.get_segment_names() + for rb_segment in segment_names: + rb_segment_obj = await rule_based_segment_storage.get(rb_segment) + segment_list.update(set(rb_segment_obj.excluded.get_excluded_standard_segments())) + segment_list.update(rb_segment_obj.get_condition_segment_names()) + + return segment_list + def get_valid_flag_sets(flag_sets, flag_set_filter): """ Check each flag set in given array, return it if exist in a given config flag set array, if config array is empty return all diff --git a/tests/api/test_splits_api.py b/tests/api/test_splits_api.py index bfb45c16..c9aeee8b 100644 --- a/tests/api/test_splits_api.py +++ b/tests/api/test_splits_api.py @@ -122,6 +122,41 @@ def get(sdk, splitChanges, sdk_key, extra_headers, query): assert self.query[2] == {'s': '1.3', 'since': 123, 'rbSince': -1} assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": 123, "t": -1}} assert split_api.clear_storage + + def test_using_old_spec_since(self, mocker): + """Test using old_spec_since variable.""" + httpclient = mocker.Mock(spec=client.HttpClient) + self.counter = 0 + self.query = [] + def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + if self.counter == 3: + return client.HttpResponse(400, 'error', {}) + if self.counter == 4: + return client.HttpResponse(200, '{"splits": [], "since": 456, "till": 456}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPI(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + response = split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 + + response = split_api.fetch_splits(456, -1, FetchOptions(False, None, None, None)) + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 1000000 + assert self.query[2] == {'s': '1.3', 'since': 456, 'rbSince': -1} + assert self.query[3] == {'s': '1.1', 'since': 456} + assert response == {"ff": {"d": [], "s": 456, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} class SplitAPIAsyncTests(object): """Split async API test cases.""" @@ -253,9 +288,45 @@ async def get(sdk, splitChanges, sdk_key, extra_headers, query): assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] assert not split_api.clear_storage - time.sleep(1) + time.sleep(1) splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) assert self.query[2] == {'s': '1.3', 'since': 123, 'rbSince': -1} assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": 123, "t": -1}} assert split_api.clear_storage + + @pytest.mark.asyncio + async def test_using_old_spec_since(self, mocker): + """Test using old_spec_since variable.""" + httpclient = mocker.Mock(spec=client.HttpClient) + self.counter = 0 + self.query = [] + async def get(sdk, splitChanges, sdk_key, extra_headers, query): + self.counter += 1 + self.query.append(query) + if self.counter == 1: + return client.HttpResponse(400, 'error', {}) + if self.counter == 2: + return client.HttpResponse(200, '{"splits": [], "since": 123, "till": 456}', {}) + if self.counter == 3: + return client.HttpResponse(400, 'error', {}) + if self.counter == 4: + return client.HttpResponse(200, '{"splits": [], "since": 456, "till": 456}', {}) + + httpclient.is_sdk_endpoint_overridden.return_value = True + httpclient.get = get + split_api = splits.SplitsAPIAsync(httpclient, 'some_api_key', SdkMetadata('1.0', 'some', '1.2.3.4'), mocker.Mock()) + response = await split_api.fetch_splits(123, -1, FetchOptions(False, None, None, None)) + assert response == {"ff": {"d": [], "s": 123, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} + assert self.query == [{'s': '1.3', 'since': 123, 'rbSince': -1}, {'s': '1.1', 'since': 123}] + assert not split_api.clear_storage + + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 10 + + response = await split_api.fetch_splits(456, -1, FetchOptions(False, None, None, None)) + time.sleep(1) + splits._PROXY_CHECK_INTERVAL_MILLISECONDS_SS = 1000000 + assert self.query[2] == {'s': '1.3', 'since': 456, 'rbSince': -1} + assert self.query[3] == {'s': '1.1', 'since': 456} + assert response == {"ff": {"d": [], "s": 456, "t": 456}, "rbs": {"d": [], "s": -1, "t": -1}} diff --git a/tests/sync/test_segments_synchronizer.py b/tests/sync/test_segments_synchronizer.py index 5a6ef849..e88db2fa 100644 --- a/tests/sync/test_segments_synchronizer.py +++ b/tests/sync/test_segments_synchronizer.py @@ -5,10 +5,11 @@ from splitio.util.backoff import Backoff from splitio.api import APIException from splitio.api.commons import FetchOptions -from splitio.storage import SplitStorage, SegmentStorage +from splitio.storage import SplitStorage, SegmentStorage, RuleBasedSegmentsStorage from splitio.storage.inmemmory import InMemorySegmentStorage, InMemorySegmentStorageAsync, InMemorySplitStorage, InMemorySplitStorageAsync from splitio.sync.segment import SegmentSynchronizer, SegmentSynchronizerAsync, LocalSegmentSynchronizer, LocalSegmentSynchronizerAsync from splitio.models.segments import Segment +from splitio.models import rule_based_segments from splitio.optional.loaders import aiofiles, asyncio import pytest @@ -23,6 +24,8 @@ def test_synchronize_segments_error(self, mocker): storage = mocker.Mock(spec=SegmentStorage) storage.get_change_number.return_value = -1 + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] api = mocker.Mock() @@ -30,7 +33,7 @@ def run(x): raise APIException("something broke") api.fetch_segment.side_effect = run - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) assert not segments_synchronizer.synchronize_segments() def test_synchronize_segments(self, mocker): @@ -38,6 +41,10 @@ def test_synchronize_segments(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = ['rbs'] + rbs_storage.get.return_value = rule_based_segments.from_raw({'name': 'rbs', 'conditions': [], 'trafficTypeName': 'user', 'changeNumber': 123, 'status': 'ACTIVE', 'excluded': {'keys': [], 'segments': [{'type': 'standard', 'name': 'segmentD'}]}}) + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -52,10 +59,14 @@ def change_number_mock(segment_name): if segment_name == 'segmentC' and change_number_mock._count_c == 0: change_number_mock._count_c = 1 return -1 + if segment_name == 'segmentD' and change_number_mock._count_d == 0: + change_number_mock._count_d = 1 + return -1 return 123 change_number_mock._count_a = 0 change_number_mock._count_b = 0 change_number_mock._count_c = 0 + change_number_mock._count_d = 0 storage.get_change_number.side_effect = change_number_mock # Setup a mocked segment api to return segments mentioned before. @@ -72,27 +83,35 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 1 return {'name': 'segmentC', 'added': ['key7', 'key8', 'key9'], 'removed': [], 'since': -1, 'till': 123} + if segment_name == 'segmentD' and fetch_segment_mock._count_d == 0: + fetch_segment_mock._count_d = 1 + return {'name': 'segmentD', 'added': ['key10'], 'removed': [], + 'since': -1, 'till': 123} return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 fetch_segment_mock._count_b = 0 fetch_segment_mock._count_c = 0 + fetch_segment_mock._count_d = 0 api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) assert segments_synchronizer.synchronize_segments() api_calls = [call for call in api.fetch_segment.mock_calls] + assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls assert mocker.call('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls assert mocker.call('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentD', -1, FetchOptions(True, None, None, None, None)) in api_calls assert mocker.call('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls assert mocker.call('segmentB', 123, FetchOptions(True, None, None, None, None)) in api_calls assert mocker.call('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert mocker.call('segmentD', 123, FetchOptions(True, None, None, None, None)) in api_calls segment_put_calls = storage.put.mock_calls - segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) + segments_to_validate = set(['segmentA', 'segmentB', 'segmentC', 'segmentD']) for call in segment_put_calls: _, positional_args, _ = call segment = positional_args[0] @@ -104,6 +123,8 @@ def test_synchronize_segment(self, mocker): """Test particular segment update.""" split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] def change_number_mock(segment_name): if change_number_mock._count_a == 0: @@ -124,7 +145,7 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) segments_synchronizer.synchronize_segment('segmentA') api_calls = [call for call in api.fetch_segment.mock_calls] @@ -137,6 +158,8 @@ def test_synchronize_segment_cdn(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] def change_number_mock(segment_name): change_number_mock._count_a += 1 @@ -170,7 +193,7 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) segments_synchronizer.synchronize_segment('segmentA') assert mocker.call('segmentA', -1, FetchOptions(True, None, None, None, None)) in api.fetch_segment.mock_calls @@ -183,7 +206,7 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): def test_recreate(self, mocker): """Test recreate logic.""" - segments_synchronizer = SegmentSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock()) + segments_synchronizer = SegmentSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) current_pool = segments_synchronizer._worker_pool segments_synchronizer.recreate() assert segments_synchronizer._worker_pool != current_pool @@ -196,6 +219,11 @@ class SegmentsSynchronizerAsyncTests(object): async def test_synchronize_segments_error(self, mocker): """On error.""" split_storage = mocker.Mock(spec=SplitStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] @@ -215,7 +243,7 @@ async def run(*args): raise APIException("something broke") api.fetch_segment = run - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) assert not await segments_synchronizer.synchronize_segments() await segments_synchronizer.shutdown() @@ -227,6 +255,15 @@ async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] split_storage.get_segment_names = get_segment_names + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + async def get_segment_names_rbs(): + return ['rbs'] + rbs_storage.get_segment_names = get_segment_names_rbs + + async def get_rbs(segment_name): + return rule_based_segments.from_raw({'name': 'rbs', 'conditions': [], 'trafficTypeName': 'user', 'changeNumber': 123, 'status': 'ACTIVE', 'excluded': {'keys': [], 'segments': [{'type': 'standard', 'name': 'segmentD'}]}}) + rbs_storage.get = get_rbs + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -241,10 +278,14 @@ async def change_number_mock(segment_name): if segment_name == 'segmentC' and change_number_mock._count_c == 0: change_number_mock._count_c = 1 return -1 + if segment_name == 'segmentD' and change_number_mock._count_d == 0: + change_number_mock._count_d = 1 + return -1 return 123 change_number_mock._count_a = 0 change_number_mock._count_b = 0 change_number_mock._count_c = 0 + change_number_mock._count_d = 0 storage.get_change_number = change_number_mock self.segment_put = [] @@ -276,25 +317,36 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_c = 1 return {'name': 'segmentC', 'added': ['key7', 'key8', 'key9'], 'removed': [], 'since': -1, 'till': 123} + if segment_name == 'segmentD' and fetch_segment_mock._count_d == 0: + fetch_segment_mock._count_d = 1 + return {'name': 'segmentD', 'added': ['key10'], 'removed': [], + 'since': -1, 'till': 123} return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 fetch_segment_mock._count_b = 0 fetch_segment_mock._count_c = 0 + fetch_segment_mock._count_d = 0 api = mocker.Mock() api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) assert await segments_synchronizer.synchronize_segments() - assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None, None)) - assert (self.segment[1], self.change[1], self.options[1]) == ('segmentA', 123, FetchOptions(True, None, None, None, None)) - assert (self.segment[2], self.change[2], self.options[2]) == ('segmentB', -1, FetchOptions(True, None, None, None, None)) - assert (self.segment[3], self.change[3], self.options[3]) == ('segmentB', 123, FetchOptions(True, None, None, None, None)) - assert (self.segment[4], self.change[4], self.options[4]) == ('segmentC', -1, FetchOptions(True, None, None, None, None)) - assert (self.segment[5], self.change[5], self.options[5]) == ('segmentC', 123, FetchOptions(True, None, None, None, None)) - - segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) + api_calls = [] + for i in range(8): + api_calls.append((self.segment[i], self.change[i], self.options[i])) + + assert ('segmentD', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentD', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls + + segments_to_validate = set(['segmentA', 'segmentB', 'segmentC', 'segmentD']) for segment in self.segment_put: assert isinstance(segment, Segment) assert segment.name in segments_to_validate @@ -307,6 +359,11 @@ async def test_synchronize_segment(self, mocker): """Test particular segment update.""" split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs async def change_number_mock(segment_name): if change_number_mock._count_a == 0: @@ -340,7 +397,7 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) await segments_synchronizer.synchronize_segment('segmentA') assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None, None)) @@ -355,6 +412,11 @@ async def test_synchronize_segment_cdn(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs async def change_number_mock(segment_name): change_number_mock._count_a += 1 @@ -400,7 +462,7 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): api = mocker.Mock() api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) await segments_synchronizer.synchronize_segment('segmentA') assert (self.segment[0], self.change[0], self.options[0]) == ('segmentA', -1, FetchOptions(True, None, None, None, None)) @@ -415,7 +477,7 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): @pytest.mark.asyncio async def test_recreate(self, mocker): """Test recreate logic.""" - segments_synchronizer = SegmentSynchronizerAsync(mocker.Mock(), mocker.Mock(), mocker.Mock()) + segments_synchronizer = SegmentSynchronizerAsync(mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) current_pool = segments_synchronizer._worker_pool await segments_synchronizer.shutdown() segments_synchronizer.recreate() diff --git a/tests/sync/test_synchronizer.py b/tests/sync/test_synchronizer.py index 6c850dd5..60ab7993 100644 --- a/tests/sync/test_synchronizer.py +++ b/tests/sync/test_synchronizer.py @@ -106,6 +106,8 @@ def test_sync_all_failed_segments(self, mocker): storage = mocker.Mock() split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] split_sync = mocker.Mock(spec=SplitSynchronizer) split_sync.synchronize_splits.return_value = None @@ -113,7 +115,7 @@ def run(x, y, c): raise APIException("something broke") api.fetch_segment.side_effect = run - segment_sync = SegmentSynchronizer(api, split_storage, storage) + segment_sync = SegmentSynchronizer(api, split_storage, storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -132,7 +134,7 @@ def test_synchronize_splits(self, mocker): segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} - segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -176,6 +178,7 @@ def sync_segments(*_): def test_sync_all(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] split_storage.get_change_number.return_value = 123 split_storage.get_segment_names.return_value = ['segmentA'] class flag_set_filter(): @@ -197,7 +200,7 @@ def intersect(sets): segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} - segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) @@ -469,7 +472,7 @@ async def test_sync_all_failed_segments(self, mocker): api = mocker.Mock() storage = mocker.Mock() split_storage = mocker.Mock(spec=SplitStorage) - split_storage.get_segment_names.return_value = ['segmentA'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) split_sync = mocker.Mock(spec=SplitSynchronizer) split_sync.synchronize_splits.return_value = None @@ -481,7 +484,11 @@ async def get_segment_names(): return ['seg'] split_storage.get_segment_names = get_segment_names - segment_sync = SegmentSynchronizerAsync(api, split_storage, storage) + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs + + segment_sync = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -515,7 +522,7 @@ async def fetch_segment(segment_name, change, options): 'key3'], 'removed': [], 'since': 123, 'till': 123} segment_api.fetch_segment = fetch_segment - segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) @@ -620,7 +627,7 @@ async def fetch_segment(segment_name, change, options): 'removed': [], 'since': 123, 'till': 123} segment_api.fetch_segment = fetch_segment - segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage) + segment_sync = SegmentSynchronizerAsync(segment_api, split_storage, segment_storage, rbs_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = SynchronizerAsync(split_synchronizers, mocker.Mock(spec=SplitTasks)) diff --git a/tests/tasks/test_segment_sync.py b/tests/tasks/test_segment_sync.py index d5640709..cc701e52 100644 --- a/tests/tasks/test_segment_sync.py +++ b/tests/tasks/test_segment_sync.py @@ -6,7 +6,7 @@ from splitio.api.commons import FetchOptions from splitio.tasks import segment_sync -from splitio.storage import SegmentStorage, SplitStorage +from splitio.storage import SegmentStorage, SplitStorage, RuleBasedSegmentsStorage from splitio.models.splits import Split from splitio.models.segments import Segment from splitio.models.grammar.condition import Condition @@ -21,6 +21,8 @@ def test_normal_operation(self, mocker): """Test the normal operation flow.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. @@ -65,7 +67,7 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTask(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -99,6 +101,8 @@ def test_that_errors_dont_stop_task(self, mocker): """Test that if fetching segments fails at some_point, the task will continue running.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + rbs_storage.get_segment_names.return_value = [] # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. @@ -142,7 +146,7 @@ def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment.side_effect = fetch_segment_mock - segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizer(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTask(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -183,6 +187,11 @@ async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] split_storage.get_segment_names = get_segment_names + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -241,7 +250,7 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTaskAsync(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -251,12 +260,16 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): await task.stop() assert not task.is_running() - assert (self.segment_name[0], self.change_number[0], self.fetch_options[0]) == ('segmentA', -1, fetch_options) - assert (self.segment_name[1], self.change_number[1], self.fetch_options[1]) == ('segmentA', 123, fetch_options) - assert (self.segment_name[2], self.change_number[2], self.fetch_options[2]) == ('segmentB', -1, fetch_options) - assert (self.segment_name[3], self.change_number[3], self.fetch_options[3]) == ('segmentB', 123, fetch_options) - assert (self.segment_name[4], self.change_number[4], self.fetch_options[4]) == ('segmentC', -1, fetch_options) - assert (self.segment_name[5], self.change_number[5], self.fetch_options[5]) == ('segmentC', 123, fetch_options) + api_calls = [] + for i in range(6): + api_calls.append((self.segment_name[i], self.change_number[i], self.fetch_options[i])) + + assert ('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) for segment in self.segments: @@ -272,6 +285,11 @@ async def get_segment_names(): return ['segmentA', 'segmentB', 'segmentC'] split_storage.get_segment_names = get_segment_names + rbs_storage = mocker.Mock(spec=RuleBasedSegmentsStorage) + async def get_segment_names_rbs(): + return [] + rbs_storage.get_segment_names = get_segment_names_rbs + # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) @@ -329,7 +347,7 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_options = FetchOptions(True, None, None, None, None) api.fetch_segment = fetch_segment_mock - segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage) + segments_synchronizer = SegmentSynchronizerAsync(api, split_storage, storage, rbs_storage) task = segment_sync.SegmentSynchronizationTaskAsync(segments_synchronizer.synchronize_segments, 0.5) task.start() @@ -338,12 +356,16 @@ async def fetch_segment_mock(segment_name, change_number, fetch_options): await task.stop() assert not task.is_running() - - assert (self.segment_name[0], self.change_number[0], self.fetch_options[0]) == ('segmentA', -1, fetch_options) - assert (self.segment_name[1], self.change_number[1], self.fetch_options[1]) == ('segmentA', 123, fetch_options) - assert (self.segment_name[2], self.change_number[2], self.fetch_options[2]) == ('segmentB', -1, fetch_options) - assert (self.segment_name[3], self.change_number[3], self.fetch_options[3]) == ('segmentC', -1, fetch_options) - assert (self.segment_name[4], self.change_number[4], self.fetch_options[4]) == ('segmentC', 123, fetch_options) + + api_calls = [] + for i in range(5): + api_calls.append((self.segment_name[i], self.change_number[i], self.fetch_options[i])) + + assert ('segmentA', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentA', 123, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentB', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', -1, FetchOptions(True, None, None, None, None)) in api_calls + assert ('segmentC', 123, FetchOptions(True, None, None, None, None)) in api_calls segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) for segment in self.segments: diff --git a/tests/util/test_storage_helper.py b/tests/util/test_storage_helper.py index 1dab0d01..5804a6fa 100644 --- a/tests/util/test_storage_helper.py +++ b/tests/util/test_storage_helper.py @@ -2,7 +2,8 @@ import pytest from splitio.util.storage_helper import update_feature_flag_storage, get_valid_flag_sets, combine_valid_flag_sets, \ - update_rule_based_segment_storage, update_rule_based_segment_storage_async, update_feature_flag_storage_async + update_rule_based_segment_storage, update_rule_based_segment_storage_async, update_feature_flag_storage_async, \ + get_standard_segment_names_in_rbs_storage_async, get_standard_segment_names_in_rbs_storage from splitio.storage.inmemmory import InMemorySplitStorage, InMemoryRuleBasedSegmentStorage, InMemoryRuleBasedSegmentStorageAsync, \ InMemorySplitStorageAsync from splitio.models import splits, rule_based_segments @@ -190,6 +191,17 @@ def clear(): segments = update_rule_based_segment_storage(storage, [self.rbs], 123, True) assert self.clear == 1 + + def test_get_standard_segment_in_rbs_storage(self, mocker): + storage = InMemoryRuleBasedSegmentStorage() + segments = update_rule_based_segment_storage(storage, [self.rbs], 123) + assert get_standard_segment_names_in_rbs_storage(storage) == {'excluded_segment', 'employees'} + + @pytest.mark.asyncio + async def test_get_standard_segment_in_rbs_storage(self, mocker): + storage = InMemoryRuleBasedSegmentStorageAsync() + segments = await update_rule_based_segment_storage_async(storage, [self.rbs], 123) + assert await get_standard_segment_names_in_rbs_storage_async(storage) == {'excluded_segment', 'employees'} @pytest.mark.asyncio async def test_update_rule_base_segment_storage_async(self, mocker):