diff --git a/localstack/services/events/events_starter.py b/localstack/services/events/events_starter.py index 47740b10aa1a7..19c568366be47 100644 --- a/localstack/services/events/events_starter.py +++ b/localstack/services/events/events_starter.py @@ -1,7 +1,9 @@ +import re import datetime import json import uuid import logging +import ipaddress from moto.events.models import Rule as rule_model from moto.events.responses import EventsHandler as events_handler from localstack import config @@ -24,6 +26,10 @@ DEFAULT_EVENT_BUS_NAME: set() } +CONTENT_BASE_FILTER_KEYWORDS = [ + 'prefix', 'anything-but', 'numeric', 'cidr', 'exists' +] + def send_event_to_sqs(event, arn): region = arn.split(':')[3] @@ -52,12 +58,39 @@ def filter_event_with_target_input_path(target, event): def filter_event_based_on_event_format(self, rule, event): + def filter_event(event_pattern, event): + for key, value in event_pattern.items(): + event_value = event.get(key.lower()) + if not event_value: + return False + + if isinstance(value, list) and not identify_content_base_parameter_in_pattern(value): + if isinstance(event_value, list) and \ + get_two_lists_intersection(value, event_value) == []: + return False + elif not isinstance(event_value, list) and \ + isinstance(event_value, (str, int)) and \ + event_value not in value: + return False + + elif isinstance(value, list) and identify_content_base_parameter_in_pattern(value): + if not filter_event_with_content_base_parameter(value, event_value): + return False + + elif isinstance(value, (str, int)): + try: + if isinstance(json.loads(value), dict) and \ + not filter_event(json.loads(value), event_value): + return False + except json.decoder.JSONDecodeError: + return False + return True + rule_information = self.events_backend.describe_rule(rule) if rule_information.event_pattern: event_pattern = json.loads(rule_information.event_pattern) - for key, value in event_pattern.items(): - if event.get(key.lower()) and event.get(key.lower()) not in value and event.get(key) != value: - return False + if not filter_event(event_pattern, event): + return False return True @@ -198,3 +231,100 @@ def start_events(port=None, asynchronous=None, update_listener=None): asynchronous=asynchronous, update_listener=update_listener ) + + +# --------------- +# HELPER METHODS +# --------------- + + +def get_two_lists_intersection(lst1, lst2): + lst3 = [value for value in lst1 if value in lst2] + return lst3 + + +def identify_content_base_parameter_in_pattern(parameters): + if any([list(param.keys())[0] in CONTENT_BASE_FILTER_KEYWORDS for param in parameters if isinstance(param, dict)]): + return True + + +def filter_event_with_content_base_parameter(pattern_value, event_value): + for element in pattern_value: + if (isinstance(element, (str, int))) \ + and (event_value == element or element in event_value): + return True + elif isinstance(element, dict): + element_key = list(element.keys())[0] + element_value = element.get(element_key) + if element_key.lower() == 'prefix': + if re.match(r'^{}'.format(element_value), event_value): + return True + elif element_key.lower() == 'exists': + if element_value and event_value: + return True + elif not element_value and not event_value: + return True + elif element_key.lower() == 'cidr': + ips = [str(ip) for ip in ipaddress.IPv4Network(element_value)] + if event_value in ips: + return True + elif element_key.lower() == 'numeric': + if check_valid_numeric_content_base_rule(element_value): + for index in range(len(element_value)): + if isinstance(element_value[index], int): + continue + if element_value[index] == '>' and \ + isinstance(element_value[index + 1], int) and \ + event_value <= element_value[index + 1]: + break + elif element_value[index] == '>=' and \ + isinstance(element_value[index + 1], int) and \ + event_value < element_value[index + 1]: + break + elif element_value[index] == '<' and \ + isinstance(element_value[index + 1], int) and \ + event_value >= element_value[index + 1]: + break + elif element_value[index] == '<=' and \ + isinstance(element_value[index + 1], int) and \ + event_value > element_value[index + 1]: + break + else: + return True + + elif element_key.lower() == 'anything-but': + if isinstance(element_value, list) and \ + event_value not in element_value: + return True + elif (isinstance(element_value, (str, int))) and \ + event_value != element_value: + return True + elif isinstance(element_value, dict): + nested_key = list(element_value)[0] + if nested_key == 'prefix' and \ + not re.match(r'^{}'.format(element_value.get(nested_key)), event_value): + return True + return False + + +def check_valid_numeric_content_base_rule(list_of_operators): + if len(list_of_operators) > 4: + return False + + if '=' in list_of_operators: + return False + + if len(list_of_operators) > 2: + upper_limit = None + lower_limit = None + for index in range(len(list_of_operators)): + if not isinstance(list_of_operators[index], int) and \ + '<' in list_of_operators[index]: + upper_limit = list_of_operators[index + 1] + if not isinstance(list_of_operators[index], int) and \ + '>' in list_of_operators[index]: + lower_limit = list_of_operators[index + 1] + if upper_limit and lower_limit and upper_limit < lower_limit: + return False + index = index + 1 + return True diff --git a/tests/integration/test_events.py b/tests/integration/test_events.py index 1c17700ac5294..7110f82430562 100644 --- a/tests/integration/test_events.py +++ b/tests/integration/test_events.py @@ -22,9 +22,9 @@ EVENT_DETAIL = '{\"command\":\"update-account\",\"payload\":{\"acc_id\":\"0a787ecb-4015\",\"sf_id\":\"baz\"}}' TEST_EVENT_PATTERN = { - 'Source': 'core.update-account-command', - 'DetailType': 'core.update-account-command', - 'Detail': EVENT_DETAIL + 'Source': ['core.update-account-command'], + 'detail-type': ['core.update-account-command'], + 'Detail': [EVENT_DETAIL] } @@ -113,24 +113,25 @@ def test_put_events_with_target_sqs(self): queue_name = 'queue-{}'.format(short_uid()) rule_name = 'rule-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) + bus_name = 'bus-{}'.format(short_uid()) sqs_client = aws_stack.connect_to_service('sqs') queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl'] queue_arn = aws_stack.sqs_queue_arn(queue_name) self.events_client.create_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) self.events_client.put_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN) ) rs = self.events_client.put_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Targets=[ { 'Id': target_id, @@ -146,10 +147,10 @@ def test_put_events_with_target_sqs(self): self.events_client.put_events( Entries=[{ - 'EventBusName': TEST_EVENT_BUS_NAME, - 'Source': TEST_EVENT_PATTERN['Source'], - 'DetailType': TEST_EVENT_PATTERN['DetailType'], - 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail']) + 'EventBusName': bus_name, + 'Source': TEST_EVENT_PATTERN['Source'][0], + 'DetailType': TEST_EVENT_PATTERN['detail-type'][0], + 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0]) }] ) @@ -162,30 +163,31 @@ def get_message(queue_url): actual_event = json.loads(messages[0]['Body']) self.assertIsValidEvent(actual_event) - self.assertEqual(actual_event['detail'], TEST_EVENT_PATTERN['Detail']) + self.assertEqual(actual_event['detail'], TEST_EVENT_PATTERN['Detail'][0]) # clean up sqs_client.delete_queue(QueueUrl=queue_url) self.events_client.remove_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Ids=[target_id], Force=True ) self.events_client.delete_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Force=True ) self.events_client.delete_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) def test_put_events_with_target_lambda(self): rule_name = 'rule-{}'.format(short_uid()) function_name = 'lambda-func-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) + bus_name = 'bus-{}'.format(short_uid()) rs = testutil.create_lambda_function(handler_file=os.path.join(THIS_FOLDER, 'lambdas', 'lambda_echo.py'), func_name=function_name, @@ -194,18 +196,18 @@ def test_put_events_with_target_lambda(self): func_arn = rs['CreateFunctionResponse']['FunctionArn'] self.events_client.create_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) self.events_client.put_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN) ) rs = self.events_client.put_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Targets=[ { 'Id': target_id, @@ -221,10 +223,10 @@ def test_put_events_with_target_lambda(self): self.events_client.put_events( Entries=[{ - 'EventBusName': TEST_EVENT_BUS_NAME, - 'Source': TEST_EVENT_PATTERN['Source'], - 'DetailType': TEST_EVENT_PATTERN['DetailType'], - 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail']) + 'EventBusName': bus_name, + 'Source': TEST_EVENT_PATTERN['Source'][0], + 'DetailType': TEST_EVENT_PATTERN['detail-type'][0], + 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0]) }] ) @@ -233,24 +235,24 @@ def test_put_events_with_target_lambda(self): sleep=1, function_name=function_name, expected_length=1) actual_event = events[0] self.assertIsValidEvent(actual_event) - self.assertDictEqual(json.loads(actual_event['detail']), json.loads(TEST_EVENT_PATTERN['Detail'])) + self.assertDictEqual(json.loads(actual_event['detail']), json.loads(TEST_EVENT_PATTERN['Detail'][0])) # clean up testutil.delete_lambda_function(function_name) self.events_client.remove_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Ids=[target_id], Force=True ) self.events_client.delete_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Force=True ) self.events_client.delete_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) def test_scheduled_expression_events(self): @@ -378,6 +380,7 @@ def test_put_events_with_target_firehose(self): stream_name = 'firehose-{}'.format(short_uid()) rule_name = 'rule-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) + bus_name = 'bus-{}'.format(short_uid()) # create firehose target bucket s3_client = aws_stack.connect_to_service('s3') @@ -396,18 +399,18 @@ def test_put_events_with_target_firehose(self): stream_arn = stream['DeliveryStreamARN'] self.events_client.create_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) self.events_client.put_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN) ) rs = self.events_client.put_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Targets=[ { 'Id': target_id, @@ -423,10 +426,10 @@ def test_put_events_with_target_firehose(self): self.events_client.put_events( Entries=[{ - 'EventBusName': TEST_EVENT_BUS_NAME, - 'Source': TEST_EVENT_PATTERN['Source'], - 'DetailType': TEST_EVENT_PATTERN['DetailType'], - 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail']) + 'EventBusName': bus_name, + 'Source': TEST_EVENT_PATTERN['Source'][0], + 'DetailType': TEST_EVENT_PATTERN['detail-type'][0], + 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0]) }] ) @@ -437,7 +440,7 @@ def test_put_events_with_target_firehose(self): s3_object = s3_client.get_object(Bucket=s3_bucket, Key=key) actual_event = json.loads(s3_object['Body'].read().decode()) self.assertIsValidEvent(actual_event) - self.assertEqual(actual_event['detail'], TEST_EVENT_PATTERN['Detail']) + self.assertEqual(actual_event['detail'], TEST_EVENT_PATTERN['Detail'][0]) # clean up firehose_client.delete_delivery_stream(DeliveryStreamName=stream_name) @@ -447,17 +450,17 @@ def test_put_events_with_target_firehose(self): self.events_client.remove_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Ids=[target_id], Force=True ) self.events_client.delete_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Force=True ) self.events_client.delete_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) def test_put_events_with_target_sqs_new_region(self): @@ -499,24 +502,25 @@ def test_put_events_with_input_path(self): queue_name = 'queue-{}'.format(short_uid()) rule_name = 'rule-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) + bus_name = 'bus-{}'.format(short_uid()) sqs_client = aws_stack.connect_to_service('sqs') queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl'] queue_arn = aws_stack.sqs_queue_arn(queue_name) self.events_client.create_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) self.events_client.put_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN) ) self.events_client.put_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Targets=[ { 'Id': target_id, @@ -528,10 +532,10 @@ def test_put_events_with_input_path(self): self.events_client.put_events( Entries=[{ - 'EventBusName': TEST_EVENT_BUS_NAME, - 'Source': TEST_EVENT_PATTERN['Source'], - 'DetailType': TEST_EVENT_PATTERN['DetailType'], - 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail']) + 'EventBusName': bus_name, + 'Source': TEST_EVENT_PATTERN['Source'][0], + 'DetailType': TEST_EVENT_PATTERN['detail-type'][0], + 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0]) }] ) @@ -545,10 +549,10 @@ def get_message(queue_url): self.events_client.put_events( Entries=[{ - 'EventBusName': TEST_EVENT_BUS_NAME, + 'EventBusName': bus_name, 'Source': 'dummySource', - 'DetailType': TEST_EVENT_PATTERN['DetailType'], - 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail']) + 'DetailType': TEST_EVENT_PATTERN['detail-type'][0], + 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0]) }] ) @@ -560,17 +564,17 @@ def get_message(queue_url): self.events_client.remove_targets( Rule=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Ids=[target_id], Force=True ) self.events_client.delete_rule( Name=rule_name, - EventBusName=TEST_EVENT_BUS_NAME, + EventBusName=bus_name, Force=True ) self.events_client.delete_event_bus( - Name=TEST_EVENT_BUS_NAME + Name=bus_name ) def test_put_event_without_source(self): @@ -585,3 +589,125 @@ def test_put_event_without_source(self): ] ) self.assertIn('Entries', response) + + def test_put_event_with_content_base_rule_in_pattern(self): + queue_name = 'queue-{}'.format(short_uid()) + rule_name = 'rule-{}'.format(short_uid()) + target_id = 'target-{}'.format(short_uid()) + + sqs_client = aws_stack.connect_to_service('sqs') + queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl'] + queue_arn = aws_stack.sqs_queue_arn(queue_name) + + pattern = { + 'Source': [{'exists': True}], + 'detail-type': [{'prefix': 'core.app'}], + 'Detail': json.dumps({ + 'decription': ['this-is-event-details'], + 'amount': [200], + 'salary': [2000, 4000], + 'env': ['dev', 'prod'], + 'user': ['user1', 'user2', 'user3'], + 'admins': ['skyli', {'prefix': 'hey'}, {'prefix': 'ad'}], + 'test1': [{'anything-but': 200}], + 'test2': [{'anything-but': 'test2'}], + 'test3': [{'anything-but': ['test3', 'test33']}], + 'test4': [{'anything-but': {'prefix': 'test4'}}], + 'ip': [{'cidr': '10.102.1.0/24'}], + 'num-test1': [{'numeric': ['<', 200]}], + 'num-test2': [{'numeric': ['<=', 200]}], + 'num-test3': [{'numeric': ['>', 200]}], + 'num-test4': [{'numeric': ['>=', 200]}], + 'num-test5': [{'numeric': ['>=', 200, '<=', 500]}], + 'num-test6': [{'numeric': ['>', 200, '<', 500]}], + 'num-test7': [{'numeric': ['>=', 200, '<', 500]}] + }) + } + + event = { + 'EventBusName': TEST_EVENT_BUS_NAME, + 'Source': 'core.update-account-command', + 'DetailType': 'core.app.backend', + 'Detail': json.dumps({ + 'decription': 'this-is-event-details', + 'amount': 200, + 'salary': 2000, + 'env': 'prod', + 'user': ['user4', 'user3'], + 'admins': 'admin', + 'test1': 300, + 'test2': 'test22', + 'test3': 'test333', + 'test4': 'this test4', + 'ip': '10.102.1.100', + 'num-test1': 100, + 'num-test2': 200, + 'num-test3': 300, + 'num-test4': 200, + 'num-test5': 500, + 'num-test6': 300, + 'num-test7': 300 + }) + } + + self.events_client.create_event_bus( + Name=TEST_EVENT_BUS_NAME + ) + + self.events_client.put_rule( + Name=rule_name, + EventBusName=TEST_EVENT_BUS_NAME, + EventPattern=json.dumps(pattern) + ) + + self.events_client.put_targets( + Rule=rule_name, + EventBusName=TEST_EVENT_BUS_NAME, + Targets=[ + { + 'Id': target_id, + 'Arn': queue_arn, + 'InputPath': '$.detail' + } + ] + ) + self.events_client.put_events( + Entries=[event] + ) + + def get_message(queue_url): + resp = sqs_client.receive_message(QueueUrl=queue_url) + return resp.get('Messages') + + messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url) + self.assertEqual(len(messages), 1) + self.assertEqual(json.loads(messages[0].get('Body')), json.loads(event['Detail'])) + + event_details = json.loads(event['Detail']) + event_details['admins'] = 'not_admin' + event['Detail'] = json.dumps(event_details) + + self.events_client.put_events( + Entries=[event] + ) + + messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url) + self.assertEqual(messages, None) + + # clean up + sqs_client.delete_queue(QueueUrl=queue_url) + + self.events_client.remove_targets( + Rule=rule_name, + EventBusName=TEST_EVENT_BUS_NAME, + Ids=[target_id], + Force=True + ) + self.events_client.delete_rule( + Name=rule_name, + EventBusName=TEST_EVENT_BUS_NAME, + Force=True + ) + self.events_client.delete_event_bus( + Name=TEST_EVENT_BUS_NAME + )