From cf63323bc9a6e935ec9264300afeb6dd6878e723 Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Sun, 11 May 2025 23:11:26 +0200 Subject: [PATCH 1/6] Add EC2 support for GetSecurityGroupsForVpc API operation (#12602) Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> --- .../localstack/services/ec2/provider.py | 27 +++++++ tests/aws/services/ec2/test_ec2.py | 81 +++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/localstack-core/localstack/services/ec2/provider.py b/localstack-core/localstack/services/ec2/provider.py index 59a560cd7295e..e7f0e8624becc 100644 --- a/localstack-core/localstack/services/ec2/provider.py +++ b/localstack-core/localstack/services/ec2/provider.py @@ -50,6 +50,8 @@ DnsOptionsSpecification, DnsRecordIpType, Ec2Api, + GetSecurityGroupsForVpcRequest, + GetSecurityGroupsForVpcResult, InstanceType, IpAddressType, LaunchTemplate, @@ -70,6 +72,7 @@ RevokeSecurityGroupEgressRequest, RevokeSecurityGroupEgressResult, RIProductDescription, + SecurityGroupForVpc, String, SubnetConfigurationsList, Tenancy, @@ -539,6 +542,30 @@ def create_flow_logs( return response + @handler("GetSecurityGroupsForVpc", expand=False) + def get_security_groups_for_vpc( + self, + context: RequestContext, + get_security_groups_for_vpc_request: GetSecurityGroupsForVpcRequest, + ) -> GetSecurityGroupsForVpcResult: + vpc_id = get_security_groups_for_vpc_request.get("VpcId") + backend = get_ec2_backend(context.account_id, context.region) + filters = {"vpc-id": [vpc_id]} + filtered_sgs = backend.describe_security_groups(filters=filters) + + sgs = [ + SecurityGroupForVpc( + Description=sg.description, + GroupId=sg.id, + GroupName=sg.name, + OwnerId=context.account_id, + PrimaryVpcId=sg.vpc_id, + Tags=[{"Key": k, "Value": v} for k, v in (sg.get_tags() or {}).items()], + ) + for sg in filtered_sgs + ] + return GetSecurityGroupsForVpcResult(SecurityGroupForVpcs=sgs, NextToken=None) + @patch(SubnetBackend.modify_subnet_attribute) def modify_subnet_attribute(fn, self, subnet_id, attr_name, attr_value): diff --git a/tests/aws/services/ec2/test_ec2.py b/tests/aws/services/ec2/test_ec2.py index f0ba136034454..09f21648d8a02 100644 --- a/tests/aws/services/ec2/test_ec2.py +++ b/tests/aws/services/ec2/test_ec2.py @@ -482,6 +482,87 @@ def test_create_vpc_with_custom_id(self, aws_client, create_vpc): assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert e.value.response["Error"]["Code"] == "InvalidVpc.DuplicateCustomId" + @markers.aws.only_localstack + def test_get_security_groups_for_vpc(self, cleanups, aws_client, create_vpc): + # Create a VPC + vpc: dict = create_vpc( + cidr_block="10.0.0.0/16", + tag_specifications=[ + { + "ResourceType": "vpc", + "Tags": [ + {"Key": "Name", "Value": "test-vpc"}, + ], + } + ], + ) + vpc_id: str = vpc["Vpc"]["VpcId"] + + # Create security groups in the VPC + sg1_response = aws_client.ec2.create_security_group( + GroupName="sg-1", Description="Test Security Group 1", VpcId=vpc_id + ) + sg1_id = sg1_response["GroupId"] + + sg2_response = aws_client.ec2.create_security_group( + GroupName="sg-2", Description="Test Security Group 2", VpcId=vpc_id + ) + sg2_id = sg2_response["GroupId"] + + # Create a security group in a different VPC (default VPC) + default_vpc_response = aws_client.ec2.describe_vpcs( + Filters=[{"Name": "isDefault", "Values": ["true"]}] + ) + default_vpc_id = default_vpc_response["Vpcs"][0]["VpcId"] + + sg3_response = aws_client.ec2.create_security_group( + GroupName="sg-3", Description="Test Security Group 3", VpcId=default_vpc_id + ) + sg3_id = sg3_response["GroupId"] + + # Call GetSecurityGroupsForVpc + response = aws_client.ec2.get_security_groups_for_vpc(VpcId=vpc_id) + + # Verify response + assert "SecurityGroupForVpcs" in response + sg_ids = [sg["GroupId"] for sg in response["SecurityGroupForVpcs"]] + + # Should only include the security groups we created in the VPC + assert sg1_id in sg_ids + assert sg2_id in sg_ids + assert sg3_id not in sg_ids + + # Should include the default security group for the VPC + default_sg = next( + (sg for sg in response["SecurityGroupForVpcs"] if sg["GroupName"] == "default"), None + ) + assert default_sg is not None + + cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg1_id)) + cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg2_id)) + cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg3_id)) + cleanups.append(lambda: aws_client.ec2.delete_vpc(VpcId=vpc_id)) + + @markers.aws.only_localstack + def test_get_security_groups_for_vpc_invalid_inputs(self, aws_client): + # Test with non-existent VPC ID + response = aws_client.ec2.get_security_groups_for_vpc(VpcId="vpc-nonexistent") + assert "SecurityGroupForVpcs" in response + assert len(response["SecurityGroupForVpcs"]) == 0 + + # Test with invalid VPC ID format + response = aws_client.ec2.get_security_groups_for_vpc(VpcId="123-invalid-vpc-format") + assert "SecurityGroupForVpcs" in response + assert len(response["SecurityGroupForVpcs"]) == 0 + + # Test with missing VPC ID parameter + with pytest.raises(Exception): + aws_client.ec2.get_security_groups_for_vpc() + + # Test with empty string for VPC ID + with pytest.raises(Exception): + aws_client.ec2.get_security_groups_for_vpc(VpcId="") + @markers.aws.only_localstack def test_create_subnet_with_tags(self, cleanups, aws_client, create_vpc): # Create a VPC. From 37330c50a3a2cd0e28baafb0c78d4dcd803e7952 Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Wed, 14 May 2025 22:11:10 +0200 Subject: [PATCH 2/6] Update ec2_create_security_group fixture to use GroupId instead of GroupName when adding ingress rules This avoids issues with security groups in different VPCs Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> --- .../localstack/testing/pytest/fixtures.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/localstack-core/localstack/testing/pytest/fixtures.py b/localstack-core/localstack/testing/pytest/fixtures.py index a127e9a94aab5..e8c76205bbc8e 100644 --- a/localstack-core/localstack/testing/pytest/fixtures.py +++ b/localstack-core/localstack/testing/pytest/fixtures.py @@ -2005,22 +2005,24 @@ def factory(ports=None, **kwargs): if "GroupName" not in kwargs: kwargs["GroupName"] = f"test-sg-{short_uid()}" security_group = aws_client.ec2.create_security_group(**kwargs) + sg_id = security_group["GroupId"] + + if ports: + permissions = [ + { + "FromPort": port, + "IpProtocol": "tcp", + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + "ToPort": port, + } + for port in ports + ] + aws_client.ec2.authorize_security_group_ingress( + GroupId=sg_id, + IpPermissions=permissions, + ) - permissions = [ - { - "FromPort": port, - "IpProtocol": "tcp", - "IpRanges": [{"CidrIp": "0.0.0.0/0"}], - "ToPort": port, - } - for port in ports or [] - ] - aws_client.ec2.authorize_security_group_ingress( - GroupName=kwargs["GroupName"], - IpPermissions=permissions, - ) - - ec2_sgs.append(security_group["GroupId"]) + ec2_sgs.append(sg_id) return security_group yield factory From 629f2803f76092b97643ddb8af39257c4209827d Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Wed, 14 May 2025 22:11:58 +0200 Subject: [PATCH 3/6] Add GetSecurityGroupsForVpc API parity tests (#12602) Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> --- tests/aws/services/ec2/test_ec2.py | 160 +++++++++--------- tests/aws/services/ec2/test_ec2.snapshot.json | 68 ++++++++ .../aws/services/ec2/test_ec2.validation.json | 3 + 3 files changed, 150 insertions(+), 81 deletions(-) diff --git a/tests/aws/services/ec2/test_ec2.py b/tests/aws/services/ec2/test_ec2.py index 09f21648d8a02..397b8cab451c3 100644 --- a/tests/aws/services/ec2/test_ec2.py +++ b/tests/aws/services/ec2/test_ec2.py @@ -482,87 +482,6 @@ def test_create_vpc_with_custom_id(self, aws_client, create_vpc): assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert e.value.response["Error"]["Code"] == "InvalidVpc.DuplicateCustomId" - @markers.aws.only_localstack - def test_get_security_groups_for_vpc(self, cleanups, aws_client, create_vpc): - # Create a VPC - vpc: dict = create_vpc( - cidr_block="10.0.0.0/16", - tag_specifications=[ - { - "ResourceType": "vpc", - "Tags": [ - {"Key": "Name", "Value": "test-vpc"}, - ], - } - ], - ) - vpc_id: str = vpc["Vpc"]["VpcId"] - - # Create security groups in the VPC - sg1_response = aws_client.ec2.create_security_group( - GroupName="sg-1", Description="Test Security Group 1", VpcId=vpc_id - ) - sg1_id = sg1_response["GroupId"] - - sg2_response = aws_client.ec2.create_security_group( - GroupName="sg-2", Description="Test Security Group 2", VpcId=vpc_id - ) - sg2_id = sg2_response["GroupId"] - - # Create a security group in a different VPC (default VPC) - default_vpc_response = aws_client.ec2.describe_vpcs( - Filters=[{"Name": "isDefault", "Values": ["true"]}] - ) - default_vpc_id = default_vpc_response["Vpcs"][0]["VpcId"] - - sg3_response = aws_client.ec2.create_security_group( - GroupName="sg-3", Description="Test Security Group 3", VpcId=default_vpc_id - ) - sg3_id = sg3_response["GroupId"] - - # Call GetSecurityGroupsForVpc - response = aws_client.ec2.get_security_groups_for_vpc(VpcId=vpc_id) - - # Verify response - assert "SecurityGroupForVpcs" in response - sg_ids = [sg["GroupId"] for sg in response["SecurityGroupForVpcs"]] - - # Should only include the security groups we created in the VPC - assert sg1_id in sg_ids - assert sg2_id in sg_ids - assert sg3_id not in sg_ids - - # Should include the default security group for the VPC - default_sg = next( - (sg for sg in response["SecurityGroupForVpcs"] if sg["GroupName"] == "default"), None - ) - assert default_sg is not None - - cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg1_id)) - cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg2_id)) - cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg3_id)) - cleanups.append(lambda: aws_client.ec2.delete_vpc(VpcId=vpc_id)) - - @markers.aws.only_localstack - def test_get_security_groups_for_vpc_invalid_inputs(self, aws_client): - # Test with non-existent VPC ID - response = aws_client.ec2.get_security_groups_for_vpc(VpcId="vpc-nonexistent") - assert "SecurityGroupForVpcs" in response - assert len(response["SecurityGroupForVpcs"]) == 0 - - # Test with invalid VPC ID format - response = aws_client.ec2.get_security_groups_for_vpc(VpcId="123-invalid-vpc-format") - assert "SecurityGroupForVpcs" in response - assert len(response["SecurityGroupForVpcs"]) == 0 - - # Test with missing VPC ID parameter - with pytest.raises(Exception): - aws_client.ec2.get_security_groups_for_vpc() - - # Test with empty string for VPC ID - with pytest.raises(Exception): - aws_client.ec2.get_security_groups_for_vpc(VpcId="") - @markers.aws.only_localstack def test_create_subnet_with_tags(self, cleanups, aws_client, create_vpc): # Create a VPC. @@ -775,6 +694,85 @@ def _create_security_group() -> dict: assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert e.value.response["Error"]["Code"] == "InvalidSecurityGroupId.DuplicateCustomId" + @markers.snapshot.skip_snapshot_verify( + paths=[ + "$..Tags", + "$..SecurityGroupForVpcs..Description", + "$..SecurityGroupForVpcs..GroupId", + "$..SecurityGroupForVpcs..GroupName" + ] +) + @markers.aws.validated + def test_get_security_groups_for_vpc( + self, snapshot, cleanups, aws_client, create_vpc, ec2_create_security_group + ): + snapshot.add_transformers_list( + [ + snapshot.transform.key_value("GroupId"), + snapshot.transform.key_value("SecurityGroupArn"), + snapshot.transform.key_value("vpc-id"), + ] + ) + + # Create a VPC + vpc: dict = create_vpc( + cidr_block="10.0.0.0/16", + tag_specifications=[ + { + "ResourceType": "vpc", + "Tags": [ + {"Key": "Name", "Value": "test-vpc"}, + ], + } + ], + ) + vpc_id: str = vpc["Vpc"]["VpcId"] + snapshot.match("create_vpc", {"vpc-id": vpc_id}) + + # Create security groups in the VPC + sg1 = ec2_create_security_group( + GroupName="test-security-group-1", + Description="Test Security Group 1 Description", + VpcId=vpc_id + ) + sg1_id = sg1["GroupId"] + snapshot.match("create_security_group_1", sg1) + + sg2 = ec2_create_security_group( + GroupName="test-security-group-2", + Description="Test Security Group 2 Description", + VpcId=vpc_id + ) + sg2_id = sg2["GroupId"] + snapshot.match("create_security_group_2", sg2) + + # Create a security group in a different VPC (default VPC) + default_vpc = aws_client.ec2.describe_vpcs( + Filters=[{"Name": "isDefault", "Values": ["true"]}] + ) + default_vpc_id = default_vpc["Vpcs"][0]["VpcId"] + + sg3 = ec2_create_security_group( + GroupName="test-security-group-3", + Description="Test Security Group 3 Description", + VpcId=default_vpc_id + ) + sg3_id = sg3["GroupId"] + snapshot.match("create_security_group_3", sg3) + + vpc_sgs = aws_client.ec2.get_security_groups_for_vpc(VpcId=vpc_id) + snapshot.match("get_security_groups_for_vpc", vpc_sgs) + + # Should only include the security groups created in the VPC + vpc_sg_ids = [sg["GroupId"] for sg in vpc_sgs["SecurityGroupForVpcs"]] + assert sg1_id in vpc_sg_ids + assert sg2_id in vpc_sg_ids + assert sg3_id not in vpc_sg_ids + + cleanups.append(lambda: aws_client.ec2.delete_vpc(VpcId=vpc_id)) + cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg1_id)) + cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg2_id)) + cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg3_id)) @markers.snapshot.skip_snapshot_verify( # Moto and LS do not return the ClientToken diff --git a/tests/aws/services/ec2/test_ec2.snapshot.json b/tests/aws/services/ec2/test_ec2.snapshot.json index 3347bf78d1bdb..a94d9039421ac 100644 --- a/tests/aws/services/ec2/test_ec2.snapshot.json +++ b/tests/aws/services/ec2/test_ec2.snapshot.json @@ -335,5 +335,73 @@ } } } + }, + "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { + "recorded-date": "14-05-2025, 20:05:26", + "recorded-content": { + "create_vpc": { + "vpc-id": "" + }, + "create_security_group_1": { + "GroupId": "", + "SecurityGroupArn": "arn::ec2::111111111111:security-group/", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create_security_group_2": { + "GroupId": "", + "SecurityGroupArn": "arn::ec2::111111111111:security-group/", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create_security_group_3": { + "GroupId": "", + "SecurityGroupArn": "arn::ec2::111111111111:security-group/", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get_security_groups_for_vpc": { + "SecurityGroupForVpcs": [ + { + "Description": "Test Security Group 2 Description", + "GroupId": "", + "GroupName": "test-security-group-2", + "OwnerId": "111111111111", + "PrimaryVpcId": "", + "Tags": [] + }, + { + "Description": "default VPC security group", + "GroupId": "", + "GroupName": "default", + "OwnerId": "111111111111", + "PrimaryVpcId": "", + "Tags": [] + }, + { + "Description": "Test Security Group 1 Description", + "GroupId": "", + "GroupName": "test-security-group-1", + "OwnerId": "111111111111", + "PrimaryVpcId": "", + "Tags": [] + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc_invalid_inputs": { + "recorded-date": "14-05-2025, 20:07:05", + "recorded-content": {} } } diff --git a/tests/aws/services/ec2/test_ec2.validation.json b/tests/aws/services/ec2/test_ec2.validation.json index 2a599a8011508..a25bba4d851ae 100644 --- a/tests/aws/services/ec2/test_ec2.validation.json +++ b/tests/aws/services/ec2/test_ec2.validation.json @@ -11,6 +11,9 @@ "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_describe_vpn_gateways_filter_by_vpc": { "last_validated_date": "2024-06-07T01:11:12+00:00" }, + "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { + "last_validated_date": "2025-05-14T20:05:19+00:00" + }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_vcp_peering_difference_regions": { "last_validated_date": "2024-06-07T21:28:25+00:00" }, From fd203161ce4f4f5db83b769491d8a0d67dedc178 Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Wed, 14 May 2025 22:58:13 +0200 Subject: [PATCH 4/6] Ensure tests work correctly with the updated ec2_create_security_group fixture (#12602) Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> --- tests/aws/services/ec2/test_ec2.py | 84 +++++++++---------- tests/aws/services/ec2/test_ec2.snapshot.json | 41 +-------- .../aws/services/ec2/test_ec2.validation.json | 2 +- 3 files changed, 40 insertions(+), 87 deletions(-) diff --git a/tests/aws/services/ec2/test_ec2.py b/tests/aws/services/ec2/test_ec2.py index c83844b69e4cb..4bb522abe2f51 100644 --- a/tests/aws/services/ec2/test_ec2.py +++ b/tests/aws/services/ec2/test_ec2.py @@ -694,85 +694,77 @@ def _create_security_group() -> dict: assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert e.value.response["Error"]["Code"] == "InvalidSecurityGroupId.DuplicateCustomId" - @markers.snapshot.skip_snapshot_verify( - paths=[ - "$..Tags", - "$..SecurityGroupForVpcs..Description", - "$..SecurityGroupForVpcs..GroupId", - "$..SecurityGroupForVpcs..GroupName", - ] - ) + @markers.snapshot.skip_snapshot_verify(paths=["$..Tags"]) @markers.aws.validated def test_get_security_groups_for_vpc( - self, snapshot, cleanups, aws_client, create_vpc, ec2_create_security_group + self, snapshot, aws_client, create_vpc, ec2_create_security_group ): - snapshot.add_transformers_list( - [ - snapshot.transform.key_value("GroupId"), - snapshot.transform.key_value("SecurityGroupArn"), - snapshot.transform.key_value("vpc-id"), - ] + snapshot.add_transformers_list([snapshot.transform.key_value("GroupId")]) + + # Get the default VPC + default_vpc = aws_client.ec2.describe_vpcs( + Filters=[{"Name": "isDefault", "Values": ["true"]}] ) + default_vpc_id = default_vpc["Vpcs"][0]["VpcId"] - # Create a VPC - vpc: dict = create_vpc( + # Create a custom VPC for testing + custom_vpc: dict = create_vpc( cidr_block="10.0.0.0/16", tag_specifications=[ { "ResourceType": "vpc", "Tags": [ - {"Key": "Name", "Value": "test-vpc"}, + {"Key": "Name", "Value": f"test-vpc-{short_uid()}"}, ], } ], ) - vpc_id: str = vpc["Vpc"]["VpcId"] - snapshot.match("create_vpc", {"vpc-id": vpc_id}) + custom_vpc_id: str = custom_vpc["Vpc"]["VpcId"] - # Create security groups in the VPC + # Create security groups in the default VPC sg1 = ec2_create_security_group( - GroupName="test-security-group-1", + GroupName=f"test-security-group-{short_uid()}", Description="Test Security Group 1 Description", - VpcId=vpc_id, + VpcId=default_vpc_id, + ports=[22], ) sg1_id = sg1["GroupId"] snapshot.match("create_security_group_1", sg1) sg2 = ec2_create_security_group( - GroupName="test-security-group-2", + GroupName=f"test-security-group-{short_uid()}", Description="Test Security Group 2 Description", - VpcId=vpc_id, + VpcId=default_vpc_id, + ports=[22], ) sg2_id = sg2["GroupId"] snapshot.match("create_security_group_2", sg2) - # Create a security group in a different VPC (default VPC) - default_vpc = aws_client.ec2.describe_vpcs( - Filters=[{"Name": "isDefault", "Values": ["true"]}] - ) - default_vpc_id = default_vpc["Vpcs"][0]["VpcId"] - + # Create a security group in the custom VPC sg3 = ec2_create_security_group( - GroupName="test-security-group-3", + GroupName=f"test-security-group-{short_uid()}", Description="Test Security Group 3 Description", - VpcId=default_vpc_id, + VpcId=custom_vpc_id, + ports=[22], ) sg3_id = sg3["GroupId"] snapshot.match("create_security_group_3", sg3) - vpc_sgs = aws_client.ec2.get_security_groups_for_vpc(VpcId=vpc_id) - snapshot.match("get_security_groups_for_vpc", vpc_sgs) - - # Should only include the security groups created in the VPC - vpc_sg_ids = [sg["GroupId"] for sg in vpc_sgs["SecurityGroupForVpcs"]] - assert sg1_id in vpc_sg_ids - assert sg2_id in vpc_sg_ids - assert sg3_id not in vpc_sg_ids - - cleanups.append(lambda: aws_client.ec2.delete_vpc(VpcId=vpc_id)) - cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg1_id)) - cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg2_id)) - cleanups.append(lambda: aws_client.ec2.delete_security_group(GroupId=sg3_id)) + # Should only include the security groups created in the default VPC + default_vpc_sgs = aws_client.ec2.get_security_groups_for_vpc(VpcId=default_vpc_id) + default_vpc_sg_ids = [sg["GroupId"] for sg in default_vpc_sgs["SecurityGroupForVpcs"]] + assert "SecurityGroupForVpcs" in default_vpc_sgs + assert sg1_id in default_vpc_sg_ids + assert sg2_id in default_vpc_sg_ids + assert sg3_id not in default_vpc_sg_ids + + # Should only include the security group created in the custom VPC + custom_vpc_sgs = aws_client.ec2.get_security_groups_for_vpc(VpcId=custom_vpc_id) + custom_vpc_sg_ids = [sg["GroupId"] for sg in custom_vpc_sgs["SecurityGroupForVpcs"]] + assert "SecurityGroupForVpcs" in custom_vpc_sgs + assert sg1_id not in custom_vpc_sg_ids + assert sg2_id not in custom_vpc_sg_ids + assert sg3_id in custom_vpc_sg_ids @markers.snapshot.skip_snapshot_verify( diff --git a/tests/aws/services/ec2/test_ec2.snapshot.json b/tests/aws/services/ec2/test_ec2.snapshot.json index a94d9039421ac..750e715d19bf5 100644 --- a/tests/aws/services/ec2/test_ec2.snapshot.json +++ b/tests/aws/services/ec2/test_ec2.snapshot.json @@ -337,11 +337,8 @@ } }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { - "recorded-date": "14-05-2025, 20:05:26", + "recorded-date": "17-05-2025, 11:57:25", "recorded-content": { - "create_vpc": { - "vpc-id": "" - }, "create_security_group_1": { "GroupId": "", "SecurityGroupArn": "arn::ec2::111111111111:security-group/", @@ -365,43 +362,7 @@ "HTTPHeaders": {}, "HTTPStatusCode": 200 } - }, - "get_security_groups_for_vpc": { - "SecurityGroupForVpcs": [ - { - "Description": "Test Security Group 2 Description", - "GroupId": "", - "GroupName": "test-security-group-2", - "OwnerId": "111111111111", - "PrimaryVpcId": "", - "Tags": [] - }, - { - "Description": "default VPC security group", - "GroupId": "", - "GroupName": "default", - "OwnerId": "111111111111", - "PrimaryVpcId": "", - "Tags": [] - }, - { - "Description": "Test Security Group 1 Description", - "GroupId": "", - "GroupName": "test-security-group-1", - "OwnerId": "111111111111", - "PrimaryVpcId": "", - "Tags": [] - } - ], - "ResponseMetadata": { - "HTTPHeaders": {}, - "HTTPStatusCode": 200 - } } } - }, - "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc_invalid_inputs": { - "recorded-date": "14-05-2025, 20:07:05", - "recorded-content": {} } } diff --git a/tests/aws/services/ec2/test_ec2.validation.json b/tests/aws/services/ec2/test_ec2.validation.json index a25bba4d851ae..feb6c88fef4d8 100644 --- a/tests/aws/services/ec2/test_ec2.validation.json +++ b/tests/aws/services/ec2/test_ec2.validation.json @@ -12,7 +12,7 @@ "last_validated_date": "2024-06-07T01:11:12+00:00" }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { - "last_validated_date": "2025-05-14T20:05:19+00:00" + "last_validated_date": "2025-05-17T11:57:23+00:00" }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_vcp_peering_difference_regions": { "last_validated_date": "2024-06-07T21:28:25+00:00" From 05ecaceb15fbb602f1ff7d39b1665817ead12993 Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Sun, 18 May 2025 17:42:51 +0200 Subject: [PATCH 5/6] Improve EC2 GetSecurityGroupsForVpc parity tests for stability (#12602) - Structure tests to validate core filtering functionality - Add SortingTransformer to normalize security group ordering in responses - Verify both initial VPC state and post-security-group addition state Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> --- .../localstack/testing/pytest/fixtures.py | 5 + tests/aws/services/ec2/test_ec2.py | 102 ++++++++---------- tests/aws/services/ec2/test_ec2.snapshot.json | 65 +++++++++-- .../aws/services/ec2/test_ec2.validation.json | 2 +- 4 files changed, 107 insertions(+), 67 deletions(-) diff --git a/localstack-core/localstack/testing/pytest/fixtures.py b/localstack-core/localstack/testing/pytest/fixtures.py index 6bddcb162632f..b89d5aedf2a87 100644 --- a/localstack-core/localstack/testing/pytest/fixtures.py +++ b/localstack-core/localstack/testing/pytest/fixtures.py @@ -2009,11 +2009,16 @@ def factory(ports=None, ip_protocol: str = "tcp", **kwargs): :param ip_protocol: the ip protocol for the permissions (tcp by default) """ if "GroupName" not in kwargs: + # FIXME: This will fail against AWS since the sg prefix is not valid for GroupName + # > "Group names may not be in the format sg-*". kwargs["GroupName"] = f"sg-{short_uid()}" # Making sure the call to CreateSecurityGroup gets the right arguments _args = select_from_typed_dict(CreateSecurityGroupRequest, kwargs) security_group = aws_client.ec2.create_security_group(**_args) security_group_id = security_group["GroupId"] + + # FIXME: If 'ports' is None or an empty list, authorize_security_group_ingress will fail due to missing IpPermissions. + # Must ensure ports are explicitly provided or skip authorization entirely if not required. permissions = [ { "FromPort": port, diff --git a/tests/aws/services/ec2/test_ec2.py b/tests/aws/services/ec2/test_ec2.py index 4bb522abe2f51..62a607bff99d0 100644 --- a/tests/aws/services/ec2/test_ec2.py +++ b/tests/aws/services/ec2/test_ec2.py @@ -3,6 +3,7 @@ import pytest from botocore.exceptions import ClientError +from localstack_snapshot.snapshots.transformer import SortingTransformer from moto.ec2 import ec2_backends from moto.ec2.utils import ( random_security_group_id, @@ -694,77 +695,62 @@ def _create_security_group() -> dict: assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 400 assert e.value.response["Error"]["Code"] == "InvalidSecurityGroupId.DuplicateCustomId" - @markers.snapshot.skip_snapshot_verify(paths=["$..Tags"]) + @markers.snapshot.skip_snapshot_verify( + paths=[ + "$..Tags", # Tags can differ between environments + "$..Vpc.IsDefault", # TODO: CreateVPC should return an IsDefault param + "$..Vpc.DhcpOptionsId", # FIXME: DhcpOptionsId uses different reference formats in AWS vs LocalStack + ] + ) @markers.aws.validated def test_get_security_groups_for_vpc( self, snapshot, aws_client, create_vpc, ec2_create_security_group ): - snapshot.add_transformers_list([snapshot.transform.key_value("GroupId")]) + group_name = f"test-security-group-{short_uid()}" + group_description = f"Description for {group_name}" - # Get the default VPC - default_vpc = aws_client.ec2.describe_vpcs( - Filters=[{"Name": "isDefault", "Values": ["true"]}] + # Returned security groups appear to be sorted by the randomly generated GroupId field, + # so we should sort snapshots by this value to mitigate flakiness for runs against AWS. + snapshot.add_transformer( + SortingTransformer("SecurityGroupForVpcs", lambda x: x["GroupName"]) ) - default_vpc_id = default_vpc["Vpcs"][0]["VpcId"] + snapshot.add_transformer(snapshot.transform.key_value("GroupId")) + snapshot.add_transformer(snapshot.transform.key_value("GroupName")) + snapshot.add_transformer(snapshot.transform.key_value("VpcId")) + snapshot.add_transformer(snapshot.transform.key_value("AssociationId")) + snapshot.add_transformer(snapshot.transform.key_value("DhcpOptionsId")) - # Create a custom VPC for testing - custom_vpc: dict = create_vpc( + # Create VPC for testing + vpc: dict = create_vpc( cidr_block="10.0.0.0/16", - tag_specifications=[ - { - "ResourceType": "vpc", - "Tags": [ - {"Key": "Name", "Value": f"test-vpc-{short_uid()}"}, - ], - } - ], ) - custom_vpc_id: str = custom_vpc["Vpc"]["VpcId"] + vpc_id: str = vpc["Vpc"]["VpcId"] + snapshot.match("create_vpc_response", vpc) - # Create security groups in the default VPC - sg1 = ec2_create_security_group( - GroupName=f"test-security-group-{short_uid()}", - Description="Test Security Group 1 Description", - VpcId=default_vpc_id, - ports=[22], - ) - sg1_id = sg1["GroupId"] - snapshot.match("create_security_group_1", sg1) + # Wait to ensure VPC is available + waiter = aws_client.ec2.get_waiter("vpc_available") + waiter.wait(VpcIds=[vpc_id]) - sg2 = ec2_create_security_group( - GroupName=f"test-security-group-{short_uid()}", - Description="Test Security Group 2 Description", - VpcId=default_vpc_id, - ports=[22], + # Get all security groups in the VPC + get_security_groups_for_vpc = aws_client.ec2.get_security_groups_for_vpc(VpcId=vpc_id) + snapshot.match("get_security_groups_for_vpc", get_security_groups_for_vpc) + + # Create new security group in the VPC + create_security_group = ec2_create_security_group( + GroupName=group_name, + Description=group_description, + VpcId=vpc_id, + ports=[22], # TODO: Handle port issues in the fixture ) - sg2_id = sg2["GroupId"] - snapshot.match("create_security_group_2", sg2) + snapshot.match("create_security_group", create_security_group) - # Create a security group in the custom VPC - sg3 = ec2_create_security_group( - GroupName=f"test-security-group-{short_uid()}", - Description="Test Security Group 3 Description", - VpcId=custom_vpc_id, - ports=[22], - ) - sg3_id = sg3["GroupId"] - snapshot.match("create_security_group_3", sg3) - - # Should only include the security groups created in the default VPC - default_vpc_sgs = aws_client.ec2.get_security_groups_for_vpc(VpcId=default_vpc_id) - default_vpc_sg_ids = [sg["GroupId"] for sg in default_vpc_sgs["SecurityGroupForVpcs"]] - assert "SecurityGroupForVpcs" in default_vpc_sgs - assert sg1_id in default_vpc_sg_ids - assert sg2_id in default_vpc_sg_ids - assert sg3_id not in default_vpc_sg_ids - - # Should only include the security group created in the custom VPC - custom_vpc_sgs = aws_client.ec2.get_security_groups_for_vpc(VpcId=custom_vpc_id) - custom_vpc_sg_ids = [sg["GroupId"] for sg in custom_vpc_sgs["SecurityGroupForVpcs"]] - assert "SecurityGroupForVpcs" in custom_vpc_sgs - assert sg1_id not in custom_vpc_sg_ids - assert sg2_id not in custom_vpc_sg_ids - assert sg3_id in custom_vpc_sg_ids + # Ensure new security group is in the VPC + get_security_groups_for_vpc_after_addition = aws_client.ec2.get_security_groups_for_vpc( + VpcId=vpc_id + ) + snapshot.match( + "get_security_groups_for_vpc_after_addition", get_security_groups_for_vpc_after_addition + ) @markers.snapshot.skip_snapshot_verify( diff --git a/tests/aws/services/ec2/test_ec2.snapshot.json b/tests/aws/services/ec2/test_ec2.snapshot.json index 750e715d19bf5..1f17ff3eb29e1 100644 --- a/tests/aws/services/ec2/test_ec2.snapshot.json +++ b/tests/aws/services/ec2/test_ec2.snapshot.json @@ -337,17 +337,50 @@ } }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { - "recorded-date": "17-05-2025, 11:57:25", + "recorded-date": "18-05-2025, 18:08:00", "recorded-content": { - "create_security_group_1": { - "GroupId": "", - "SecurityGroupArn": "arn::ec2::111111111111:security-group/", + "create_vpc_response": { + "Vpc": { + "CidrBlock": "10.0.0.0/16", + "CidrBlockAssociationSet": [ + { + "AssociationId": "", + "CidrBlock": "10.0.0.0/16", + "CidrBlockState": { + "State": "associated" + } + } + ], + "DhcpOptionsId": "", + "InstanceTenancy": "", + "Ipv6CidrBlockAssociationSet": [], + "IsDefault": false, + "OwnerId": "111111111111", + "State": "pending", + "VpcId": "" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get_security_groups_for_vpc": { + "SecurityGroupForVpcs": [ + { + "Description": " VPC security group", + "GroupId": "", + "GroupName": "", + "OwnerId": "111111111111", + "PrimaryVpcId": "", + "Tags": [] + } + ], "ResponseMetadata": { "HTTPHeaders": {}, "HTTPStatusCode": 200 } }, - "create_security_group_2": { + "create_security_group": { "GroupId": "", "SecurityGroupArn": "arn::ec2::111111111111:security-group/", "ResponseMetadata": { @@ -355,9 +388,25 @@ "HTTPStatusCode": 200 } }, - "create_security_group_3": { - "GroupId": "", - "SecurityGroupArn": "arn::ec2::111111111111:security-group/", + "get_security_groups_for_vpc_after_addition": { + "SecurityGroupForVpcs": [ + { + "Description": " VPC security group", + "GroupId": "", + "GroupName": "", + "OwnerId": "111111111111", + "PrimaryVpcId": "", + "Tags": [] + }, + { + "Description": "Description for ", + "GroupId": "", + "GroupName": "", + "OwnerId": "111111111111", + "PrimaryVpcId": "", + "Tags": [] + } + ], "ResponseMetadata": { "HTTPHeaders": {}, "HTTPStatusCode": 200 diff --git a/tests/aws/services/ec2/test_ec2.validation.json b/tests/aws/services/ec2/test_ec2.validation.json index feb6c88fef4d8..28af944182ca3 100644 --- a/tests/aws/services/ec2/test_ec2.validation.json +++ b/tests/aws/services/ec2/test_ec2.validation.json @@ -12,7 +12,7 @@ "last_validated_date": "2024-06-07T01:11:12+00:00" }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { - "last_validated_date": "2025-05-17T11:57:23+00:00" + "last_validated_date": "2025-05-18T18:07:59+00:00" }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_vcp_peering_difference_regions": { "last_validated_date": "2024-06-07T21:28:25+00:00" From 6bac29d5904ccc748eda2e6f827ef94c325a2e1d Mon Sep 17 00:00:00 2001 From: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> Date: Mon, 19 May 2025 15:03:25 +0200 Subject: [PATCH 6/6] Normalize tag format in GetSecurityGroupsForVpc response (#12602) Signed-off-by: Ramtin Mesgari <26694963+iamramtin@users.noreply.github.com> --- localstack-core/localstack/services/ec2/provider.py | 2 +- tests/aws/services/ec2/test_ec2.py | 8 ++++++++ tests/aws/services/ec2/test_ec2.snapshot.json | 8 +++++++- tests/aws/services/ec2/test_ec2.validation.json | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/localstack-core/localstack/services/ec2/provider.py b/localstack-core/localstack/services/ec2/provider.py index e7f0e8624becc..401276a0da7bd 100644 --- a/localstack-core/localstack/services/ec2/provider.py +++ b/localstack-core/localstack/services/ec2/provider.py @@ -560,7 +560,7 @@ def get_security_groups_for_vpc( GroupName=sg.name, OwnerId=context.account_id, PrimaryVpcId=sg.vpc_id, - Tags=[{"Key": k, "Value": v} for k, v in (sg.get_tags() or {}).items()], + Tags=[{"Key": tag.get("key"), "Value": tag.get("value")} for tag in sg.get_tags()], ) for sg in filtered_sgs ] diff --git a/tests/aws/services/ec2/test_ec2.py b/tests/aws/services/ec2/test_ec2.py index 62a607bff99d0..cd006fa90abb7 100644 --- a/tests/aws/services/ec2/test_ec2.py +++ b/tests/aws/services/ec2/test_ec2.py @@ -723,6 +723,14 @@ def test_get_security_groups_for_vpc( # Create VPC for testing vpc: dict = create_vpc( cidr_block="10.0.0.0/16", + tag_specifications=[ + { + "ResourceType": "vpc", + "Tags": [ + {"Key": "test-key", "Value": "test-value"}, + ], + } + ], ) vpc_id: str = vpc["Vpc"]["VpcId"] snapshot.match("create_vpc_response", vpc) diff --git a/tests/aws/services/ec2/test_ec2.snapshot.json b/tests/aws/services/ec2/test_ec2.snapshot.json index 1f17ff3eb29e1..c76d1b6969e9a 100644 --- a/tests/aws/services/ec2/test_ec2.snapshot.json +++ b/tests/aws/services/ec2/test_ec2.snapshot.json @@ -337,7 +337,7 @@ } }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { - "recorded-date": "18-05-2025, 18:08:00", + "recorded-date": "19-05-2025, 13:53:56", "recorded-content": { "create_vpc_response": { "Vpc": { @@ -357,6 +357,12 @@ "IsDefault": false, "OwnerId": "111111111111", "State": "pending", + "Tags": [ + { + "Key": "test-key", + "Value": "test-value" + } + ], "VpcId": "" }, "ResponseMetadata": { diff --git a/tests/aws/services/ec2/test_ec2.validation.json b/tests/aws/services/ec2/test_ec2.validation.json index 28af944182ca3..19e617efc962a 100644 --- a/tests/aws/services/ec2/test_ec2.validation.json +++ b/tests/aws/services/ec2/test_ec2.validation.json @@ -12,7 +12,7 @@ "last_validated_date": "2024-06-07T01:11:12+00:00" }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_get_security_groups_for_vpc": { - "last_validated_date": "2025-05-18T18:07:59+00:00" + "last_validated_date": "2025-05-19T13:54:09+00:00" }, "tests/aws/services/ec2/test_ec2.py::TestEc2Integrations::test_vcp_peering_difference_regions": { "last_validated_date": "2024-06-07T21:28:25+00:00"