Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 946b273

Browse files
bentskubentsku
authored
add parsing and routing to APIGW next gen (#11051)
1 parent 4d38399 commit 946b273

File tree

10 files changed

+665
-6
lines changed

10 files changed

+665
-6
lines changed

‎localstack-core/localstack/services/apigateway/models.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,17 @@ def __init__(self, rest_api: RestApi):
5858

5959

6060
class RestApiDeployment:
61-
def __init__(self, localstack_rest_api: RestApiContainer, moto_rest_api: MotoRestAPI):
61+
def __init__(
62+
self,
63+
account_id: str,
64+
region: str,
65+
localstack_rest_api: RestApiContainer,
66+
moto_rest_api: MotoRestAPI,
67+
):
6268
self.localstack_rest_api = localstack_rest_api
6369
self.moto_rest_api = moto_rest_api
70+
self.account_id = account_id
71+
self.region = region
6472

6573

6674
class ApiGatewayStore(BaseStore):

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/context.py

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,67 @@
1-
from typing import Optional
1+
from http import HTTPMethod
2+
from typing import Optional, TypedDict
23

34
from rolo import Request
45
from rolo.gateway import RequestContext
6+
from werkzeug.datastructures import Headers
57

8+
from localstack.aws.api.apigateway import Method, Resource
69
from localstack.services.apigateway.models import RestApiDeployment
710

811

12+
class InvocationRequest(TypedDict, total=False):
13+
http_method: Optional[HTTPMethod]
14+
"""HTTP Method of the incoming request"""
15+
raw_path: Optional[str]
16+
"""Raw path of the incoming request with no modification, needed to keep double forward slashes"""
17+
path: Optional[str]
18+
"""Path of the request with no URL decoding"""
19+
path_parameters: Optional[dict[str, str]]
20+
"""Path parameters of the request"""
21+
query_string_parameters: Optional[dict[str, str]]
22+
"""Query string parameters of the request"""
23+
# TODO: need to check if we need the raw headers (as it's practical for casing reasons)
24+
raw_headers: Optional[Headers]
25+
"""Raw headers using the Headers datastructure which allows access with no regards to casing"""
26+
headers: Optional[dict[str, str]]
27+
"""Headers of the request"""
28+
multi_value_query_string_parameters: Optional[dict[str, list[str]]]
29+
"""Multi value query string parameters of the request"""
30+
multi_value_headers: Optional[dict[str, list[str]]]
31+
"""Multi value headers of the request"""
32+
body: Optional[bytes]
33+
"""Body content of the request"""
34+
35+
936
class RestApiInvocationContext(RequestContext):
1037
"""
1138
This context is going to be used to pass relevant information across an API Gateway invocation.
1239
"""
1340

41+
invocation_request: Optional[InvocationRequest]
42+
"""Contains the data relative to the invocation request"""
1443
deployment: Optional[RestApiDeployment]
44+
"""Contains the invoked REST API Resources"""
1545
api_id: Optional[str]
46+
"""The REST API identifier of the invoked API"""
1647
stage: Optional[str]
48+
"""The REST API stage linked to this invocation"""
49+
region: Optional[str]
50+
"""The region the REST API is living in."""
51+
account_id: Optional[str]
52+
"""The account the REST API is living in."""
53+
resource: Optional[Resource]
54+
"""The resource the invocation matched"""
55+
resource_method: Optional[Method]
56+
"""The method of the resource the invocation matched"""
1757

1858
def __init__(self, request: Request):
1959
super().__init__(request)
2060
self.deployment = None
2161
self.api_id = None
2262
self.stage = None
63+
self.account_id = None
64+
self.region = None
65+
self.invocation_request = None
66+
self.resource = None
67+
self.resource_method = None

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/gateway.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ def __init__(self):
2121
self.request_handlers.extend(
2222
[
2323
handlers.parse_request,
24+
handlers.route_request,
2425
handlers.preprocess_request,
2526
handlers.method_request_handler,
2627
handlers.integration_request_handler,

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/handlers/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
from .method_request import MethodRequestHandler
88
from .method_response import MethodResponseHandler
99
from .parse import InvocationRequestParser
10+
from .resource_router import InvocationRequestRouter
1011

1112
legacy_handler = LegacyHandler()
1213
parse_request = InvocationRequestParser()
14+
route_request = InvocationRequestRouter()
1315
preprocess_request = CompositeHandler()
1416
method_request_handler = MethodRequestHandler()
1517
integration_request_handler = IntegrationRequestHandler()

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/handlers/parse.py

Lines changed: 85 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
import logging
2+
from collections import defaultdict
3+
from urllib.parse import urlparse
4+
5+
from rolo.request import Request, restore_payload
6+
from werkzeug.datastructures import Headers, MultiDict
27

38
from localstack.http import Response
49

510
from ..api import RestApiGatewayHandler, RestApiGatewayHandlerChain
6-
from ..context import RestApiInvocationContext
11+
from ..context import InvocationRequest, RestApiInvocationContext
712

813
LOG = logging.getLogger(__name__)
914

@@ -15,5 +20,82 @@ def __call__(
1520
context: RestApiInvocationContext,
1621
response: Response,
1722
):
18-
# populate context with more data from the deployment
19-
pass
23+
context.account_id = context.deployment.account_id
24+
context.region = context.deployment.region
25+
self.parse_and_enrich(context)
26+
27+
def parse_and_enrich(self, context: RestApiInvocationContext):
28+
# first, create the InvocationRequest with the incoming request
29+
context.invocation_request = self.create_invocation_request(context.request)
30+
31+
def create_invocation_request(self, request: Request) -> InvocationRequest:
32+
params, multi_value_params = self._get_single_and_multi_values_from_multidict(request.args)
33+
headers, multi_value_headers = self._get_single_and_multi_values_from_headers(
34+
request.headers
35+
)
36+
invocation_request = InvocationRequest(
37+
http_method=request.method,
38+
query_string_parameters=params,
39+
multi_value_query_string_parameters=multi_value_params,
40+
raw_headers=request.headers,
41+
headers=headers,
42+
multi_value_headers=multi_value_headers,
43+
body=restore_payload(request),
44+
)
45+
46+
self._enrich_with_raw_path(request, invocation_request)
47+
48+
return invocation_request
49+
50+
@staticmethod
51+
def _enrich_with_raw_path(request: Request, invocation_request: InvocationRequest):
52+
# Base path is not URL-decoded, so we need to get the `RAW_URI` from the request
53+
raw_uri = request.environ.get("RAW_URI") or request.path
54+
55+
# if the request comes from the LocalStack only `_user_request_` route, we need to remove this prefix from the
56+
# path, in order to properly route the request
57+
if "_user_request_" in raw_uri:
58+
raw_uri = raw_uri.partition("_user_request_")[2]
59+
60+
if raw_uri.startswith("//"):
61+
# if the RAW_URI starts with double slashes, `urlparse` will fail to decode it as path only
62+
# it also means that we already only have the path, so we just need to remove the query string
63+
raw_uri = raw_uri.split("?")[0]
64+
raw_path = "/" + raw_uri.lstrip("/")
65+
66+
else:
67+
# we need to make sure we have a path here, sometimes RAW_URI can be a full URI (when proxied)
68+
raw_path = raw_uri = urlparse(raw_uri).path
69+
70+
invocation_request["path"] = raw_path
71+
invocation_request["raw_path"] = raw_uri
72+
73+
@staticmethod
74+
def _get_single_and_multi_values_from_multidict(
75+
multi_dict: MultiDict,
76+
) -> tuple[dict[str, str], dict[str, list[str]]]:
77+
single_values = {}
78+
multi_values = defaultdict(list)
79+
80+
for key, value in multi_dict.items(multi=True):
81+
multi_values[key].append(value)
82+
# for the single value parameters, AWS only keeps the last value of the list
83+
single_values[key] = value
84+
85+
return single_values, dict(multi_values)
86+
87+
@staticmethod
88+
def _get_single_and_multi_values_from_headers(
89+
headers: Headers,
90+
) -> tuple[dict[str, str], dict[str, list[str]]]:
91+
single_values = {}
92+
multi_values = {}
93+
94+
for key in dict(headers).keys():
95+
# TODO: AWS verify multi value headers to see which one AWS keeps (first or last)
96+
if key not in single_values:
97+
single_values[key] = headers[key]
98+
99+
multi_values[key] = headers.getlist(key)
100+
101+
return single_values, multi_values

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/handlers/resource_router.py

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import logging
2+
from functools import cache
3+
4+
from werkzeug.exceptions import MethodNotAllowed, NotFound
5+
from werkzeug.routing import Map, MapAdapter
6+
7+
from localstack.aws.api.apigateway import ListOfResource, Resource
8+
from localstack.aws.protocol.op_router import (
9+
GreedyPathConverter,
10+
# TODO: all under are private, not sure exactly how we should proceed for them to be re-usable
11+
_path_param_regex,
12+
_post_process_arg_name,
13+
_StrictMethodRule,
14+
_transform_path_params_to_rule_vars,
15+
)
16+
from localstack.http import Response
17+
from localstack.services.apigateway.models import RestApiDeployment
18+
19+
from ..api import RestApiGatewayHandler, RestApiGatewayHandlerChain
20+
from ..context import RestApiInvocationContext
21+
from ..helpers import get_resources_from_deployment
22+
23+
LOG = logging.getLogger(__name__)
24+
25+
26+
class RestAPIResourceRouter:
27+
"""
28+
A router implementation which abstracts the routing of incoming REST API Context to a specific
29+
resource of the Deployment.
30+
"""
31+
32+
_map: Map
33+
34+
def __init__(self, deployment: RestApiDeployment):
35+
resources = get_resources_from_deployment(deployment)
36+
self._resources_map = {resource["id"]: resource for resource in resources}
37+
self._map = get_rule_map_for_resources(resources)
38+
39+
def match(self, context: RestApiInvocationContext) -> tuple[Resource, dict[str, str]]:
40+
"""
41+
Matches the given request to the resource it targets (or raises an exception if no resource matches).
42+
43+
:param context:
44+
:return: A tuple with the matched resource and the (already parsed) path params
45+
:raises: TODO: Gateway exception in case the given request does not match any operation
46+
"""
47+
48+
request = context.request
49+
# bind the map to get the actual matcher
50+
matcher: MapAdapter = self._map.bind(context.request.host)
51+
52+
# perform the matching
53+
try:
54+
# trailing slashes are ignored in APIGW
55+
path = context.invocation_request["path"].rstrip("/")
56+
57+
rule, args = matcher.match(path, method=request.method, return_rule=True)
58+
except (MethodNotAllowed, NotFound) as e:
59+
# MethodNotAllowed (405) exception is raised if a path is matching, but the method does not.
60+
# Our router might handle this as a 404, validate with AWS.
61+
# TODO: raise proper Gateway exception
62+
raise Exception("Not found") from e
63+
64+
# post process the arg keys and values
65+
# - the path param keys need to be "un-sanitized", i.e. sanitized rule variable names need to be reverted
66+
# - the path param values might still be url-encoded
67+
args = {_post_process_arg_name(k): v for k, v in args.items()}
68+
69+
# extract the operation model from the rule
70+
resource_id: str = rule.endpoint
71+
resource = self._resources_map[resource_id]
72+
73+
return resource, args
74+
75+
76+
class InvocationRequestRouter(RestApiGatewayHandler):
77+
def __call__(
78+
self,
79+
chain: RestApiGatewayHandlerChain,
80+
context: RestApiInvocationContext,
81+
response: Response,
82+
):
83+
self.route_and_enrich(context)
84+
85+
def route_and_enrich(self, context: RestApiInvocationContext):
86+
router = self.get_router_for_deployment(context.deployment)
87+
88+
resource, path_parameters = router.match(context)
89+
90+
context.invocation_request["path_parameters"] = path_parameters
91+
context.resource = resource
92+
93+
method = (
94+
resource["resourceMethods"].get(context.request.method)
95+
or resource["resourceMethods"]["ANY"]
96+
)
97+
context.resource_method = method
98+
99+
@staticmethod
100+
@cache
101+
def get_router_for_deployment(deployment: RestApiDeployment) -> RestAPIResourceRouter:
102+
return RestAPIResourceRouter(deployment)
103+
104+
105+
def get_rule_map_for_resources(resources: ListOfResource) -> Map:
106+
rules = []
107+
for resource in resources:
108+
for method, resource_method in resource.get("resourceMethods", {}).items():
109+
path = resource["path"]
110+
# translate the requestUri to a Werkzeug rule string
111+
rule_string = _path_param_regex.sub(_transform_path_params_to_rule_vars, path)
112+
rules.append(
113+
_StrictMethodRule(string=rule_string, method=method, endpoint=resource["id"])
114+
) # type: ignore
115+
116+
return Map(
117+
rules=rules,
118+
# don't be strict about trailing slashes when matching
119+
strict_slashes=False,
120+
# we can't really use werkzeug's merge-slashes since it uses HTTP redirects to solve it
121+
merge_slashes=False,
122+
# get service-specific converters
123+
converters={"path": GreedyPathConverter},
124+
)
125+
126+
127+
# TODO: there are private imports from `localstack/aws/protocol/op_router.py`, we could also copy paste them for now

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/helpers.py

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,44 @@
22

33
from moto.apigateway.models import RestAPI as MotoRestAPI
44

5+
from localstack.aws.api.apigateway import ListOfResource, Resource
56
from localstack.services.apigateway.models import RestApiContainer, RestApiDeployment
67

78

89
def freeze_rest_api(
9-
moto_rest_api: MotoRestAPI, localstack_rest_api: RestApiContainer
10+
account_id: str, region: str, moto_rest_api: MotoRestAPI, localstack_rest_api: RestApiContainer
1011
) -> RestApiDeployment:
1112
"""Snapshot a REST API in time to create a deployment"""
1213
return RestApiDeployment(
14+
account_id=account_id,
15+
region=region,
1316
moto_rest_api=copy.deepcopy(moto_rest_api),
1417
localstack_rest_api=copy.deepcopy(localstack_rest_api),
1518
)
19+
20+
21+
def get_resources_from_deployment(deployment: RestApiDeployment) -> ListOfResource:
22+
"""
23+
This returns the `Resources` from a deployment
24+
This allows to decouple the underlying split of resources between Moto and LocalStack, and always return the right
25+
format.
26+
"""
27+
moto_resources = deployment.moto_rest_api.resources
28+
29+
resources: ListOfResource = []
30+
for moto_resource in moto_resources.values():
31+
resource = Resource(
32+
id=moto_resource.id,
33+
parentId=moto_resource.parent_id,
34+
pathPart=moto_resource.path_part,
35+
path=moto_resource.get_path(),
36+
resourceMethods={
37+
# TODO: check if resource_methods.to_json() returns everything we need/want
38+
k: v.to_json()
39+
for k, v in moto_resource.resource_methods.items()
40+
},
41+
)
42+
43+
resources.append(resource)
44+
45+
return resources

‎localstack-core/localstack/services/apigateway/next_gen/provider.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ def create_deployment(
103103
moto_rest_api = get_moto_rest_api(context, rest_api_id)
104104
rest_api_container = get_rest_api_container(context, rest_api_id=rest_api_id)
105105
frozen_deployment = freeze_rest_api(
106+
account_id=context.account_id,
107+
region=context.region,
106108
moto_rest_api=moto_rest_api,
107109
localstack_rest_api=rest_api_container,
108110
)

‎tests/unit/services/apigateway/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)