diff --git a/TODO-Feature.txt b/TODO-Feature.txt new file mode 100644 index 000000000..cdd5603a5 --- /dev/null +++ b/TODO-Feature.txt @@ -0,0 +1,115 @@ + + +-->> Criar issue no github para iniciar discussao. + -> Melhora para tornar mais flexivel o built in custom authorizer. + * type + The authorizer type. Valid values are TOKEN for a Lambda function using a single authorization token submitted in a custom header, REQUEST for a Lambda function using incoming request parameters, and COGNITO_USER_POOLS for using an Amazon Cognito user pool. + Issue = Extend Custom Authorizer Support API Gateway Request Parameter #948 + -> Feature para permitir deploys de rest api com multiplos lambdas. + * motivacao eh poder segregar rotas do api gateway para multiplos lambdas + - Porque? + - Melhorar o monitoramento e controle (permissoes e throtling especificos por handler) + - Permitir otimizar o pacote .zip + - Se algum handler estiver com problema ele nao impacta os demais + + + +-->> Possibilidade de virar uma extensao. + +* definir boiler plate (organização dos diretorios) desse novo tipo de projeto + + + ├── helloworld + │ ├── functionlib + │ │ ├── __init__.py + │ │ └── utils.py + │ ├── __init__.py + │ └── helloworld.py + ├── chalicelib + │ ├── __init__.py + │ └── generics.py + ├── app.py + ├── requirements.txt + └── etc ... + +* definir novas regras do framework e seus impactos + user_blueprint.registrate_route( + name='user_delete', + path='/v1/user/delete', + function=UserDeleteHandler, + ) + +* dificuldades de design de codigo: + * model DeploymentPackage é esperado por ser unico, hoje em dia. Uma instancia compartilhada. + - Possibilidades para resolver o problema: + - verificar a nessecidade de criar um DeploymentPackage dentro de + cada um dos metodos que cria os recursos. + - utilizar ou não a instancia global dentro dos metodos que criam os modelos. (utilizada) -> (done) + - iniciar pelo Rest Api + + + + ---->>> Dificuldade de design de meio de campo resolvida <<<---- + +* Criar projeto baseado na estrutura de pasta definida, para testar casos de uso. + +* Abordagem Top-Down: + - Criar regras e logica no app: + - Metodo de interface entre a view do Chalice e um handler padrão de lambda: + - Estudar inputs do handler do chalice vs o handler padrao da aws + - chalice input: + reimplementar o metodo __call__ da classe Chalice + class Request + - local runner: + - garantir que o handler seja direcionado localmente + mas que não seja apontado nos templates do api gateway + (ver uma maneira de bloquear) + - LambdaHandler: + Parecido com o atual BaseHandler da autenticação porem de uso mais genérico. + - Classe para traduzir os inputs, similar ao BifrostChalice + + - Traduzir o import do Handler para o function_path (conseguir achar a pasta a ser buildada a partir do import) + - Traduzir os methods a partir dos metodos da classe. + + - segregar o metodo _create_rest_api_model em dois outros metodos dependendo do tipo de rest api (?) + - Nao sei se ainda é necessário depois de ter resolvido a questão do deployment object. + - Pode se criar metodos apenas para organizar caso seja preciso. + + - segregar o metodo create_deployment_package em dois outros metodos dependendo do tipo de package + - Para separar: + - diretorio do projeto a ser deployado. + - logica para pegar arquivos. + - logica de build do requirements + + - Outras funcionalidades adicionais + + +* template do cloudformation: + - ? + +* template do terraform: + - adicionar lambda functions no template (done) + - referenciar o filename correto no template do terraform (done) + - Referenciar os lambdas corretos nas determinadas rotas (done) + + - corrigir o bug da duplicacao no nome api_handler_handler + +* adicionar arquivos nos pacotes dos lambdas + +* variaveis de ambiente (?) + +* Ver diferenca entre nosso api gateway e gerado + - como implementar vpc link + - mock de resposta para o options + +* instalar dependencias que nao foram possiveis pelo pip do chalice + +* ver como ele faz o deploy de fato do terraform + +* como deployar apenas parte do detalhe do terraform + +** Mesclar funcionalidades anteriores com alteracao ** + +** testes unitarios ** + +** padroes de codigo ** \ No newline at end of file diff --git a/chalice/app.py b/chalice/app.py index c403c49a4..0cccf3048 100644 --- a/chalice/app.py +++ b/chalice/app.py @@ -474,7 +474,7 @@ class RouteEntry(object): def __init__(self, view_function, view_name, path, method, api_key_required=None, content_types=None, - cors=False, authorizer=None): + cors=False, authorizer=None, function=None): self.view_function = view_function self.view_name = view_name self.uri_pattern = path @@ -495,6 +495,7 @@ def __init__(self, view_function, view_name, path, method, cors = None self.cors = cors self.authorizer = authorizer + self.function = function def _parse_view_args(self): if '{' not in self.uri_pattern: @@ -590,12 +591,14 @@ def info(self, connection_id): class DecoratorAPI(object): - def authorizer(self, ttl_seconds=None, execution_role=None, name=None): + def authorizer(self, ttl_seconds=None, execution_role=None, name=None, function=None): return self._create_registration_function( handler_type='authorizer', name=name, registration_kwargs={ - 'ttl_seconds': ttl_seconds, 'execution_role': execution_role, + 'ttl_seconds': ttl_seconds, + 'execution_role': execution_role, + 'function': function } ) @@ -864,6 +867,7 @@ def _register_authorizer(self, name, handler_string, wrapped_handler, actual_kwargs = kwargs.copy() ttl_seconds = actual_kwargs.pop('ttl_seconds', None) execution_role = actual_kwargs.pop('execution_role', None) + function = actual_kwargs.pop('function', None) if actual_kwargs: raise TypeError( 'TypeError: authorizer() got unexpected keyword ' @@ -873,6 +877,7 @@ def _register_authorizer(self, name, handler_string, wrapped_handler, handler_string=handler_string, ttl_seconds=ttl_seconds, execution_role=execution_role, + function=function, ) wrapped_handler.config = auth_config self.builtin_auth_handlers.append(auth_config) @@ -891,6 +896,7 @@ def _register_route(self, name, user_handler, kwargs, **unused): 'content_types': actual_kwargs.pop('content_types', ['application/json']), 'cors': actual_kwargs.pop('cors', self.api.cors), + 'function': actual_kwargs.pop('function', 'api_handler'), } if route_kwargs['cors'] is None: route_kwargs['cors'] = self.api.cors @@ -1151,12 +1157,13 @@ def _add_cors_headers(self, response, cors_headers): class BuiltinAuthConfig(object): def __init__(self, name, handler_string, ttl_seconds=None, - execution_role=None): + execution_role=None, function=None): # We'd also support all the misc config options you can set. self.name = name self.handler_string = handler_string self.ttl_seconds = ttl_seconds self.execution_role = execution_role + self.function = function # ChaliceAuthorizer is unique in that the runtime component (the thing diff --git a/chalice/deploy/deployer.py b/chalice/deploy/deployer.py index 433950f31..1eaadd7cc 100644 --- a/chalice/deploy/deployer.py +++ b/chalice/deploy/deployer.py @@ -94,6 +94,7 @@ from botocore.session import Session # noqa from typing import Optional, Dict, List, Any, Set, Tuple, cast # noqa +import chalice from chalice import app from chalice.config import Config # noqa from chalice.config import DeployedResources # noqa @@ -380,6 +381,7 @@ def __init__(self): def build(self, config, stage_name): # type: (Config, str) -> models.Application + print('build') resources = [] # type: List[models.Model] deployment = models.DeploymentPackage(models.Placeholder.BUILD_STAGE) for function in config.chalice_app.pure_lambda_functions: @@ -444,10 +446,36 @@ def _create_rest_api_model(self, ): # type: (...) -> models.RestAPI # Need to mess with the function name for back-compat. - lambda_function = self._create_lambda_model( - config=config, deployment=deployment, name='api_handler', - handler_name='app.app', stage_name=stage_name - ) + print("_create_rest_api_model") + + lambdas_to_build = set() + for route_key, route_value in config.chalice_app.routes.items(): + for method_key, method_value in route_value.items(): + lambdas_to_build.add(method_value.function) + + # Check here if the deployment instance must be used and appended to + # lambdas_functions list + lambdas_functions = [] + if 'api_handler' in lambdas_to_build: + lambdas_to_build.remove('api_handler') + lambda_function = self._create_lambda_model( + config=config, deployment=deployment, name='api_handler', + handler_name='app.app', stage_name=stage_name + ) + lambdas_functions.append(lambda_function) + + for lambda_path in lambdas_to_build: + name = '%s_handler' % ('_'.join(lambda_path.split('/'))) + + deployment = models.DeploymentPackage( + models.Placeholder.BUILD_STAGE, + lambda_path + ) + lambda_function = self._create_lambda_model( + config=config, deployment=deployment, name=name, + handler_name='app.app', stage_name=stage_name + ) + lambdas_functions.append(lambda_function) # For backwards compatibility with the old deployer, the # lambda function for the API handler doesn't have the # resource_name appended to its complete function_name, @@ -485,6 +513,7 @@ def _create_rest_api_model(self, api_gateway_stage=config.api_gateway_stage, lambda_function=lambda_function, authorizers=authorizers, + lambdas_functions=lambdas_functions, policy=policy ) @@ -614,6 +643,7 @@ def _create_lambda_model(self, handler_name, # type: str stage_name, # type: str ): + print("_create_lambda_model") # type: (...) -> models.LambdaFunction new_config = config.scope( chalice_stage=config.chalice_stage, @@ -736,7 +766,8 @@ def _build_lambda_function(self, security_group_ids=security_group_ids, subnet_ids=subnet_ids, reserved_concurrency=config.reserved_concurrency, - layers=lambda_layers + layers=lambda_layers, + function_path=deployment.function_path ) self._inject_role_traits(function, role) return function @@ -875,9 +906,16 @@ def handle_deploymentpackage(self, config, resource): # type: (Config, models.DeploymentPackage) -> None if isinstance(resource.filename, models.Placeholder): zip_filename = self._packager.create_deployment_package( - config.project_dir, config.lambda_python_version) + config.project_dir, config.lambda_python_version, function_path=resource.function_path) resource.filename = zip_filename + # def handle_functionpackage(self, config, resource): + # # type: (Config, models.DeploymentPackage) -> None + # if isinstance(resource.filename, models.Placeholder): + # zip_filename = self._packager.create_deployment_package( + # config.project_dir, config.lambda_python_version, function_path=resource.function_path) + # resource.filename = zip_filename + class SwaggerBuilder(BaseDeployStep): def __init__(self, swagger_generator): diff --git a/chalice/deploy/models.py b/chalice/deploy/models.py index 5110bf1eb..c06602ebb 100644 --- a/chalice/deploy/models.py +++ b/chalice/deploy/models.py @@ -111,6 +111,13 @@ def dependencies(self): @attrs class DeploymentPackage(Model): filename = attrib() # type: DV[str] + function_path = attrib(default='') # type: str + + +@attrs +class FunctionPackage(DeploymentPackage): + filename = attrib() # type: DV[str] + function_path = attrib() # type: str @attrs @@ -166,6 +173,7 @@ class LambdaFunction(ManagedModel): subnet_ids = attrib() # type: List[str] reserved_concurrency = attrib() # type: int layers = attrib() # type: List[str] + function_path = attrib(default='') # type: str def dependencies(self): # type: () -> List[Model] @@ -209,10 +217,11 @@ class RestAPI(ManagedModel): lambda_function = attrib() # type: LambdaFunction policy = attrib(default=None) # type: Optional[IAMPolicy] authorizers = attrib(default=Factory(list)) # type: List[LambdaFunction] + lambdas_functions = attrib(default=Factory(list)) # type: List[LambdaFunction] def dependencies(self): # type: () -> List[Model] - return cast(List[Model], [self.lambda_function] + self.authorizers) + return cast(List[Model], self.lambdas_functions + self.authorizers) @attrs diff --git a/chalice/deploy/packager.py b/chalice/deploy/packager.py index c425128e1..52005a1b6 100644 --- a/chalice/deploy/packager.py +++ b/chalice/deploy/packager.py @@ -28,6 +28,7 @@ OptBytes = Optional[bytes] logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class InvalidSourceDistributionNameError(Exception): @@ -54,14 +55,13 @@ class PackageDownloadError(Exception): class LambdaDeploymentPackager(object): - _CHALICE_LIB_DIR = 'chalicelib' + _CHALICE_LIB_DIR = 'bifrost' _VENDOR_DIR = 'vendor' _RUNTIME_TO_ABI = { 'python2.7': 'cp27mu', 'python3.6': 'cp36m', 'python3.7': 'cp37m', - 'python3.8': 'cp38', } def __init__(self, osutils, dependency_builder, ui): @@ -76,9 +76,15 @@ def _get_requirements_filename(self, project_dir): return self._osutils.joinpath(project_dir, 'requirements.txt') def create_deployment_package(self, project_dir, python_version, - package_filename=None): + package_filename=None, function_path=None): # type: (str, str, Optional[str]) -> str msg = "Creating deployment package." + + if function_path: + project_dir = f'{project_dir}/aws_lambda/{function_path}' + + print(project_dir) + self._ui.write("%s\n" % msg) logger.debug(msg) # Now we need to create a zip file and add in the site-packages @@ -88,6 +94,7 @@ def create_deployment_package(self, project_dir, python_version, if package_filename is None: package_filename = deployment_package_filename requirements_filepath = self._get_requirements_filename(project_dir) + print(requirements_filepath) with self._osutils.tempdir() as site_packages_dir: try: abi = self._RUNTIME_TO_ABI[python_version] @@ -105,7 +112,7 @@ def create_deployment_package(self, project_dir, python_version, with self._osutils.open_zip(package_filename, 'w', self._osutils.ZIP_DEFLATED) as z: self._add_py_deps(z, site_packages_dir) - self._add_app_files(z, project_dir) + # self._add_app_files(z, project_dir) self._add_vendor_files(z, self._osutils.joinpath( project_dir, self._VENDOR_DIR)) return package_filename @@ -230,8 +237,9 @@ def inject_latest_app(self, deployment_package_filename, project_dir): for el in inzip.infolist(): if self._needs_latest_version(el.filename): continue - contents = inzip.read(el.filename) - outzip.writestr(el, contents) + else: + contents = inzip.read(el.filename) + outzip.writestr(el, contents) # Then at the end, add back the app.py, chalicelib, # and runtime files. self._add_app_files(outzip, project_dir) @@ -337,20 +345,6 @@ def _download_binary_wheels(self, abi, packages, directory): self._pip.download_manylinux_wheels( abi, [pkg.identifier for pkg in packages], directory) - def _download_sdists(self, packages, directory): - # type: (Set[Package], str) -> None - logger.debug("Downloading missing sdists: %s", packages) - self._pip.download_sdists( - [pkg.identifier for pkg in packages], directory) - - def _find_sdists(self, directory): - # type: (str) -> Set[Package] - packages = [Package(directory, filename) for filename - in self._osutils.get_directory_contents(directory)] - sdists = {package for package in packages - if package.dist_type == 'sdist'} - return sdists - def _build_sdists(self, sdists, directory, compile_c=True): # type: (Set[Package], str, bool) -> None logger.debug("Build missing wheels from sdists " @@ -373,21 +367,6 @@ def _categorize_wheel_files(self, abi, directory): incompatible_wheels.add(wheel) return compatible_wheels, incompatible_wheels - def _categorize_deps(self, abi, deps): - # type: (str, Set[Package]) -> Any - compatible_wheels = set() - incompatible_wheels = set() - sdists = set() - for package in deps: - if package.dist_type == 'sdist': - sdists.add(package) - else: - if self._is_compatible_wheel_filename(abi, package.filename): - compatible_wheels.add(package) - else: - incompatible_wheels.add(package) - return sdists, compatible_wheels, incompatible_wheels - def _download_dependencies(self, abi, directory, requirements_filename): # type: (str, str, str) -> Tuple[Set[Package], Set[Package]] # Download all dependencies we can, letting pip choose what to @@ -407,8 +386,17 @@ def _download_dependencies(self, abi, directory, requirements_filename): # platform lambda runs on (linux_x86_64/manylinux) then the downloaded # wheel file may not be compatible with lambda. Pure python wheels # still will be compatible because they have no platform dependencies. - sdists, compatible_wheels, incompatible_wheels = self._categorize_deps( - abi, deps) + compatible_wheels = set() + incompatible_wheels = set() + sdists = set() + for package in deps: + if package.dist_type == 'sdist': + sdists.add(package) + else: + if self._is_compatible_wheel_filename(abi, package.filename): + compatible_wheels.add(package) + else: + incompatible_wheels.add(package) logger.debug("initial compatible: %s", compatible_wheels) logger.debug("initial incompatible: %s", incompatible_wheels | sdists) @@ -423,14 +411,6 @@ def _download_dependencies(self, abi, directory, requirements_filename): # that has an sdist but not a valid wheel file is still not going to # work on lambda and we must now try and build the sdist into a wheel # file ourselves. - # There also may be the case where no sdist was ever downloaded. For - # example if we are on MacOS, and the package in question has a mac - # compatible wheel file but no linux ones, we will only have an - # incompatible wheel file and no sdist. So we need to get any missing - # sdists before we can build them. - missing_sdists = incompatible_wheels - sdists - self._download_sdists(missing_sdists, directory) - sdists = self._find_sdists(directory) compatible_wheels, incompatible_wheels = self._categorize_wheel_files( abi, directory) logger.debug( @@ -801,10 +781,3 @@ def download_manylinux_wheels(self, abi, packages, directory): 'manylinux1_x86_64', '--implementation', 'cp', '--abi', abi, '--dest', directory, package] self._execute('download', arguments) - - def download_sdists(self, packages, directory): - # type: (List[str], str) -> None - for package in packages: - arguments = ["--no-binary=:all:", "--no-deps", "--dest", - directory, package] - self._execute('download', arguments) diff --git a/chalice/deploy/swagger.py b/chalice/deploy/swagger.py index 588188fa3..85a50b87a 100644 --- a/chalice/deploy/swagger.py +++ b/chalice/deploy/swagger.py @@ -168,7 +168,7 @@ def _generate_precanned_responses(self): } return responses - def _uri(self, lambda_arn=None): + def _uri(self, lambda_arn=None, view=None): # type: (Optional[str]) -> Any if lambda_arn is None: lambda_arn = self._deployed_resources['api_handler_arn'] @@ -184,7 +184,7 @@ def _generate_apig_integ(self, view): 'statusCode': "200", } }, - 'uri': self._uri(), + 'uri': self._uri(view=view), 'passthroughBehavior': 'when_no_match', 'httpMethod': 'POST', 'contentHandling': 'CONVERT_TO_TEXT', @@ -294,9 +294,14 @@ def __init__(self): # type: () -> None pass - def _uri(self, lambda_arn=None): + def _uri(self, lambda_arn=None, view=None): # type: (Optional[str]) -> Any - return '${aws_lambda_function.api_handler.invoke_arn}' + if view: + function_name = '%s_handler' % ('_'.join( + view.function.split('/'))) + return '${aws_lambda_function.%s.invoke_arn}' % (function_name) + else: + return '${aws_lambda_function.api_handler.invoke_arn}' def _auth_uri(self, authorizer): # type: (ChaliceAuthorizer) -> Any diff --git a/chalice/package.py b/chalice/package.py index 87963b50a..cd950b8fa 100644 --- a/chalice/package.py +++ b/chalice/package.py @@ -782,6 +782,7 @@ def _cwe_helper(self, resource, template): def _generate_lambdafunction(self, resource, template): # type: (models.LambdaFunction, Dict[str, Any]) -> None + print('_generate_lambdafunction') func_definition = { 'function_name': resource.function_name, 'runtime': resource.runtime, @@ -968,17 +969,31 @@ def _fixup_deployment_package(self, template, outdir): class TerraformCodeLocationPostProcessor(TemplatePostProcessor): + + # TODO: Trigar esse para rest api normal + # def process(self, template, config, outdir, chalice_stage_name): + # # type: (Dict[str, Any], Config, str, str) -> None + + # copied = False + # for r in template['resource'].get('aws_lambda_function', {}).values(): + # if not copied: + # asset_path = os.path.join(outdir, 'deployment.zip') + # self._osutils.copy(r['filename'], asset_path) + # copied = True + # r['filename'] = "./deployment.zip" + # r['source_code_hash'] = '${filebase64sha256("./deployment.zip")}' + + # TODO: Trigar esse para rest api com multiplos lambdas def process(self, template, config, outdir, chalice_stage_name): # type: (Dict[str, Any], Config, str, str) -> None - copied = False for r in template['resource'].get('aws_lambda_function', {}).values(): - if not copied: - asset_path = os.path.join(outdir, 'deployment.zip') - self._osutils.copy(r['filename'], asset_path) - copied = True - r['filename'] = "./deployment.zip" - r['source_code_hash'] = '${filebase64sha256("./deployment.zip")}' + asset_filename = "%s_deployment.zip" % r['function_name'] + asset_path = os.path.join(outdir, asset_filename) + self._osutils.copy(r['filename'], asset_path) + + r['filename'] = "./%s" % asset_filename + r['source_code_hash'] = '${filebase64sha256("./%s")}' % asset_filename class TemplateMergePostProcessor(TemplatePostProcessor):