diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f883039..ba9286f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [2.36.0](https://github.com/terraform-aws-modules/terraform-aws-lambda/compare/v2.35.1...v2.36.0) (2022-03-26) + + +### Features + +* Add support to build automatically npm dependencies ([#293](https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/293)) ([ecb3807](https://github.com/terraform-aws-modules/terraform-aws-lambda/commit/ecb38076b0408982183ebb8070aff7c7e01c4b82)) + ### [2.35.1](https://github.com/terraform-aws-modules/terraform-aws-lambda/compare/v2.35.0...v2.35.1) (2022-03-18) diff --git a/README.md b/README.md index 1f89893f..005c2a38 100644 --- a/README.md +++ b/README.md @@ -403,6 +403,11 @@ source_path = [ "!vendor/colorful-.+.dist-info/.*", "!vendor/colorful/__pycache__/?.*", ] + }, { + path = "src/nodejs14.x-app1", + npm_requirements = true, + npm_tmp_dir = "/tmp/dir/location" + prefix_in_zip = "foo/bar1", }, { path = "src/python3.8-app3", commands = [ @@ -424,8 +429,9 @@ source_path = [ ] ``` -Few notes: +*Few notes:* +- If you specify a source path as a string that references a folder and the runtime begins with `python` or `nodejs`, the build process will automatically build python and nodejs dependencies if `requirements.txt` or `package.json` file will be found in the source folder. If you want to customize this behavior, please use the object notation as explained below. - All arguments except `path` are optional. - `patterns` - List of Python regex filenames should satisfy. Default value is "include everything" which is equal to `patterns = [".*"]`. This can also be specified as multiline heredoc string (no comments allowed). Some examples of valid patterns: @@ -442,10 +448,12 @@ Few notes: !abc/def/hgk/.* # Filter out again in abc/def/hgk sub folder ``` -- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements`. +- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements` and `npm_requirements`. - `:zip [source] [destination]` is a special command which creates content of current working directory (first argument) and places it inside of path (second argument). - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. +- `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. +- `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. ### Building in Docker @@ -455,7 +463,7 @@ If your Lambda Function or Layer uses some dependencies you can build them in Do build_in_docker = true docker_file = "src/python3.8-app1/docker/Dockerfile" docker_build_root = "src/python3.8-app1/docker" - docker_image = "lambci/lambda:build-python3.8" + docker_image = "public.ecr.aws/sam/build-python3.8" runtime = "python3.8" # Setting runtime is required when building package in Docker and Lambda Layer resource. Using this module you can install dependencies from private hosts. To do this, you need for forward SSH agent: diff --git a/examples/build-package/README.md b/examples/build-package/README.md index 7ee75c69..0d1f4a4f 100644 --- a/examples/build-package/README.md +++ b/examples/build-package/README.md @@ -38,11 +38,14 @@ Note that this example may create resources which cost money. Run `terraform des | [lambda\_layer\_pip\_requirements](#module\_lambda\_layer\_pip\_requirements) | ../.. | n/a | | [package\_dir](#module\_package\_dir) | ../../ | n/a | | [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a | +| [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a | +| [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a | | [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a | | [package\_file](#module\_package\_file) | ../../ | n/a | | [package\_file\_with\_pip\_requirements](#module\_package\_file\_with\_pip\_requirements) | ../../ | n/a | | [package\_with\_commands\_and\_patterns](#module\_package\_with\_commands\_and\_patterns) | ../../ | n/a | | [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a | +| [package\_with\_npm\_requirements\_in\_docker](#module\_package\_with\_npm\_requirements\_in\_docker) | ../../ | n/a | | [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a | | [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a | diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index 636ac8d8..61dd0102 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -220,7 +220,45 @@ module "package_with_docker" { docker_with_ssh_agent = true # docker_file = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile" docker_build_root = "${path.module}/../../docker" - docker_image = "lambci/lambda:build-python3.8" + docker_image = "public.ecr.aws/sam/build-python3.8" +} + +# Create zip-archive of a single directory where "npm install" will also be executed (default for nodejs runtime) +module "package_dir_with_npm_install" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = "${path.module}/../fixtures/nodejs14.x-app1" +} + +# Create zip-archive of a single directory without running "npm install" (which is the default for nodejs runtime) +module "package_dir_without_npm_install" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = [ + { + path = "${path.module}/../fixtures/nodejs14.x-app1" + npm_requirements = false + # npm_requirements = true # Will run "npm install" with package.json + } + ] +} + +# Create zip-archive of a single directory where "npm install" will also be executed using docker +module "package_with_npm_requirements_in_docker" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = "${path.module}/../fixtures/nodejs14.x-app1" + build_in_docker = true + hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install" } ################################ @@ -240,6 +278,7 @@ module "lambda_layer" { build_in_docker = true runtime = "python3.8" + docker_image = "public.ecr.aws/sam/build-python3.8" docker_file = "${path.module}/../fixtures/python3.8-app1/docker/Dockerfile" } diff --git a/examples/fixtures/nodejs14.x-app1/index.js b/examples/fixtures/nodejs14.x-app1/index.js new file mode 100644 index 00000000..97968e4a --- /dev/null +++ b/examples/fixtures/nodejs14.x-app1/index.js @@ -0,0 +1,16 @@ +'use strict'; + +module.exports.hello = async (event) => { + console.log(event); + return { + statusCode: 200, + body: JSON.stringify( + { + message: `Go Serverless.tf! Your Nodejs function executed successfully!`, + input: event, + }, + null, + 2 + ), + }; +}; diff --git a/examples/fixtures/nodejs14.x-app1/package.json b/examples/fixtures/nodejs14.x-app1/package.json new file mode 100644 index 00000000..89c23f36 --- /dev/null +++ b/examples/fixtures/nodejs14.x-app1/package.json @@ -0,0 +1,8 @@ +{ + "name": "nodejs14.x-app1", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "requests": "^0.3.0" + } +} diff --git a/package.py b/package.py index b9fccf28..7bd8dd2d 100644 --- a/package.py +++ b/package.py @@ -660,6 +660,18 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step('pip', runtime, requirements, prefix, tmp_dir) hash(requirements) + def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): + requirements = path + if os.path.isdir(path): + requirements = os.path.join(path, 'package.json') + if not os.path.isfile(requirements): + if required: + raise RuntimeError( + 'File not found: {}'.format(requirements)) + else: + step('npm', runtime, requirements, prefix, tmp_dir) + hash(requirements) + def commands_step(path, commands): if not commands: return @@ -717,6 +729,9 @@ def commands_step(path, commands): if runtime.startswith('python'): pip_requirements_step( os.path.join(path, 'requirements.txt')) + elif runtime.startswith('nodejs'): + npm_requirements_step( + os.path.join(path, 'package.json')) step('zip', path, None) hash(path) @@ -731,6 +746,7 @@ def commands_step(path, commands): else: prefix = claim.get('prefix_in_zip') pip_requirements = claim.get('pip_requirements') + npm_requirements = claim.get('npm_package_json') runtime = claim.get('runtime', query.runtime) if pip_requirements and runtime.startswith('python'): @@ -740,6 +756,13 @@ def commands_step(path, commands): pip_requirements_step(pip_requirements, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir')) + if npm_requirements and runtime.startswith('nodejs'): + if isinstance(npm_requirements, bool) and path: + npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir')) + else: + npm_requirements_step(npm_requirements, prefix, + required=True, tmp_dir=claim.get('npm_tmp_dir')) + if path: step('zip', path, prefix) if patterns: @@ -793,6 +816,16 @@ def execute(self, build_plan, zip_stream, query): else: # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) + elif cmd == 'npm': + runtime, npm_requirements, prefix, tmp_dir = action[1:] + with install_npm_requirements(query, npm_requirements, tmp_dir) as rd: + if rd: + if pf: + self._zip_write_with_filter(zs, pf, rd, prefix, + timestamp=0) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == 'sh': r, w = os.pipe() side_ch = os.fdopen(r) @@ -934,6 +967,89 @@ def install_pip_requirements(query, requirements_file, tmp_dir): yield temp_dir +@contextmanager +def install_npm_requirements(query, requirements_file, tmp_dir): + # TODO: + # 1. Emit files instead of temp_dir + + if not os.path.exists(requirements_file): + yield + return + + runtime = query.runtime + artifacts_dir = query.artifacts_dir + temp_dir = query.temp_dir + docker = query.docker + docker_image_tag_id = None + + if docker: + docker_file = docker.docker_file + docker_image = docker.docker_image + docker_build_root = docker.docker_build_root + + if docker_image: + ok = False + while True: + output = check_output(docker_image_id_command(docker_image)) + if output: + docker_image_tag_id = output.decode().strip() + log.debug("DOCKER TAG ID: %s -> %s", + docker_image, docker_image_tag_id) + ok = True + if ok: + break + docker_cmd = docker_build_command( + build_root=docker_build_root, + docker_file=docker_file, + tag=docker_image, + ) + check_call(docker_cmd) + ok = True + elif docker_file or docker_build_root: + raise ValueError('docker_image must be specified ' + 'for a custom image future references') + + log.info('Installing npm requirements: %s', requirements_file) + with tempdir(tmp_dir) as temp_dir: + requirements_filename = os.path.basename(requirements_file) + target_file = os.path.join(temp_dir, requirements_filename) + shutil.copyfile(requirements_file, target_file) + + subproc_env = None + if not docker and OSX: + subproc_env = os.environ.copy() + + # Install dependencies into the temporary directory. + with cd(temp_dir): + npm_command = ['npm', 'install'] + if docker: + with_ssh_agent = docker.with_ssh_agent + chown_mask = '{}:{}'.format(os.getuid(), os.getgid()) + shell_command = [shlex_join(npm_command), '&&', + shlex_join(['chown', '-R', + chown_mask, '.'])] + shell_command = [' '.join(shell_command)] + check_call(docker_run_command( + '.', shell_command, runtime, + image=docker_image_tag_id, + shell=True, ssh_agent=with_ssh_agent + )) + else: + cmd_log.info(shlex_join(npm_command)) + log_handler and log_handler.flush() + try: + check_call(npm_command, env=subproc_env) + except FileNotFoundError as e: + raise RuntimeError( + "Nodejs interpreter version equal " + "to defined lambda runtime ({}) should be " + "available in system PATH".format(runtime) + ) from e + + os.remove(target_file) + yield temp_dir + + def docker_image_id_command(tag): """""" docker_cmd = ['docker', 'images', '--format={{.ID}}', tag] @@ -1011,7 +1127,7 @@ def docker_run_command(build_root, command, runtime, ]) if not image: - image = 'lambci/lambda:build-{}'.format(runtime) + image = 'public.ecr.aws/sam/build-{}'.format(runtime) docker_cmd.append(image) @@ -1128,7 +1244,7 @@ def prepare_command(args): def build_command(args): """ Builds a zip file from the source_dir or source_file. - Installs dependencies with pip automatically. + Installs dependencies with pip or npm automatically. """ log = logging.getLogger('build')