diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 8f6235d2f..dcf438365 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.18.0" +__version__ = "1.19.0" RPC_PROTOCOL_VERSION = "0.3" diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index a7154f66a..7cb65ceac 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -5,8 +5,10 @@ import logging import os import shutil +from pathlib import Path from typing import Set, Iterator, Tuple +from aws_lambda_builders import utils from aws_lambda_builders.utils import copytree LOG = logging.getLogger(__name__) @@ -31,6 +33,9 @@ class Purpose(object): # Action is copying source code COPY_SOURCE = "COPY_SOURCE" + # Action is linking source code + LINK_SOURCE = "LINK_SOURCE" + # Action is copying dependencies COPY_DEPENDENCIES = "COPY_DEPENDENCIES" @@ -111,6 +116,31 @@ def execute(self): copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes)) +class LinkSourceAction(BaseAction): + + NAME = "LinkSource" + + DESCRIPTION = "Linking source code to the target folder" + + PURPOSE = Purpose.LINK_SOURCE + + def __init__(self, source_dir, dest_dir): + self._source_dir = source_dir + self._dest_dir = dest_dir + + def execute(self): + source_files = set(os.listdir(self._source_dir)) + + for source_file in source_files: + source_path = Path(self._source_dir, source_file) + destination_path = Path(self._dest_dir, source_file) + if destination_path.exists(): + os.remove(destination_path) + else: + os.makedirs(destination_path.parent, exist_ok=True) + utils.create_symlink_or_copy(str(source_path), str(destination_path)) + + class CopyDependenciesAction(BaseAction): NAME = "CopyDependencies" @@ -175,10 +205,10 @@ def __init__(self, target_dir): def execute(self): if not os.path.isdir(self.target_dir): - LOG.info("Clean up action: %s does not exist and will be skipped.", str(self.target_dir)) + LOG.debug("Clean up action: %s does not exist and will be skipped.", str(self.target_dir)) return targets = os.listdir(self.target_dir) - LOG.info("Clean up action: folder %s will be cleaned", str(self.target_dir)) + LOG.debug("Clean up action: folder %s will be cleaned", str(self.target_dir)) for name in targets: target_path = os.path.join(self.target_dir, name) diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index 2ed3f3f8e..f1c24eb1a 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -19,7 +19,7 @@ class MisMatchRuntimeError(LambdaBuilderError): MESSAGE = ( "{language} executable found in your path does not " "match runtime. " - "\n Expected version: {required_runtime}, Found version: {runtime_path}. " + "\n Expected version: {required_runtime}, Found a different version at {runtime_path}. " "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30" ) diff --git a/aws_lambda_builders/path_resolver.py b/aws_lambda_builders/path_resolver.py index e8f9736ee..6f15bdcbe 100644 --- a/aws_lambda_builders/path_resolver.py +++ b/aws_lambda_builders/path_resolver.py @@ -6,10 +6,14 @@ class PathResolver(object): - def __init__(self, binary, runtime, executable_search_paths=None): + def __init__(self, binary, runtime, additional_binaries=None, executable_search_paths=None): self.binary = binary self.runtime = runtime self.executables = [self.runtime, self.binary] + self.additional_binaries = additional_binaries + if isinstance(additional_binaries, list): + self.executables = self.executables + self.additional_binaries + self.executable_search_paths = executable_search_paths def _which(self): diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index 791b68669..0e5e7447b 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -6,6 +6,7 @@ import sys import os import logging +from pathlib import Path from aws_lambda_builders.architecture import X86_64, ARM64 @@ -182,3 +183,16 @@ def get_goarch(architecture): returns a valid GO Architecture value """ return "arm64" if architecture == ARM64 else "amd64" + + +def create_symlink_or_copy(source: str, destination: str) -> None: + """Tries to create symlink, if it fails it will copy source into destination""" + LOG.debug("Creating symlink; source: %s, destination: %s", source, destination) + try: + os.symlink(Path(source).absolute(), Path(destination).absolute()) + except OSError as ex: + LOG.warning( + "Symlink operation is failed, falling back to copying files", + exc_info=ex if LOG.isEnabledFor(logging.DEBUG) else None, + ) + copytree(source, destination) diff --git a/aws_lambda_builders/workflows/custom_make/actions.py b/aws_lambda_builders/workflows/custom_make/actions.py index 67579a74a..e689012e5 100644 --- a/aws_lambda_builders/workflows/custom_make/actions.py +++ b/aws_lambda_builders/workflows/custom_make/actions.py @@ -22,7 +22,16 @@ class CustomMakeAction(BaseAction): DESCRIPTION = "Running build target on Makefile" PURPOSE = Purpose.COMPILE_SOURCE - def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subprocess_make, build_logical_id): + def __init__( + self, + artifacts_dir, + scratch_dir, + manifest_path, + osutils, + subprocess_make, + build_logical_id, + working_directory=None, + ): """ :type artifacts_dir: str :param artifacts_dir: directory where artifacts needs to be stored. @@ -38,6 +47,13 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subproces :type subprocess_make aws_lambda_builders.workflows.custom_make.make.SubprocessMake :param subprocess_make: An instance of the Make process wrapper + + :type build_logical_id: str + :param build_logical_id: the lambda resource logical id that will be built by the custom action. + + :type working_directory: str + :param working_directory: path to the working directory where the Makefile will be executed. Use the scratch_dir + as the working directory if the input working_directory is None """ super(CustomMakeAction, self).__init__() self.artifacts_dir = artifacts_dir @@ -46,6 +62,7 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subproces self.osutils = osutils self.subprocess_make = subprocess_make self.build_logical_id = build_logical_id + self.working_directory = working_directory if working_directory else scratch_dir @property def artifact_dir_path(self): @@ -91,7 +108,7 @@ def execute(self): "build-{logical_id}".format(logical_id=self.build_logical_id), ], env=current_env, - cwd=self.scratch_dir, + cwd=self.working_directory, ) except MakeExecutionError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/custom_make/workflow.py b/aws_lambda_builders/workflows/custom_make/workflow.py index 04c680237..627125667 100644 --- a/aws_lambda_builders/workflows/custom_make/workflow.py +++ b/aws_lambda_builders/workflows/custom_make/workflow.py @@ -34,6 +34,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim # Find the logical id of the function to be built. options = kwargs.get("options") or {} build_logical_id = options.get("build_logical_id", None) + working_directory = options.get("working_directory", scratch_dir) if not build_logical_id: raise WorkflowFailedError( @@ -51,6 +52,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim osutils=self.os_utils, subprocess_make=subprocess_make, build_logical_id=build_logical_id, + working_directory=working_directory, ) self.actions = [CopySourceAction(source_dir, scratch_dir, excludes=self.EXCLUDED_FILES), make_action] diff --git a/aws_lambda_builders/workflows/nodejs_npm/actions.py b/aws_lambda_builders/workflows/nodejs_npm/actions.py index 423a18faf..d74e7088e 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm/actions.py @@ -80,7 +80,7 @@ class NodejsNpmInstallAction(BaseAction): DESCRIPTION = "Installing dependencies from NPM" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - def __init__(self, artifacts_dir, subprocess_npm, is_production=True): + def __init__(self, artifacts_dir, subprocess_npm): """ :type artifacts_dir: str :param artifacts_dir: an existing (writable) directory with project source files. @@ -96,7 +96,6 @@ def __init__(self, artifacts_dir, subprocess_npm, is_production=True): super(NodejsNpmInstallAction, self).__init__() self.artifacts_dir = artifacts_dir self.subprocess_npm = subprocess_npm - self.is_production = is_production def execute(self): """ @@ -104,14 +103,11 @@ def execute(self): :raises lambda_builders.actions.ActionFailedError: when NPM execution fails """ - - mode = "--production" if self.is_production else "--production=false" - try: LOG.debug("NODEJS installing in: %s", self.artifacts_dir) self.subprocess_npm.run( - ["install", "-q", "--no-audit", "--no-save", mode, "--unsafe-perm"], cwd=self.artifacts_dir + ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"], cwd=self.artifacts_dir ) except NpmExecutionError as ex: diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index 6cf515ef4..7c8ffe897 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -145,7 +145,7 @@ def get_resolvers(self): return [PathResolver(runtime=self.runtime, binary="npm")] @staticmethod - def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build_options, is_production=True): + def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build_options): """ Get the install action used to install dependencies at artifacts_dir @@ -180,4 +180,4 @@ def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build if (osutils.file_exists(lockfile_path) or osutils.file_exists(shrinkwrap_path)) and npm_ci_option: return NodejsNpmCIAction(artifacts_dir, subprocess_npm=subprocess_npm) - return NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm, is_production=is_production) + return NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm) diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/DESIGN.md b/aws_lambda_builders/workflows/nodejs_npm_esbuild/DESIGN.md index 948eacef2..66f37faee 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/DESIGN.md +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/DESIGN.md @@ -117,24 +117,23 @@ testing flow before invoking `sam build`. For additional typescript caveats with #### Configuring the bundler -The Lambda builder invokes `esbuild` with sensible defaults that will work for the majority of cases. Importantly, the following three parameters are set by default +The Lambda builder invokes `esbuild` with sensible defaults that will work for the majority of cases. Importantly, the following parameters are set by default * `--minify`, as it [produces a smaller runtime package](https://esbuild.github.io/api/#minify) -* `--sourcemap`, as it generates a [source map that allows for correct stack trace reporting](https://esbuild.github.io/api/#sourcemap) in case of errors (see the [Error reporting](#error-reporting) section above) * `--target es2020`, as it allows for javascript features present in Node 14 Users might want to tweak some of these runtime arguments for a specific project, for example not including the source map to further reduce the package size, or restricting javascript features to an older version. The Lambda builder allows this with optional sub-properties of the `aws_sam` configuration property. * `target`: string, corresponding to a supported [esbuild target](https://esbuild.github.io/api/#target) property * `minify`: boolean, defaulting to `true` -* `sourcemap`: boolean, defaulting to `true` +* `sourcemap`: boolean, defaulting to `false` -Here is an example that deactivates minification and source maps, and supports JavaScript features compatible with Node.js version 10. +Here is an example that deactivates minification, enables source maps, and supports JavaScript features compatible with Node.js version 10. ```json { "entry_points": ["included.ts"], "target": "node10", "minify": false, - "sourcemap": false + "sourcemap": true } diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py index 76e104193..810b242dc 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py @@ -2,15 +2,17 @@ Actions specific to the esbuild bundler """ import logging -from tempfile import NamedTemporaryFile - -from pathlib import Path +from typing import Any, Dict from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError -from .esbuild import EsbuildExecutionError +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils +from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import EsbuildCommandBuilder, SubprocessEsbuild +from aws_lambda_builders.workflows.nodejs_npm_esbuild.exceptions import EsbuildExecutionError LOG = logging.getLogger(__name__) +EXTERNAL_KEY = "external" + class EsbuildBundleAction(BaseAction): @@ -23,16 +25,14 @@ class EsbuildBundleAction(BaseAction): DESCRIPTION = "Packaging source using Esbuild" PURPOSE = Purpose.COPY_SOURCE - ENTRY_POINTS = "entry_points" - def __init__( self, - scratch_dir, - artifacts_dir, - bundler_config, - osutils, - subprocess_esbuild, - subprocess_nodejs=None, + scratch_dir: str, + artifacts_dir: str, + bundler_config: Dict[str, Any], + osutils: OSUtils, + subprocess_esbuild: SubprocessEsbuild, + manifest: str, skip_deps=False, ): """ @@ -49,168 +49,58 @@ def __init__( :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild :param subprocess_esbuild: An instance of the Esbuild process wrapper - :type subprocess_nodejs: aws_lambda_builders.workflows.nodejs_npm_esbuild.node.SubprocessNodejs - :param subprocess_nodejs: An instance of the nodejs process wrapper - :type skip_deps: bool :param skip_deps: if dependencies should be omitted from bundling :type bundler_config: Dict[str,Any] :param bundler_config: the bundler configuration + + :type manifest: str + :param manifest: path to package.json file contents to read """ super(EsbuildBundleAction, self).__init__() - self.scratch_dir = scratch_dir - self.artifacts_dir = artifacts_dir - self.bundler_config = bundler_config - self.osutils = osutils - self.subprocess_esbuild = subprocess_esbuild - self.skip_deps = skip_deps - self.subprocess_nodejs = subprocess_nodejs + self._scratch_dir = scratch_dir + self._artifacts_dir = artifacts_dir + self._bundler_config = bundler_config + self._osutils = osutils + self._subprocess_esbuild = subprocess_esbuild + self._skip_deps = skip_deps + self._manifest = manifest - def execute(self): + def execute(self) -> None: """ Runs the action. :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails """ + esbuild_command = EsbuildCommandBuilder( + self._scratch_dir, self._artifacts_dir, self._bundler_config, self._osutils, self._manifest + ) - explicit_entry_points = self._construct_esbuild_entry_points() - - args = explicit_entry_points + ["--bundle", "--platform=node", "--format=cjs"] - minify = self.bundler_config.get("minify", True) - sourcemap = self.bundler_config.get("sourcemap", True) - target = self.bundler_config.get("target", "es2020") - external = self.bundler_config.get("external", []) - loader = self.bundler_config.get("loader", []) - if minify: - args.append("--minify") - if sourcemap: - args.append("--sourcemap") - if external: - args.extend(map(lambda x: f"--external:{x}", external)) - if loader: - args.extend(map(lambda x: f"--loader:{x}", loader)) - - args.append("--target={}".format(target)) - args.append("--outdir={}".format(self.artifacts_dir)) - - if self.skip_deps: - LOG.info("Running custom esbuild using Node.js") - # Don't pass externals because the esbuild.js template makes everything external - script = EsbuildBundleAction._get_node_esbuild_template( - explicit_entry_points, target, self.artifacts_dir, minify, sourcemap - ) - self._run_external_esbuild_in_nodejs(script) - return + if self._should_bundle_deps_externally(): + esbuild_command.build_with_no_dependencies() + if EXTERNAL_KEY in self._bundler_config: + # Already marking everything as external, + # shouldn't attempt to do it again when building args from config + self._bundler_config.pop(EXTERNAL_KEY) + + args = ( + esbuild_command.build_entry_points().build_default_values().build_esbuild_args_from_config().get_command() + ) try: - self.subprocess_esbuild.run(args, cwd=self.scratch_dir) + self._subprocess_esbuild.run(args, cwd=self._scratch_dir) except EsbuildExecutionError as ex: raise ActionFailedError(str(ex)) - def _run_external_esbuild_in_nodejs(self, script): - """ - Run esbuild in a separate process through Node.js - Workaround for https://github.com/evanw/esbuild/issues/1958 - - :type script: str - :param script: Node.js script to execute - - :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails - """ - with NamedTemporaryFile(dir=self.scratch_dir, mode="w") as tmp: - tmp.write(script) - tmp.flush() - try: - self.subprocess_nodejs.run([tmp.name], cwd=self.scratch_dir) - except EsbuildExecutionError as ex: - raise ActionFailedError(str(ex)) - - def _construct_esbuild_entry_points(self): - """ - Construct the list of explicit entry points - """ - if self.ENTRY_POINTS not in self.bundler_config: - raise ActionFailedError(f"{self.ENTRY_POINTS} not set ({self.bundler_config})") - - entry_points = self.bundler_config[self.ENTRY_POINTS] - - if not isinstance(entry_points, list): - raise ActionFailedError(f"{self.ENTRY_POINTS} must be a list ({self.bundler_config})") - - if not entry_points: - raise ActionFailedError(f"{self.ENTRY_POINTS} must not be empty ({self.bundler_config})") - - entry_paths = [self.osutils.joinpath(self.scratch_dir, entry_point) for entry_point in entry_points] - - LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self.artifacts_dir) - - explicit_entry_points = [] - for entry_path, entry_point in zip(entry_paths, entry_points): - explicit_entry_points.append(self._get_explicit_file_type(entry_point, entry_path)) - return explicit_entry_points - - @staticmethod - def _get_node_esbuild_template(entry_points, target, out_dir, minify, sourcemap): + def _should_bundle_deps_externally(self) -> bool: """ - Get the esbuild nodejs plugin template - - :type entry_points: List[str] - :param entry_points: list of entry points - - :type target: str - :param target: target version - - :type out_dir: str - :param out_dir: output directory to bundle into - - :type minify: bool - :param minify: if bundled code should be minified - - :type sourcemap: bool - :param sourcemap: if esbuild should produce a sourcemap + Checks if all dependencies should be marked as external and not bundled with source code - :rtype: str - :return: formatted template + :rtype: boolean + :return: True if all dependencies should be marked as external """ - curr_dir = Path(__file__).resolve().parent - with open(str(Path(curr_dir, "esbuild-plugin.js.template")), "r") as f: - input_str = f.read() - result = input_str.format( - target=target, - minify="true" if minify else "false", - sourcemap="true" if sourcemap else "false", - out_dir=repr(out_dir), - entry_points=entry_points, - ) - return result - - def _get_explicit_file_type(self, entry_point, entry_path): - """ - Get an entry point with an explicit .ts or .js suffix. - - :type entry_point: str - :param entry_point: path to entry file from code uri - - :type entry_path: str - :param entry_path: full path of entry file - - :rtype: str - :return: entry point with appropriate file extension - - :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails - """ - if Path(entry_point).suffix: - if self.osutils.file_exists(entry_path): - return entry_point - raise ActionFailedError("entry point {} does not exist".format(entry_path)) - - for ext in [".ts", ".js"]: - entry_path_with_ext = entry_path + ext - if self.osutils.file_exists(entry_path_with_ext): - return entry_point + ext - - raise ActionFailedError("entry point {} does not exist".format(entry_path)) + return self._skip_deps or "./node_modules/*" in self._bundler_config.get(EXTERNAL_KEY, []) class EsbuildCheckVersionAction(BaseAction): diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild-plugin.js.template b/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild-plugin.js.template deleted file mode 100644 index f3d2beb99..000000000 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild-plugin.js.template +++ /dev/null @@ -1,19 +0,0 @@ -let skipBundleNodeModules = {{ - name: 'make-all-packages-external', - setup(build) {{ - let filter = /^[^.\/]|^\.[^.\/]|^\.\.[^\/]/ // Must not start with "/" or "./" or "../" - build.onResolve({{ filter }}, args => ({{ path: args.path, external: true }})) - }}, -}} - -require('esbuild').build({{ - entryPoints: {entry_points}, - bundle: true, - platform: 'node', - format: 'cjs', - target: '{target}', - sourcemap: {sourcemap}, - outdir: {out_dir}, - minify: {minify}, - plugins: [skipBundleNodeModules], -}}).catch(() => process.exit(1)) diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild.py index d1bf92b25..209727720 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/esbuild.py @@ -1,26 +1,19 @@ """ Wrapper around calling esbuild through a subprocess. """ +from pathlib import Path import logging +from typing import Dict, Any, List -from aws_lambda_builders.exceptions import LambdaBuilderError +from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils +from aws_lambda_builders.workflows.nodejs_npm_esbuild.exceptions import EsbuildCommandError, EsbuildExecutionError LOG = logging.getLogger(__name__) -class EsbuildExecutionError(LambdaBuilderError): - - """ - Exception raised in case esbuild execution fails. - It will pass on the standard error output from the esbuild console. - """ - - MESSAGE = "Esbuild Failed: {message}" - - class SubprocessEsbuild(object): - """ Wrapper around the Esbuild command line utility, making it easy to consume execution results. @@ -61,7 +54,11 @@ def esbuild_binary(self): if binaries: return binaries[0] else: - raise EsbuildExecutionError(message="cannot find esbuild") + raise EsbuildExecutionError( + message="Cannot find esbuild. esbuild must be installed on the host machine to use this feature. " + "It is recommended to be installed on the PATH, " + "but can also be included as a project dependency." + ) def run(self, args, cwd=None): @@ -102,3 +99,212 @@ def run(self, args, cwd=None): raise EsbuildExecutionError(message=err.decode("utf8").strip()) return out.decode("utf8").strip() + + +# The esbuild API flags are broken up into three forms (https://esbuild.github.io/api/): +# Multi-word arguments are expected to be passed down using snake case e.g. entry_points +# Boolean types (--minify) +SUPPORTED_ESBUILD_APIS_BOOLEAN = [ + "minify", + "sourcemap", +] + +# single value types (--target=es2020) +SUPPORTED_ESBUILD_APIS_SINGLE_VALUE = [ + "target", + "format", + "main_fields", +] + +# Multi-value types (--external:axios --external:aws-sdk) +SUPPORTED_ESBUILD_APIS_MULTI_VALUE = [ + "external", + "loader", +] + + +class EsbuildCommandBuilder: + ENTRY_POINTS = "entry_points" + + def __init__( + self, scratch_dir: str, artifacts_dir: str, bundler_config: Dict[Any, Any], osutils: OSUtils, manifest: str + ): + self._scratch_dir = scratch_dir + self._artifacts_dir = artifacts_dir + self._bundler_config = bundler_config + self._osutils = osutils + self._manifest = manifest + self._command: List[str] = [] + + def get_command(self) -> List[str]: + """ + Get all of the commands flags created by the command builder + + :rtype: List[str] + :return: List of esbuild commands to be executed + """ + return self._command + + def build_esbuild_args_from_config(self) -> "EsbuildCommandBuilder": + """ + Build arguments configured in the command config (e.g. template.yaml) + + :rtype: EsbuildCommandBuilder + :return: An instance of the command builder + """ + args = [] + + args.extend(self._get_boolean_args()) + args.extend(self._get_single_value_args()) + args.extend(self._get_multi_value_args()) + + LOG.debug("Found the following args in the config: %s", str(args)) + + self._command.extend(args) + return self + + def build_entry_points(self) -> "EsbuildCommandBuilder": + """ + Build the entry points to the command + + :rtype: EsbuildCommandBuilder + :return: An instance of the command builder + """ + if self.ENTRY_POINTS not in self._bundler_config: + raise EsbuildCommandError(f"{self.ENTRY_POINTS} not set ({self._bundler_config})") + + entry_points = self._bundler_config[self.ENTRY_POINTS] + + if not isinstance(entry_points, list): + raise EsbuildCommandError(f"{self.ENTRY_POINTS} must be a list ({self._bundler_config})") + + if not entry_points: + raise EsbuildCommandError(f"{self.ENTRY_POINTS} must not be empty ({self._bundler_config})") + + entry_paths = [self._osutils.joinpath(self._scratch_dir, entry_point) for entry_point in entry_points] + + LOG.debug("NODEJS building %s using esbuild to %s", entry_paths, self._artifacts_dir) + + for entry_path, entry_point in zip(entry_paths, entry_points): + self._command.append(self._get_explicit_file_type(entry_point, entry_path)) + + return self + + def build_default_values(self) -> "EsbuildCommandBuilder": + """ + Build the default values that each call to esbuild should contain + + :rtype: EsbuildCommandBuilder + :return: An instance of the command builder + """ + args = ["--bundle", "--platform=node", "--outdir={}".format(self._artifacts_dir)] + + if "target" not in self._bundler_config: + args.append("--target=es2020") + + if "format" not in self._bundler_config: + args.append("--format=cjs") + + if "minify" not in self._bundler_config: + args.append("--minify") + + LOG.debug("Using the following default args: %s", str(args)) + + self._command.extend(args) + return self + + def build_with_no_dependencies(self) -> "EsbuildCommandBuilder": + """ + Set all dependencies located in the package.json to + external so as to not bundle them with the source code + + :rtype: EsbuildCommandBuilder + :return: An instance of the command builder + """ + package = self._osutils.parse_json(self._manifest) + dependencies = package.get("dependencies", {}).keys() + args = ["--external:{}".format(dep) for dep in dependencies] + self._command.extend(args) + return self + + def _get_boolean_args(self) -> List[str]: + """ + Get a list of all the boolean value flag types (e.g. --minify) + + :rtype: List[str] + :return: Arguments to be appended to the command list + """ + args = [] + for param in SUPPORTED_ESBUILD_APIS_BOOLEAN: + if param in self._bundler_config and self._bundler_config[param] is True: + args.append(f"--{self._convert_snake_to_kebab_case(param)}") + return args + + def _get_single_value_args(self) -> List[str]: + """ + Get a list of all the single value flag types (e.g. --target=es2020) + + :rtype: List[str] + :return: Arguments to be appended to the command list + """ + args = [] + for param in SUPPORTED_ESBUILD_APIS_SINGLE_VALUE: + if param in self._bundler_config: + value = self._bundler_config.get(param) + args.append(f"--{self._convert_snake_to_kebab_case(param)}={value}") + return args + + def _get_multi_value_args(self) -> List[str]: + """ + Get a list of all the multi-value flag types (e.g. --external:aws-sdk) + + :rtype: List[str] + :return: Arguments to be appended to the command list + """ + args = [] + for param in SUPPORTED_ESBUILD_APIS_MULTI_VALUE: + if param in self._bundler_config: + values = self._bundler_config.get(param) + if not isinstance(values, list): + raise EsbuildCommandError(f"Invalid type for property {param}, must be a dict.") + for param_item in values: + args.append(f"--{self._convert_snake_to_kebab_case(param)}:{param_item}") + return args + + def _get_explicit_file_type(self, entry_point, entry_path): + """ + Get an entry point with an explicit .ts or .js suffix. + + :type entry_point: str + :param entry_point: path to entry file from code uri + + :type entry_path: str + :param entry_path: full path of entry file + + :rtype: str + :return: entry point with appropriate file extension + + :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails + """ + if Path(entry_point).suffix: + if self._osutils.file_exists(entry_path): + return entry_point + raise ActionFailedError("entry point {} does not exist".format(entry_path)) + + for ext in [".ts", ".js"]: + entry_path_with_ext = entry_path + ext + if self._osutils.file_exists(entry_path_with_ext): + return entry_point + ext + + raise ActionFailedError("entry point {} does not exist".format(entry_path)) + + @staticmethod + def _convert_snake_to_kebab_case(arg: str) -> str: + """ + The configuration properties passed down to Lambda Builders are done so using snake case + e.g. "main_fields" but esbuild expects them using kebab-case "main-fields" + + :rtype: str + :return: mutated string to match the esbuild argument format + """ + return arg.replace("_", "-") diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/exceptions.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/exceptions.py new file mode 100644 index 000000000..56bbf1f4a --- /dev/null +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/exceptions.py @@ -0,0 +1,21 @@ +""" +Esbuild specific exceptions +""" +from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.exceptions import LambdaBuilderError + + +class EsbuildExecutionError(LambdaBuilderError): + """ + Exception raised in case esbuild execution fails. + It will pass on the standard error output from the esbuild console. + """ + + MESSAGE = "Esbuild Failed: {message}" + + +class EsbuildCommandError(ActionFailedError): + """ + Exception raised in case esbuild can't build a valid esbuild command from the given config. + It will pass on the standard error output from the esbuild console. + """ diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/node.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/node.py deleted file mode 100644 index 6a50ea495..000000000 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/node.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -Wrapper around calling nodejs through a subprocess. -""" - -import logging - -from aws_lambda_builders.exceptions import LambdaBuilderError - -LOG = logging.getLogger(__name__) - - -class NodejsExecutionError(LambdaBuilderError): - - """ - Exception raised in case nodejs execution fails. - It will pass on the standard error output from the Node.js console. - """ - - MESSAGE = "Nodejs Failed: {message}" - - -class SubprocessNodejs(object): - - """ - Wrapper around the nodejs command line utility, making it - easy to consume execution results. - """ - - def __init__(self, osutils, executable_search_paths, which): - """ - :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils - :param osutils: An instance of OS Utilities for file manipulation - - :type executable_search_paths: list - :param executable_search_paths: List of paths to the node package binary utilities. This will - be used to find embedded Nodejs at runtime if present in the package - - :type which: aws_lambda_builders.utils.which - :param which: Function to get paths which conform to the given mode on the PATH - with the prepended additional search paths - """ - self.osutils = osutils - self.executable_search_paths = executable_search_paths - self.which = which - - def nodejs_binary(self): - """ - Finds the Nodejs binary at runtime. - - The utility may be present as a package dependency of the Lambda project, - or in the global path. If there is one in the Lambda project, it should - be preferred over a global utility. The check has to be executed - at runtime, since nodejs dependencies will be installed by the workflow - using one of the previous actions. - """ - - LOG.debug("checking for nodejs in: %s", self.executable_search_paths) - binaries = self.which("node", executable_search_paths=self.executable_search_paths) - LOG.debug("potential nodejs binaries: %s", binaries) - - if binaries: - return binaries[0] - else: - raise NodejsExecutionError(message="cannot find nodejs") - - def run(self, args, cwd=None): - - """ - Runs the action. - - :type args: list - :param args: Command line arguments to pass to Nodejs - - :type cwd: str - :param cwd: Directory where to execute the command (defaults to current dir) - - :rtype: str - :return: text of the standard output from the command - - :raises aws_lambda_builders.workflows.nodejs_npm.npm.NodejsExecutionError: - when the command executes with a non-zero return code. The exception will - contain the text of the standard error output from the command. - - :raises ValueError: if arguments are not provided, or not a list - """ - - if not isinstance(args, list): - raise ValueError("args must be a list") - - if not args: - raise ValueError("requires at least one arg") - - invoke_nodejs = [self.nodejs_binary()] + args - - LOG.debug("executing Nodejs: %s", invoke_nodejs) - - p = self.osutils.popen(invoke_nodejs, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd) - - out, err = p.communicate() - - if p.returncode != 0: - raise NodejsExecutionError(message=err.decode("utf8").strip()) - - return out.decode("utf8").strip() diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py deleted file mode 100644 index 2d1aadb57..000000000 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -esbuild specific utilities and feature flag -""" - -EXPERIMENTAL_FLAG_ESBUILD = "experimentalEsbuild" - - -def is_experimental_esbuild_scope(experimental_flags): - """ - A function which will determine if experimental esbuild scope is active - """ - return bool(experimental_flags) and EXPERIMENTAL_FLAG_ESBUILD in experimental_flags diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py index fcd5de062..06171c2dc 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py @@ -10,17 +10,15 @@ from aws_lambda_builders.actions import ( CopySourceAction, CleanUpAction, - CopyDependenciesAction, MoveDependenciesAction, BaseAction, + LinkSourceAction, ) from aws_lambda_builders.utils import which from .actions import ( EsbuildBundleAction, EsbuildCheckVersionAction, ) -from .node import SubprocessNodejs -from .utils import is_experimental_esbuild_scope from .esbuild import SubprocessEsbuild, EsbuildExecutionError from ..nodejs_npm import NodejsNpmWorkflow from ..nodejs_npm.npm import SubprocessNpm @@ -61,12 +59,13 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim if not osutils.file_exists(manifest_path): LOG.warning("package.json file not found. Bundling source without dependencies.") - self.actions = [EsbuildBundleAction(source_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild)] + self.actions = [ + EsbuildBundleAction( + source_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild, self.manifest_path + ) + ] return - if not is_experimental_esbuild_scope(self.experimental_flags): - raise EsbuildExecutionError(message="Feature flag must be enabled to use this workflow") - self.actions = self.actions_with_bundler( source_dir, scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_npm, subprocess_esbuild ) @@ -105,8 +104,6 @@ def actions_with_bundler( CopySourceAction(source_dir, scratch_dir, excludes=self.EXCLUDED_FILES + tuple(["node_modules"])) ] - subprocess_node = SubprocessNodejs(osutils, self.executable_search_paths, which=which) - # Bundle dependencies separately in a dependency layer. We need to check the esbuild # version here to ensure that it supports skipping dependency bundling esbuild_no_deps = [ @@ -117,14 +114,20 @@ def actions_with_bundler( bundler_config, osutils, subprocess_esbuild, - subprocess_node, + self.manifest_path, skip_deps=True, ), ] - esbuild_with_deps = EsbuildBundleAction(scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild) + esbuild_with_deps = EsbuildBundleAction( + scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild, self.manifest_path + ) install_action = NodejsNpmWorkflow.get_install_action( - source_dir, scratch_dir, subprocess_npm, osutils, self.options, is_production=False + source_dir, + scratch_dir, + subprocess_npm, + osutils, + self.options, ) if self.download_dependencies and not self.dependencies_dir: @@ -165,13 +168,18 @@ def _accelerate_workflow_actions( actions += [install_action, CleanUpAction(self.dependencies_dir)] if self.combine_dependencies: # Auto dependency layer disabled, first build - actions += [esbuild_with_deps, CopyDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)] + actions += [ + esbuild_with_deps, + MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir), + LinkSourceAction(self.dependencies_dir, scratch_dir), + ] else: # Auto dependency layer enabled, first build actions += esbuild_no_deps + [MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)] else: if self.dependencies_dir: - actions.append(CopySourceAction(self.dependencies_dir, scratch_dir)) + actions.append(LinkSourceAction(self.dependencies_dir, scratch_dir)) + if self.combine_dependencies: # Auto dependency layer disabled, subsequent builds actions += [esbuild_with_deps] @@ -180,7 +188,10 @@ def _accelerate_workflow_actions( actions += esbuild_no_deps else: # Invalid workflow, can't have no dependency dir and no installation - raise EsbuildExecutionError(message="Lambda Builders encountered and invalid workflow") + raise EsbuildExecutionError( + message="Lambda Builders encountered an invalid workflow. A workflow can't " + "include a dependencies directory without installing dependencies." + ) return actions @@ -203,7 +214,10 @@ def get_resolvers(self): return [PathResolver(runtime=self.runtime, binary="npm")] def _get_esbuild_subprocess(self, subprocess_npm, scratch_dir, osutils) -> SubprocessEsbuild: - npm_bin_path = subprocess_npm.run(["bin"], cwd=scratch_dir) + try: + npm_bin_path = subprocess_npm.run(["bin"], cwd=scratch_dir) + except FileNotFoundError: + raise EsbuildExecutionError(message="The esbuild workflow couldn't find npm installed on your system.") executable_search_paths = [npm_bin_path] if self.executable_search_paths is not None: executable_search_paths = executable_search_paths + self.executable_search_paths diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index 19ee7656d..f5eea5882 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -11,6 +11,9 @@ import tarfile import subprocess import sys +from typing import Optional, List + +EXPERIMENTAL_FLAG_BUILD_PERFORMANCE = "experimentalBuildPerformance" class OSUtils(object): @@ -106,3 +109,10 @@ def pipe(self): def basename(self, path): # type: (str) -> str return os.path.basename(path) + + +def is_experimental_build_improvements_enabled(experimental_flags: Optional[List[str]]) -> bool: + """ + A function which will determine if experimental build improvements is active + """ + return bool(experimental_flags) and EXPERIMENTAL_FLAG_BUILD_PERFORMANCE in experimental_flags diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index 080725097..844d02fbd 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -4,11 +4,12 @@ import logging from aws_lambda_builders.workflow import BaseWorkflow, Capability -from aws_lambda_builders.actions import CopySourceAction, CleanUpAction +from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator +from aws_lambda_builders.path_resolver import PathResolver from .actions import PythonPipBuildAction -from .utils import OSUtils +from .utils import OSUtils, is_experimental_build_improvements_enabled LOG = logging.getLogger(__name__) @@ -64,6 +65,8 @@ class PythonPipWorkflow(BaseWorkflow): ".idea", ) + PYTHON_VERSION_THREE = "3" + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): super(PythonPipWorkflow, self).__init__( @@ -106,9 +109,31 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim # folder if self.dependencies_dir and self.combine_dependencies: # when copying downloaded dependencies back to artifacts folder, don't exclude anything - self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) + if is_experimental_build_improvements_enabled(self.experimental_flags): + self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir)) + else: + self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + def get_resolvers(self): + """ + Specialized Python path resolver that looks for additional binaries in addition to the language specific binary. + """ + return [ + PathResolver( + runtime=self.runtime, + binary=self.CAPABILITY.language, + additional_binaries=self._get_additional_binaries(), + executable_search_paths=self.executable_search_paths, + ) + ] + + def _get_additional_binaries(self): + # python3 is an additional binary that has to be considered in addition to the original python binary, when + # the specified python runtime is 3.x + major, _ = self.runtime.replace(self.CAPABILITY.language, "").split(".") + return [f"{self.CAPABILITY.language}{major}"] if major == self.PYTHON_VERSION_THREE else None + def get_validators(self): return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/tests/integration/workflows/custom_make/test_custom_make.py b/tests/integration/workflows/custom_make/test_custom_make.py index 1f641178e..4bd31bef0 100644 --- a/tests/integration/workflows/custom_make/test_custom_make.py +++ b/tests/integration/workflows/custom_make/test_custom_make.py @@ -57,6 +57,48 @@ def test_must_build_python_project_through_makefile(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + def test_build_python_project_failed_through_makefile_no_python_source_in_default_working_directory(self): + source_code = os.path.join(os.path.dirname(__file__), "testdata", "makefile-in-different-working-directory") + manifest_path_valid = os.path.join(source_code, "Makefile") + with self.assertRaises(WorkflowFailedError): + self.builder.build( + source_code, + self.artifacts_dir, + self.scratch_dir, + manifest_path_valid, + runtime=self.runtime, + options={"build_logical_id": "HelloWorldFunction"}, + ) + + def test_must_build_python_project_through_makefile_with_custom_working_directory(self): + source_code = os.path.join(os.path.dirname(__file__), "testdata", "makefile-in-different-working-directory") + manifest_path_valid = os.path.join(source_code, "Makefile") + working_directory = os.path.join(source_code, "source_code") + self.builder.build( + source_code, + self.artifacts_dir, + self.scratch_dir, + manifest_path_valid, + runtime=self.runtime, + options={"build_logical_id": "HelloWorldFunction", "working_directory": working_directory}, + ) + dependencies_installed = { + "chardet", + "urllib3", + "idna", + "urllib3-1.25.11.dist-info", + "chardet-3.0.4.dist-info", + "certifi-2020.4.5.2.dist-info", + "certifi", + "idna-2.10.dist-info", + "requests", + "requests-2.23.0.dist-info", + } + + expected_files = self.test_data_files.union(dependencies_installed) + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + def test_must_build_python_project_through_makefile_unknown_target(self): with self.assertRaises(WorkflowFailedError): self.builder.build( diff --git a/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/Makefile b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/Makefile new file mode 100644 index 000000000..0133698ab --- /dev/null +++ b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/Makefile @@ -0,0 +1,5 @@ +build-HelloWorldFunction: + cp *.py $(ARTIFACTS_DIR) + cp requirements-requests.txt $(ARTIFACTS_DIR) + python -m pip install -r requirements-requests.txt -t $(ARTIFACTS_DIR) + rm -rf $(ARTIFACTS_DIR)/bin diff --git a/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/__init__.py b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/main.py b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/main.py new file mode 100644 index 000000000..dc2b47454 --- /dev/null +++ b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/main.py @@ -0,0 +1,6 @@ +import requests + + +def lambda_handler(event, context): + # Just return the requests version. + return "{}".format(requests.__version__) diff --git a/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/requirements-requests.txt b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/requirements-requests.txt new file mode 100644 index 000000000..a0111f46d --- /dev/null +++ b/tests/integration/workflows/custom_make/testdata/makefile-in-different-working-directory/source_code/requirements-requests.txt @@ -0,0 +1,4 @@ +requests==2.23.0 + +# Pinning so the test can expect a given version +certifi==2020.4.5.2 # dep of requests \ No newline at end of file diff --git a/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py b/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py index 38448a990..a22ecb025 100644 --- a/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py +++ b/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py @@ -2,12 +2,13 @@ import shutil import tempfile from unittest import TestCase +from unittest.mock import patch + from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError from aws_lambda_builders.workflows.nodejs_npm.npm import SubprocessNpm from aws_lambda_builders.workflows.nodejs_npm.utils import OSUtils from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import EsbuildExecutionError -from aws_lambda_builders.workflows.nodejs_npm_esbuild.utils import EXPERIMENTAL_FLAG_ESBUILD from parameterized import parameterized @@ -22,29 +23,21 @@ def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() self.dependencies_dir = tempfile.mkdtemp() - self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps-esbuild") - self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm-esbuild", application_framework=None) + self.osutils = OSUtils() + self._set_esbuild_binary_path() + + def _set_esbuild_binary_path(self): + npm = SubprocessNpm(self.osutils) + esbuild_dir = os.path.join(self.TEST_DATA_FOLDER, "esbuild-binary") + npm.run(["ci"], cwd=esbuild_dir) + self.binpath = npm.run(["bin"], cwd=esbuild_dir) def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) - @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)]) - def test_doesnt_build_without_feature_flag(self, runtime): - source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") - - with self.assertRaises(EsbuildExecutionError) as context: - self.builder.build( - source_dir, - self.artifacts_dir, - self.scratch_dir, - os.path.join(source_dir, "package.json"), - runtime=runtime, - ) - self.assertEqual(str(context.exception), "Esbuild Failed: Feature flag must be enabled to use this workflow") - @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)]) def test_builds_javascript_project_with_dependencies(self, runtime): source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") @@ -58,10 +51,11 @@ def test_builds_javascript_project_with_dependencies(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js", "included.js.map"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -78,10 +72,11 @@ def test_builds_javascript_project_with_multiple_entrypoints(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js", "included.js.map", "included2.js", "included2.js.map"} + expected_files = {"included.js", "included2.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -98,24 +93,17 @@ def test_builds_typescript_projects(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js", "included.js.map"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)]) def test_builds_with_external_esbuild(self, runtime): - osutils = OSUtils() - npm = SubprocessNpm(osutils) source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-esbuild") - esbuild_dir = os.path.join(self.TEST_DATA_FOLDER, "esbuild-binary") - - npm.run(["ci"], cwd=esbuild_dir) - - binpath = npm.run(["bin"], cwd=esbuild_dir) - options = {"entry_points": ["included.js"]} self.builder.build( @@ -125,11 +113,11 @@ def test_builds_with_external_esbuild(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - executable_search_paths=[binpath], - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js", "included.js.map"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -144,7 +132,8 @@ def test_no_options_passed_to_esbuild(self, runtime): self.scratch_dir, os.path.join(source_dir, "package.json"), runtime=runtime, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) self.assertEqual(str(context.exception), "NodejsNpmEsbuildBuilder:EsbuildBundle - entry_points not set ({})") @@ -162,10 +151,11 @@ def test_bundle_with_implicit_file_types(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js.map", "implicit.js.map", "implicit.js", "included.js"} + expected_files = {"implicit.js", "included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -187,11 +177,11 @@ def test_bundles_project_without_dependencies(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], executable_search_paths=[binpath], ) - expected_files = {"included.js.map", "included.js"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -215,11 +205,11 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_w runtime=runtime, dependencies_dir=self.dependencies_dir, download_dependencies=False, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], executable_search_paths=[binpath], ) - expected_files = {"included.js.map", "included.js"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -237,10 +227,11 @@ def test_builds_project_with_remote_dependencies_with_download_dependencies_and_ options=options, dependencies_dir=self.dependencies_dir, download_dependencies=True, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js.map", "included.js"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -267,10 +258,15 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_w runtime=runtime, dependencies_dir=None, download_dependencies=False, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - self.assertEqual(str(context.exception), "Esbuild Failed: Lambda Builders encountered and invalid workflow") + self.assertEqual( + str(context.exception), + "Esbuild Failed: Lambda Builders encountered an invalid workflow. A" + " workflow can't include a dependencies directory without installing dependencies.", + ) @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)]) def test_builds_project_without_combine_dependencies(self, runtime): @@ -287,10 +283,11 @@ def test_builds_project_without_combine_dependencies(self, runtime): dependencies_dir=self.dependencies_dir, download_dependencies=True, combine_dependencies=False, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js.map", "included.js"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -315,10 +312,11 @@ def test_builds_javascript_project_with_external(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js", "included.js.map"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) with open(str(os.path.join(self.artifacts_dir, "included.js"))) as f: @@ -340,10 +338,11 @@ def test_builds_javascript_project_with_loader(self, runtime): os.path.join(source_dir, "package.json"), runtime=runtime, options=options, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], + executable_search_paths=[self.binpath], ) - expected_files = {"included.js", "included.js.map"} + expected_files = {"included.js"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -366,3 +365,24 @@ def test_builds_javascript_project_with_loader(self, runtime): "\turania: astronomy and astrology" ), ) + + @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",)]) + def test_includes_sourcemap_if_requested(self, runtime): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") + + options = {"entry_points": ["included.js"], "sourcemap": True} + + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=runtime, + options=options, + experimental_flags=[], + executable_search_paths=[self.binpath], + ) + + expected_files = {"included.js", "included.js.map"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) diff --git a/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package-lock.json b/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package-lock.json index 9ce0cc19a..9a03e1592 100644 --- a/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package-lock.json +++ b/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package-lock.json @@ -5,27 +5,518 @@ "requires": true, "packages": { "": { + "name": "esbuild-binary", "version": "1.0.0", "license": "ISC", "dependencies": { - "esbuild": "^0.11.23" + "esbuild": "^0.14.13" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", + "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", + "cpu": [ + "loong64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" } }, "node_modules/esbuild": { - "version": "0.11.23", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", - "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==", + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.14.54.tgz", + "integrity": "sha512-Cy9llcy8DvET5uznocPyqL3BFRrFXSVqbgpMJ9Wz8oVjZlh/zUSNbPRbov0VX7VxN2JH1Oa0uNxZ7eLRb62pJA==", "hasInstallScript": true, "bin": { "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/linux-loong64": "0.14.54", + "esbuild-android-64": "0.14.54", + "esbuild-android-arm64": "0.14.54", + "esbuild-darwin-64": "0.14.54", + "esbuild-darwin-arm64": "0.14.54", + "esbuild-freebsd-64": "0.14.54", + "esbuild-freebsd-arm64": "0.14.54", + "esbuild-linux-32": "0.14.54", + "esbuild-linux-64": "0.14.54", + "esbuild-linux-arm": "0.14.54", + "esbuild-linux-arm64": "0.14.54", + "esbuild-linux-mips64le": "0.14.54", + "esbuild-linux-ppc64le": "0.14.54", + "esbuild-linux-riscv64": "0.14.54", + "esbuild-linux-s390x": "0.14.54", + "esbuild-netbsd-64": "0.14.54", + "esbuild-openbsd-64": "0.14.54", + "esbuild-sunos-64": "0.14.54", + "esbuild-windows-32": "0.14.54", + "esbuild-windows-64": "0.14.54", + "esbuild-windows-arm64": "0.14.54" + } + }, + "node_modules/esbuild-android-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.14.54.tgz", + "integrity": "sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-android-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.54.tgz", + "integrity": "sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-darwin-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.14.54.tgz", + "integrity": "sha512-jtdKWV3nBviOd5v4hOpkVmpxsBy90CGzebpbO9beiqUYVMBtSc0AL9zGftFuBon7PNDcdvNCEuQqw2x0wP9yug==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-darwin-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.54.tgz", + "integrity": "sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-freebsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.54.tgz", + "integrity": "sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-freebsd-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.54.tgz", + "integrity": "sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.54.tgz", + "integrity": "sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.54.tgz", + "integrity": "sha512-EgjAgH5HwTbtNsTqQOXWApBaPVdDn7XcK+/PtJwZLT1UmpLoznPd8c5CxqsH2dQK3j05YsB3L17T8vE7cp4cCg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-arm": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.54.tgz", + "integrity": "sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.54.tgz", + "integrity": "sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-mips64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.54.tgz", + "integrity": "sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw==", + "cpu": [ + "mips64el" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-ppc64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.54.tgz", + "integrity": "sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ==", + "cpu": [ + "ppc64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-riscv64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.54.tgz", + "integrity": "sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg==", + "cpu": [ + "riscv64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-s390x": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.54.tgz", + "integrity": "sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-netbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.54.tgz", + "integrity": "sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-openbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.54.tgz", + "integrity": "sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-sunos-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.54.tgz", + "integrity": "sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.54.tgz", + "integrity": "sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.54.tgz", + "integrity": "sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.54.tgz", + "integrity": "sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" } } }, "dependencies": { + "@esbuild/linux-loong64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", + "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", + "optional": true + }, "esbuild": { - "version": "0.11.23", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.11.23.tgz", - "integrity": "sha512-iaiZZ9vUF5wJV8ob1tl+5aJTrwDczlvGP0JoMmnpC2B0ppiMCu8n8gmy5ZTGl5bcG081XBVn+U+jP+mPFm5T5Q==" + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.14.54.tgz", + "integrity": "sha512-Cy9llcy8DvET5uznocPyqL3BFRrFXSVqbgpMJ9Wz8oVjZlh/zUSNbPRbov0VX7VxN2JH1Oa0uNxZ7eLRb62pJA==", + "requires": { + "@esbuild/linux-loong64": "0.14.54", + "esbuild-android-64": "0.14.54", + "esbuild-android-arm64": "0.14.54", + "esbuild-darwin-64": "0.14.54", + "esbuild-darwin-arm64": "0.14.54", + "esbuild-freebsd-64": "0.14.54", + "esbuild-freebsd-arm64": "0.14.54", + "esbuild-linux-32": "0.14.54", + "esbuild-linux-64": "0.14.54", + "esbuild-linux-arm": "0.14.54", + "esbuild-linux-arm64": "0.14.54", + "esbuild-linux-mips64le": "0.14.54", + "esbuild-linux-ppc64le": "0.14.54", + "esbuild-linux-riscv64": "0.14.54", + "esbuild-linux-s390x": "0.14.54", + "esbuild-netbsd-64": "0.14.54", + "esbuild-openbsd-64": "0.14.54", + "esbuild-sunos-64": "0.14.54", + "esbuild-windows-32": "0.14.54", + "esbuild-windows-64": "0.14.54", + "esbuild-windows-arm64": "0.14.54" + } + }, + "esbuild-android-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.14.54.tgz", + "integrity": "sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==", + "optional": true + }, + "esbuild-android-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.54.tgz", + "integrity": "sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg==", + "optional": true + }, + "esbuild-darwin-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.14.54.tgz", + "integrity": "sha512-jtdKWV3nBviOd5v4hOpkVmpxsBy90CGzebpbO9beiqUYVMBtSc0AL9zGftFuBon7PNDcdvNCEuQqw2x0wP9yug==", + "optional": true + }, + "esbuild-darwin-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.54.tgz", + "integrity": "sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw==", + "optional": true + }, + "esbuild-freebsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.54.tgz", + "integrity": "sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg==", + "optional": true + }, + "esbuild-freebsd-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.54.tgz", + "integrity": "sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q==", + "optional": true + }, + "esbuild-linux-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.54.tgz", + "integrity": "sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw==", + "optional": true + }, + "esbuild-linux-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.54.tgz", + "integrity": "sha512-EgjAgH5HwTbtNsTqQOXWApBaPVdDn7XcK+/PtJwZLT1UmpLoznPd8c5CxqsH2dQK3j05YsB3L17T8vE7cp4cCg==", + "optional": true + }, + "esbuild-linux-arm": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.54.tgz", + "integrity": "sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw==", + "optional": true + }, + "esbuild-linux-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.54.tgz", + "integrity": "sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig==", + "optional": true + }, + "esbuild-linux-mips64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.54.tgz", + "integrity": "sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw==", + "optional": true + }, + "esbuild-linux-ppc64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.54.tgz", + "integrity": "sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ==", + "optional": true + }, + "esbuild-linux-riscv64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.54.tgz", + "integrity": "sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg==", + "optional": true + }, + "esbuild-linux-s390x": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.54.tgz", + "integrity": "sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA==", + "optional": true + }, + "esbuild-netbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.54.tgz", + "integrity": "sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w==", + "optional": true + }, + "esbuild-openbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.54.tgz", + "integrity": "sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw==", + "optional": true + }, + "esbuild-sunos-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.54.tgz", + "integrity": "sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw==", + "optional": true + }, + "esbuild-windows-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.54.tgz", + "integrity": "sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w==", + "optional": true + }, + "esbuild-windows-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.54.tgz", + "integrity": "sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ==", + "optional": true + }, + "esbuild-windows-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.54.tgz", + "integrity": "sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg==", + "optional": true } } } diff --git a/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package.json b/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package.json index eb70cca6e..d1cd0a40c 100644 --- a/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package.json +++ b/tests/integration/workflows/nodejs_npm_esbuild/testdata/esbuild-binary/package.json @@ -9,6 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "esbuild": "^0.11.23" + "esbuild": "^0.14.13" } } diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index ce6bc4aba..3e1fe6cb3 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -1,27 +1,34 @@ import os +import pathlib import shutil import sys import platform import tempfile from unittest import TestCase, skipIf import mock +from parameterized import parameterized_class from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError import logging +from aws_lambda_builders.utils import which +from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE + logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") IS_WINDOWS = platform.system().lower() == "windows" NOT_ARM = platform.processor() != "aarch64" ARM_RUNTIMES = {"python3.8", "python3.9"} +@parameterized_class(("experimental_flags",), [([]), ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE])]) class TestPythonPipWorkflow(TestCase): """ Verifies that `python_pip` workflow works by building a Lambda that requires Numpy """ TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") + experimental_flags = [] def setUp(self): self.source_dir = self.TEST_DATA_FOLDER @@ -42,7 +49,10 @@ def setUp(self): "local-dependencies", } - self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None) + self.dependency_manager = "pip" + self.builder = LambdaBuilder( + language="python", dependency_manager=self.dependency_manager, application_framework=None + ) self.runtime = "{language}{major}.{minor}".format( language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor ) @@ -75,7 +85,38 @@ def check_architecture_in(self, library, architectures): def test_must_build_python_project(self): self.builder.build( - self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + ) + + if self.runtime == "python3.6": + self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) + else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + def test_must_build_python_project_python3_binary(self): + python_paths = which("python") + executable_dir = pathlib.Path(tempfile.gettempdir()) + new_python_path = executable_dir.joinpath("python3") + os.symlink(python_paths[0], new_python_path) + # Build with access to the newly symlinked python3 binary. + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + executable_search_paths=[executable_dir], ) if self.runtime == "python3.6": @@ -87,6 +128,7 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + os.unlink(new_python_path) @skipIf(NOT_ARM, "Skip if not running on ARM64") def test_must_build_python_project_from_sdist_with_arm(self): @@ -100,6 +142,7 @@ def test_must_build_python_project_from_sdist_with_arm(self): self.manifest_path_sdist, runtime=self.runtime, architecture="arm64", + experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files.union({"wrapt", "wrapt-1.13.3.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) @@ -118,6 +161,7 @@ def test_must_build_python_project_with_arm_architecture(self): self.manifest_path_valid, runtime=self.runtime, architecture="arm64", + experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) @@ -135,6 +179,7 @@ def test_mismatch_runtime_python_project(self): self.scratch_dir, self.manifest_path_valid, runtime=self.runtime_mismatch[self.runtime], + experimental_flags=self.experimental_flags, ) except WorkflowFailedError as ex: # handle both e.g. missing /usr/bin/python2.7 and situation where @@ -162,7 +207,14 @@ def test_must_resolve_local_dependency(self): # need to make sure the correct path is used in the requirements file locally and in CI with open(manifest, "w") as f: f.write(str(path_to_package)) - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest, runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + manifest, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + ) expected_files = { "local_package", "local_package-0.0.0.dist-info", @@ -179,7 +231,12 @@ def test_must_fail_to_resolve_dependencies(self): with self.assertRaises(WorkflowFailedError) as ctx: self.builder.build( - self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid, runtime=self.runtime + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_invalid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, ) message_in_exception = "Invalid requirement: 'boto3=1.19.99'" in str(ctx.exception) @@ -193,6 +250,7 @@ def test_must_log_warning_if_requirements_not_found(self): self.scratch_dir, os.path.join("non", "existent", "manifest"), runtime=self.runtime, + experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files output_files = set(os.listdir(self.artifacts_dir)) @@ -218,6 +276,7 @@ def test_without_download_dependencies_with_dependencies_dir(self): runtime=self.runtime, download_dependencies=False, dependencies_dir=self.dependencies_dir, + experimental_flags=self.experimental_flags, ) # if download_dependencies is False and dependencies is empty, the artifacts_dir should just copy files from @@ -243,6 +302,7 @@ def test_with_download_dependencies_and_dependencies_dir(self): runtime=self.runtime, download_dependencies=True, dependencies_dir=self.dependencies_dir, + experimental_flags=self.experimental_flags, ) # build artifact should be same as usual @@ -287,6 +347,7 @@ def test_without_download_dependencies_without_dependencies_dir(self): runtime=self.runtime, download_dependencies=False, dependencies_dir=None, + experimental_flags=self.experimental_flags, ) # if download_dependencies is False and dependencies is None, the artifacts_dir should just copy files from @@ -318,6 +379,7 @@ def test_without_combine_dependencies(self): download_dependencies=True, dependencies_dir=self.dependencies_dir, combine_dependencies=False, + experimental_flags=self.experimental_flags, ) expected_files = os.listdir(source_dir) diff --git a/tests/unit/test_path_resolver.py b/tests/unit/test_path_resolver.py index 20d52ae7b..b576af7e4 100644 --- a/tests/unit/test_path_resolver.py +++ b/tests/unit/test_path_resolver.py @@ -9,11 +9,12 @@ class TestPathResolver(TestCase): def setUp(self): - self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti") + self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti", additional_binaries=["chitti2"]) def test_inits(self): self.assertEqual(self.path_resolver.runtime, "chitti2.0") self.assertEqual(self.path_resolver.binary, "chitti") + self.assertEqual(self.path_resolver.executables, ["chitti2.0", "chitti", "chitti2"]) def test_which_fails(self): with self.assertRaises(ValueError): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 000000000..97d2ec16f --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,32 @@ +from unittest import TestCase +from unittest.mock import patch + +from aws_lambda_builders import utils + + +class Test_create_symlink_or_copy(TestCase): + @patch("aws_lambda_builders.utils.Path") + @patch("aws_lambda_builders.utils.os") + @patch("aws_lambda_builders.utils.copytree") + def test_must_create_symlink_with_absolute_path(self, patched_copy_tree, pathced_os, patched_path): + source_path = "source/path" + destination_path = "destination/path" + utils.create_symlink_or_copy(source_path, destination_path) + + pathced_os.symlink.assert_called_with( + patched_path(source_path).absolute(), patched_path(destination_path).absolute() + ) + patched_copy_tree.assert_not_called() + + @patch("aws_lambda_builders.utils.Path") + @patch("aws_lambda_builders.utils.os") + @patch("aws_lambda_builders.utils.copytree") + def test_must_copy_if_symlink_fails(self, patched_copy_tree, pathced_os, patched_path): + pathced_os.symlink.side_effect = OSError("Unable to create symlink") + + source_path = "source/path" + destination_path = "destination/path" + utils.create_symlink_or_copy(source_path, destination_path) + + pathced_os.symlink.assert_called_once() + patched_copy_tree.assert_called_with(source_path, destination_path) diff --git a/tests/unit/workflows/custom_make/test_actions.py b/tests/unit/workflows/custom_make/test_actions.py index 7bcc4d83f..6f1bdd432 100644 --- a/tests/unit/workflows/custom_make/test_actions.py +++ b/tests/unit/workflows/custom_make/test_actions.py @@ -43,6 +43,32 @@ def test_call_makefile_target(self, OSUtilMock, SubprocessMakeMock): ["--makefile", "manifest", "build-logical_id"], cwd="scratch_dir", env=ANY ) + @patch("aws_lambda_builders.workflows.custom_make.utils.OSUtils") + @patch("aws_lambda_builders.workflows.custom_make.make.SubProcessMake") + def test_call_makefile_target_with_working_directory(self, OSUtilMock, SubprocessMakeMock): + osutils = OSUtilMock.return_value + subprocess_make = SubprocessMakeMock.return_value + + action = CustomMakeAction( + "artifacts", + "scratch_dir", + "manifest", + osutils=osutils, + subprocess_make=subprocess_make, + build_logical_id="logical_id", + working_directory="working_dir", + ) + + osutils.dirname.side_effect = lambda value: "/dir:{}".format(value) + osutils.abspath.side_effect = lambda value: "/abs:{}".format(value) + osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) + + action.execute() + + subprocess_make.run.assert_called_with( + ["--makefile", "manifest", "build-logical_id"], cwd="working_dir", env=ANY + ) + @patch("aws_lambda_builders.workflows.custom_make.utils.OSUtils") @patch("aws_lambda_builders.workflows.custom_make.make.SubProcessMake") def test_makefile_target_fails(self, OSUtilMock, SubprocessMakeMock): diff --git a/tests/unit/workflows/custom_make/test_workflow.py b/tests/unit/workflows/custom_make/test_workflow.py index 8775c11b0..39f58abcd 100644 --- a/tests/unit/workflows/custom_make/test_workflow.py +++ b/tests/unit/workflows/custom_make/test_workflow.py @@ -1,4 +1,5 @@ from unittest import TestCase +from unittest.mock import patch from aws_lambda_builders.architecture import X86_64, ARM64 from aws_lambda_builders.actions import CopySourceAction @@ -46,3 +47,20 @@ def test_must_validate_architecture(self): self.assertEqual(workflow.architecture, "x86_64") self.assertEqual(workflow_with_arm.architecture, "arm64") + + def test_use_scratch_dir_as_working_directory(self): + workflow = CustomMakeWorkflow( + "source", "artifacts", "scratch_dir", "manifest", options={"build_logical_id": "hello"} + ) + self.assertEqual(workflow.actions[1].working_directory, "scratch_dir") + + def test_use_working_directory(self): + workflow = CustomMakeWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + options={"build_logical_id": "hello", "working_directory": "working/dir/path"}, + ) + + self.assertEqual(workflow.actions[1].working_directory, "working/dir/path") diff --git a/tests/unit/workflows/nodejs_npm/test_actions.py b/tests/unit/workflows/nodejs_npm/test_actions.py index 7b0ff51ba..60ff528ef 100644 --- a/tests/unit/workflows/nodejs_npm/test_actions.py +++ b/tests/unit/workflows/nodejs_npm/test_actions.py @@ -64,19 +64,7 @@ def test_installs_npm_production_dependencies_for_npm_project(self, SubprocessNp action.execute() - expected_args = ["install", "-q", "--no-audit", "--no-save", "--production", "--unsafe-perm"] - - subprocess_npm.run.assert_called_with(expected_args, cwd="artifacts") - - @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") - def test_can_set_mode(self, SubprocessNpmMock): - subprocess_npm = SubprocessNpmMock.return_value - - action = NodejsNpmInstallAction("artifacts", subprocess_npm=subprocess_npm, is_production=False) - - action.execute() - - expected_args = ["install", "-q", "--no-audit", "--no-save", "--production=false", "--unsafe-perm"] + expected_args = ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"] subprocess_npm.run.assert_called_with(expected_args, cwd="artifacts") diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py b/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py index c6751a6e4..d08db7d4b 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py @@ -1,5 +1,3 @@ -import tempfile -from pathlib import Path from unittest import TestCase from unittest.mock import Mock @@ -13,16 +11,16 @@ class TestEsbuildBundleAction(TestCase): @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild") - @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.node.SubprocessNodejs") - def setUp(self, OSUtilMock, SubprocessEsbuildMock, SubprocessNodejsMock): + def setUp(self, OSUtilMock, SubprocessEsbuildMock): self.osutils = OSUtilMock.return_value self.subprocess_esbuild = SubprocessEsbuildMock.return_value - self.subprocess_nodejs = SubprocessNodejsMock.return_value self.osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) self.osutils.file_exists.side_effect = [True, True] def test_raises_error_if_entrypoints_not_specified(self): - action = EsbuildBundleAction("source", "artifacts", {"config": "param"}, self.osutils, self.subprocess_esbuild) + action = EsbuildBundleAction( + "source", "artifacts", {"config": "param"}, self.osutils, self.subprocess_esbuild, "package.json" + ) with self.assertRaises(ActionFailedError) as raised: action.execute() @@ -30,7 +28,12 @@ def test_raises_error_if_entrypoints_not_specified(self): def test_raises_error_if_entrypoints_not_a_list(self): action = EsbuildBundleAction( - "source", "artifacts", {"config": "param", "entry_points": "abc"}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"config": "param", "entry_points": "abc"}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) with self.assertRaises(ActionFailedError) as raised: action.execute() @@ -41,7 +44,12 @@ def test_raises_error_if_entrypoints_not_a_list(self): def test_raises_error_if_entrypoints_empty_list(self): action = EsbuildBundleAction( - "source", "artifacts", {"config": "param", "entry_points": []}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"config": "param", "entry_points": []}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) with self.assertRaises(ActionFailedError) as raised: action.execute() @@ -52,7 +60,12 @@ def test_raises_error_if_entrypoints_empty_list(self): def test_packages_javascript_with_minification_and_sourcemap(self): action = EsbuildBundleAction( - "source", "artifacts", {"entry_points": ["x.js"]}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"entry_points": ["x.js"], "sourcemap": True}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) action.execute() @@ -61,11 +74,11 @@ def test_packages_javascript_with_minification_and_sourcemap(self): "x.js", "--bundle", "--platform=node", + "--outdir=artifacts", + "--target=es2020", "--format=cjs", "--minify", "--sourcemap", - "--target=es2020", - "--outdir=artifacts", ], cwd="source", ) @@ -77,6 +90,7 @@ def test_packages_with_externals(self): {"entry_points": ["x.js"], "external": ["fetch", "aws-sdk"]}, self.osutils, self.subprocess_esbuild, + "", ) action.execute() self.subprocess_esbuild.run.assert_called_with( @@ -84,13 +98,12 @@ def test_packages_with_externals(self): "x.js", "--bundle", "--platform=node", + "--outdir=artifacts", + "--target=es2020", "--format=cjs", "--minify", - "--sourcemap", "--external:fetch", "--external:aws-sdk", - "--target=es2020", - "--outdir=artifacts", ], cwd="source", ) @@ -102,6 +115,7 @@ def test_packages_with_custom_loaders(self): {"entry_points": ["x.js"], "loader": [".proto=text", ".json=js"]}, self.osutils, self.subprocess_esbuild, + "", ) action.execute() self.subprocess_esbuild.run.assert_called_with( @@ -109,13 +123,12 @@ def test_packages_with_custom_loaders(self): "x.js", "--bundle", "--platform=node", + "--outdir=artifacts", + "--target=es2020", "--format=cjs", "--minify", - "--sourcemap", "--loader:.proto=text", "--loader:.json=js", - "--target=es2020", - "--outdir=artifacts", ], cwd="source", ) @@ -123,7 +136,7 @@ def test_packages_with_custom_loaders(self): def test_checks_if_single_entrypoint_exists(self): action = EsbuildBundleAction( - "source", "artifacts", {"entry_points": ["x.js"]}, self.osutils, self.subprocess_esbuild + "source", "artifacts", {"entry_points": ["x.js"]}, self.osutils, self.subprocess_esbuild, "package.json" ) self.osutils.file_exists.side_effect = [False] @@ -138,7 +151,12 @@ def test_checks_if_multiple_entrypoints_exist(self): self.osutils.file_exists.side_effect = [True, False] action = EsbuildBundleAction( - "source", "artifacts", {"entry_points": ["x.js", "y.js"]}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"entry_points": ["x.js", "y.js"]}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) with self.assertRaises(ActionFailedError) as raised: @@ -152,7 +170,12 @@ def test_checks_if_multiple_entrypoints_exist(self): def test_excludes_sourcemap_if_requested(self): action = EsbuildBundleAction( - "source", "artifacts", {"entry_points": ["x.js"], "sourcemap": False}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"entry_points": ["x.js"], "sourcemap": False}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) action.execute() self.subprocess_esbuild.run.assert_called_with( @@ -160,17 +183,22 @@ def test_excludes_sourcemap_if_requested(self): "x.js", "--bundle", "--platform=node", + "--outdir=artifacts", + "--target=es2020", "--format=cjs", "--minify", - "--target=es2020", - "--outdir=artifacts", ], cwd="source", ) def test_does_not_minify_if_requested(self): action = EsbuildBundleAction( - "source", "artifacts", {"entry_points": ["x.js"], "minify": False}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"entry_points": ["x.js"], "minify": False}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) action.execute() self.subprocess_esbuild.run.assert_called_with( @@ -178,17 +206,21 @@ def test_does_not_minify_if_requested(self): "x.js", "--bundle", "--platform=node", - "--format=cjs", - "--sourcemap", - "--target=es2020", "--outdir=artifacts", + "--target=es2020", + "--format=cjs", ], cwd="source", ) def test_uses_specified_target(self): action = EsbuildBundleAction( - "source", "artifacts", {"entry_points": ["x.js"], "target": "node14"}, self.osutils, self.subprocess_esbuild + "source", + "artifacts", + {"entry_points": ["x.js"], "target": "node14"}, + self.osutils, + self.subprocess_esbuild, + "package.json", ) action.execute() self.subprocess_esbuild.run.assert_called_with( @@ -196,11 +228,10 @@ def test_uses_specified_target(self): "x.js", "--bundle", "--platform=node", + "--outdir=artifacts", "--format=cjs", "--minify", - "--sourcemap", "--target=node14", - "--outdir=artifacts", ], cwd="source", ) @@ -212,6 +243,7 @@ def test_includes_multiple_entry_points_if_requested(self): {"entry_points": ["x.js", "y.js"], "target": "node14"}, self.osutils, self.subprocess_esbuild, + "package.json", ) action.execute() self.subprocess_esbuild.run.assert_called_with( @@ -220,91 +252,46 @@ def test_includes_multiple_entry_points_if_requested(self): "y.js", "--bundle", "--platform=node", + "--outdir=artifacts", "--format=cjs", "--minify", - "--sourcemap", "--target=node14", - "--outdir=artifacts", ], cwd="source", ) - def test_runs_node_subprocess_if_deps_skipped(self): + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_includes_building_with_external_dependencies(self, osutils_mock): + osutils_mock.parse_json.return_value = { + "dependencies": {"@faker-js/faker": "7.1.0", "uuidv4": "^6.2.12", "axios": "0.0.0"} + } action = EsbuildBundleAction( - tempfile.mkdtemp(), + "source", "artifacts", - {"entry_points": ["app.ts"]}, - self.osutils, + {"entry_points": ["x.js", "y.js"], "target": "node14", "external": "./node_modules/*"}, + osutils_mock, self.subprocess_esbuild, - self.subprocess_nodejs, - True, + "package.json", ) action.execute() - self.subprocess_nodejs.run.assert_called() - - def test_reads_nodejs_bundle_template_file(self): - template = EsbuildBundleAction._get_node_esbuild_template(["app.ts"], "es2020", "outdir", False, True) - expected_template = """let skipBundleNodeModules = { - name: 'make-all-packages-external', - setup(build) { - let filter = /^[^.\/]|^\.[^.\/]|^\.\.[^\/]/ // Must not start with "/" or "./" or "../" - build.onResolve({ filter }, args => ({ path: args.path, external: true })) - }, -} - -require('esbuild').build({ - entryPoints: ['app.ts'], - bundle: true, - platform: 'node', - format: 'cjs', - target: 'es2020', - sourcemap: true, - outdir: 'outdir', - minify: false, - plugins: [skipBundleNodeModules], -}).catch(() => process.exit(1)) -""" - self.assertEqual(template, expected_template) - - -class TestImplicitFileTypeResolution(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") - @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild") - def setUp(self, OSUtilMock, SubprocessEsbuildMock): - self.osutils = OSUtilMock.return_value - self.subprocess_esbuild = SubprocessEsbuildMock.return_value - self.action = EsbuildBundleAction( - "source", - "artifacts", - {}, - self.osutils, - self.subprocess_esbuild, + self.assertNotIn("external", action._bundler_config) + self.subprocess_esbuild.run.assert_called_with( + [ + "--external:@faker-js/faker", + "--external:uuidv4", + "--external:axios", + "x.js", + "y.js", + "--bundle", + "--platform=node", + "--outdir=artifacts", + "--format=cjs", + "--minify", + "--target=node14", + ], + cwd="source", ) - @parameterized.expand( - [ - ([True], "file.ts", "file.ts"), - ([False, True], "file", "file.js"), - ([True], "file", "file.ts"), - ] - ) - def test_implicit_and_explicit_file_types(self, file_exists, entry_point, expected): - self.osutils.file_exists.side_effect = file_exists - explicit_entry_point = self.action._get_explicit_file_type(entry_point, "") - self.assertEqual(expected, explicit_entry_point) - - @parameterized.expand( - [ - ([False], "file.ts"), - ([False, False], "file"), - ] - ) - def test_throws_exception_entry_point_not_found(self, file_exists, entry_point): - self.osutils.file_exists.side_effect = file_exists - with self.assertRaises(ActionFailedError) as context: - self.action._get_explicit_file_type(entry_point, "invalid") - self.assertEqual(str(context.exception), "entry point invalid does not exist") - class TestEsbuildVersionCheckerAction(TestCase): @parameterized.expand(["0.14.0", "0.0.0", "0.14.12"]) diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_esbuild.py b/tests/unit/workflows/nodejs_npm_esbuild/test_esbuild.py index 44f94387e..f38b81b8f 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_esbuild.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_esbuild.py @@ -1,7 +1,13 @@ from unittest import TestCase from mock import patch +from parameterized import parameterized -from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import SubprocessEsbuild, EsbuildExecutionError +from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import ( + SubprocessEsbuild, + EsbuildExecutionError, + EsbuildCommandBuilder, +) class FakePopen: @@ -64,7 +70,11 @@ def test_raises_EsbuildExecutionError_if_which_returns_no_results(self): with self.assertRaises(EsbuildExecutionError) as raised: self.under_test.run(["pack"]) - self.assertEqual(raised.exception.args[0], "Esbuild Failed: cannot find esbuild") + self.assertEqual( + raised.exception.args[0], + "Esbuild Failed: Cannot find esbuild. esbuild must be installed on the host machine to use this feature. " + "It is recommended to be installed on the PATH, but can also be included as a project dependency.", + ) def test_raises_ValueError_if_args_not_a_list(self): with self.assertRaises(ValueError) as raised: @@ -77,3 +87,173 @@ def test_raises_ValueError_if_args_empty(self): self.under_test.run([]) self.assertEqual(raised.exception.args[0], "requires at least one arg") + + +class TestImplicitFileTypeResolution(TestCase): + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild") + def setUp(self, OSUtilMock, SubprocessEsbuildMock): + self.osutils = OSUtilMock.return_value + self.subprocess_esbuild = SubprocessEsbuildMock.return_value + self.esbuild_command_builder = EsbuildCommandBuilder( + "source", + "artifacts", + {}, + self.osutils, + "package.json", + ) + + @parameterized.expand( + [ + ([True], "file.ts", "file.ts"), + ([False, True], "file", "file.js"), + ([True], "file", "file.ts"), + ] + ) + def test_implicit_and_explicit_file_types(self, file_exists, entry_point, expected): + self.osutils.file_exists.side_effect = file_exists + explicit_entry_point = self.esbuild_command_builder._get_explicit_file_type(entry_point, "") + self.assertEqual(expected, explicit_entry_point) + + @parameterized.expand( + [ + ([False], "file.ts"), + ([False, False], "file"), + ] + ) + def test_throws_exception_entry_point_not_found(self, file_exists, entry_point): + self.osutils.file_exists.side_effect = file_exists + with self.assertRaises(ActionFailedError) as context: + self.esbuild_command_builder._get_explicit_file_type(entry_point, "invalid") + self.assertEqual(str(context.exception), "entry point invalid does not exist") + + +class TestEsbuildCommandBuilder(TestCase): + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_builds_entry_points(self, osutils_mock): + bundler_config = {"entry_points": ["x.js", "y.ts"]} + args = ( + EsbuildCommandBuilder("scratch", "artifacts", bundler_config, osutils_mock, "") + .build_entry_points() + .get_command() + ) + self.assertEqual(args, ["x.js", "y.ts"]) + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_builds_default_values(self, osutils_mock): + bundler_config = {} + args = ( + EsbuildCommandBuilder("scratch", "artifacts", bundler_config, osutils_mock, "") + .build_default_values() + .get_command() + ) + self.assertEqual( + args, + [ + "--bundle", + "--platform=node", + "--outdir=artifacts", + "--target=es2020", + "--format=cjs", + "--minify", + ], + ) + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_combined_builder_exclude_all_dependencies(self, osutils_mock): + bundler_config = {"entry_points": ["x.js"], "loader": [".proto=text", ".json=js"]} + osutils_mock.parse_json.return_value = { + "dependencies": {"@faker-js/faker": "7.1.0", "uuidv4": "^6.2.12", "axios": "0.0.0"} + } + args = ( + EsbuildCommandBuilder("scratch", "artifacts", bundler_config, osutils_mock, "") + .build_entry_points() + .build_default_values() + .build_esbuild_args_from_config() + .build_with_no_dependencies() + .get_command() + ) + self.assertEqual( + args, + [ + "x.js", + "--bundle", + "--platform=node", + "--outdir=artifacts", + "--target=es2020", + "--format=cjs", + "--minify", + "--loader:.proto=text", + "--loader:.json=js", + "--external:@faker-js/faker", + "--external:uuidv4", + "--external:axios", + ], + ) + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_builds_args_from_config(self, osutils_mock): + bundler_config = { + "minify": True, + "sourcemap": False, + "format": "esm", + "target": "node14", + "loader": [".proto=text", ".json=js"], + "external": ["aws-sdk", "axios"], + "main_fields": "module,main", + } + + args = ( + EsbuildCommandBuilder("scratch", "artifacts", bundler_config, osutils_mock, "") + .build_esbuild_args_from_config() + .get_command() + ) + self.assertEqual( + args, + [ + "--minify", + "--target=node14", + "--format=esm", + "--main-fields=module,main", + "--external:aws-sdk", + "--external:axios", + "--loader:.proto=text", + "--loader:.json=js", + ], + ) + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + def test_combined_builder_with_dependencies(self, osutils_mock): + bundler_config = {"entry_points": ["x.js"], "loader": [".proto=text", ".json=js"], "format": "esm"} + args = ( + EsbuildCommandBuilder("scratch", "artifacts", bundler_config, osutils_mock, "") + .build_entry_points() + .build_default_values() + .build_esbuild_args_from_config() + .get_command() + ) + self.assertEqual( + args, + [ + "x.js", + "--bundle", + "--platform=node", + "--outdir=artifacts", + "--target=es2020", + "--minify", + "--format=esm", + "--loader:.proto=text", + "--loader:.json=js", + ], + ) + + @parameterized.expand( + [ + ("main_fields", "main-fields"), + ("entry_points", "entry-points"), + ("main-fields", "main-fields"), + ("bundle", "bundle"), + ] + ) + def test_convert_snake_case_to_kebab_case(self, field, expected): + self.assertEqual(EsbuildCommandBuilder._convert_snake_to_kebab_case(field), expected) diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_node.py b/tests/unit/workflows/nodejs_npm_esbuild/test_node.py deleted file mode 100644 index d9a7c89c2..000000000 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_node.py +++ /dev/null @@ -1,77 +0,0 @@ -from unittest import TestCase -from mock import patch - -from aws_lambda_builders.workflows.nodejs_npm_esbuild.node import SubprocessNodejs, NodejsExecutionError - - -class FakePopen: - def __init__(self, out=b"out", err=b"err", retcode=0): - self.out = out - self.err = err - self.returncode = retcode - - def communicate(self): - return self.out, self.err - - -class TestSubprocessNodejs(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") - def setUp(self, OSUtilMock): - self.osutils = OSUtilMock.return_value - self.osutils.pipe = "PIPE" - self.popen = FakePopen() - self.osutils.popen.side_effect = [self.popen] - - which = lambda cmd, executable_search_paths: ["{}/{}".format(executable_search_paths[0], cmd)] - - self.under_test = SubprocessNodejs(self.osutils, ["/a/b", "/c/d"], which) - - def test_run_executes_binary_found_in_exec_paths(self): - - self.under_test.run(["arg-a", "arg-b"]) - - self.osutils.popen.assert_called_with(["/a/b/node", "arg-a", "arg-b"], cwd=None, stderr="PIPE", stdout="PIPE") - - def test_uses_cwd_if_supplied(self): - self.under_test.run(["arg-a", "arg-b"], cwd="/a/cwd") - - self.osutils.popen.assert_called_with( - ["/a/b/node", "arg-a", "arg-b"], cwd="/a/cwd", stderr="PIPE", stdout="PIPE" - ) - - def test_returns_popen_out_decoded_if_retcode_is_0(self): - self.popen.out = b"some encoded text\n\n" - - result = self.under_test.run(["pack"]) - - self.assertEqual(result, "some encoded text") - - def test_raises_NodejsExecutionError_with_err_text_if_retcode_is_not_0(self): - self.popen.returncode = 1 - self.popen.err = b"some error text\n\n" - - with self.assertRaises(NodejsExecutionError) as raised: - self.under_test.run(["pack"]) - - self.assertEqual(raised.exception.args[0], "Nodejs Failed: some error text") - - def test_raises_NodejsExecutionError_if_which_returns_no_results(self): - - which = lambda cmd, executable_search_paths: [] - self.under_test = SubprocessNodejs(self.osutils, ["/a/b", "/c/d"], which) - with self.assertRaises(NodejsExecutionError) as raised: - self.under_test.run(["pack"]) - - self.assertEqual(raised.exception.args[0], "Nodejs Failed: cannot find nodejs") - - def test_raises_ValueError_if_args_not_a_list(self): - with self.assertRaises(ValueError) as raised: - self.under_test.run(("pack")) - - self.assertEqual(raised.exception.args[0], "args must be a list") - - def test_raises_ValueError_if_args_empty(self): - with self.assertRaises(ValueError) as raised: - self.under_test.run([]) - - self.assertEqual(raised.exception.args[0], "requires at least one arg") diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_utils.py b/tests/unit/workflows/nodejs_npm_esbuild/test_utils.py deleted file mode 100644 index f0dce1e2f..000000000 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_utils.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase -from parameterized import parameterized - -from aws_lambda_builders.workflows.nodejs_npm_esbuild.utils import ( - EXPERIMENTAL_FLAG_ESBUILD, - is_experimental_esbuild_scope, -) - - -class TestNodejsUtils(TestCase): - @parameterized.expand( - [ - (None, False), - ([], False), - ([EXPERIMENTAL_FLAG_ESBUILD], True), - ([EXPERIMENTAL_FLAG_ESBUILD, "SomeOtherFlag"], True), - ] - ) - def test_experimental_esbuild_scope_check(self, experimental_flags, expected): - self.assertEqual(is_experimental_esbuild_scope(experimental_flags), expected) diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py index 493318e52..cabdd55ab 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py @@ -1,13 +1,19 @@ from unittest import TestCase +from unittest.mock import ANY + from mock import patch, call -from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, CopyDependenciesAction, MoveDependenciesAction +from aws_lambda_builders.actions import ( + CopySourceAction, + CleanUpAction, + MoveDependenciesAction, + LinkSourceAction, +) from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.nodejs_npm.actions import NodejsNpmInstallAction, NodejsNpmCIAction from aws_lambda_builders.workflows.nodejs_npm_esbuild import NodejsNpmEsbuildWorkflow from aws_lambda_builders.workflows.nodejs_npm_esbuild.actions import EsbuildBundleAction, EsbuildCheckVersionAction from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import SubprocessEsbuild -from aws_lambda_builders.workflows.nodejs_npm_esbuild.utils import EXPERIMENTAL_FLAG_ESBUILD class FakePopen: @@ -46,7 +52,7 @@ def test_workflow_sets_up_npm_actions_with_bundler_if_manifest_requests_it(self) "scratch_dir", "manifest", osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(len(workflow.actions), 3) @@ -67,11 +73,11 @@ def test_sets_up_esbuild_search_path_from_npm_bin(self): "scratch_dir", "manifest", osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.osutils.popen.assert_called_with(["npm", "bin"], stdout="PIPE", stderr="PIPE", cwd="scratch_dir") - esbuild = workflow.actions[2].subprocess_esbuild + esbuild = workflow.actions[2]._subprocess_esbuild self.assertIsInstance(esbuild, SubprocessEsbuild) self.assertEqual(esbuild.executable_search_paths, ["project/bin"]) @@ -87,11 +93,11 @@ def test_sets_up_esbuild_search_path_with_workflow_executable_search_paths_after "manifest", osutils=self.osutils, executable_search_paths=["other/bin"], - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.osutils.popen.assert_called_with(["npm", "bin"], stdout="PIPE", stderr="PIPE", cwd="scratch_dir") - esbuild = workflow.actions[2].subprocess_esbuild + esbuild = workflow.actions[2]._subprocess_esbuild self.assertIsInstance(esbuild, SubprocessEsbuild) self.assertEqual(esbuild.executable_search_paths, ["project/bin", "other/bin"]) @@ -105,7 +111,7 @@ def test_workflow_uses_npm_ci_if_lockfile_exists(self): "scratch_dir", "manifest", osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], options={"use_npm_ci": True}, ) @@ -125,7 +131,7 @@ def test_workflow_uses_npm_ci_if_shrinkwrap_exists(self): "scratch_dir", "manifest", osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], options={"use_npm_ci": True}, ) @@ -147,7 +153,7 @@ def test_workflow_doesnt_use_npm_ci_no_options_config(self): "scratch_dir", "manifest", osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(len(workflow.actions), 3) @@ -169,7 +175,7 @@ def test_must_validate_architecture(self): "manifest", options={"artifact_executable_name": "foo"}, osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) workflow_with_arm = NodejsNpmEsbuildWorkflow( @@ -179,7 +185,7 @@ def test_must_validate_architecture(self): "manifest", architecture=ARM64, osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(workflow.architecture, "x86_64") @@ -194,7 +200,7 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_without_dep "scratch_dir", "manifest", osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(len(workflow.actions), 3) @@ -214,12 +220,12 @@ def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dep download_dependencies=False, combine_dependencies=True, osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(len(workflow.actions), 3) self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], CopySourceAction) + self.assertIsInstance(workflow.actions[1], LinkSourceAction) self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dependencies_dir_no_combine_deps(self): @@ -234,12 +240,12 @@ def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dep download_dependencies=False, combine_dependencies=False, osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(len(workflow.actions), 4) self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], CopySourceAction) + self.assertIsInstance(workflow.actions[1], LinkSourceAction) self.assertIsInstance(workflow.actions[2], EsbuildCheckVersionAction) self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) @@ -255,16 +261,17 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_depende dependencies_dir="dep", download_dependencies=True, osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) - self.assertEqual(len(workflow.actions), 5) + self.assertEqual(len(workflow.actions), 6) self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], NodejsNpmInstallAction) self.assertIsInstance(workflow.actions[2], CleanUpAction) self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) - self.assertIsInstance(workflow.actions[4], CopyDependenciesAction) + self.assertIsInstance(workflow.actions[4], MoveDependenciesAction) + self.assertIsInstance(workflow.actions[5], LinkSourceAction) def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_dependencies_dir_no_combine_deps(self): workflow = NodejsNpmEsbuildWorkflow( @@ -276,7 +283,7 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_depende download_dependencies=True, combine_dependencies=False, osutils=self.osutils, - experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], + experimental_flags=[], ) self.assertEqual(len(workflow.actions), 6) @@ -287,3 +294,23 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_depende self.assertIsInstance(workflow.actions[3], EsbuildCheckVersionAction) self.assertIsInstance(workflow.actions[4], EsbuildBundleAction) self.assertIsInstance(workflow.actions[5], MoveDependenciesAction) + + @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.workflow.NodejsNpmWorkflow") + def test_workflow_uses_production_npm_version(self, get_workflow_mock): + workflow = NodejsNpmEsbuildWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + dependencies_dir=None, + download_dependencies=True, + combine_dependencies=False, + osutils=self.osutils, + experimental_flags=[], + ) + + self.assertEqual(len(workflow.actions), 3) + self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) + + get_workflow_mock.get_install_action.assert_called_with("source", "scratch_dir", ANY, ANY, None) diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index b57200efd..0c653f330 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -2,20 +2,41 @@ from mock import patch, ANY, Mock from unittest import TestCase -from aws_lambda_builders.actions import CopySourceAction, CleanUpAction -from aws_lambda_builders.workflows.python_pip.utils import OSUtils +from parameterized import parameterized_class + +from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction +from aws_lambda_builders.path_resolver import PathResolver +from aws_lambda_builders.workflows.python_pip.utils import OSUtils, EXPERIMENTAL_FLAG_BUILD_PERFORMANCE from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from aws_lambda_builders.workflows.python_pip.workflow import PythonPipBuildAction, PythonPipWorkflow +@parameterized_class( + ("experimental_flags",), + [ + ([]), + ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE]), + ], +) class TestPythonPipWorkflow(TestCase): + experimental_flags = [] + def setUp(self): self.osutils = OSUtils() self.osutils_mock = Mock(spec=self.osutils) self.osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock + "source", + "artifacts", + "scratch_dir", + "manifest", + runtime="python3.9", + osutils=self.osutils_mock, + experimental_flags=self.experimental_flags, ) + self.python_major_version = "3" + self.python_minor_version = "9" + self.language = "python" def test_workflow_sets_up_actions(self): self.assertEqual(len(self.workflow.actions), 2) @@ -25,7 +46,13 @@ def test_workflow_sets_up_actions(self): def test_workflow_sets_up_actions_without_requirements(self): self.osutils_mock.file_exists.return_value = False self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock + "source", + "artifacts", + "scratch_dir", + "manifest", + runtime="python3.9", + osutils=self.osutils_mock, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) @@ -34,6 +61,18 @@ def test_workflow_validator(self): for validator in self.workflow.get_validators(): self.assertTrue(isinstance(validator, PythonRuntimeValidator)) + def test_workflow_resolver(self): + for resolver in self.workflow.get_resolvers(): + self.assertTrue(isinstance(resolver, PathResolver)) + self.assertTrue( + resolver.executables, + [ + self.language, + f"{self.language}{self.python_major_version}.{self.python_minor_version}", + f"{self.language}{self.python_major_version}", + ], + ) + def test_workflow_sets_up_actions_without_download_dependencies_with_dependencies_dir(self): osutils_mock = Mock(spec=self.osutils) osutils_mock.file_exists.return_value = True @@ -42,13 +81,17 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir="dep", download_dependencies=False, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 2) - self.assertIsInstance(self.workflow.actions[0], CopySourceAction) + if self.experimental_flags: + self.assertIsInstance(self.workflow.actions[0], LinkSourceAction) + else: + self.assertIsInstance(self.workflow.actions[0], CopySourceAction) self.assertIsInstance(self.workflow.actions[1], CopySourceAction) def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_dir(self): @@ -59,18 +102,22 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir="dep", download_dependencies=True, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 4) self.assertIsInstance(self.workflow.actions[0], CleanUpAction) self.assertIsInstance(self.workflow.actions[1], PythonPipBuildAction) - self.assertIsInstance(self.workflow.actions[2], CopySourceAction) + if self.experimental_flags: + self.assertIsInstance(self.workflow.actions[2], LinkSourceAction) + else: + self.assertIsInstance(self.workflow.actions[2], CopySourceAction) + # check copying dependencies does not have any exclude + self.assertEqual(self.workflow.actions[2].excludes, []) self.assertIsInstance(self.workflow.actions[3], CopySourceAction) - # check copying dependencies does not have any exclude - self.assertEqual(self.workflow.actions[2].excludes, []) def test_workflow_sets_up_actions_without_download_dependencies_without_dependencies_dir(self): osutils_mock = Mock(spec=self.osutils) @@ -80,10 +127,11 @@ def test_workflow_sets_up_actions_without_download_dependencies_without_dependen "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir=None, download_dependencies=False, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) @@ -96,11 +144,12 @@ def test_workflow_sets_up_actions_without_combine_dependencies(self): "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir="dep", download_dependencies=True, combine_dependencies=False, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 3) self.assertIsInstance(self.workflow.actions[0], CleanUpAction) @@ -114,7 +163,7 @@ def test_must_build_with_architecture(self, PythonPipBuildActionMock): "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", architecture="ARM64", osutils=self.osutils_mock, ) @@ -122,7 +171,7 @@ def test_must_build_with_architecture(self, PythonPipBuildActionMock): "artifacts", "scratch_dir", "manifest", - "python3.7", + "python3.9", None, binaries=ANY, architecture="ARM64",