From 4c12b411aefc0aea86bb87dea5ea8c3a34143f8f Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Thu, 8 Sep 2022 11:26:49 -0700 Subject: [PATCH 1/4] chore: Updated maintainers (#386) * Updated maintainers * Removed memebers --- .github/workflows/pr-labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index ad29070e9..f6389b8ab 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -21,7 +21,7 @@ jobs: with: github-token: ${{secrets.GITHUB_TOKEN}} script: | - const maintainers = ['jfuss', 'c2tarun', 'hoffa', 'awood45', 'CoshUS', 'aahung', 'hawflau', 'mndeveci', 'ssenchenko', 'wchengru', 'mingkun2020', 'qingchm', 'moelasmar', 'xazhao', 'mildaniel', 'marekaiv', 'torresxb1'] + const maintainers = ['jfuss', 'hoffa', 'awood45', 'aahung', 'hawflau', 'mndeveci', 'ssenchenko', 'qingchm', 'moelasmar', 'xazhao', 'mildaniel', 'marekaiv', 'torresxb1', 'lucashuy', 'hnnasit', 'sriram-mv'] if (maintainers.includes(context.payload.sender.login)) { github.rest.issues.addLabels({ issue_number: context.issue.number, From 94030f08308cc2ea895a0979f1a55df779017d0b Mon Sep 17 00:00:00 2001 From: Collen Byer Date: Tue, 11 Oct 2022 15:50:15 -0400 Subject: [PATCH 2/4] feat: add `-trimpath` to GoModulesBuilder (#389) * add trimpath to GoModulesBuilder with tests * add trim_go_path parameter comment in GoModulesBuilder * fix lint * add tests for trim path showing equality * fix variable names --- .../workflows/go_modules/builder.py | 9 ++- .../workflows/go_modules/workflow.py | 5 +- .../workflows/go_modules/test_go.py | 68 ++++++++++++++++++- .../go_modules/testdata/no-deps-copy/go.mod | 1 + .../go_modules/testdata/no-deps-copy/main.go | 4 ++ .../integration/workflows/go_modules/utils.py | 20 ++++++ .../unit/workflows/go_modules/test_builder.py | 11 +++ 7 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 tests/integration/workflows/go_modules/testdata/no-deps-copy/go.mod create mode 100644 tests/integration/workflows/go_modules/testdata/no-deps-copy/main.go diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index fb9247c46..22d51b4e8 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -21,7 +21,7 @@ class GoModulesBuilder(object): LANGUAGE = "go" - def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_64): + def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_64, trim_go_path=False): """Initialize a GoModulesBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -33,11 +33,15 @@ def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_6 :type architecture: str :param architecture: name of the type of architecture + + :type trim_go_path: bool + :param trim_go_path: should go build use -trimpath flag """ self.osutils = osutils self.binaries = binaries self.mode = mode self.goarch = get_goarch(architecture) + self.trim_go_path = trim_go_path def build(self, source_dir_path, output_path): """Builds a go project onto an output path. @@ -53,6 +57,9 @@ def build(self, source_dir_path, output_path): env.update({"GOOS": "linux", "GOARCH": self.goarch}) runtime_path = self.binaries[self.LANGUAGE].binary_path cmd = [runtime_path, "build"] + if self.trim_go_path: + LOG.debug("Trimpath requested: Setting go build configuration to -trimpath") + cmd += ["-trimpath"] if self.mode and self.mode.lower() == BuildMode.DEBUG: LOG.debug("Debug build requested: Setting configuration to Debug") cmd += ["-gcflags", "all=-N -l"] diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index ceafdbd7a..fa780a5ad 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -28,10 +28,13 @@ def __init__( options = kwargs.get("options") or {} handler = options.get("artifact_executable_name", None) + trim_go_path = options.get("trim_go_path", False) output_path = osutils.joinpath(artifacts_dir, handler) - builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode, architecture=self.architecture) + builder = GoModulesBuilder( + osutils, binaries=self.binaries, mode=mode, architecture=self.architecture, trim_go_path=trim_go_path + ) self.actions = [GoModulesBuildAction(source_dir, output_path, builder)] def get_validators(self): diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py index 17f0aad46..64ff05d49 100644 --- a/tests/integration/workflows/go_modules/test_go.py +++ b/tests/integration/workflows/go_modules/test_go.py @@ -13,6 +13,7 @@ from aws_lambda_builders.exceptions import WorkflowFailedError from tests.integration.workflows.go_modules.utils import get_executable_arch +from tests.integration.workflows.go_modules.utils import get_md5_hexdigest class TestGoWorkflow(TestCase): @@ -112,6 +113,71 @@ def test_builds_arm64_architecture(self): options={"artifact_executable_name": "no-deps-main-arm64"}, architecture="arm64", ) - pathname = Path(self.artifacts_dir, "no-deps-main-arm64") self.assertEqual(get_executable_arch(pathname), "AArch64") + + def test_builds_with_trimpath(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_trimpath = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-trimpath", "trim_go_path": True}, + architecture="x86_64", + ) + pathname = Path(self.artifacts_dir, "no-deps-main-trimpath") + self.assertEqual(get_executable_arch(pathname), "x64") + + def test_builds_without_trimpath_are_not_equal(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_no_trimpath = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main", "trim_go_path": False}, + architecture="x86_64", + ) + + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-copy") + built_no_trimpath_copy = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-copy", "trim_go_path": False}, + architecture="x86_64", + ) + pathname = Path(self.artifacts_dir, "no-deps-main") + pathnameOfCopy = Path(self.artifacts_dir, "no-deps-main-copy") + self.assertNotEqual(get_md5_hexdigest(pathname), get_md5_hexdigest(pathnameOfCopy)) + + def test_builds_with_trimpath_are_equal(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + built_with_trimpath = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main", "trim_go_path": True}, + architecture="x86_64", + ) + + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-copy") + built_with_trimpath_copy = self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main-copy", "trim_go_path": True}, + architecture="x86_64", + ) + pathname = Path(self.artifacts_dir, "no-deps-main") + pathnameOfCopy = Path(self.artifacts_dir, "no-deps-main-copy") + self.assertEqual(get_md5_hexdigest(pathname), get_md5_hexdigest(pathnameOfCopy)) diff --git a/tests/integration/workflows/go_modules/testdata/no-deps-copy/go.mod b/tests/integration/workflows/go_modules/testdata/no-deps-copy/go.mod new file mode 100644 index 000000000..846a0e06b --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/no-deps-copy/go.mod @@ -0,0 +1 @@ +module github.com/awslabs/aws-lambda-builders diff --git a/tests/integration/workflows/go_modules/testdata/no-deps-copy/main.go b/tests/integration/workflows/go_modules/testdata/no-deps-copy/main.go new file mode 100644 index 000000000..da29a2cad --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/no-deps-copy/main.go @@ -0,0 +1,4 @@ +package main + +func main() { +} diff --git a/tests/integration/workflows/go_modules/utils.py b/tests/integration/workflows/go_modules/utils.py index a5caae61e..648ea4f51 100644 --- a/tests/integration/workflows/go_modules/utils.py +++ b/tests/integration/workflows/go_modules/utils.py @@ -1,4 +1,5 @@ from elftools.elf.elffile import ELFFile +import hashlib def get_executable_arch(path): @@ -18,3 +19,22 @@ def get_executable_arch(path): with open(str(path), "rb") as f: e = ELFFile(f) return e.get_machine_arch() + + +def get_md5_hexdigest(path): + """ + Returns the hexdigest of a binary + + Parameters + ---------- + path : str + path to the Go binaries generated + + Returns + ------- + str + Hex digest of the binaries + """ + with open(str(path), "rb") as f: + hashed = hashlib.md5(f.read()) + return hashed.hexdigest() diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index bc3d24bbb..d821a4a74 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -61,6 +61,17 @@ def test_debug_configuration_set(self): stdout="PIPE", ) + def test_trimpath_configuration_set(self): + self.under_test = GoModulesBuilder(self.osutils, self.binaries, "release", "x86_64", True) + self.under_test.build("source_dir", "output_path") + self.osutils.popen.assert_called_with( + ["/path/to/go", "build", "-trimpath", "-o", "output_path", "source_dir"], + cwd="source_dir", + env={"GOOS": "linux", "GOARCH": "amd64"}, + stderr="PIPE", + stdout="PIPE", + ) + def test_debug_configuration_set_with_arm_architecture(self): self.under_test = GoModulesBuilder(self.osutils, self.binaries, "Debug", "arm64") self.under_test.build("source_dir", "output_path") From 46e0895084a90e19da7d7466bd3d603f0af5e0eb Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:09:56 -0700 Subject: [PATCH 3/4] fix: disable symlinking for python dependencies (#391) * fix: disable symlinking for python dependencies * update unit tests --- aws_lambda_builders/workflows/python_pip/workflow.py | 3 ++- tests/unit/workflows/python_pip/test_workflow.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index 844d02fbd..183e6835f 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -109,7 +109,8 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim # folder if self.dependencies_dir and self.combine_dependencies: # when copying downloaded dependencies back to artifacts folder, don't exclude anything - if is_experimental_build_improvements_enabled(self.experimental_flags): + # symlinking python dependencies is disabled for now since it is breaking sam local commands + if False and is_experimental_build_improvements_enabled(self.experimental_flags): self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir)) else: self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index 0c653f330..83994acad 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -88,7 +88,8 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 2) - if self.experimental_flags: + # symlinking python dependencies is disabled for now since it is breaking sam local commands + if False and self.experimental_flags: self.assertIsInstance(self.workflow.actions[0], LinkSourceAction) else: self.assertIsInstance(self.workflow.actions[0], CopySourceAction) @@ -111,7 +112,8 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di self.assertEqual(len(self.workflow.actions), 4) self.assertIsInstance(self.workflow.actions[0], CleanUpAction) self.assertIsInstance(self.workflow.actions[1], PythonPipBuildAction) - if self.experimental_flags: + # symlinking python dependencies is disabled for now since it is breaking sam local commands + if False and self.experimental_flags: self.assertIsInstance(self.workflow.actions[2], LinkSourceAction) else: self.assertIsInstance(self.workflow.actions[2], CopySourceAction) From 09115eeed9c52ad778314a50ad642ba1c7d7a7b8 Mon Sep 17 00:00:00 2001 From: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Date: Mon, 17 Oct 2022 16:14:48 -0700 Subject: [PATCH 4/4] chore: bump version to 1.20.0 (#392) --- aws_lambda_builders/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index dcf438365..241f0a31e 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,8 @@ """ AWS Lambda Builder Library """ -__version__ = "1.19.0" + +# Changing version will trigger a new release! +# Please make the version change as the last step of your development. +__version__ = "1.20.0" RPC_PROTOCOL_VERSION = "0.3"