Merge branch 'develop'

aws_lambda_builders/__init__.py

@@ -4,5 +4,5 @@
 
 # Changing version will trigger a new release!
 # Please make the version change as the last step of your development.
-__version__ = "1.21.0"
+__version__ = "1.22.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/workflows/dotnet_clipackage/actions.py

@@ -108,7 +108,7 @@ def execute(self):
 
             # The dotnet lambda package command outputs a zip file for the package. To make this compatible
             # with the workflow, unzip the zip file into the artifacts directory and then delete the zip archive.
-            self.os_utils.expand_zip(zipfullpath, self.artifacts_dir)
+            self.os_utils.unzip(zipfullpath, self.artifacts_dir)
 
         except DotnetCLIExecutionError as ex:
             raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/dotnet_clipackage/utils.py

@@ -1,14 +1,15 @@
 """
 Commonly used utilities
 """
-
+import logging
 import os
 import platform
-import shutil
 import subprocess
 import zipfile
 from aws_lambda_builders.utils import which
 
+LOG = logging.getLogger(__name__)
+
 
 class OSUtils(object):
     """
@@ -25,11 +26,126 @@ def is_windows(self):
     def which(self, executable, executable_search_paths=None):
         return which(executable, executable_search_paths=executable_search_paths)
 
-    def expand_zip(self, zipfullpath, destination_dir):
-        ziparchive = zipfile.ZipFile(zipfullpath, "r")
-        ziparchive.extractall(destination_dir)
-        ziparchive.close()
-        os.remove(zipfullpath)
+    def unzip(self, zip_file_path, output_dir, permission=None):
+        """
+        This method and dependent methods were copied from SAM CLI, but with the addition of deleting the zip file
+        https://github.com/aws/aws-sam-cli/blob/458076265651237a662a372f54d5b3df49fd6797/samcli/local/lambdafn/zip.py#L81
+
+        Unzip the given file into the given directory while preserving file permissions in the process.
+        Parameters
+        ----------
+        zip_file_path : str
+            Path to the zip file
+        output_dir : str
+            Path to the directory where the it should be unzipped to
+        permission : int
+            Permission to set in an octal int form
+        """
+
+        with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+
+            # For each item in the zip file, extract the file and set permissions if available
+            for file_info in zip_ref.infolist():
+                extracted_path = self._extract(file_info, output_dir, zip_ref)
+
+                # If the extracted_path is a symlink, do not set the permissions. If the target of the symlink does not
+                # exist, then os.chmod will fail with FileNotFoundError
+                if not os.path.islink(extracted_path):
+                    self._set_permissions(file_info, extracted_path)
+                    self._override_permissions(extracted_path, permission)
+
+        if not os.path.islink(extracted_path):
+            self._override_permissions(output_dir, permission)
+
+        os.remove(zip_file_path)
+
+    def _is_symlink(self, file_info):
+        """
+        Check the upper 4 bits of the external attribute for a symlink.
+        See: https://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute
+        Parameters
+        ----------
+        file_info : zipfile.ZipInfo
+            The ZipInfo for a ZipFile
+        Returns
+        -------
+        bool
+            A response regarding whether the ZipInfo defines a symlink or not.
+        """
+
+        return (file_info.external_attr >> 28) == 0xA
+
+    def _extract(self, file_info, output_dir, zip_ref):
+        """
+        Unzip the given file into the given directory while preserving file permissions in the process.
+        Parameters
+        ----------
+        file_info : zipfile.ZipInfo
+            The ZipInfo for a ZipFile
+        output_dir : str
+            Path to the directory where the it should be unzipped to
+        zip_ref : zipfile.ZipFile
+            The ZipFile we are working with.
+        Returns
+        -------
+        string
+            Returns the target path the Zip Entry was extracted to.
+        """
+
+        # Handle any regular file/directory entries
+        if not self._is_symlink(file_info):
+            return zip_ref.extract(file_info, output_dir)
+
+        source = zip_ref.read(file_info.filename).decode("utf8")
+        link_name = os.path.normpath(os.path.join(output_dir, file_info.filename))
+
+        # make leading dirs if needed
+        leading_dirs = os.path.dirname(link_name)
+        if not os.path.exists(leading_dirs):
+            os.makedirs(leading_dirs)
+
+        # If the link already exists, delete it or symlink() fails
+        if os.path.lexists(link_name):
+            os.remove(link_name)
+
+        # Create a symbolic link pointing to source named link_name.
+        os.symlink(source, link_name)
+
+        return link_name
+
+    def _override_permissions(self, path, permission):
+        """
+        Forcefully override the permissions on the path
+        Parameters
+        ----------
+        path str
+            Path where the file or directory
+        permission octal int
+            Permission to set
+        """
+        if permission:
+            os.chmod(path, permission)
+
+    def _set_permissions(self, zip_file_info, extracted_path):
+        """
+        Sets permissions on the extracted file by reading the ``external_attr`` property of given file info.
+        Parameters
+        ----------
+        zip_file_info : zipfile.ZipInfo
+            Object containing information about a file within a zip archive
+        extracted_path : str
+            Path where the file has been extracted to
+        """
+
+        # Permission information is stored in first two bytes.
+        permission = zip_file_info.external_attr >> 16
+        if not permission:
+            # Zips created on certain Windows machines, however, might not have any permission information on them.
+            # Skip setting a permission on these files.
+            LOG.debug("File %s in zipfile does not have permission information", zip_file_info.filename)
+            return
+
+        os.chmod(extracted_path, permission)
 
     @property
     def pipe(self):

tests/integration/workflows/dotnet_clipackage/test_dotnet.py

@@ -45,6 +45,10 @@ def verify_architecture(self, deps_file_name, expected_architecture, version=Non
 
         self.assertEqual(target, target_name)
 
+    def verify_execute_permissions(self, entrypoint_file_name):
+        entrypoint_file_path = os.path.join(self.artifacts_dir, entrypoint_file_name)
+        self.assertTrue(os.access(entrypoint_file_path, os.X_OK))
+
 
 class TestDotnet31(TestDotnetBase):
     """
@@ -192,3 +196,29 @@ def test_with_defaults_file_arm64(self):
 
         self.assertEqual(expected_files, output_files)
         self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64", version="6.0")
+
+    def test_with_custom_runtime(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "CustomRuntime6")
+
+        self.builder.build(
+            source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=X86_64
+        )
+
+        expected_files = {
+            "Amazon.Lambda.Core.dll",
+            "Amazon.Lambda.RuntimeSupport.dll",
+            "Amazon.Lambda.Serialization.SystemTextJson.dll",
+            "bootstrap",
+            "bootstrap.deps.json",
+            "bootstrap.dll",
+            "bootstrap.pdb",
+            "bootstrap.runtimeconfig.json",
+        }
+
+        output_files = set(os.listdir(self.artifacts_dir))
+
+        self.assertEqual(expected_files, output_files)
+        self.verify_architecture("bootstrap.deps.json", "linux-x64", version="6.0")
+        # Execute permissions are required for custom runtimes which bootstrap themselves, otherwise `sam local invoke`
+        # won't have permission to run the file
+        self.verify_execute_permissions("bootstrap")

tests/integration/workflows/dotnet_clipackage/testdata/CustomRuntime6/CustomRuntime6.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AWSProjectType>Lambda</AWSProjectType>
+    <AssemblyName>bootstrap</AssemblyName>
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <PublishReadyToRun>true</PublishReadyToRun>
+  </PropertyGroup>
+  <!-- 
+  When publishing Lambda functions for ARM64 to the provided.al2 runtime a newer version of libicu needs to be included
+  in the deployment bundle because .NET requires a newer version of libicu then is preinstalled with Amazon Linux 2.
+  -->
+  <ItemGroup Condition="'$(RuntimeIdentifier)' == 'linux-arm64'">
+    <RuntimeHostConfigurationOption Include="System.Globalization.AppLocalIcu" Value="68.2.0.9" />
+    <PackageReference Include="Microsoft.ICU.ICU4C.Runtime" Version="68.2.0.9" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Amazon.Lambda.RuntimeSupport" Version="1.8.2" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.1.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.3.0" />
+  </ItemGroup>
+</Project>

tests/integration/workflows/dotnet_clipackage/testdata/CustomRuntime6/Function.cs

@@ -0,0 +1,21 @@
+using Amazon.Lambda.Core;
+using Amazon.Lambda.RuntimeSupport;
+using Amazon.Lambda.Serialization.SystemTextJson;
+
+namespace CustomRuntime6;
+
+public class Function
+{
+    private static async Task Main(string[] args)
+    {
+        Func<string, ILambdaContext, string> handler = FunctionHandler;
+        await LambdaBootstrapBuilder.Create(handler, new DefaultLambdaJsonSerializer())
+            .Build()
+            .RunAsync();
+    }
+
+    public static string FunctionHandler(string input, ILambdaContext context)
+    {
+        return input.ToUpper();
+    }
+}

tests/integration/workflows/dotnet_clipackage/testdata/CustomRuntime6/aws-lambda-tools-defaults.json

@@ -0,0 +1,16 @@
+{
+  "Information": [
+    "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.",
+    "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.",
+    "dotnet lambda help",
+    "All the command line options for the Lambda command can be specified in this file."
+  ],
+  "profile": "",
+  "region": "",
+  "configuration": "Release",
+  "function-runtime": "provided.al2",
+  "function-memory-size": 256,
+  "function-timeout": 30,
+  "function-handler": "bootstrap",
+  "msbuild-parameters": "--self-contained true"
+}

tests/unit/workflows/dotnet_clipackage/test_utils.py

@@ -0,0 +1,34 @@
+import os
+import stat
+import tempfile
+from unittest import TestCase
+from zipfile import ZipFile
+
+from aws_lambda_builders.workflows.dotnet_clipackage.utils import OSUtils
+
+
+class TestDotnetCliPackageWorkflow(TestCase):
+    def test_unzip_keeps_execute_permission_on_linux(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            with tempfile.TemporaryDirectory(dir=temp_dir) as output_dir:
+                test_file_name = "myFileToZip"
+                path_to_file_to_zip = os.path.join(temp_dir, test_file_name)
+                path_to_zip_file = os.path.join(temp_dir, "myZip.zip")
+                expected_output_file = os.path.join(output_dir, test_file_name)
+                with open(path_to_file_to_zip, "a") as the_file:
+                    the_file.write("Hello World!")
+
+                # Set execute permissions on the file before zipping (won't do anything on Windows)
+                st = os.stat(path_to_file_to_zip)
+                os.chmod(path_to_file_to_zip, st.st_mode | stat.S_IEXEC | stat.S_IXOTH | stat.S_IXGRP)
+
+                # Zip the file
+                with ZipFile(path_to_zip_file, "w") as myzip:
+                    myzip.write(path_to_file_to_zip, test_file_name)
+
+                # Unzip the file
+                OSUtils().unzip(path_to_zip_file, output_dir)
+                self.assertTrue(os.path.exists(expected_output_file))
+
+                # Assert that execute permissions are still applied
+                self.assertTrue(os.access(expected_output_file, os.X_OK))