From 5b952a09aabe70aa1908e5ba4a01c1d427b71f00 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Thu, 23 Mar 2023 16:34:48 -0700 Subject: [PATCH 1/4] Remove py as dependency which cause security vulnerability check fail --- smsparkbuild/py39/Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/smsparkbuild/py39/Pipfile b/smsparkbuild/py39/Pipfile index f72699c..04197db 100644 --- a/smsparkbuild/py39/Pipfile +++ b/smsparkbuild/py39/Pipfile @@ -35,7 +35,6 @@ importlib-metadata = "==4.11.3" pytest-parallel = "==0.1.1" pytest-rerunfailures = "10.0" numpy = "==1.22.2" -py = "==1.11.0" [requires] python_version = "3.9" From 17ae6a781f25c620d7787903b994605916637f3d Mon Sep 17 00:00:00 2001 From: Can Sun Date: Mon, 27 Mar 2023 09:30:34 -0700 Subject: [PATCH 2/4] Add module py back and ignore the vulnerability 51457 --- Makefile | 2 +- smsparkbuild/py39/Pipfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8b2be39..e3560d1 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ build-container-library: init install-container-library: init # temporarily bypass urllib3 because circular dependency will be introduced if bumped up urllib3 version - pipenv run safety check -i 43975 # https://github.com/pyupio/safety + pipenv run safety check -i 43975 -i 51457 # https://github.com/pyupio/safety build-static-config: ./scripts/fetch-ec2-instance-type-info.sh --region ${REGION} --use-case ${USE_CASE} --spark-version ${SPARK_VERSION} \ diff --git a/smsparkbuild/py39/Pipfile b/smsparkbuild/py39/Pipfile index 04197db..f72699c 100644 --- a/smsparkbuild/py39/Pipfile +++ b/smsparkbuild/py39/Pipfile @@ -35,6 +35,7 @@ importlib-metadata = "==4.11.3" pytest-parallel = "==0.1.1" pytest-rerunfailures = "10.0" numpy = "==1.22.2" +py = "==1.11.0" [requires] python_version = "3.9" From 79d692d920b9cc7e2aff9d6df3939dbfde411303 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Mon, 27 Mar 2023 09:44:06 -0700 Subject: [PATCH 3/4] Add some comments --- Pipfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Pipfile b/Pipfile index 08f9cfc..f72699c 100644 --- a/Pipfile +++ b/Pipfile @@ -14,20 +14,20 @@ waitress = "==2.1.2" types-waitress = "==2.0.6" requests = "==2.27.1" types-requests = "==2.27.16" -rsa = "==4.3" +rsa = "==4.9" pyasn1 = "==0.4.8" boto3 = "==1.21.33" -safety = "==2.3.1" +safety = "==2.3.5" black = "==22.3.0" mypy = "==0.942" flake8 = "==4.0.1" flake8-docstrings = "==1.5.0" -pytest = "==7.1.1" +pytest = "==7.2.2" pytest-cov = "==2.10.0" -pytest-xdist = "==2.5.0" +pytest-xdist = "==3.2.1" docker = "==5.0.3" docker-compose = "==1.29.2" -cryptography = "==36.0.2" +cryptography = "==39.0.2" typing-extensions = "==4.1.1" sagemaker = "==2.117.0" smspark = {editable = true, path = "."} @@ -35,6 +35,7 @@ importlib-metadata = "==4.11.3" pytest-parallel = "==0.1.1" pytest-rerunfailures = "10.0" numpy = "==1.22.2" +py = "==1.11.0" [requires] python_version = "3.9" From eb3f1968513726bea9d5a85e7aad1a73db9e1b48 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Mon, 27 Mar 2023 09:47:47 -0700 Subject: [PATCH 4/4] Add comments --- Makefile | 3 +++ Pipfile | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e3560d1..3e026c7 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ build-container-library: init install-container-library: init # temporarily bypass urllib3 because circular dependency will be introduced if bumped up urllib3 version + # temporarily bypass py=1.1.0 because pytest-parallel has a dependency on it however the module is no longer maitained. + # In the future the pylib will be removed from pytest-parallel dependency and 51457 should only impact the local tests. + # For more info, https://github.com/pytest-dev/py/issues/287 pipenv run safety check -i 43975 -i 51457 # https://github.com/pyupio/safety build-static-config: diff --git a/Pipfile b/Pipfile index f72699c..08f9cfc 100644 --- a/Pipfile +++ b/Pipfile @@ -14,20 +14,20 @@ waitress = "==2.1.2" types-waitress = "==2.0.6" requests = "==2.27.1" types-requests = "==2.27.16" -rsa = "==4.9" +rsa = "==4.3" pyasn1 = "==0.4.8" boto3 = "==1.21.33" -safety = "==2.3.5" +safety = "==2.3.1" black = "==22.3.0" mypy = "==0.942" flake8 = "==4.0.1" flake8-docstrings = "==1.5.0" -pytest = "==7.2.2" +pytest = "==7.1.1" pytest-cov = "==2.10.0" -pytest-xdist = "==3.2.1" +pytest-xdist = "==2.5.0" docker = "==5.0.3" docker-compose = "==1.29.2" -cryptography = "==39.0.2" +cryptography = "==36.0.2" typing-extensions = "==4.1.1" sagemaker = "==2.117.0" smspark = {editable = true, path = "."} @@ -35,7 +35,6 @@ importlib-metadata = "==4.11.3" pytest-parallel = "==0.1.1" pytest-rerunfailures = "10.0" numpy = "==1.22.2" -py = "==1.11.0" [requires] python_version = "3.9"