From bb4952f5575e71c19473982a0287db8773b4fd5a Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:00:25 +0200 Subject: [PATCH 1/3] Revert "Python: Disable failing integration tests" This reverts commit 8c2455fc113f3d0ee91b5d264c96782e06a6c274. --- .../force-enable-library-extraction/{disabled-test.sh => test.sh} | 0 .../ignore-venv/{disabled-test.sh => test.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/extractor/cli-integration-test/force-enable-library-extraction/{disabled-test.sh => test.sh} (100%) rename python/extractor/cli-integration-test/ignore-venv/{disabled-test.sh => test.sh} (100%) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh similarity index 100% rename from python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh rename to python/extractor/cli-integration-test/force-enable-library-extraction/test.sh diff --git a/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh similarity index 100% rename from python/extractor/cli-integration-test/ignore-venv/disabled-test.sh rename to python/extractor/cli-integration-test/ignore-venv/test.sh From a0d6324f685ccfbada6d682379101f6822dfef44 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:01:10 +0200 Subject: [PATCH 2/3] Python: Fix ignore-venv integration test Now that we no longer support the fallback option (https://github.com/github/codeql/pull/16127) --- .../cli-integration-test/ignore-venv/test.sh | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/python/extractor/cli-integration-test/ignore-venv/test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh index e13680088915..8735203ec32d 100755 --- a/python/extractor/cli-integration-test/ignore-venv/test.sh +++ b/python/extractor/cli-integration-test/ignore-venv/test.sh @@ -17,63 +17,33 @@ mkdir dbs # set up venvs cd repo_dir +# make venv with some package in it (so we show that our ignore logic is correct) python3 -m venv venv venv/bin/pip install flask -python3 -m venv venv2 - cd "$SCRIPTDIR" -# In 2.16.0 we stop extracting libraries by default, so to test this functionality we -# need to force enable it. Once we release 2.17.0 and turn off library extraction for -# good, we can remove the part of this test ensuring that dependencies in an active -# venv are still extracted (since that will no longer be the case). -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 - -# Create DBs with venv2 active (that does not have flask installed) -source repo_dir/venv2/bin/activate - export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= $CODEQL database create dbs/normal --language python --source-root repo_dir/ export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1 $CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/ -# Create DB with venv active that has flask installed. We want to ensure that we're -# still able to resolve imports to flask, but don't want to extract EVERYTHING from -# within the venv. Important note is that the test-file in the repo_dir actually imports -# flask :D -source repo_dir/venv/bin/activate -export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= -$CODEQL database create dbs/normal-with-flask-venv --language python --source-root repo_dir/ - # --- set +x EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l) -EXTRACTED_ACTIVE_FLASK=$(unzip -l dbs/normal-with-flask-venv/src.zip | wc -l) exitcode=0 echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE" -echo "EXTRACTED_ACTIVE_FLASK=$EXTRACTED_ACTIVE_FLASK" if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE" exitcode=1 fi -if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_ACTIVE_FLASK ]]; then - echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_ACTIVE_FLASK" - exitcode=1 -fi - -if [[ ! $EXTRACTED_ACTIVE_FLASK -lt $EXTRACTED_NO_VENV_IGNORE ]]; then - echo "ERROR: EXTRACTED_ACTIVE_FLASK not smaller EXTRACTED_NO_VENV_IGNORE" - exitcode=1 -fi - exit $exitcode From e9e7ccddcea1f00326522457c5a5d816564d55df Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 9 Apr 2024 14:02:34 +0200 Subject: [PATCH 3/3] Python: delete `force-enable-library-extraction` integration test --- .../repo_dir/foo.py | 3 -- .../force-enable-library-extraction/test.sh | 41 ------------------- 2 files changed, 44 deletions(-) delete mode 100644 python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py delete mode 100755 python/extractor/cli-integration-test/force-enable-library-extraction/test.sh diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py b/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py deleted file mode 100644 index cf0cd77a1087..000000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py +++ /dev/null @@ -1,3 +0,0 @@ -import pip - -print(42) diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh deleted file mode 100755 index 9d74cfaca4b2..000000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/test.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ - -set -x - -CODEQL=${CODEQL:-codeql} - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$SCRIPTDIR" - -# start on clean slate -rm -rf dbs -mkdir dbs - -cd "$SCRIPTDIR" - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0= -$CODEQL database create dbs/normal --language python --source-root repo_dir/ - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 -$CODEQL database create dbs/with-lib-extraction --language python --source-root repo_dir/ - -# --- - -set +x - -EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) -EXTRACTED_WITH_LIB_EXTRACTION=$(unzip -l dbs/with-lib-extraction/src.zip | wc -l) - -exitcode=0 - -echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" -echo "EXTRACTED_WITH_LIB_EXTRACTION=$EXTRACTED_WITH_LIB_EXTRACTION" - -if [[ ! $EXTRACTED_WITH_LIB_EXTRACTION -gt $EXTRACTED_NORMAL ]]; then - echo "ERROR: EXTRACTED_WITH_LIB_EXTRACTION not greater than EXTRACTED_NORMAL" - exitcode=1 -fi - -exit $exitcode