[157] Update package manager to poetry

* update poetry install action * update poetry install method * update unit tests pipeline * update makefile test command * update unit test workflow * run unit tests with multiple python versions * cleanup files & update Makefile * update dependency installation with poetry for integration tests * update unit test workflow * update release workflows * update docs * fix typo * remove verbose argument in terraform integration tests * update toolbox image version * fix build command * add supported python versions & doc * update poetry lock file * add support for python 3.11 * update docs
binbashar · Apr 17, 2024 · 6c080b0 · 6c080b0
1 parent 00cbdd1
commit 6c080b0
Show file tree

Hide file tree

Showing 17 changed files with 2,094 additions and 1,609 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,6 @@
+[run]
+branch = True
+
+[report]
+show_missing = True
+skip_covered = True
diff --git a/.github/workflows/release-pypi-build-push-package.yml b/.github/workflows/release-pypi-build-push-package.yml
@@ -15,41 +15,40 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.8'
-          
-      - name: install_dependencies
+
+      - name: Install Poetry
         run: |
-          pip3 install --user --upgrade pipenv && pipenv install --dev --python 3.8
-        shell: bash
+          echo "[INFO] Installing Poetry..."
+          curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
+          echo "[INFO] Configuring Poetry..."
+          poetry config virtualenvs.create false  # This prevents poetry from creating a virtual environment
 
-      - name: bump_version
+      - name: Install dependencies using Poetry
         run: |
-          echo "[INFO] Get latest tag"
-          git fetch --all --tags
-          RELEASE_VERSION=$(git tag --sort version:refname | tail -1)
-          RELEASE_VERSION=$(echo $RELEASE_VERSION | sed s/v//)
-          echo $RELEASE_VERSION
+          echo "[INFO] Installing dependencies..."
+          export ENV POETRY_VIRTUALENVS_CREATE=false
+          export PATH="${PATH}:${HOME}/.poetry/bin"
+          poetry install --with=dev
 
-          echo "[INFO] Write version"
-          sed -i s/$PLACEHOLDER/$RELEASE_VERSION/ $INIT_FILE
-        env:
-          INIT_FILE: leverage/__init__.py
-          PLACEHOLDER: 0.0.0
+      - name: bump_version
+        run: |
+          make bump-version-ci
 
       - name: clean
         run: |
-          rm -rf ./build/ &&	rm -rf ./dist/
+          make clean
 
       - name: build
         run: |
-          pipenv run python setup.py sdist bdist_wheel
+          make build
 
       - name: check
         run:
-          pipenv run twine check dist/*
+          make check
 
       - name: upload
         run: |
-          pipenv run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
+          make push-ci
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

diff --git a/.github/workflows/release-pypi-build-push-test-package.yml b/.github/workflows/release-pypi-build-push-test-package.yml
@@ -20,46 +20,39 @@ jobs:
         with:
           python-version: '3.8'
 
-      - name: install_dependencies
+      - name: Install Poetry
         run: |
-          pip3 install --user --upgrade pipenv && pipenv install --dev --python 3.8
-        shell: bash
+          echo "[INFO] Installing Poetry..."
+          curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
+          echo "[INFO] Configuring Poetry..."
+          poetry config virtualenvs.create false  # This prevents poetry from creating a virtual environment
 
-      - name: bump_version
+      - name: Install dependencies using Poetry
         run: |
-          echo "[INFO] Get current version"
-          CURRENT_VERSION=$(cat $INIT_FILE | grep "__version__" | grep -oP "([0-9]*\.[0-9]*\.[0-9]*)")
-          echo "[INFO] Current version: $CURRENT_VERSION"
+          echo "[INFO] Installing dependencies..."
+          export ENV POETRY_VIRTUALENVS_CREATE=false
+          export PATH="${PATH}:${HOME}/.poetry/bin"
+          poetry install --with=dev
 
-          [ -z "$RELEASE_VERSION" ] && \
-            echo "[INFO] Get latest version from TestPypi." && \
-            LATEST_VERSION=$(curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]") && \
-            echo "[INFO] Latest version: $LATEST_VERSION" && \
-            RELEASE_VERSION="$(echo $LATEST_VERSION | awk 'BEGIN{FS="."; OFS="."} {print $1,$2,$3+1}')rc.1"
-
-          echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
-          ([ $(echo $RELEASE_VERSION | grep -oP "([0-9]*\.[0-9]*\.[0-9]*\-(rc|alpha|beta)[0-9]+)") ] && echo "[INFO] Version ok" ) || (echo "[ERROR] Version is wrong" && exit 1)
-          echo "[INFO] Bump version to $RELEASE_VERSION"
-          sed -i 's/'$CURRENT_VERSION'/'$RELEASE_VERSION'/' $INIT_FILE
-        env:
-          INIT_FILE: leverage/__init__.py
-          RELEASE_VERSION: ${{ github.event.inputs.version }}
+      - name: bump_test_version
+        run: |
+          make bump-test-version RELEASE_VERSION=${{ github.event.inputs.version }}
 
       - name: clean
         run: |
-          rm -rf ./build/ &&	rm -rf ./dist/
+          make clean
 
       - name: build
         run: |
-          pipenv run python setup.py sdist bdist_wheel
+          make build
 
       - name: check
         run:
-          pipenv run twine check dist/*
+          make check
 
       - name: upload
         run: |
-          pipenv run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
+          make push-ci
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml
@@ -25,7 +25,7 @@ jobs:
     strategy:
       max-parallel: 1
       matrix:
-        python-version: [3.8.14,3.9.15,3.10.8]
+        python-version: ['3.8.14', '3.9.15', '3.10.8', '3.11.8']
     steps:
       - name: Checkout base branch
         uses: actions/checkout@v3
@@ -35,11 +35,24 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install Poetry
+        run: |
+          echo "[INFO] Installing Poetry..."
+          curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
+          echo "[INFO] Configuring Poetry..."
+          poetry config virtualenvs.create false  # This prevents poetry from creating a virtual environment
+
+      - name: Install dependencies using Poetry
+        run: |
+          echo "[INFO] Installing dependencies..."
+          export ENV POETRY_VIRTUALENVS_CREATE=false
+          export PATH="${PATH}:${HOME}/.poetry/bin"
+          poetry install --with=dev
+
       - name: Build Leverage CLI
         run: |
-          printf "[INFO] Building Leverage CLI\n"
-          if ! (which pipenv 2>/dev/null); then printf "Installing pipenv..." && pip install pipenv && pipenv --python $(which python) ; else printf "pipenv is already installed"; fi
-          printf "Working with python version $(python --version)"
+          echo "[INFO] Building Leverage CLI"
+          echo "[INFO] Working with python version $(python --version)"
           make build
           pip install -e .
 
@@ -175,7 +188,7 @@ jobs:
           EOF
           echo "[INFO] Disable MFA\n"
           sed -i "s/^\(MFA_ENABLED=\)true/\1false/" build.env
-          sed -E -i 's/^TERRAFORM_IMAGE_TAG=.+$/TERRAFORM_IMAGE_TAG=1.2.7-0.0.5/' build.env;
+          sed -E -i 's/^TERRAFORM_IMAGE_TAG=.+$/TERRAFORM_IMAGE_TAG=1.3.5-0.1.15/' build.env;
         working-directory: ../theblairwitchproject
 
       - name: Test Testing Reference Architecture

diff --git a/.github/workflows/tests-unit.yml b/.github/workflows/tests-unit.yml
@@ -5,19 +5,40 @@ on: [push, pull_request]
 jobs:
   unit_tests:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
     steps:
       - uses: actions/checkout@v3
 
-      - name: install_requirements
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: |
+          echo "[INFO] Installing Poetry..."
+          curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
+          echo "[INFO] Configuring Poetry..."
+          poetry config virtualenvs.create false  # This prevents poetry from creating a virtual environment
+
+      - name: Install dependencies using Poetry
         run: |
-          echo "[INFO] Installing requirements..."
-          pip install -r dev-requirements.txt
+          echo "[INFO] Installing dependencies..."
+          export ENV POETRY_VIRTUALENVS_CREATE=false
+          export PATH="${PATH}:${HOME}/.poetry/bin"
+          poetry install --with=dev
 
-      - name: run_unit_tests
+      - name: Run unit tests
         run: |
           echo "[INFO] Running unit tests and generate coverage report"
-          pytest --verbose
+          export PATH="${PATH}:${HOME}/.poetry/bin"
+          poetry run pytest --verbose --cov=./ --cov-report=xml
         shell: bash
 
       - name: Report Coveralls
+        if: matrix.python-version == '3.8'  # Run this for the first python version only!
         uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/Dockerfile b/Dockerfile
@@ -5,6 +5,9 @@ LABEL vendor="Binbash Leverage ([email protected])"
 RUN apk update &&\
     apk add --no-cache bash bash-completion ncurses git curl gcc musl-dev python3 python3-dev py3-pip
 
+ENV POETRY_VIRTUALENVS_CREATE=false
+ENV PATH="${PATH}:/root/.poetry/bin"
+
 # Install bats from source
 RUN git clone https://github.com/bats-core/bats-core.git && ./bats-core/install.sh /usr/local
 # Install other bats modules
@@ -16,19 +19,16 @@ RUN mkdir /root/.ssh
 # Needed for git to run propertly
 RUN touch /root/.gitconfig
 
-WORKDIR /leverage
-RUN git config --global --add safe.directory /leverage
-# Install requirements for running unit tests
-COPY ./dev-requirements.txt .
-RUN pip install -r dev-requirements.txt
+RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local POETRY_VERSION=1.8.2 python3 -
+
+RUN git config --global --add safe.directory /workdir
+
+# Copying all necessary files to /workdir directory
+COPY . /workdir
+WORKDIR /workdir
+
+RUN poetry install --with=dev --with=main
 
+COPY entrypoint.sh /
 # Make script to configure and start docker daemon the default entrypoint
-RUN echo $'#!/bin/bash                                                                     \n\
-# Configure docker daemon to listen through socket                                         \n\
-mkdir /etc/docker                                                                          \n\
-echo \'{"tls": false, "hosts": ["unix:///var/run/docker.sock"]}\' > /etc/docker/daemon.json\n\
-# Start daemon silently                                                                    \n\
-dockerd > /dev/null 2>&1 &                                                                 \n\
-exec "$@"                                                                                  \n' >> /entrypoint.sh
-RUN chmod +x /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]
diff --git a/Makefile b/Makefile
@@ -2,6 +2,10 @@
 LEVERAGE_TESTING_IMAGE := binbash/leverage-cli-testing
 LEVERAGE_TESTING_TAG   := 2.5.0
 LEVERAGE_IMAGE_TAG     := 1.2.7-0.0.5
+PYPROJECT_FILE := pyproject.toml
+INIT_FILE := leverage/__init__.py
+RELEASE_VERSION ?= $(shell curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]" | awk 'BEGIN{FS="."; OFS="."} {print $$1,$$2,$$3+1}' )rc.1
+PLACEHOLDER := 0.0.0
 
 help:
 	@echo 'Available Commands:'
@@ -11,31 +15,59 @@ build-image: ## Build docker image for testing
 	docker build . -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG}
 
 test-unit: ## Run unit tests and create a coverage report
-	docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose
+	docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose --cov=./ --cov-report=xml
 
 test-unit-no-cov: ## Run unit tests with no coverage report
 	docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose --no-cov
 
 test-int: ## Run integration tests
-	docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage --env LEVERAGE_IMAGE_TAG=${LEVERAGE_IMAGE_TAG} -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} bash -c "pip3 install -e . > /dev/null && bats --verbose-run --show-output-of-passing-tests -p -r tests/bats"
+	docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage --env LEVERAGE_IMAGE_TAG=${LEVERAGE_IMAGE_TAG} -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} bash -c "bats --verbose-run --show-output-of-passing-tests --print-output-on-failure -T -t -p -r tests/bats"
 
 tests: test-unit-no-cov test-int ## Run full set of tests
 
 setup: ## Set up requirements
-	python3 -m pip3 install --user --upgrade pipenv && pipenv install --dev
+	poetry install --with=dev --with=main
 
 clean: ## Clean build files
 	rm -rf ./build/
 	rm -rf ./dist/
 
 build: clean ## Build distributables
-	pipenv run python setup.py sdist bdist_wheel
+	poetry build
 
 check: ## Check distributables
-	pipenv run twine check dist/*
+	poetry run twine check dist/*
 
 push: ## Push distributables to PyPi
-	pipenv run twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
+	poetry run twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
+
+push-ci: ## Push distributables to PyPi (to be used in CI)
+	poetry run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
 
 push-test: ## Push distributables to Pypi test
-	pipenv run twine upload --repository testpypi dist/*
+	poetry run twine upload --repository testpypi dist/*
+
+bump-test-version: ## Bump version based on TestPyPI or provided input
+	@echo "[INFO] Get current version from __init__.py"
+	$(eval CURRENT_VERSION=$(shell awk '/__version__/ {print $$3}' $(INIT_FILE) | tr -d '"' | tr -d "'"))
+	@echo "[INFO] Current version: $(CURRENT_VERSION)"
+	@echo "[INFO] Get latest version from TestPypi."
+	$(eval LATEST_VERSION=$(shell curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]"))
+	@echo "[INFO] Latest version: $(LATEST_VERSION)"
+	$(eval RELEASE_VERSION=$(shell echo $(LATEST_VERSION) | awk 'BEGIN{FS="."; OFS="."} {print $$1,$$2,$$3+1}')rc.1)
+	@echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
+	@echo $(RELEASE_VERSION) | awk '/[0-9]+\.[0-9]+\.[0-9]+-(rc|alpha|beta)[0-9]+/ {print "[INFO] Version ok"}' || (echo "[ERROR] Version is wrong" && exit 1)
+	@echo "[INFO] Bump version to $(RELEASE_VERSION)"
+	@sed -i '' 's/__version__ = "$(CURRENT_VERSION)"/__version__ = "$(RELEASE_VERSION)"/' $(INIT_FILE)
+	@sed -i '' 's/version = "$(CURRENT_VERSION)"/version = "$(RELEASE_VERSION)"/' $(PYPROJECT_FILE)
+
+bump-version-ci: ## Fetch latest tag, update versions in __init__.py and pyproject.toml
+	@echo "[INFO] Get latest tag"
+	$(eval RELEASE_VERSION=$(shell git fetch --all --tags && git tag --sort=version:refname | tail -1 | sed 's/v//'))
+	@echo $(RELEASE_VERSION)
+
+	@echo "[INFO] Write version to __init__.py"
+	@sed -i '' 's/$(PLACEHOLDER)/$(RELEASE_VERSION)/' $(INIT_FILE)
+
+	@echo "[INFO] Update version in pyproject.toml"
+	@sed -i '' 's/version = ".*"/version = "$(RELEASE_VERSION)"/' $(PYPROJECT_FILE)
diff --git a/Pipfile b/Pipfile