Skip to content

Commit

Permalink
[157] Update package manager to poetry
Browse files Browse the repository at this point in the history
* update poetry install action
* update poetry install method
* update unit tests pipeline
* update makefile test command
* update unit test workflow
* run unit tests with multiple python versions
* cleanup files & update Makefile
* update dependency installation with poetry for integration tests
* update unit test workflow
* update release workflows
* update docs
* fix typo
* remove verbose argument in terraform integration tests
* update toolbox image version
* fix build command
* add supported python versions & doc
* update poetry lock file
* add support for python 3.11
  • Loading branch information
borland667 authored and Osvaldo Demo committed Apr 17, 2024
1 parent 00cbdd1 commit e2a3f6b
Show file tree
Hide file tree
Showing 17 changed files with 2,093 additions and 1,609 deletions.
6 changes: 6 additions & 0 deletions .coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[run]
branch = True

[report]
show_missing = True
skip_covered = True
37 changes: 18 additions & 19 deletions .github/workflows/release-pypi-build-push-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,40 @@ jobs:
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: install_dependencies

- name: Install Poetry
run: |
pip3 install --user --upgrade pipenv && pipenv install --dev --python 3.8
shell: bash
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment
- name: bump_version
- name: Install dependencies using Poetry
run: |
echo "[INFO] Get latest tag"
git fetch --all --tags
RELEASE_VERSION=$(git tag --sort version:refname | tail -1)
RELEASE_VERSION=$(echo $RELEASE_VERSION | sed s/v//)
echo $RELEASE_VERSION
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev
echo "[INFO] Write version"
sed -i s/$PLACEHOLDER/$RELEASE_VERSION/ $INIT_FILE
env:
INIT_FILE: leverage/__init__.py
PLACEHOLDER: 0.0.0
- name: bump_version
run: |
make bump-version-ci
- name: clean
run: |
rm -rf ./build/ && rm -rf ./dist/
make clean
- name: build
run: |
pipenv run python setup.py sdist bdist_wheel
make build
- name: check
run:
pipenv run twine check dist/*
make check

- name: upload
run: |
pipenv run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
make push-ci
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
Expand Down
41 changes: 17 additions & 24 deletions .github/workflows/release-pypi-build-push-test-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,46 +20,39 @@ jobs:
with:
python-version: '3.8'

- name: install_dependencies
- name: Install Poetry
run: |
pip3 install --user --upgrade pipenv && pipenv install --dev --python 3.8
shell: bash
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment
- name: bump_version
- name: Install dependencies using Poetry
run: |
echo "[INFO] Get current version"
CURRENT_VERSION=$(cat $INIT_FILE | grep "__version__" | grep -oP "([0-9]*\.[0-9]*\.[0-9]*)")
echo "[INFO] Current version: $CURRENT_VERSION"
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev
[ -z "$RELEASE_VERSION" ] && \
echo "[INFO] Get latest version from TestPypi." && \
LATEST_VERSION=$(curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]") && \
echo "[INFO] Latest version: $LATEST_VERSION" && \
RELEASE_VERSION="$(echo $LATEST_VERSION | awk 'BEGIN{FS="."; OFS="."} {print $1,$2,$3+1}')rc.1"
echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
([ $(echo $RELEASE_VERSION | grep -oP "([0-9]*\.[0-9]*\.[0-9]*\-(rc|alpha|beta)[0-9]+)") ] && echo "[INFO] Version ok" ) || (echo "[ERROR] Version is wrong" && exit 1)
echo "[INFO] Bump version to $RELEASE_VERSION"
sed -i 's/'$CURRENT_VERSION'/'$RELEASE_VERSION'/' $INIT_FILE
env:
INIT_FILE: leverage/__init__.py
RELEASE_VERSION: ${{ github.event.inputs.version }}
- name: bump_test_version
run: |
make bump-test-version RELEASE_VERSION=${{ github.event.inputs.version }}
- name: clean
run: |
rm -rf ./build/ && rm -rf ./dist/
make clean
- name: build
run: |
pipenv run python setup.py sdist bdist_wheel
make build
- name: check
run:
pipenv run twine check dist/*
make check

- name: upload
run: |
pipenv run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
make push-ci
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
23 changes: 18 additions & 5 deletions .github/workflows/tests-integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
strategy:
max-parallel: 1
matrix:
python-version: [3.8.14,3.9.15,3.10.8]
python-version: ['3.8.14', '3.9.15', '3.10.8', '3.11.8']
steps:
- name: Checkout base branch
uses: actions/checkout@v3
Expand All @@ -35,11 +35,24 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Install Poetry
run: |
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment
- name: Install dependencies using Poetry
run: |
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev
- name: Build Leverage CLI
run: |
printf "[INFO] Building Leverage CLI\n"
if ! (which pipenv 2>/dev/null); then printf "Installing pipenv..." && pip install pipenv && pipenv --python $(which python) ; else printf "pipenv is already installed"; fi
printf "Working with python version $(python --version)"
echo "[INFO] Building Leverage CLI"
echo "[INFO] Working with python version $(python --version)"
make build
pip install -e .
Expand Down Expand Up @@ -175,7 +188,7 @@ jobs:
EOF
echo "[INFO] Disable MFA\n"
sed -i "s/^\(MFA_ENABLED=\)true/\1false/" build.env
sed -E -i 's/^TERRAFORM_IMAGE_TAG=.+$/TERRAFORM_IMAGE_TAG=1.2.7-0.0.5/' build.env;
sed -E -i 's/^TERRAFORM_IMAGE_TAG=.+$/TERRAFORM_IMAGE_TAG=1.3.5-0.1.15/' build.env;
working-directory: ../theblairwitchproject

- name: Test Testing Reference Architecture
Expand Down
31 changes: 26 additions & 5 deletions .github/workflows/tests-unit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,40 @@ on: [push, pull_request]
jobs:
unit_tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ '3.8', '3.9', '3.10', '3.11' ]
steps:
- uses: actions/checkout@v3

- name: install_requirements
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}

- name: Install Poetry
run: |
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment
- name: Install dependencies using Poetry
run: |
echo "[INFO] Installing requirements..."
pip install -r dev-requirements.txt
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev
- name: run_unit_tests
- name: Run unit tests
run: |
echo "[INFO] Running unit tests and generate coverage report"
pytest --verbose
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry run pytest --verbose --cov=./ --cov-report=xml
shell: bash

- name: Report Coveralls
if: matrix.python-version == '3.8' # Run this for the first python version only!
uses: coverallsapp/github-action@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
26 changes: 13 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ LABEL vendor="Binbash Leverage ([email protected])"
RUN apk update &&\
apk add --no-cache bash bash-completion ncurses git curl gcc musl-dev python3 python3-dev py3-pip

ENV POETRY_VIRTUALENVS_CREATE=false
ENV PATH="${PATH}:/root/.poetry/bin"

# Install bats from source
RUN git clone https://github.com/bats-core/bats-core.git && ./bats-core/install.sh /usr/local
# Install other bats modules
Expand All @@ -16,19 +19,16 @@ RUN mkdir /root/.ssh
# Needed for git to run propertly
RUN touch /root/.gitconfig

WORKDIR /leverage
RUN git config --global --add safe.directory /leverage
# Install requirements for running unit tests
COPY ./dev-requirements.txt .
RUN pip install -r dev-requirements.txt
RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local POETRY_VERSION=1.8.2 python3 -

RUN git config --global --add safe.directory /workdir

# Copying all necessary files to /workdir directory
COPY . /workdir
WORKDIR /workdir

RUN poetry install --with=dev --with=main

COPY entrypoint.sh /
# Make script to configure and start docker daemon the default entrypoint
RUN echo $'#!/bin/bash \n\
# Configure docker daemon to listen through socket \n\
mkdir /etc/docker \n\
echo \'{"tls": false, "hosts": ["unix:///var/run/docker.sock"]}\' > /etc/docker/daemon.json\n\
# Start daemon silently \n\
dockerd > /dev/null 2>&1 & \n\
exec "$@" \n' >> /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
46 changes: 39 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
LEVERAGE_TESTING_IMAGE := binbash/leverage-cli-testing
LEVERAGE_TESTING_TAG := 2.5.0
LEVERAGE_IMAGE_TAG := 1.2.7-0.0.5
PYPROJECT_FILE := pyproject.toml
INIT_FILE := leverage/__init__.py
RELEASE_VERSION ?= $(shell curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]" | awk 'BEGIN{FS="."; OFS="."} {print $$1,$$2,$$3+1}' )rc.1
PLACEHOLDER := 0.0.0

help:
@echo 'Available Commands:'
Expand All @@ -11,31 +15,59 @@ build-image: ## Build docker image for testing
docker build . -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG}

test-unit: ## Run unit tests and create a coverage report
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose --cov=./ --cov-report=xml

test-unit-no-cov: ## Run unit tests with no coverage report
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose --no-cov

test-int: ## Run integration tests
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage --env LEVERAGE_IMAGE_TAG=${LEVERAGE_IMAGE_TAG} -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} bash -c "pip3 install -e . > /dev/null && bats --verbose-run --show-output-of-passing-tests -p -r tests/bats"
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage --env LEVERAGE_IMAGE_TAG=${LEVERAGE_IMAGE_TAG} -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} bash -c "bats --verbose-run --show-output-of-passing-tests --print-output-on-failure -T -t -p -r tests/bats"

tests: test-unit-no-cov test-int ## Run full set of tests

setup: ## Set up requirements
python3 -m pip3 install --user --upgrade pipenv && pipenv install --dev
poetry install --with=dev --with=main

clean: ## Clean build files
rm -rf ./build/
rm -rf ./dist/

build: clean ## Build distributables
pipenv run python setup.py sdist bdist_wheel
poetry build

check: ## Check distributables
pipenv run twine check dist/*
poetry run twine check dist/*

push: ## Push distributables to PyPi
pipenv run twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
poetry run twine upload --repository-url https://upload.pypi.org/legacy/ dist/*

push-ci: ## Push distributables to PyPi (to be used in CI)
poetry run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*

push-test: ## Push distributables to Pypi test
pipenv run twine upload --repository testpypi dist/*
poetry run twine upload --repository testpypi dist/*

bump-test-version: ## Bump version based on TestPyPI or provided input
@echo "[INFO] Get current version from __init__.py"
$(eval CURRENT_VERSION=$(shell awk '/__version__/ {print $$3}' $(INIT_FILE) | tr -d '"' | tr -d "'"))
@echo "[INFO] Current version: $(CURRENT_VERSION)"
@echo "[INFO] Get latest version from TestPypi."
$(eval LATEST_VERSION=$(shell curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]"))
@echo "[INFO] Latest version: $(LATEST_VERSION)"
$(eval RELEASE_VERSION=$(shell echo $(LATEST_VERSION) | awk 'BEGIN{FS="."; OFS="."} {print $$1,$$2,$$3+1}')rc.1)
@echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
@echo $(RELEASE_VERSION) | awk '/[0-9]+\.[0-9]+\.[0-9]+-(rc|alpha|beta)[0-9]+/ {print "[INFO] Version ok"}' || (echo "[ERROR] Version is wrong" && exit 1)
@echo "[INFO] Bump version to $(RELEASE_VERSION)"
@sed -i '' 's/__version__ = "$(CURRENT_VERSION)"/__version__ = "$(RELEASE_VERSION)"/' $(INIT_FILE)
@sed -i '' 's/version = "$(CURRENT_VERSION)"/version = "$(RELEASE_VERSION)"/' $(PYPROJECT_FILE)

bump-version-ci: ## Fetch latest tag, update versions in __init__.py and pyproject.toml
@echo "[INFO] Get latest tag"
$(eval RELEASE_VERSION=$(shell git fetch --all --tags && git tag --sort=version:refname | tail -1 | sed 's/v//'))
@echo $(RELEASE_VERSION)

@echo "[INFO] Write version to __init__.py"
@sed -i '' 's/$(PLACEHOLDER)/$(RELEASE_VERSION)/' $(INIT_FILE)

@echo "[INFO] Update version in pyproject.toml"
@sed -i '' 's/version = ".*"/version = "$(RELEASE_VERSION)"/' $(PYPROJECT_FILE)
30 changes: 0 additions & 30 deletions Pipfile

This file was deleted.

Loading

0 comments on commit e2a3f6b

Please sign in to comment.