Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[157] Refactor of build processes and dependency management using Poetry #261

Merged
merged 1 commit into from
Apr 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[run]
branch = True

[report]
show_missing = True
skip_covered = True
37 changes: 18 additions & 19 deletions .github/workflows/release-pypi-build-push-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,40 @@ jobs:
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: install_dependencies

- name: Install Poetry
run: |
pip3 install --user --upgrade pipenv && pipenv install --dev --python 3.8
shell: bash
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment

- name: bump_version
- name: Install dependencies using Poetry
run: |
echo "[INFO] Get latest tag"
git fetch --all --tags
RELEASE_VERSION=$(git tag --sort version:refname | tail -1)
RELEASE_VERSION=$(echo $RELEASE_VERSION | sed s/v//)
echo $RELEASE_VERSION
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev

echo "[INFO] Write version"
sed -i s/$PLACEHOLDER/$RELEASE_VERSION/ $INIT_FILE
env:
INIT_FILE: leverage/__init__.py
PLACEHOLDER: 0.0.0
- name: bump_version
run: |
make bump-version-ci

- name: clean
run: |
rm -rf ./build/ && rm -rf ./dist/
make clean

- name: build
run: |
pipenv run python setup.py sdist bdist_wheel
make build

- name: check
run:
pipenv run twine check dist/*
make check

- name: upload
run: |
pipenv run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
make push-ci
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
Expand Down
41 changes: 17 additions & 24 deletions .github/workflows/release-pypi-build-push-test-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,46 +20,39 @@ jobs:
with:
python-version: '3.8'

- name: install_dependencies
- name: Install Poetry
run: |
pip3 install --user --upgrade pipenv && pipenv install --dev --python 3.8
shell: bash
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment

- name: bump_version
- name: Install dependencies using Poetry
run: |
echo "[INFO] Get current version"
CURRENT_VERSION=$(cat $INIT_FILE | grep "__version__" | grep -oP "([0-9]*\.[0-9]*\.[0-9]*)")
echo "[INFO] Current version: $CURRENT_VERSION"
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev

[ -z "$RELEASE_VERSION" ] && \
echo "[INFO] Get latest version from TestPypi." && \
LATEST_VERSION=$(curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]") && \
echo "[INFO] Latest version: $LATEST_VERSION" && \
RELEASE_VERSION="$(echo $LATEST_VERSION | awk 'BEGIN{FS="."; OFS="."} {print $1,$2,$3+1}')rc.1"

echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
([ $(echo $RELEASE_VERSION | grep -oP "([0-9]*\.[0-9]*\.[0-9]*\-(rc|alpha|beta)[0-9]+)") ] && echo "[INFO] Version ok" ) || (echo "[ERROR] Version is wrong" && exit 1)
echo "[INFO] Bump version to $RELEASE_VERSION"
sed -i 's/'$CURRENT_VERSION'/'$RELEASE_VERSION'/' $INIT_FILE
env:
INIT_FILE: leverage/__init__.py
RELEASE_VERSION: ${{ github.event.inputs.version }}
- name: bump_test_version
run: |
make bump-test-version RELEASE_VERSION=${{ github.event.inputs.version }}

- name: clean
run: |
rm -rf ./build/ && rm -rf ./dist/
make clean

- name: build
run: |
pipenv run python setup.py sdist bdist_wheel
make build

- name: check
run:
pipenv run twine check dist/*
make check

- name: upload
run: |
pipenv run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*
make push-ci
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
21 changes: 17 additions & 4 deletions .github/workflows/tests-integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
strategy:
max-parallel: 1
matrix:
python-version: [3.8.14,3.9.15,3.10.8]
python-version: ['3.8.14', '3.9.15', '3.10.8', '3.11.8']
toolbox-image-tag: ['1.3.5-0.1.15', '1.5.0-0.1.15', '1.6.0-0.1.15']
steps:
- name: Checkout base branch
Expand All @@ -36,11 +36,24 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Install Poetry
run: |
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment

- name: Install dependencies using Poetry
run: |
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev

- name: Build Leverage CLI
run: |
printf "[INFO] Building Leverage CLI\n"
if ! (which pipenv 2>/dev/null); then printf "Installing pipenv..." && pip install pipenv && pipenv --python $(which python) ; else printf "pipenv is already installed"; fi
printf "Working with python version $(python --version)"
echo "[INFO] Building Leverage CLI"
echo "[INFO] Working with python version $(python --version)"
make build
pip install -e .

Expand Down
31 changes: 26 additions & 5 deletions .github/workflows/tests-unit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,40 @@ on: [push, pull_request]
jobs:
unit_tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ '3.8', '3.9', '3.10', '3.11' ]
steps:
- uses: actions/checkout@v3

- name: install_requirements
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}

- name: Install Poetry
run: |
echo "[INFO] Installing Poetry..."
curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.2 python -
echo "[INFO] Configuring Poetry..."
poetry config virtualenvs.create false # This prevents poetry from creating a virtual environment

- name: Install dependencies using Poetry
run: |
echo "[INFO] Installing requirements..."
pip install -r dev-requirements.txt
echo "[INFO] Installing dependencies..."
export ENV POETRY_VIRTUALENVS_CREATE=false
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry install --with=dev

- name: run_unit_tests
- name: Run unit tests
run: |
echo "[INFO] Running unit tests and generate coverage report"
pytest --verbose
export PATH="${PATH}:${HOME}/.poetry/bin"
poetry run pytest --verbose --cov=./ --cov-report=xml
shell: bash

- name: Report Coveralls
if: matrix.python-version == '3.8' # Run this for the first python version only!
uses: coverallsapp/github-action@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
26 changes: 13 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ LABEL vendor="Binbash Leverage ([email protected])"
RUN apk update &&\
apk add --no-cache bash bash-completion ncurses git curl gcc musl-dev python3 python3-dev py3-pip

ENV POETRY_VIRTUALENVS_CREATE=false
ENV PATH="${PATH}:/root/.poetry/bin"

# Install bats from source
RUN git clone https://github.com/bats-core/bats-core.git && ./bats-core/install.sh /usr/local
# Install other bats modules
Expand All @@ -16,19 +19,16 @@ RUN mkdir /root/.ssh
# Needed for git to run propertly
RUN touch /root/.gitconfig

WORKDIR /leverage
RUN git config --global --add safe.directory /leverage
# Install requirements for running unit tests
COPY ./dev-requirements.txt .
RUN pip install -r dev-requirements.txt
RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local POETRY_VERSION=1.8.2 python3 -

RUN git config --global --add safe.directory /workdir

# Copying all necessary files to /workdir directory
COPY . /workdir
WORKDIR /workdir

RUN poetry install --with=dev --with=main

COPY entrypoint.sh /
# Make script to configure and start docker daemon the default entrypoint
RUN echo $'#!/bin/bash \n\
# Configure docker daemon to listen through socket \n\
mkdir /etc/docker \n\
echo \'{"tls": false, "hosts": ["unix:///var/run/docker.sock"]}\' > /etc/docker/daemon.json\n\
# Start daemon silently \n\
dockerd > /dev/null 2>&1 & \n\
exec "$@" \n' >> /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
46 changes: 39 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
LEVERAGE_TESTING_IMAGE := binbash/leverage-cli-testing
LEVERAGE_TESTING_TAG := 2.5.0
LEVERAGE_IMAGE_TAG := 1.2.7-0.0.5
PYPROJECT_FILE := pyproject.toml
INIT_FILE := leverage/__init__.py
RELEASE_VERSION ?= $(shell curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]" | awk 'BEGIN{FS="."; OFS="."} {print $$1,$$2,$$3+1}' )rc.1
PLACEHOLDER := 0.0.0

help:
@echo 'Available Commands:'
Expand All @@ -11,31 +15,59 @@ build-image: ## Build docker image for testing
docker build . -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG}

test-unit: ## Run unit tests and create a coverage report
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose --cov=./ --cov-report=xml

test-unit-no-cov: ## Run unit tests with no coverage report
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} pytest --verbose --no-cov

test-int: ## Run integration tests
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage --env LEVERAGE_IMAGE_TAG=${LEVERAGE_IMAGE_TAG} -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} bash -c "pip3 install -e . > /dev/null && bats --verbose-run --show-output-of-passing-tests -p -r tests/bats"
docker run --rm --privileged --mount type=bind,src=$(shell pwd),dst=/leverage --env LEVERAGE_IMAGE_TAG=${LEVERAGE_IMAGE_TAG} -t ${LEVERAGE_TESTING_IMAGE}:${LEVERAGE_TESTING_TAG} bash -c "bats --verbose-run --show-output-of-passing-tests --print-output-on-failure -T -t -p -r tests/bats"

tests: test-unit-no-cov test-int ## Run full set of tests

setup: ## Set up requirements
python3 -m pip3 install --user --upgrade pipenv && pipenv install --dev
poetry install --with=dev --with=main

clean: ## Clean build files
rm -rf ./build/
rm -rf ./dist/

build: clean ## Build distributables
pipenv run python setup.py sdist bdist_wheel
poetry build

check: ## Check distributables
pipenv run twine check dist/*
poetry run twine check dist/*

push: ## Push distributables to PyPi
pipenv run twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
poetry run twine upload --repository-url https://upload.pypi.org/legacy/ dist/*

push-ci: ## Push distributables to PyPi (to be used in CI)
poetry run twine upload --non-interactive --repository-url https://upload.pypi.org/legacy/ dist/*

push-test: ## Push distributables to Pypi test
pipenv run twine upload --repository testpypi dist/*
poetry run twine upload --repository testpypi dist/*

bump-test-version: ## Bump version based on TestPyPI or provided input
@echo "[INFO] Get current version from __init__.py"
$(eval CURRENT_VERSION=$(shell awk '/__version__/ {print $$3}' $(INIT_FILE) | tr -d '"' | tr -d "'"))
@echo "[INFO] Current version: $(CURRENT_VERSION)"
@echo "[INFO] Get latest version from TestPypi."
$(eval LATEST_VERSION=$(shell curl -sL "https://test.pypi.org/pypi/leverage/json" | jq -r ".releases | keys | sort | .[-1]"))
@echo "[INFO] Latest version: $(LATEST_VERSION)"
$(eval RELEASE_VERSION=$(shell echo $(LATEST_VERSION) | awk 'BEGIN{FS="."; OFS="."} {print $$1,$$2,$$3+1}')rc.1)
@echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
@echo $(RELEASE_VERSION) | awk '/[0-9]+\.[0-9]+\.[0-9]+-(rc|alpha|beta)[0-9]+/ {print "[INFO] Version ok"}' || (echo "[ERROR] Version is wrong" && exit 1)
@echo "[INFO] Bump version to $(RELEASE_VERSION)"
@sed -i '' 's/__version__ = "$(CURRENT_VERSION)"/__version__ = "$(RELEASE_VERSION)"/' $(INIT_FILE)
@sed -i '' 's/version = "$(CURRENT_VERSION)"/version = "$(RELEASE_VERSION)"/' $(PYPROJECT_FILE)

bump-version-ci: ## Fetch latest tag, update versions in __init__.py and pyproject.toml
@echo "[INFO] Get latest tag"
$(eval RELEASE_VERSION=$(shell git fetch --all --tags && git tag --sort=version:refname | tail -1 | sed 's/v//'))
@echo $(RELEASE_VERSION)

@echo "[INFO] Write version to __init__.py"
@sed -i '' 's/$(PLACEHOLDER)/$(RELEASE_VERSION)/' $(INIT_FILE)

@echo "[INFO] Update version in pyproject.toml"
@sed -i '' 's/version = ".*"/version = "$(RELEASE_VERSION)"/' $(PYPROJECT_FILE)
30 changes: 0 additions & 30 deletions Pipfile

This file was deleted.

Loading
Loading