From 1d63cf7f68852a14b1b79c30d37aafcb2beb6003 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Thu, 7 Mar 2024 13:28:07 +0200 Subject: [PATCH 1/6] Add pyinstaller .spec file --- commcare-export.spec | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 commcare-export.spec diff --git a/commcare-export.spec b/commcare-export.spec new file mode 100644 index 0000000..7aed69c --- /dev/null +++ b/commcare-export.spec @@ -0,0 +1,40 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['commcare_export/cli.py'], + pathex=[], + binaries=[], + datas=[ + ('./commcare_export', './commcare_export'), + ('./migrations', './migrations'), + ], + hiddenimports=[ + 'sqlalchemy.sql.default_comparator', + ], + hookspath=[], + runtime_hooks=[], + excludes=[], +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.datas, + [], + name='commcare-export', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) From eb641e4047fc7e88fa32ada675aeb4f123e02540 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Thu, 7 Mar 2024 13:28:59 +0200 Subject: [PATCH 2/6] Add build folder for compiling exe --- build_exe/README.md | 54 +++++++++++++++++++++++++ build_exe/linux/Dockerfile-py3-amd64 | 59 ++++++++++++++++++++++++++++ build_exe/linux/entrypoint-linux.sh | 14 +++++++ build_exe/requirements.txt | 8 ++++ 4 files changed, 135 insertions(+) create mode 100644 build_exe/README.md create mode 100644 build_exe/linux/Dockerfile-py3-amd64 create mode 100644 build_exe/linux/entrypoint-linux.sh create mode 100644 build_exe/requirements.txt diff --git a/build_exe/README.md b/build_exe/README.md new file mode 100644 index 0000000..afebee0 --- /dev/null +++ b/build_exe/README.md @@ -0,0 +1,54 @@ +# Compiling DET to running executable +This folder contains relevant files needed (dockerfiles and scripts) for compiling the DET into an executable file. +The file structure is segmented into the different operating systems the resultant executable will +be compatible on. + +(Currently only Linux is supported; Windows coming soon) + + +## How it works +In order to compile the DET script into a working executable we use [pyinstaller](https://github.com/pyinstaller/pyinstaller) in a containerized +environment. The dockerfile is an edited version from [cdrx/docker-pyinstaller](https://github.com/cdrx/docker-pyinstaller) +which is slightly modified to suit our use-case. + +When a new release of the DET is published, a workflow is triggered which automatically compiles an executable from the latest +code using the custom built docker image, `dimagi/commcare-export-pyinstaller-linux`, then uploads it to the release as an asset. + +If you ever have to compile the executable yourself you can follow the section below, *Compiling executable files locally*, on how to compile an executable locally. + + +Compiling executable files locally +----------------------------------- +The DET executable files are compiled using a tool called [pyinstaller](https://pyinstaller.org/en/stable/). +Pyinstaller is very easy to use, but only works out-of-the-box for Linux as support for cross-compilation was +dropped in earlier releases. Another tool, [wine](https://www.winehq.org/), can be used in conjuction with +pyinstaller to compile the Windows exe files (not yet supported). + +Luckily in the world we live containerization is a thing. We use a docker container, `dimagi/commcare-export-pyinstaller-linux` +(based on [docker-pyinstaller](https://github.com/cdrx/docker-pyinstaller)), which allows you to seamlessly compile the Linux binary, so we don't ever have to worry about installing any additional packages ourselves. + +To compile a new linux binary, first make sure you have the docker image used to generate the executable: +> docker pull dimagi/commcare-export-pyinstaller-linux:latest + +Now it's really as simple as running +> docker run -v "$(pwd):/src/" dimagi/commcare-export-pyinstaller-linux + +Once you're done, the compiled file can be located at `./dist/linux/commcare-export`. + +The tool needs two files to make the process work: +1. `commcare-export.spec`: this file is used by `pyinstaller` and is already defined and sits at the top of this project. +It shouldn't be necessary for you to change any parameters in the file. +2. `requirements.txt`: this file lists all the necessary packages needed for running commcare-export. + + +## Updating the docker image +Are you sure you need to update the image? + +Just checking... + + +If it's needed to make any changes (for whatever reason) to the docker image you can rebuild the image as follows: +> docker build -f ./build_exe/linux/Dockerfile-py3-amd64 -t dimagi/commcare-export-pyinstaller-linux:latest . + +Now upload the new image to dockerhub (remember to log in to the account first!): +> docker image push dimagi/commcare-export-pyinstaller-linux:latest diff --git a/build_exe/linux/Dockerfile-py3-amd64 b/build_exe/linux/Dockerfile-py3-amd64 new file mode 100644 index 0000000..1c33b42 --- /dev/null +++ b/build_exe/linux/Dockerfile-py3-amd64 @@ -0,0 +1,59 @@ +FROM ubuntu:20.04 +SHELL ["/bin/bash", "-i", "-c"] + +ARG PYTHON_VERSION=3.9.18 +ARG PYINSTALLER_VERSION=6.4 + +ENV PYPI_URL=https://pypi.python.org/ +ENV PYPI_INDEX_URL=https://pypi.python.org/simple +ENV PYENV_VERSION=${PYTHON_VERSION} + +COPY ./build_exe/linux/entrypoint-linux.sh /entrypoint.sh + +RUN \ + set -x \ + # update system + && apt-get update \ + # install requirements + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + wget \ + git \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + zlib1g-dev \ + libffi-dev \ + # required because openSSL on Ubuntu 12.04 and 14.04 run out of support versions of OpenSSL + && mkdir openssl \ + && cd openssl \ + # latest version, there won't be anything newer for this + && wget https://www.openssl.org/source/openssl-1.0.2u.tar.gz \ + && tar -xzvf openssl-1.0.2u.tar.gz \ + && cd openssl-1.0.2u \ + && ./config --prefix=$HOME/openssl --openssldir=$HOME/openssl shared zlib \ + && make \ + && make install \ + # install pyenv + && echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc \ + && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc \ + && source ~/.bashrc \ + && curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \ + && echo 'eval "$(pyenv init -)"' >> ~/.bashrc \ + && source ~/.bashrc \ + # install python + && PATH="$HOME/openssl:$PATH" CPPFLAGS="-O2 -I$HOME/openssl/include" CFLAGS="-I$HOME/openssl/include/" LDFLAGS="-L$HOME/openssl/lib -Wl,-rpath,$HOME/openssl/lib" LD_LIBRARY_PATH=$HOME/openssl/lib:$LD_LIBRARY_PATH LD_RUN_PATH="$HOME/openssl/lib" CONFIGURE_OPTS="--with-openssl=$HOME/openssl" PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYTHON_VERSION \ + && pyenv global $PYTHON_VERSION \ + && pip install --upgrade pip \ + # install pyinstaller + && pip install pyinstaller==$PYINSTALLER_VERSION \ + && mkdir /src/ \ + && chmod +x /entrypoint.sh + +VOLUME /src/ +WORKDIR /src/ + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/build_exe/linux/entrypoint-linux.sh b/build_exe/linux/entrypoint-linux.sh new file mode 100644 index 0000000..ba02df5 --- /dev/null +++ b/build_exe/linux/entrypoint-linux.sh @@ -0,0 +1,14 @@ +#!/bin/bash -i + +# Fail on errors. +set -e + +# Make sure .bashrc is sourced +. /root/.bashrc + +cd /src + +pip install . +pip install -r build_exe/requirements.txt + +pyinstaller --clean -y --dist ./dist/linux --workpath /tmp *.spec diff --git a/build_exe/requirements.txt b/build_exe/requirements.txt new file mode 100644 index 0000000..5231b26 --- /dev/null +++ b/build_exe/requirements.txt @@ -0,0 +1,8 @@ +# This file is only used by pyinstaller to create the executable DET instance +chardet +psycopg2-binary +pymysql +pyodbc +urllib3==1.26.7 +xlwt +openpyxl From da90bef971d32d8dda0bda62ef091ad2c69fbd5c Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Thu, 7 Mar 2024 13:30:33 +0200 Subject: [PATCH 3/6] Add github workflow action --- .github/workflows/release_actions.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/release_actions.yml diff --git a/.github/workflows/release_actions.yml b/.github/workflows/release_actions.yml new file mode 100644 index 0000000..cf7f7f8 --- /dev/null +++ b/.github/workflows/release_actions.yml @@ -0,0 +1,26 @@ +name: commcare-export release actions +on: + release: + types: [published] + +jobs: + generate_release_assets: + name: Generate release assets + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Pull pyinstaller docker image + run: | + docker pull dimagi/commcare-export-pyinstaller-linux + + - name: Compile linux binary + run: | + docker run -v "$(pwd):/src/" dimagi/commcare-export-pyinstaller-linux + + - name: Upload release assets + uses: AButler/upload-release-assets@v3.0 + with: + files: "./dist/linux/*" + repo-token: ${{ secrets.GITHUB_TOKEN }} From 860b048892e16b48b284cdd02b83b03d553bfc92 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Thu, 7 Mar 2024 13:30:42 +0200 Subject: [PATCH 4/6] Update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 639110a..f12418f 100644 --- a/README.md +++ b/README.md @@ -602,6 +602,12 @@ https://pypi.python.org/pypi/commcare-export https://github.com/dimagi/commcare-export/releases +Once the release is published a GitHub workflow is kicked off that compiles an executable of the DET compatible with +running on a Linux machine (Windows coming soon), adding it as a release asset. + +If you decide to download and use the executable file, please make sure the file has the executable permission enabled, +after which it can be invoked like any other executable though the command line. + Testing and Test Databases -------------------------- From 95d3e923ac399e92193406624da58c47b1d36a20 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Thu, 7 Mar 2024 13:33:20 +0200 Subject: [PATCH 5/6] Use sys.exit instead of exit --- commcare_export/cli.py | 6 +++--- commcare_export/utils_cli.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commcare_export/cli.py b/commcare_export/cli.py index 8fba090..fc8214a 100644 --- a/commcare_export/cli.py +++ b/commcare_export/cli.py @@ -197,14 +197,14 @@ def main(argv): if args.version: print('commcare-export version {}'.format(__version__)) - exit(0) + sys.exit(0) if not args.project: print( 'commcare-export: error: argument --project is required', file=sys.stderr ) - exit(1) + sys.exit(1) if args.profile: # hotshot is gone in Python 3 @@ -214,7 +214,7 @@ def main(argv): profile.start() try: - exit(main_with_args(args)) + sys.exit(main_with_args(args)) finally: if args.profile: profile.close() diff --git a/commcare_export/utils_cli.py b/commcare_export/utils_cli.py index b5ef1da..fc58e27 100644 --- a/commcare_export/utils_cli.py +++ b/commcare_export/utils_cli.py @@ -151,7 +151,7 @@ def main(argv): format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s' ) - exit(main_with_args(args)) + sys.exit(main_with_args(args)) def main_with_args(args): From 65b2bcb2d35ce92732deca649982ab72c3e1ba6d Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Thu, 7 Mar 2024 15:08:36 +0200 Subject: [PATCH 6/6] Add runtime hook to set bundled env variable --- build_exe/linux/entrypoint-linux.sh | 2 +- build_exe/runtime_hook.py | 4 ++++ commcare-export.spec | 2 +- commcare_export/version.py | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 build_exe/runtime_hook.py diff --git a/build_exe/linux/entrypoint-linux.sh b/build_exe/linux/entrypoint-linux.sh index ba02df5..5d781fb 100644 --- a/build_exe/linux/entrypoint-linux.sh +++ b/build_exe/linux/entrypoint-linux.sh @@ -8,7 +8,7 @@ set -e cd /src -pip install . +pip install commcare-export pip install -r build_exe/requirements.txt pyinstaller --clean -y --dist ./dist/linux --workpath /tmp *.spec diff --git a/build_exe/runtime_hook.py b/build_exe/runtime_hook.py new file mode 100644 index 0000000..d226247 --- /dev/null +++ b/build_exe/runtime_hook.py @@ -0,0 +1,4 @@ +import os + +# This env variable is used to alter bundled behaviour +os.environ['DET_EXECUTABLE'] = '1' diff --git a/commcare-export.spec b/commcare-export.spec index 7aed69c..428f7f4 100644 --- a/commcare-export.spec +++ b/commcare-export.spec @@ -13,7 +13,7 @@ a = Analysis( 'sqlalchemy.sql.default_comparator', ], hookspath=[], - runtime_hooks=[], + runtime_hooks=['build_exe/runtime_hook.py'], excludes=[], ) pyz = PYZ(a.pure) diff --git a/commcare_export/version.py b/commcare_export/version.py index b4f2a63..5f3c336 100644 --- a/commcare_export/version.py +++ b/commcare_export/version.py @@ -17,6 +17,9 @@ def stored_version(): def git_version(): + if os.environ.get('DET_EXECUTABLE'): + return None + described_version_bytes = subprocess.Popen( ['git', 'describe'], stdout=subprocess.PIPE