From 80a266547828b4ee03bf38634f3c0bf91932e50c Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Tue, 30 Jan 2024 17:22:21 +0200 Subject: [PATCH 1/6] Create spec file for pyinstaller --- commcare-export.spec | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 commcare-export.spec diff --git a/commcare-export.spec b/commcare-export.spec new file mode 100644 index 0000000..127c627 --- /dev/null +++ b/commcare-export.spec @@ -0,0 +1,37 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['commcare_export/cli.py'], + pathex=[], + binaries=[], + datas=[('./commcare_export', '.')], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.datas, + [], + name='commcare-export', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) From 134240c77bdaea3073ee10466220e65b4b3560a4 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Mon, 5 Feb 2024 14:15:36 +0200 Subject: [PATCH 2/6] Generate exe files on release and add as release assets --- .github/workflows/release_actions.yml | 33 +++++++++++++++++++++++++++ commcare-export.spec | 1 - 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release_actions.yml diff --git a/.github/workflows/release_actions.yml b/.github/workflows/release_actions.yml new file mode 100644 index 0000000..26466fb --- /dev/null +++ b/.github/workflows/release_actions.yml @@ -0,0 +1,33 @@ +name: commcare-export release actions +on: + release: + types: [published] + +jobs: + generate_release_assets: + name: Generate release assets + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull docker images for pyinstaller/wine + run: | + docker pull cdrx/pyinstaller-linux + docker pull cdrx/pyinstaller-windows + + - name: Generate linux binary + run: | + docker run -v "$(pwd):/src/" cdrx/pyinstaller-linux + + - name: Generate windows exe + run: | + docker run -v "$(pwd):/src/" cdrx/pyinstaller-windows + + - name: Upload release assets + uses: AButler/upload-release-assets@v3.0 + with: + files: "./dist/linux/*;./dist/windows/*" + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/commcare-export.spec b/commcare-export.spec index 127c627..b0d3c37 100644 --- a/commcare-export.spec +++ b/commcare-export.spec @@ -8,7 +8,6 @@ a = Analysis( datas=[('./commcare_export', '.')], hiddenimports=[], hookspath=[], - hooksconfig={}, runtime_hooks=[], excludes=[], noarchive=False, From 15db124721369c736a0bdc3ac361fcee3c8abe9f Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Mon, 5 Feb 2024 14:40:49 +0200 Subject: [PATCH 3/6] Update readme --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 271acf6..24ccd28 100644 --- a/README.md +++ b/README.md @@ -546,6 +546,11 @@ https://pypi.python.org/pypi/commcare-export https://github.com/dimagi/commcare-export/releases +Once the release is created, a GitHub workflow is kicked off that generates two release artifacts, +which constitute executable files of the DET compatible with the following operating systems, namely +1. Linux +2. Windows + Testing and Test Databases -------------------------- @@ -656,3 +661,18 @@ export HQ_API_KEY= For Travis builds these are included as encrypted vars in the travis config. + + +Compiling executable files locally +----------------------------------- +The DET executable files are compiled using a tool called [pyinstaller](https://pyinstaller.org/en/stable/). +Pyinstaller is very easy to use, but only works out-of-the-box for Linux as support for cross-compilation was +dropped in earlier releases. Another tool, [wine](https://www.winehq.org/), can be used in conjuction with +pyinstaller to compile the Windows exe files. + +Luckily in the world we live there's a repo out there called [docker-pyinstaller](https://github.com/cdrx/docker-pyinstaller) +which takes you through very simple steps to pull and use docker images to generate both the Linux binary and +Windows .exe files, so we don't ever have to worry about installing any additional packages ourselves. + +Please note that the `commcare-export.spec` file used by the docker containers is already defined and sits at the top of this project. +It shouldn't be necessary for you to change any parameters in the file. \ No newline at end of file From df17b9fb94c5cb24e462b17c459d98d67bb9085d Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Mon, 5 Feb 2024 19:17:58 +0200 Subject: [PATCH 4/6] Remove redundant repo-token and update text --- .github/workflows/release_actions.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_actions.yml b/.github/workflows/release_actions.yml index 26466fb..d29bf4b 100644 --- a/.github/workflows/release_actions.yml +++ b/.github/workflows/release_actions.yml @@ -10,19 +10,17 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Pull docker images for pyinstaller/wine run: | docker pull cdrx/pyinstaller-linux docker pull cdrx/pyinstaller-windows - - name: Generate linux binary + - name: Compile linux binary run: | docker run -v "$(pwd):/src/" cdrx/pyinstaller-linux - - name: Generate windows exe + - name: Compile windows exe run: | docker run -v "$(pwd):/src/" cdrx/pyinstaller-windows From f50c09a51e4012ee9bb903ef8740825bb2f5b727 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Tue, 13 Feb 2024 14:40:11 +0200 Subject: [PATCH 5/6] Add requirements.txt file for pyinstaller --- commcare-export.spec | 4 +++- requirements.txt | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/commcare-export.spec b/commcare-export.spec index b0d3c37..0be00f0 100644 --- a/commcare-export.spec +++ b/commcare-export.spec @@ -6,7 +6,9 @@ a = Analysis( pathex=[], binaries=[], datas=[('./commcare_export', '.')], - hiddenimports=[], + hiddenimports=[ + 'sqlalchemy.sql.default_comparator', + ], hookspath=[], runtime_hooks=[], excludes=[], diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..240ac40 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +# This file is only used by pyinstaller to create the executable DET instance +alembic +argparse +chardet +backoff>=2.0 +jsonpath-ng~=1.6.0 +ndg-httpsclient +openpyxl==2.5.12 +python-dateutil +pytz +psycopg2-binary +pymysql +pyodbc +requests +simplejson +sqlalchemy~=1.4 +sqlalchemy-migrate +urllib3==1.26.7 +xlwt From 05de6f2c1719191c5bb3ceb81d1595c2a30702a0 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Tue, 13 Feb 2024 14:40:30 +0200 Subject: [PATCH 6/6] Change exit() to sys.exit() --- commcare_export/cli.py | 6 +++--- commcare_export/utils_cli.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commcare_export/cli.py b/commcare_export/cli.py index 8fba090..fc8214a 100644 --- a/commcare_export/cli.py +++ b/commcare_export/cli.py @@ -197,14 +197,14 @@ def main(argv): if args.version: print('commcare-export version {}'.format(__version__)) - exit(0) + sys.exit(0) if not args.project: print( 'commcare-export: error: argument --project is required', file=sys.stderr ) - exit(1) + sys.exit(1) if args.profile: # hotshot is gone in Python 3 @@ -214,7 +214,7 @@ def main(argv): profile.start() try: - exit(main_with_args(args)) + sys.exit(main_with_args(args)) finally: if args.profile: profile.close() diff --git a/commcare_export/utils_cli.py b/commcare_export/utils_cli.py index b5ef1da..fc58e27 100644 --- a/commcare_export/utils_cli.py +++ b/commcare_export/utils_cli.py @@ -151,7 +151,7 @@ def main(argv): format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s' ) - exit(main_with_args(args)) + sys.exit(main_with_args(args)) def main_with_args(args):