Build for pypi instead of conda

.github/workflows/condarise.yaml → .github/workflows/build_package.yaml

@@ -1,5 +1,6 @@
-name: Condarise
+name: Build pip package
 on:
+  pull_request:
   push:
     branches:
       - main
@@ -14,33 +15,39 @@ jobs:
         shell: bash -l {0}
     steps:
       - uses: actions/checkout@main
-
-      - name: Fix meta YAML
+      
+      - name: Get version
         run: |
           make -C hail python-version-info
           VERSION=$(cat hail/python/hail/hail_version)
           VERSION=${VERSION/-/.dev}
-          cat conda/hail/meta-template.yaml \
-          | sed s/{version}/${VERSION}/ > conda/hail/meta.yaml
           echo "version=$VERSION" >> $GITHUB_ENV
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: buildenv
           channels: cpg,conda-forge,bioconda,defaults
           channel-priority: true
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Setup build env
         run: conda install pip conda-build anaconda-client
 
-      - name: Build package
-        run: conda build conda/hail
+      # Reading pip requirements and adding them into the conda recipe,
+      # also setting the package version and pinning pyspark:
+      - name: Render conda recipe
+        run: |
+          make -C hail env/SPARK_VERSION
+          python conda/render_recipe.py ${{ env.version }}
+
+      - name: Build conda package
+        run: |
+          conda-build conda/hail --variants "{'python': ['3.9', '3.10']}"
 
       # Remove old packages to save space on anaconda
       - name: Remove old versions
-        # Only run this step once.
-        if: ${{ matrix.os == 'ubuntu-latest' }}
+        # Only run this step once - that's why limiting to the ubuntu run:
+        if: ${{ matrix.os == 'ubuntu-latest' && github.event_name == 'push' }}
         run: |
           # Keep 8 latest versions
           KEEP=8
@@ -49,13 +56,14 @@ jobs:
           | xargs -I '{}' -L1 echo anaconda -t ${{ secrets.ANACONDA_TOKEN }} remove -f cpg/hail/{}
 
       - name: Upload to anaconda package repository
+        if: ${{ github.event_name == 'push' }}
         run: |
           anaconda -t ${{ secrets.ANACONDA_TOKEN }} \
-          upload ${CONDA_PREFIX}/conda-bld/**/*.tar.bz2
-      
+          upload ${CONDA_PREFIX}/conda-bld/**/*.tar.bz2  
+
       - name: Redeploy the analysis-server
-        # Only run this step once.
-        if: ${{ matrix.os == 'ubuntu-latest' }}
+        # Only run this step once - that's why limiting to the ubuntu run:
+        if: ${{ matrix.os == 'ubuntu-latest' && github.event_name == 'push' }}
         run: |
           curl --fail --silent --show-error -X POST \
             -H "Authorization: token ${{ secrets.ANALYSIS_SERVER_GITHUB_TOKEN }}" \

conda/README.md

@@ -1,22 +1,20 @@
 # Conda package
 
-This folder contains a conda recipe to build the `hail` package for
+This folder contains a conda recipe template to build the `hail` package for
 the [`cpg` Anaconda channel](https://anaconda.org/cpg/hail).
 
+The recipe is rendered and built in a GitHub CI workflow
+[.github/workflows/build_package.yaml](.github/workflows/build_package.yaml), 
+and pushes it to Anaconda on every push event to the `main` branch.
+
+## Installation
+
 Note that there is also a `hail` package in the
 [`bioconda` channel](https://github.com/bioconda/bioconda-recipes/tree/master/recipes/hail)
 synced with the [official PyPI release](https://pypi.org/project/hail). However, having
 a separate conda package in the `cpg` channel allows us to build it against the codebase
 in our fork.
 
-We don't control versioning of original Hail project, so our `cpg` conda release name 
-is the official version tag appended with the git commit has, e.g. `0.2.62.dev289c163`.
-
-[GitHub Actions CI](../.github/workflows/condarise.yaml) is set up to build the package
-using this recipe and push it to Anaconda on every push event to the `main` branch in
-the
-[CPG hail fork](https://github.com/populationgenomics/hail).
-
 When installing the package, list the `cpg` channel before `bioconda` to prioritize
 the channel order:
 
@@ -25,9 +23,7 @@ conda create --name hail -c cpg -c bioconda -c conda-forge hail
 conda activate hail
 ```
 
-You can also install Hail into an existing environment. However, note that Hail requires
-Python of versions 3.6 or 3.7, so conda might downgrade Python in that environment,
-which may affect other installed packages.
+You can also install Hail into an existing environment.
 
 Note that if you don't have `conda` installed, here are handy commands to do that:
 
@@ -39,13 +35,3 @@ else
 fi
 bash miniconda.sh
 ```
-
-When installing, to prioritize the CPG package, list the `cpg` channel before `bioconda`:
-
-```
-conda create --name hail -c cpg -c conda-forge hail
-conda activate hail
-```
-
-You can also install Hail into an existing environment; however note that Hail requires Python of versions 3.6 or 3.7, so conda might downgrade Python in that environment, which may affect other installed packages.
-

conda/hail/meta-template.yaml

@@ -7,54 +7,38 @@ source:
 
 build:
   number: 0
+  entry_points:
+    - hailctl = hailtop.hailctl.__main__:main
 
 requirements:
   build:
     - {{ compiler('cxx') }}
     - make
     - rsync
   host:
-    - python
-    - pyspark >=3.1.1,<3.2.0
-    - openjdk 8.*
+    - python {{ python }}
+    - openjdk =8
+    - pyspark {pyspark-pin}
     - lz4
     - pytest-runner
     - pip
     - google-cloud-sdk
     - google-cloud-storage
   run:
-    - python
-    - openjdk 8.*
-    - pyspark >=3.1.1,<3.2.0
-    - aiohttp
-    - aiohttp-session
-    - bokeh >1.3,<2.0
-    - decorator <5
-    - deprecated
-    - gcsfs
-    - humanize
-    - hurry.filesize
-    - nest-asyncio
-    - parsimonious
-    - pyjwt
-    - python-json-logger ==0.1.11
-    - requests
-    - scipy
-    - tabulate ==0.8.3
-    - tqdm ==4.42.1
-    - dill
-    - asyncinit
-    - google-cloud-sdk
-    - google-cloud-storage
-    - google-api-core
-    - janus >=0.6,<0.7
-    - botocore >=1.20,<2.0
+    - python {{ python }}
+    - pyspark {pyspark-pin}
+    - openjdk =8
+{other-requirements}
 
 test:
   imports:
     - hail
     - hailtop.batch
-
+    - hailtop.hailctl
+    - hailtop.hailctl.dataproc
+  commands:
+    - hailctl --help
+
 about:
   home: https://hail.is
   dev_url: https://github.com/populationgenomics/hail

conda/render_recipe.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+"""
+Render recipe:
+- populate dependencies from the pip requirements
+- set the package version,
+- pin spark version.
+"""
+import sys
+
+SPARK_VERSION = 'hail/env/SPARK_VERSION'
+PIP_REQS = 'hail/python/requirements.txt'
+META_TEMPLATE = 'conda/hail/meta-template.yaml'
+META_RENDERED = 'conda/hail/meta.yaml'
+
+if len(sys.argv) <= 1:
+    raise ValueError(
+        'Please, provide package version after as the first positional argument'
+    )
+package_version = sys.argv[1]
+
+with open(SPARK_VERSION, 'r') as file:
+    spark_version = file.read()
+
+dependencies = []
+with open(PIP_REQS, 'r') as f:
+    for line in f:
+        stripped = line.strip()
+        if stripped.startswith('#') or len(stripped) == 0:
+            continue
+        pkg = stripped
+        if pkg.startswith('pyspark'):
+            # already in the template
+            continue
+        # Removing all >, >= and == version pins, that conda-build can't 
+        # always reliably resolve:
+        pkg = pkg.split('>')[0].strip()
+        pkg = pkg.split('==')[0].strip()
+        # conda uses dashes where pip uses underscores
+        pkg = pkg.replace('_', '-')
+        if pkg == 'avro':
+            pkg = 'python-avro'
+        dependencies.append(pkg)
+
+with open(META_TEMPLATE) as inp, open(META_RENDERED, 'w') as out:
+    for line in inp:
+        if line.startswith('{other-requirements}'):
+            for dep in dependencies:
+                out.write(f'    - {dep}\n')
+        else:
+            # setting the package's version:
+            line = line.replace('{version}', package_version)
+            # adding the spark pin in the host section:
+            line = line.replace('{pyspark-pin}', f'=={spark_version}')
+            out.write(line)