Merge branch 'master' into master

.github/workflows/unit_tests.yml

@@ -0,0 +1,179 @@
+name: easyconfigs unit tests
+on: [push, pull_request]
+jobs:
+  test-suite:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.5, 3.6, 3.7]
+        modules_tool: [Lmod-6.6.3, Lmod-7.8.22, Lmod-8.1.14]
+        module_syntax: [Lua, Tcl]
+        # exclude some configurations: only test Tcl module syntax with Lmod 7.x and Python 2.7 & 3.5
+        exclude:
+          - modules_tool: Lmod-6.6.3
+            module_syntax: Tcl
+          - modules_tool: Lmod-8.1.14
+            module_syntax: Tcl
+          - python: 3.6
+            module_syntax: Tcl
+          - python: 3.7
+            module_syntax: Tcl
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install OS & Python packages
+      run: |
+        # disable apt-get update, we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
+        # sudo apt-get update
+        # for modules tool
+        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+        sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        # for testing OpenMPI-system*eb we need to have Open MPI installed
+        sudo apt-get install libopenmpi-dev openmpi-bin
+        # required for test_dep_graph
+        pip install pep8 python-graph-core python-graph-dot
+
+    - name: install EasyBuild framework
+      run: |
+          cd $HOME
+          # first determine which branch of easybuild-framework repo to install
+          BRANCH=develop
+          if [ "x$GITHUB_BASE_REF" = 'xmaster' ]; then BRANCH=master; fi
+          if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi
+          echo "Using easybuild-framework branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)"
+
+          git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-framework.git
+          cd easybuild-framework; git log -n 1; cd -
+          pip install $PWD/easybuild-framework
+
+          git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-easyblocks.git
+          cd easybuild-easyblocks; git log -n 1; cd -
+          pip install $PWD/easybuild-easyblocks
+
+    - name: install modules tool
+      run: |
+          cd $HOME
+          # use install_eb_dep.sh script provided with easybuild-framework
+          export INSTALL_DEP=$(which install_eb_dep.sh)
+          echo "Found install_eb_dep.sh script: $INSTALL_DEP"
+
+          # install modules tool
+          source $INSTALL_DEP ${{matrix.modules_tool}} $HOME
+
+          # changes in environment are not passed to other steps, so need to create files...
+          echo $MOD_INIT > mod_init
+          echo $PATH > path
+          if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
+
+    - name: run test suite
+      env:
+        EB_VERBOSE: 1
+        EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
+      run: |
+          # pull in target so we can diff against it to obtain list of touched files
+          if [ "x$GITHUB_BASE_REF" != 'xmaster' ]; then git fetch -v origin ${GITHUB_BASE_REF}:${GITHUB_BASE_REF}; fi
+
+          # initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
+          # also pick up changes to $PATH set by sourcing $MOD_INIT
+          WORKDIR=$GITHUB_WORKSPACE/easybuild-easyconfigs
+          export PATH=$WORKDIR/test/bin:$(cat $HOME/path)
+          export PYTHONPATH=$WORKDIR
+
+          # tell EasyBuild which modules tool is available
+          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
+          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
+          elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+            export EASYBUILD_MODULES_TOOL=EnvironmentModules
+          else
+            export EASYBUILD_MODULES_TOOL=Lmod
+          fi
+
+          eb --version
+          eb --show-config
+          # gather some useful info on test system
+          eb --show-system-info
+
+          # run test suite
+          python -O -m test.easyconfigs.suite
+
+          unset PYTHONPATH
+
+          # install easyconfigs via distribution package
+          python setup.py sdist > /dev/null
+          ls dist
+          pip install dist/easybuild-easyconfigs*tar.gz > /dev/null
+
+          # robot-paths value should not be empty, but have an entry that includes easybuild/easyconfigs subdir
+          echo "eb --show-config"
+          eb --show-config | tee eb_show_config.out
+          grep "^robot-paths .*/easybuild/easyconfigs" eb_show_config.out
+
+          # check whether some specific easyconfig files are found
+          echo "eb --search 'TensorFlow-1.14.*.eb'"
+          eb --search 'TensorFlow-1.14.*.eb' | tee eb_search_TF.out
+          grep '/TensorFlow-1.14.0-foss-2019a-Python-3.7.2.eb$' eb_search_TF.out
+
+          echo "eb --search '^foss-2019b.eb'"
+          eb --search '^foss-2019b.eb' | tee eb_search_foss.out
+          grep '/foss-2019b.eb$' eb_search_foss.out
+
+          # make sure CVS easyconfigs are included in installation (cfr. issue #10325)
+          echo "Searching for CVS easyconfigs..."
+          eb --search '^CVS-' | grep '/CVS-'
+
+          # try installing M4 with system toolchain (requires ConfigureMake easyblock + easyconfig)
+          eb --prefix /tmp/$USER/$GITHUB_SHA M4-1.4.18.eb
+
+  test-sdist:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.6, 3.7]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{matrix.python}}
+
+    - name: Create source distribution
+      run: python setup.py sdist
+
+    - name: Inspect files included in source distribution
+      working-directory: dist
+      run: |
+          tar xfz easybuild-easyconfigs*tar.gz
+          cd easybuild-easyconfigs-*/
+
+          # .git folder should not be there in source tarball
+          dot_git_files=$(find . -name .git)
+          if [ -n "$dot_git_files" ]; then
+            echo "Found .git folders in source tarball: $dot_git_files" && false
+          else
+            echo "No .git folders found in source tarball: OK"
+          fi
+
+          # CVS easyconfigs must be included in source tarball,
+          # see https://github.com/easybuilders/easybuild-easyconfigs/issues/10325
+          cvs_easyconfigs=$(find . -name 'CVS-*.eb')
+          if [ -z "$cvs_easyconfigs" ]; then
+            echo "CVS easyconfigs not found" && false
+          else
+            echo "Found CVS easyconfigs: $cvs_easyconfigs"
+          fi

.gitignore

@@ -10,3 +10,4 @@ dist/
 *.swp
 *.ropeproject/
 eb-*.log
+/MANIFEST

.travis.yml

@@ -1,20 +1,14 @@
 language: python
-python: 2.6
-env:
-  matrix:
-    - ENV_MOD_VERSION=3.2.10 EASYBUILD_MODULES_TOOL=EnvironmentModulesC EASYBUILD_MODULE_SYNTAX=Tcl
-    - LMOD_VERSION=6.6.3
-    - LMOD_VERSION=6.6.3 EASYBUILD_MODULE_SYNTAX=Tcl
-    - LMOD_VERSION=7.7.16
-    - LMOD_VERSION=7.7.16 EASYBUILD_MODULE_SYNTAX=Tcl
-    - ENV_MOD_VERSION=4.0.0 EASYBUILD_MODULES_TOOL=EnvironmentModules EASYBUILD_MODULE_SYNTAX=Tcl
 matrix:
   # mark build as finished as soon as job has failed
   fast_finish: true
   include:
-    # also test default configuration with Python 2.7
+    # only test with Python 2.7 & 3.6 + Lmod 7.x
+    # other test configurations in GitHub Actions (see .github/workflows/unit_tests.yml)
     - python: 2.7
-      env: LMOD_VERSION=6.6.3
+      env: LMOD_VERSION=7.8.22
+    - python: 3.6
+      env: LMOD_VERSION=7.8.22
 addons:
   apt:
     packages:
@@ -28,15 +22,17 @@ install:
     # required for test_dep_graph
     - pip install pep8 python-graph-core python-graph-dot
     # install easybuild-framework/easybuild-easyblocks (and dependencies)
-    # use 'develop' branch of framework/easyblocks, except when testing 'master'
-    - if [ "x$TRAVIS_BRANCH" = 'xmaster' ]; then BRANCH=master; else BRANCH=develop; fi
+    # use 'develop' branch of framework/easyblocks, except when testing 'master' or '4.x' branches
+    - BRANCH=develop
+    - if [ "x$TRAVIS_BRANCH" = 'xmaster' ]; then BRANCH=master; fi
+    - if [ "x$TRAVIS_BRANCH" = 'x4.x' ]; then BRANCH=4.x; fi
     - cd $HOME
-    - git clone -b $BRANCH --depth 10 --single-branch https://github.com/hpcugent/easybuild-framework.git
+    - git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-framework.git
     - cd easybuild-framework; git log -n 1; cd -
-    - easy_install $PWD/easybuild-framework
-    - git clone -b $BRANCH --depth 10 --single-branch https://github.com/hpcugent/easybuild-easyblocks.git
+    - pip install $PWD/easybuild-framework
+    - git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-easyblocks.git
     - cd easybuild-easyblocks; git log -n 1; cd -
-    - easy_install $PWD/easybuild-easyblocks
+    - pip install $PWD/easybuild-easyblocks
     # install environment modules tool using 'install_eb_dep.sh' script provided by easybuild-framework
     - if [ ! -z $ENV_MOD_VERSION ]; then source $(which install_eb_dep.sh) modules-${ENV_MOD_VERSION} $HOME; fi
     - if [ ! -z $LMOD_VERSION ]; then source $(which install_eb_dep.sh) lua-5.1.4.8 $HOME; fi
@@ -54,3 +50,20 @@ script:
     - export PYTHONPATH=$TRAVIS_BUILD_DIR
     - export PATH=$TRAVIS_BUILD_DIR/test/bin:$PATH
     - python -O -m test.easyconfigs.suite
+    # check for packaging issues by installing easybuild-easyconfigs repo,
+    # and checking whether easyconfigs are found, by verifying whether "eb --search" works as expected
+    - unset PYTHONPATH
+    - cd $HOME; pip install $TRAVIS_BUILD_DIR
+    # make sure correct 'python' command is used
+    # 'python2' may also be available, and is picked prior to 'python'
+    - export EB_PYTHON=python
+    # robot-paths value should not be empty, but have an entry that includes easybuild/easyconfigs subdir
+    - eb --show-config | tee eb_show_config.out
+    - grep "^robot-paths .*/easybuild/easyconfigs" eb_show_config.out
+    # check whether some specific easyconfig files are found
+    - eb --search 'TensorFlow-1.14.*.eb' | tee eb_search_TF.out
+    - grep '/TensorFlow-1.14.0-foss-2019a-Python-3.7.2.eb$' eb_search_TF.out
+    - eb --search '^foss-2018b.eb' | tee eb_search_foss.out
+    - grep '/foss-2018b.eb$' eb_search_foss.out
+    # try installing M4 with system toolchain (requires ConfigureMake easyblock + easyconfig)
+    - eb --prefix /tmp/$USER M4-1.4.18.eb

MANIFEST.in

@@ -1,6 +1,4 @@
-recursive-include easybuild *
 include CONTRIBUTING.md
 include LICENSE
 include README.rst
 include RELEASE_NOTES
-include setup.py