Merge pull request #77 from eth-cscs/release-0.6.0

Release 0.6.0

.coveragerc

@@ -0,0 +1,16 @@
+[run]
+branch = True
+source = abcpy
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    if False:
+    except ImportError:
+    if not has_torch:
+ignore_errors = True
+omit =
+    tests/*

.travis.yml

@@ -1,7 +1,9 @@
-dist: xenial
+dist: bionic
 language: python
-python:
-- '3.7'
+stages:
+  - name: test
+  - name: deploy
+    if: tag IS present AND branch = master
 addons:
   apt:
     sources:
@@ -14,22 +16,60 @@ addons:
     - swig
     - libmpich-dev
     - mpich
+jobs:
+    include:
+        # Unit tests on different python versions, all on Ubuntu
+        - stage: test
+          name: "Python 3.6"
+          python: "3.6"
+          env:
+            - UNIT_TEST=true
+        - name: "Python 3.7"
+          python: "3.7"
+          env:
+            - UNIT_TEST=true
+        - name: "Python 3.8"
+          python: "3.8"
+          env:
+            - UNIT_TEST=true
+      # Test coverage and without Pytorch for a single version
+        - name: "Coverage"
+          python: "3.8"
+          env:
+            - COVERAGE=true
+        - name: "No pytorch"
+          python: "3.8"
+          env:
+            - NO_PYTORCH=true
+        - stage: deploy
+          script:
+            - echo "Required dummy override of default 'script' in .travis.yml."
+          after_success:
+            - echo "Required dummy override of default 'after_success' in .travis.yml."
+          before_deploy:
+          - make clean
+          - mkdir dist
+          deploy:
+          - provider: pypi
+            user: mschoengens
+            password:
+              secure: tLopsTVkRfraHb/T1qfNvXk4L3StqpqFTflK0iAq/V+WSdARy7PDccj3P13aDo+Qvd2XYDPTSIVTveTOSHj46oser7+OmqrUYH9jQt681bmJ5aooHhw+3+NHa+fVBxMgzvCqJ+4Gvbf+3eDKowXICfPlTj5UrEil7s1jv91bSIm0HdI+mLyg1YstHOGt0O2Y6QEDPyEVRmFtyq7hB7EPheUvaJAfEl70LxV9fHiOuuQNcp9pnGRO6t9Sx4NIfIPIYzSdBoLaMSwgjy6ua1wF4iyMdKaDhMSajYb2+fWY1iyDJnmFj0/olpYUiZTrfWfQqz2j+uGT/YbmfZmSCcBTQI9ixJCtawqExoZODSq34uzc+N61riXdLEMOroxMobeBhuNj+bykp1IKaE99vYL/q8ta5dID15MtWIjWbLDVYQTQPkJ7fWllyxqOVRwa2rN37QbCctPbKpCs7WvEE7mJAaWJuOprw0AYjd2IH76YULkzbk3nR/v1nwyM2hGTYjePAy6Ue9jPgfeu9jEWu23O4u7+KMa1+scuLRP4DB1nlMStixjAJdiPMIo4OrvAiC8+ocntgi8t9+Quu5N8deyr9nM1pvWQyaNuHt3Yxd5oey3Q5UMtQFRCl5IyQKMTKttBg2p2L4wd0RdfrLgJXWkw/s6SBIyylCCDROr5gMEnPfY=
+            distributions: "sdist bdist_wheel"
+            on:
+              tags: true
+              branch: master
+
 install:
+- sudo apt-get install -y r-base  # install R for testing example
+- pip install rpy2  # install the rpy2 library for testing example with R
 - pip install -r requirements.txt
-- pip install -r requirements/backend-mpi.txt
 - pip install -r requirements/backend-spark.txt
-- pip install -r requirements/optional-requirements.txt
+- if [[ ! $NO_PYTORCH == true ]]; then pip install -r requirements/neural_networks_requirements.txt; fi;
+- if [[ $COVERAGE == true ]]; then pip install -r requirements/coverage.txt; fi;
+before_script:
+- python --version
 script:
-- make test
-before_deploy:
-- make clean
-- mkdir dist
-deploy:
-- provider: pypi
-  user: mschoengens
-  password:
-    secure: tLopsTVkRfraHb/T1qfNvXk4L3StqpqFTflK0iAq/V+WSdARy7PDccj3P13aDo+Qvd2XYDPTSIVTveTOSHj46oser7+OmqrUYH9jQt681bmJ5aooHhw+3+NHa+fVBxMgzvCqJ+4Gvbf+3eDKowXICfPlTj5UrEil7s1jv91bSIm0HdI+mLyg1YstHOGt0O2Y6QEDPyEVRmFtyq7hB7EPheUvaJAfEl70LxV9fHiOuuQNcp9pnGRO6t9Sx4NIfIPIYzSdBoLaMSwgjy6ua1wF4iyMdKaDhMSajYb2+fWY1iyDJnmFj0/olpYUiZTrfWfQqz2j+uGT/YbmfZmSCcBTQI9ixJCtawqExoZODSq34uzc+N61riXdLEMOroxMobeBhuNj+bykp1IKaE99vYL/q8ta5dID15MtWIjWbLDVYQTQPkJ7fWllyxqOVRwa2rN37QbCctPbKpCs7WvEE7mJAaWJuOprw0AYjd2IH76YULkzbk3nR/v1nwyM2hGTYjePAy6Ue9jPgfeu9jEWu23O4u7+KMa1+scuLRP4DB1nlMStixjAJdiPMIo4OrvAiC8+ocntgi8t9+Quu5N8deyr9nM1pvWQyaNuHt3Yxd5oey3Q5UMtQFRCl5IyQKMTKttBg2p2L4wd0RdfrLgJXWkw/s6SBIyylCCDROr5gMEnPfY=
-  distributions: "sdist bdist_wheel"
-  on:
-    tags: true
-    branch: master
+- if [[ $UNIT_TEST == true ]]; then make test; fi;
+- if [[ $COVERAGE == true ]]; then make coveragetest; fi;
+after_success:
+- if [[ $COVERAGE == true ]]; then bash <(curl -s https://codecov.io/bash); fi;

MANIFEST.in

@@ -0,0 +1,2 @@
+include requirements.txt
+include requirements/*

Makefile

@@ -24,32 +24,43 @@ $(MAKEDIRS):
 test: unittest unittest_mpi exampletest exampletest_mpi doctest
 
 unittest:
-	echo "Running standard unit tests.."
+	@echo "Running standard unit tests.."
 	python3 -m unittest discover -s tests -v -p "*_tests.py" || (echo "Error in standard unit tests."; exit 1)
 
 unittest_mpi:
-	echo "Running MPI backend unit tests.."
+	@echo "Running MPI backend unit tests.."
 	mpirun -np 2 python3 -m unittest discover -s tests -v -p "backend_tests_mpi.py" || (echo "Error in MPI unit tests."; exit 1)
 	mpirun -np 3 python3 -m unittest discover -s tests -v -p "backend_tests_mpi_model_mpi.py" || (echo "Error in MPI unit tests."; exit 1)
 
 exampletest: $(MAKEDIRS)
-	echo "Testing standard examples.."
-	python3 -m unittest discover -s examples -v -p "*.py" || (echo "Error in example tests."; exit 1)
+	@echo "Testing standard examples.."
+	python3 -m unittest -v tests/test_examples.py || (echo "Error in example tests."; exit 1)
 
 exampletest_mpi:
-	echo "Testing MPI backend examples.."
-	mpirun -np 2 python3 -m unittest -v examples/backends/mpi/pmcabc_gaussian.py || (echo "Error in MPI example tests."; exit 1)
+	@echo "Testing MPI backend examples.."
+	mpirun -np 2 python3 -m unittest -v tests/test_examples_mpi.py || (echo "Error in MPI example tests."; exit 1)
 
 doctest:
 	make -C doc html || (echo "Error in documentation generator."; exit 1)
 
-coveragetest:
-	command -v coverage >/dev/null 2>&1 || { echo >&2 "Python package 'coverage' has to been installed. Please, run 'pip3 install coverage'."; exit;}
+coveragetest: $(MAKEDIRS)  # compile models here as well as we check them for codecov as well.
+	@command -v coverage >/dev/null 2>&1 || { echo >&2 "Python package 'coverage' has to been installed. Please, run 'pip3 install coverage'."; exit;}
+	# unittests
 	@- $(foreach TEST, $(UNITTESTS), \
 		echo === Testing code coverage: $(TEST); \
-		python3 -m unittest $(TEST); \
 		coverage run -a --branch --source abcpy --omit \*__init__.py -m unittest $(TEST); \
 	)
+#	 unittest_mpi
+	@echo === Testing code coverage: tests/backend_tests_mpi.py
+	mpirun -np 2 coverage run -a --branch --source abcpy --omit \*__init__.py -m unittest tests/backend_tests_mpi.py
+	@echo === Testing code coverage: tests/backend_tests_mpi_model_mpi.py
+	mpirun -np 3 coverage run -a --branch --source abcpy --omit \*__init__.py -m unittest tests/backend_tests_mpi_model_mpi.py
+	# exampletest
+	@echo === Testing code coverage: tests/test_examples.py
+	@coverage run -a --branch --source abcpy --omit \*__init__.py -m unittest tests/test_examples.py
+	# exampletest_mpi
+	@echo === Testing code coverage: tests/examples_tests_mpi.py
+	mpirun -np 2 coverage run -a --branch --source abcpy --omit \*__init__.py -m unittest -v tests/test_examples_mpi.py
 	coverage html -d build/testcoverage
 	coverage report
 	@echo

README.md

@@ -1,4 +1,4 @@
-# ABCpy [![Documentation Status](https://readthedocs.org/projects/abcpy/badge/?version=latest)](http://abcpy.readthedocs.io/en/latest/?badge=latest) [![Build Status](https://travis-ci.org/eth-cscs/abcpy.svg?branch=master)](https://travis-ci.org/eth-cscs/abcpy)
+# ABCpy [![Documentation Status](https://readthedocs.org/projects/abcpy/badge/?version=latest)](http://abcpy.readthedocs.io/en/latest/?badge=latest) [![Build Status](https://travis-ci.org/eth-cscs/abcpy.svg?branch=master)](https://travis-ci.org/eth-cscs/abcpy) [![codecov](https://codecov.io/gh/eth-cscs/abcpy/branch/master/graph/badge.svg)](https://codecov.io/gh/eth-cscs/abcpy) [![DOI](https://zenodo.org/badge/doi/10.1145/3093172.3093233.svg)](http://dx.doi.org/10.1145/3093172.3093233) [![GitHub license](https://img.shields.io/github/license/eth-cscs/abcpy.svg)](https://github.com/eth-cscs/abcpy/blob/master/LICENSE) [![PyPI version shields.io](https://img.shields.io/pypi/v/abcpy.svg)](https://pypi.python.org/pypi/abcpy/) [![PyPI pyversions](https://img.shields.io/pypi/pyversions/abcpy.svg)](https://pypi.python.org/pypi/abcpy/)  [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/eth-cscs/abcpy/master?filepath=examples)
 
 ABCpy is a scientific library written in Python for Bayesian uncertainty quantification in
 absence of likelihood function, which parallelizes existing approximate Bayesian computation (ABC) 
@@ -27,21 +27,52 @@ scientists by providing
 # Documentation
 For more information, check out the
 
-* [Documentation](http://abcpy.readthedocs.io/en/v0.5.7) 
-* [Examples](https://github.com/eth-cscs/abcpy/tree/v0.5.7/examples) directory and
-* [Reference](http://abcpy.readthedocs.io/en/v0.5.7/abcpy.html)
+* [Documentation](http://abcpy.readthedocs.io/en/v0.6.0) 
+* [Examples](https://github.com/eth-cscs/abcpy/tree/v0.6.0/examples) directory and
+* [Reference](http://abcpy.readthedocs.io/en/v0.6.0/abcpy.html)
+
 
 Further, we provide a
 [collection of models](https://github.com/eth-cscs/abcpy-models) for which ABCpy
 has been applied successfully. This is a good place to look at more complicated inference setups.
 
+# Quick installation and requirements
+
+
+ABCpy can be installed from `pip`: 
+
+    pip install abcpy
+
+Check [here](https://abcpy.readthedocs.io/en/latest/installation.html) for more details.
+
+Basic requirements are listed in `requirements.txt`. That also includes packages required for MPI parallelization there, which is very often used. However, we also provide support for parallelization with Apache Spark (see below).
+
+ Additional packages are required for additional features: 
+
+- `torch` is needed in order to use neural networks to learn summary statistics. It can be installed by running `pip install -r requirements/neural_networks_requirements.txt`
+- In order to use Apache Spark for parallelization, `findspark` and `pyspark` are required; install them by `pip install -r requirements/backend-spark.txt`  
+
+## Troubleshooting `mpi4py` installation
+
+`mpi4py` requires a working MPI implementation to be installed; check the [official docs]((https://mpi4py.readthedocs.io/en/stable/install.html)) for more info. On Ubuntu, that can be installed with:
+
+    sudo apt-get install libopenmpi-dev
+
+Even when that is present, running `pip install mpi4py` can sometimes lead to errors. In fact, as specified in the [official docs]((https://mpi4py.readthedocs.io/en/stable/install.html)), the `mpicc` compiler needs to be in the search path. If that is not the case, a workaround is: 
+
+    env MPICC=/path/to/mpicc pip install mpi4py
+
+In some cases, even the above may not be enough. A possibility is using `conda` (`conda install mpi4py`) which usually handles package dependencies better than `pip`. Alternatively, you can try by installing directly `mpi4py` from the package manager; in Ubuntu, you can do:
+
+    sudo apt install python3-mpi4py 
+
+which however does not work with virtual environments.
+
+
 # Author 
 ABCpy was written by [Ritabrata Dutta, Warwick
-University](https://warwick.ac.uk/fac/sci/statistics/staff/academic-research/dutta/)
-and [Marcel Schoengens]([email protected]), CSCS, ETH Zurich, and we're
-actively developing it. Please feel free to submit any bugs or feature requests.
-We'd also love to hear about your experiences with ABCpy in general. Drop us an
-email!
+University](https://warwick.ac.uk/fac/sci/statistics/staff/academic-research/dutta/) and [Marcel Schoengens]([email protected]), CSCS, ETH Zurich, and presently actively maintained by [Lorenzo Pacchiardi, Oxford University](https://github.com/LoryPack) and [Ritabrata Dutta, Warwick
+University](https://warwick.ac.uk/fac/sci/statistics/staff/academic-research/dutta/). Please feel free to submit any bugs or feature requests. We'd also love to hear about your experiences with ABCpy in general. Drop us an email!
 
 We want to thank [Prof. Antonietta Mira, Università della svizzera
 italiana](https://search.usi.ch/en/people/f8960de6d60dd08a79b6c1eb20b7442b/Mira-Antonietta),
@@ -53,19 +84,24 @@ finally CSCS (Swiss National Super Computing Center) for their generous support.
 
 ## Citation
 
-There is a paper in the proceedings of the 2017 PASC conference. In case you use
+There is a [paper](http://dx.doi.org/10.1145/3093172.3093233) in the proceedings of the 2017 PASC conference. In case you use
 ABCpy for your publication, we would appreciate a citation. You can use
-[this](https://github.com/eth-cscs/abcpy/blob/v0.5.6/doc/literature/DuttaS-ABCpy-PASC-2017.bib)
-
-BibTex reference.
+[this](https://github.com/eth-cscs/abcpy/blob/v0.5.6/doc/literature/DuttaS-ABCpy-PASC-2017.bib) BibTex reference.
 
 
-## Other Refernces
+## Other References
 
 Publications in which ABCpy was applied:
 
-* L. Pacchiardi, P. Künzli, M. Schöngens, B. Chopard, R. Dutta, "Distance-Learning for Approximate Bayesian
-  Computation to Model a Volcanic Eruption", 2020, Sankhya B, ISSN 0976-8394, 
+* R. Dutta, K. Zouaoui-Boudjeltia, C. Kotsalos, A. Rousseau, D. Ribeiro de Sousa, J. M. Desmet, 
+A. Van Meerhaeghe, A. Mira, and B. Chopard. "Interpretable pathological test for Cardio-vascular 
+disease: Approximate Bayesian computation with distance learning.", 2020, arXiv:2010.06465.
+
+* R. Dutta, S. Gomes, D. Kalise, L. Pacchiardi. "Using mobility data in the design of optimal 
+lockdown strategies for the COVID-19 pandemic in England.", 2020, arXiv:2006.16059.
+
+* L. Pacchiardi, P. Künzli, M. Schöngens, B. Chopard, R. Dutta, "Distance-Learning for 
+Approximate Bayesian Computation to Model a Volcanic Eruption", 2020, Sankhya B, ISSN 0976-8394, 
   [DOI: 10.1007/s13571-019-00208-8](https://doi.org/10.1007/s13571-019-00208-8).
 
 * R. Dutta, J. P.  Onnela, A. Mira, "Bayesian Inference of Spreading Processes
@@ -79,11 +115,10 @@ Publications in which ABCpy was applied:
   Computation with High Performance Computing", 2018, Frontiers in physiology, 9.
 
 * A. Ebert, R. Dutta, P. Wu, K. Mengersen and A. Mira, "Likelihood-Free
-  Parameter Estimation for Dynamic Queueing Networks", 2018, arXiv:1804.02526
+  Parameter Estimation for Dynamic Queueing Networks", 2018, arXiv:1804.02526.
 
 * R. Dutta, M. Schoengens, A. Ummadisingu, N. Widerman, J. P.  Onnela, A. Mira, "ABCpy: A
-  High-Performance Computing Perspective to Approximate Bayesian Computation",
-  2017, arXiv:1711.04694
+  High-Performance Computing Perspective to Approximate Bayesian Computation", 2017, arXiv:1711.04694.
 
 ## License
 ABCpy is published under the BSD 3-clause license, see [here](LICENSE).

VERSION

@@ -1 +1 @@
-0.5.7
+0.6.0