diff --git a/.gitignore b/.gitignore index 58ac1e8..576ddac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Files output by the notebooks +notebooks/data/metadata.star + # Files output by the tests 2dhb.* diff --git a/environment.yml b/environment.yml index 029ea9a..571cc09 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - mrcfile - numba - numpy - - h5py>=2.10.0 + - h5py>=3.6.0 - pandas - pillow>=8.2.0 - pip @@ -17,8 +17,8 @@ dependencies: - pytest-cov - pytorch - pyyaml - - osfclient - pip : + - osfclient - jupyter - starfile diff --git a/notebooks/particle_metadata.ipynb b/notebooks/particle_metadata.ipynb index b8199c6..9c23e66 100644 --- a/notebooks/particle_metadata.ipynb +++ b/notebooks/particle_metadata.ipynb @@ -1,274 +1,354 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# Particle Metadata\n", - "``ioSPI`` library provides functionalities to work with cryo-EM data. To play with the data, we first need to store the data in a way that `ioSPI`can understand. To do that, ``ioSPI`` uses the STAR (Self-defining Text Archiving and Retrieval) format (Hall, Allen and Brown, 1991) which is used by RELION for the storage of label-value pairs for all kinds of input and output metadata. In ``ioSPI``, the module `particle_metatdata` is used to create a STAR file `.star`. This module formats and writes particle metadata as `.star` files, following RELION conventions. This tutorial shows you how to create a `.star` file using `particle_detadata`." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "import warnings\n", - "\n", - "sys.path.append(os.path.dirname(os.getcwd()))\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "In order to create a `.star` file, it is necessary to provide information about the experiment, such as the image pixel size and image center shift. This information is passed in the form of a list and a `Config` object." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "from ioSPI import particle_metadata\n", - "\n", - "class Config:\n", - " \"\"\"Class to instantiate the config object.\"\"\"\n", - " def __init__(self, ctf, shift):\n", - " self.ctf = ctf\n", - " self.shift = shift" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "data_list = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]]\n", - "config = Config(ctf=True, shift=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "The names of the metadata for the ``.star`` file (in RELION conventions) can be accessed using the function `get_starfile_metadata_names` passing a `Config` object." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['__rlnImageName', '__rlnAngleRot', '__rlnAngleTilt', '__rlnAnglePsi', '__rlnOriginX', '__rlnOriginY', '__rlnDefocusU', '__rlnDefocusV', '__rlnDefocusAngle', '__rlnVoltage', '__rlnImagePixelSize', '__rlnSphericalAberration', '__rlnAmplitudeContrast', '__rlnCtfBfactor']\n" - ] - } - ], - "source": [ - "variable_names = particle_metadata.get_starfile_metadata_names(config)\n", - "print(variable_names)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "Using the list of values and the `Config` object, we can format the data using `format_metadata_for_writing_cryoem_convention` function, which creates a dataframe with the data." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "data": { - "text/plain": " __rlnImageName __rlnAngleRot __rlnAngleTilt __rlnAnglePsi __rlnOriginX \\\n0 1 2 3 4 5 \n\n __rlnOriginY __rlnDefocusU __rlnDefocusV __rlnDefocusAngle \\\n0 6 7 8 9 \n\n __rlnVoltage __rlnImagePixelSize __rlnSphericalAberration \\\n0 10 11 12 \n\n __rlnAmplitudeContrast __rlnCtfBfactor \n0 13 14 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
__rlnImageName__rlnAngleRot__rlnAngleTilt__rlnAnglePsi__rlnOriginX__rlnOriginY__rlnDefocusU__rlnDefocusV__rlnDefocusAngle__rlnVoltage__rlnImagePixelSize__rlnSphericalAberration__rlnAmplitudeContrast__rlnCtfBfactor
01234567891011121314
\n
" - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "metadata_df = particle_metadata.format_metadata_for_writing_cryoem_convention(data_list=data_list, config=config)\n", - "metadata_df" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "After formatting the data, we can use the function `write_metadata_to_starfile` providing the path, dataframe and name of the star file, to save the `.star` file." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "metadata_path = os.path.join(os.getcwd(), \"data\")\n", - "filename = \"metadata.star\"\n", - "particle_metadata.write_metadata_to_starfile(path=metadata_path, metadata=metadata_df, filename=filename)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "Finally, we check whether a `.star` file with the name `metadata.star` was created or not, using the function `check_star_file` function which will raise an exception if the file is not found." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "particle_metadata.check_star_file(os.path.join(metadata_path, filename))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "The file was successfully created as there is no exception raised. Finally, let's print out the content of the `.star` file we created to verify!" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Created by the starfile Python package (version 0.4.11) at 17:31:20 on 02/05/2022\n", - "\n", - "data_\n", - "\n", - "loop_\n", - "___rlnImageName #1\n", - "___rlnAngleRot #2\n", - "___rlnAngleTilt #3\n", - "___rlnAnglePsi #4\n", - "___rlnOriginX #5\n", - "___rlnOriginY #6\n", - "___rlnDefocusU #7\n", - "___rlnDefocusV #8\n", - "___rlnDefocusAngle #9\n", - "___rlnVoltage #10\n", - "___rlnImagePixelSize #11\n", - "___rlnSphericalAberration #12\n", - "___rlnAmplitudeContrast #13\n", - "___rlnCtfBfactor #14\n", - "1\t2\t3\t4\t5\t6\t7\t8\t9\t10\t11\t12\t13\t14\n", - "\n", - "\n", - "\n" - ] - } - ], - "source": [ - "with open(os.path.join(metadata_path, filename)) as star_file:\n", - " print(star_file.read())\n", - " star_file.close()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv_iospi", - "language": "python", - "name": "venv_iospi" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Particle Metadata\n", + "The ``ioSPI`` library provides functionalities to work with cryo-EM data. To play with the data, we first need to store the data in a way that `ioSPI`can understand. \n", + "\n", + "To do that, ``ioSPI`` uses the STAR (Self-defining Text Archiving and Retrieval) format (Hall, Allen and Brown, 1991) which is used by RELION for the storage of label-value pairs for all kinds of input and output metadata. In ``ioSPI``, the module `particle_metatdata` is used to create a STAR file `.star`. This module formats and writes particle metadata as `.star` files, following RELION conventions. This tutorial shows you how to create a `.star` file using `particle_detadata`." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import warnings\n", + "\n", + "sys.path.append(os.path.dirname(os.getcwd()))\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "In order to create a `.star` file, it is necessary to provide information about the experiment, such as the image pixel size and image center shift. This information is passed in the form of a list and a `Config` object." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from ioSPI.ioSPI import particle_metadata\n", + "\n", + "class Config:\n", + " \"\"\"Class to instantiate the config object.\"\"\"\n", + " def __init__(self, ctf, shift):\n", + " self.ctf = ctf\n", + " self.shift = shift" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "data_list = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]]\n", + "config = Config(ctf=True, shift=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "The names of the metadata for the ``.star`` file (in RELION conventions) can be accessed using the function `get_starfile_metadata_names` passing a `Config` object." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['__rlnImageName', '__rlnAngleRot', '__rlnAngleTilt', '__rlnAnglePsi', '__rlnOriginX', '__rlnOriginY', '__rlnDefocusU', '__rlnDefocusV', '__rlnDefocusAngle', '__rlnVoltage', '__rlnImagePixelSize', '__rlnSphericalAberration', '__rlnAmplitudeContrast', '__rlnCtfBfactor']\n" + ] + } + ], + "source": [ + "variable_names = particle_metadata.get_starfile_metadata_names(config)\n", + "print(variable_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Using the list of values and the `Config` object, we can format the data using `format_metadata_for_writing_cryoem_convention` function, which creates a dataframe with the data." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
__rlnImageName__rlnAngleRot__rlnAngleTilt__rlnAnglePsi__rlnOriginX__rlnOriginY__rlnDefocusU__rlnDefocusV__rlnDefocusAngle__rlnVoltage__rlnImagePixelSize__rlnSphericalAberration__rlnAmplitudeContrast__rlnCtfBfactor
01234567891011121314
\n", + "
" + ], + "text/plain": [ + " __rlnImageName __rlnAngleRot __rlnAngleTilt __rlnAnglePsi __rlnOriginX \\\n", + "0 1 2 3 4 5 \n", + "\n", + " __rlnOriginY __rlnDefocusU __rlnDefocusV __rlnDefocusAngle \\\n", + "0 6 7 8 9 \n", + "\n", + " __rlnVoltage __rlnImagePixelSize __rlnSphericalAberration \\\n", + "0 10 11 12 \n", + "\n", + " __rlnAmplitudeContrast __rlnCtfBfactor \n", + "0 13 14 " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metadata_df = particle_metadata.format_metadata_for_writing_cryoem_convention(data_list=data_list, config=config)\n", + "metadata_df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "After formatting the data, we can use the function `write_metadata_to_starfile` providing the path, dataframe and name of the star file, to save the `.star` file." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "metadata_path = os.path.join(os.getcwd(), \"data\")\n", + "filename = \"metadata.star\"\n", + "particle_metadata.write_metadata_to_starfile(path=metadata_path, metadata=metadata_df, filename=filename)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Finally, we check whether a `.star` file with the name `metadata.star` was created or not, using the function `check_star_file` function which will raise an exception if the file is not found." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "particle_metadata.check_star_file(os.path.join(metadata_path, filename))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "The file was successfully created as there is no exception raised. Finally, let's print out the content of the `.star` file we created to verify!" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Created by the starfile Python package (version 0.4.11) at 09:14:33 on 04/05/2022\n", + "\n", + "data_\n", + "\n", + "loop_\n", + "___rlnImageName #1\n", + "___rlnAngleRot #2\n", + "___rlnAngleTilt #3\n", + "___rlnAnglePsi #4\n", + "___rlnOriginX #5\n", + "___rlnOriginY #6\n", + "___rlnDefocusU #7\n", + "___rlnDefocusV #8\n", + "___rlnDefocusAngle #9\n", + "___rlnVoltage #10\n", + "___rlnImagePixelSize #11\n", + "___rlnSphericalAberration #12\n", + "___rlnAmplitudeContrast #13\n", + "___rlnCtfBfactor #14\n", + "1\t2\t3\t4\t5\t6\t7\t8\t9\t10\t11\t12\t13\t14\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "with open(os.path.join(metadata_path, filename)) as star_file:\n", + " print(star_file.read())\n", + " star_file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "6a7e0f634e221a7f903f822614955d5166c853555c4499bafa7ce417375f7059" + }, + "kernelspec": { + "display_name": "venv_iospi", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py new file mode 100644 index 0000000..45f0df1 --- /dev/null +++ b/tests/test_notebooks.py @@ -0,0 +1,47 @@ +"""Unit tests for the notebooks.""" + +import glob +import os +import subprocess +import tempfile + +import pytest + +NOTEBOOKS_DIR = "notebooks" +NOTEBOOKS_TO_SKIP = os.path.join(NOTEBOOKS_DIR, "download_and_upload_with_osf.ipynb") + + +def _exec_notebook(path): + """Execute notebook at path. + + Parameters + ---------- + path : str + Relative path of the notebook. + E.g. notebooks/particle_metadata.ipynb + """ + file_name = tempfile.NamedTemporaryFile(suffix=".ipynb").name + args = ( + f"jupyter nbconvert --to notebook --execute " + f"--ExecutePreprocessor.timeout=1000 " + f"--ExecutePreprocessor.kernel_name=python3 --output {file_name} {path}" + ) + subprocess.run(args, shell=True) + + +paths = sorted(glob.glob(f"{NOTEBOOKS_DIR}/*.ipynb")) + + +@pytest.mark.parametrize("path", paths) +def test_notebook(path): + """Test the notebook at path by executing it. + + Parameters + ---------- + path : str + Relative path of the notebooks. + E.g. notebooks/particle_metadata.ipynb + """ + if path in NOTEBOOKS_TO_SKIP: + pytest.skip() + _exec_notebook(path)