From 421fad8aa47de89b868519e3c487b6e51e4c1361 Mon Sep 17 00:00:00 2001 From: Anthony Galassi <28850131+bendhouseart@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:45:56 -0500 Subject: [PATCH 001/255] [MISC][SCHEMA] Add schema-code sphinx generated read the docs to CI; pushes to bids-standard/schema-docs (#1372) --- .github/workflows/schema_docs.yaml | 70 +++++++++++++++++++++++ .gitignore | 3 + tools/schemacode/docs/Makefile | 20 +++++++ tools/schemacode/docs/bidsschematools.rst | 41 +++++++++++++ tools/schemacode/docs/conf.py | 61 ++++++++++++++++++++ tools/schemacode/docs/index.rst | 15 +++++ tools/schemacode/docs/make.bat | 35 ++++++++++++ tools/schemacode/docs/modules.rst | 7 +++ 8 files changed, 252 insertions(+) create mode 100644 .github/workflows/schema_docs.yaml create mode 100644 tools/schemacode/docs/Makefile create mode 100644 tools/schemacode/docs/bidsschematools.rst create mode 100644 tools/schemacode/docs/conf.py create mode 100644 tools/schemacode/docs/index.rst create mode 100644 tools/schemacode/docs/make.bat create mode 100644 tools/schemacode/docs/modules.rst diff --git a/.github/workflows/schema_docs.yaml b/.github/workflows/schema_docs.yaml new file mode 100644 index 0000000000..a9b6fc0f14 --- /dev/null +++ b/.github/workflows/schema_docs.yaml @@ -0,0 +1,70 @@ +name: "schemacode_docs" + +on: + push: + branches: + - "master" + tags: + - "schema-*" + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + push_schema_for_docs: + runs-on: ${{ matrix.os }} + name: Push Schema to Schema Docs + strategy: + matrix: + os: ["ubuntu-latest"] + steps: + - uses: actions/checkout@v3 + + + - name: Pull schema-docs and reqs from bids-specification + run: | + + pushd $HOME + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor}}@noreply.com" + git clone https://${{ github.repository_owner }}:${{ secrets.SCHEMA_DOCS_GITHUB_API_TOKEN }}@github.com/${{ github.repository_owner }}/schema-docs + popd + rsync --recursive --copy-links ${{ github.workspace }}/tools/schemacode/bidsschematools $HOME/schema-docs/ + rsync --recursive --copy-links ${{ github.workspace }}/tools/schemacode/docs $HOME/schema-docs/ + cat ${{ github.workspace }}/requirements.txt | grep -v tools/schemacode >> $HOME/schema-docs/requirements.txt + sort $HOME/schema-docs/requirements.txt | uniq > $HOME/schema-docs/requirements.txt.tmp + mv $HOME/schema-docs/requirements.txt.tmp $HOME/schema-docs/requirements.txt + + - name: Debug + uses: mxschmitt/action-tmate@v3 + if: ${{ inputs.debug_enabled }} + timeout-minutes: 15 + with: + limit-access-to-actor: true + + - name: Push Updates to schema-docs + run: | + + pushd $HOME/schema-docs + + git diff --exit-code > $HOME/gitdiff.log + if [[ $? == 0 ]]; then + echo "No changes found schema code, this workflow should only run when changes are made to schema-*." + echo "You must be up to no good." + else + git add --all + git commit -m "Auto update from bids-specification ${{ github.sha }}" + git push origin main + fi diff --git a/.gitignore b/.gitignore index 2ef07dacbc..59fa1a1de3 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,9 @@ venv.bak/ # mkdocs documentation /site +# schema docs documentation +tools/schemacode/build/* + # mypy .mypy_cache/ .dmypy.json diff --git a/tools/schemacode/docs/Makefile b/tools/schemacode/docs/Makefile new file mode 100644 index 0000000000..b7b8584379 --- /dev/null +++ b/tools/schemacode/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/tools/schemacode/docs/bidsschematools.rst b/tools/schemacode/docs/bidsschematools.rst new file mode 100644 index 0000000000..5965ad4b9b --- /dev/null +++ b/tools/schemacode/docs/bidsschematools.rst @@ -0,0 +1,41 @@ +.. bidsschematools: + +render +------ + +.. automodule:: bidsschematools.render + :members: + :undoc-members: + :show-inheritance: + +schema +------ + +.. automodule:: bidsschematools.schema + :members: + :undoc-members: + :show-inheritance: + +utils +----- + +.. automodule:: bidsschematools.utils + :members: + :undoc-members: + :show-inheritance: + +validator +--------- + +.. automodule:: bidsschematools.validator + :members: + :undoc-members: + :show-inheritance: + +types +----- + +.. automodule:: bidsschematools.types + :members: + :undoc-members: + :show-inheritance: diff --git a/tools/schemacode/docs/conf.py b/tools/schemacode/docs/conf.py new file mode 100644 index 0000000000..2cf1172ed8 --- /dev/null +++ b/tools/schemacode/docs/conf.py @@ -0,0 +1,61 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import pathlib +import sys + +schemacode_path = pathlib.Path(os.path.abspath("bidsschematools")) +sys.path.insert(0, str(schemacode_path)) +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = "bidsschematools" +copyright = "2022, bids-specification" +author = "bids-specification" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.duration", + "sphinx.ext.doctest", + "sphinx_rtd_theme", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "tests"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] diff --git a/tools/schemacode/docs/index.rst b/tools/schemacode/docs/index.rst new file mode 100644 index 0000000000..e344ad2340 --- /dev/null +++ b/tools/schemacode/docs/index.rst @@ -0,0 +1,15 @@ +Welcome to bidsschematools's documentation! +=========================================== + +.. toctree:: + modules + :maxdepth: 2 + :caption: Contents: + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/schemacode/docs/make.bat b/tools/schemacode/docs/make.bat new file mode 100644 index 0000000000..edfa0666ee --- /dev/null +++ b/tools/schemacode/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR= +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/tools/schemacode/docs/modules.rst b/tools/schemacode/docs/modules.rst new file mode 100644 index 0000000000..6515d1311c --- /dev/null +++ b/tools/schemacode/docs/modules.rst @@ -0,0 +1,7 @@ +bidsschematools +=============== + +.. toctree:: + :maxdepth: 4 + + bidsschematools \ No newline at end of file From b21bffb4baa2d6817ab655be3879c06ee58a81b1 Mon Sep 17 00:00:00 2001 From: Anthony Galassi <28850131+bendhouseart@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:25:46 -0500 Subject: [PATCH 002/255] update var to runner env from actions env syntax (#1381) --- .github/workflows/schema_docs.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/schema_docs.yaml b/.github/workflows/schema_docs.yaml index a9b6fc0f14..70b1ac4bc9 100644 --- a/.github/workflows/schema_docs.yaml +++ b/.github/workflows/schema_docs.yaml @@ -32,6 +32,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Debug + uses: mxschmitt/action-tmate@v3 + if: ${{ inputs.debug_enabled }} + timeout-minutes: 15 + with: + limit-access-to-actor: true - name: Pull schema-docs and reqs from bids-specification run: | @@ -46,13 +52,6 @@ jobs: cat ${{ github.workspace }}/requirements.txt | grep -v tools/schemacode >> $HOME/schema-docs/requirements.txt sort $HOME/schema-docs/requirements.txt | uniq > $HOME/schema-docs/requirements.txt.tmp mv $HOME/schema-docs/requirements.txt.tmp $HOME/schema-docs/requirements.txt - - - name: Debug - uses: mxschmitt/action-tmate@v3 - if: ${{ inputs.debug_enabled }} - timeout-minutes: 15 - with: - limit-access-to-actor: true - name: Push Updates to schema-docs run: | @@ -65,6 +64,6 @@ jobs: echo "You must be up to no good." else git add --all - git commit -m "Auto update from bids-specification ${{ github.sha }}" + git commit -m "Auto update from bids-specification $GITHUB_SHA" git push origin main fi From 1c6cc9966101d1da4f8b0c1193229268ae3455cb Mon Sep 17 00:00:00 2001 From: VisLab <1189050+VisLab@users.noreply.github.com> Date: Wed, 18 Jan 2023 04:14:58 -0600 Subject: [PATCH 003/255] [FIX] Updated links and fixed typos in hed appendix (#1383) * Corrected HED appendix * Fixed style issue in markdown * Fixed a link and changed error to warning --- src/appendices/hed.md | 73 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/src/appendices/hed.md b/src/appendices/hed.md index 37bb9a3594..604b445e74 100644 --- a/src/appendices/hed.md +++ b/src/appendices/hed.md @@ -4,7 +4,7 @@ Hierarchical Event Descriptors (HED) are a controlled vocabulary of terms descri events in a machine-actionable form so that algorithms can use the information without manual recoding. HED annotation can be used to describe any experimental events by combining -information from the dataset's `_events.tsv` files and `_events.json` sidecars. +information from the dataset's `events.tsv` files and `events.json` sidecars. ## HED annotations and vocabulary @@ -14,8 +14,7 @@ Individual terms are comma-separated and may be grouped using parentheses to ind association. See [https://www.hedtags.org/display_hed.html](https://www.hedtags.org/display_hed.html) to view the HED schema and the -[HED documentation](https://hed-specification.readthedocs.io/en/latest/index.html) -for additional resources. +[HED resources](https://www.hed-resources.org) site for additional information. Starting with HED version 8.0.0, HED allows users to annotate using individual terms or partial paths in the HED vocabulary (for example `Red` or `Visual-presentation`) @@ -24,8 +23,8 @@ rather than the full paths in the HED hierarchy ( or `Property/Sensory-property/Sensory-presentation/Visual-presentation`). -HED specific tools MUST treat the short and long HED tag forms interchangeably, -converting between the forms when necessary, based on the HED schema. +HED specific tools MUST treat the short (single term) and long (full path) HED tag forms interchangeably, +converting between the forms when necessary based on the HED schema. Examples of test datasets using the various forms can be found in [hed-examples/datasets](https://github.com/hed-standard/hed-examples/tree/main/datasets) on GitHub. @@ -83,12 +82,12 @@ The `duration` column is also annotated as a value column. "LongName": "Event category", "Description": "Indicator of type of action that is expected", "Levels": { - "go": "A red square is displayed to indicate starting", - "stop": "A blue square is displayed to indicate stopping" + "go": "A blue square is displayed to indicate starting", + "stop": "A red square is displayed to indicate stopping" }, "HED": { - "go": "Sensory-event, Visual-presentation, ((Square, Blue),(Computer-screen, Center-of))", - "stop": "Sensory-event, Visual-presentation, ((Square, Blue), (Computer-screen, Center-of))" + "go": "Sensory-event, Visual-presentation, (Square, Blue)", + "stop": "Sensory-event, Visual-presentation, (Square, Red)" } }, "response_time": { @@ -105,17 +104,16 @@ The `duration` column is also annotated as a value column. } ``` -Dedicated HED tools MUST assemble an annotation for each event by concatenating the -annotations for each column. +Dedicated HED tools MUST assemble the HED annotation for each event (row) by concatenating the +annotations for each column, along with the annotation contained directly in a `HED` column +of that row as described in the next section. Example: The fully assembled annotation for the first event in the above `events.tsv` file with onset `1.2` (the first row) is: ```Text -Duration/0.6 s, Sensory-event, Visual-presentation, -((Square, Blue), (Computer-screen, Center-of)), -(Delay/1.435 ms, Agent-action, -(Experiment-participant, (Press, Mouse-button))), +Duration/0.6 s, Sensory-event, Visual-presentation, (Square, Blue), +(Delay/1.435 ms, Agent-action, (Experiment-participant, (Press, Mouse-button))), Pathname/images/red_square.jpg ``` @@ -124,21 +122,22 @@ Pathname/images/red_square.jpg Another tagging strategy is to annotate individual events directly by including a `HED` column in the `events.tsv` file. This approach is necessary when each event has annotations that are unique -and do not fit into a standard set of patterns. +and do not fit into a standard set of patterns, +such as during manual annotation of artifacts or signal features. -Some acquisition or presentation software systems directly -write annotations during the experiment, and these MAY also be placed in the -`HED` column of the `events.tsv` file. +Some acquisition or presentation software systems may produce +individual annotations for each event during the experiment. +These individualized annotations may be placed into the `HED` column of the `events.tsv` file +when the data is converted into BIDS format. -Dedicated HED tools that assemble the full annotation for events treat MUST not distinguish -between HED annotations extracted from `_events.json` sidecars and those -appearing in the `HED` column of `_events.tsv` files. +Dedicated HED tools that assemble the full annotation for events MUST not distinguish +between HED annotations extracted from `events.json` sidecars and those +appearing in the `HED` column of `events.tsv` files. The HED strings from all sources are concatenated to form the final event annotations. -Annotations placed in sidecars are the RECOMMENDED way -to annotate data using HED. -These annnotations are preferred to those placed +Annotations placed in sidecars are the RECOMMENDED way to annotate data using HED. +These annotations are preferred to those placed directly in the `HED` column, because they are simpler, more compact, more easily edited, and less prone to inconsistencies. @@ -167,9 +166,9 @@ This allows for a proper validation of the HED annotations (for example using the `bids-validator`). Example: The following `dataset_description.json` file specifies that the -[`HED8.1.0.xml`](https://github.com/hed-standard/hed-specification/tree/master/hedxml/HED8.1.0.xml) -file from the `hedxml` directory of the -[`hed-specification`](https://github.com/hed-standard/hed-specification) +[`HED8.1.0.xml`](https://github.com/hed-standard/hed-schemas/blob/main/standard_schema/hedxml/HED8.1.0.xml) +file from the `standard_schema/hedxml` directory of the +[`hed-schemas`](https://github.com/hed-standard/hed-schemas) repository on GitHub should be used to validate the study event annotations. ```JSON @@ -181,6 +180,7 @@ repository on GitHub should be used to validate the study event annotations. ``` If you omit the `HEDVersion` field from the dataset description file, +a warning will be generated and any present HED information will be validated using the latest version of the HED schema. This is bound to result in problems, and hence, it is strongly RECOMMENDED that the `HEDVersion` field be included when using HED in a BIDS dataset. @@ -190,27 +190,26 @@ This is bound to result in problems, and hence, it is strongly RECOMMENDED that HED also allows you to use one or more specialized vocabularies along with or instead of the standard vocabulary. These specialized vocabularies are developed by communities of users and are available in the GitHub -[hed-schema-library](https://github.com/hed-standard/hed-schema-library) repository. +[hed-schemas](https://github.com/hed-standard/hed-schemas) repository. Library schema are specified in the form `library-version>`. Example: The following `dataset_description.json` file specifies that the -[HED8.1.0.xml](https://github.com/hed-standard/hed-specification/tree/master/hedxml/HED8.1.0.xml) +[HED8.1.0.xml](https://github.com/hed-standard/hed-schemas/blob/main/standard_schema/hedxml/HED8.1.0.xml) standard schema should be used along with the -SCORE library for clinical neurological annotation and a test library. -These later schemas are located at -[HED_score_0.0.1.xml](https://github.com/hed-standard/hed-schema-library/blob/main/library_schemas/score/hedxml/HED_score_0.0.1.xml) and -[HED_testlib_1.0.2.xml](https://github.com/hed-standard/hed-schema-library/blob/main/library_schemas/testlib/hedxml/HED_testlib_1.0.2.xml), respectively. +SCORE library for clinical neurological annotation located at +[HED_score_0.0.1.xml](https://github.com/hed-standard/hed-schemas/blob/main/library_schemas/score/hedxml/HED_score_0.0.1.xml). ```JSON { "Name": "A great experiment", "BIDSVersion": "1.7.0", - "HEDVersion": ["8.1.0", "sc:score_0.0.1", "ts:testlib_1.0.2"] + "HEDVersion": ["8.1.0", "sc:score_0.0.1"] } ``` -The `sc:` and `ts:` are user-chosen prefixes used to distinguish the sources +The `sc:` is a user-chosen prefixes used to distinguish the sources of the terms in the HED annotation. -These prefixes MUST be alphanumeric. +The prefixes MUST be alphanumeric. +Any number of prefixed schemas may be used in addition to a non-prefixed one. The following HED annotation from this dataset uses the `sc:` prefix with `Photomyogenic-response` and `Wicket-spikes` because these terms are from the From 111875cdacbb6a5eb55b4fa273d1bce6629999d7 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 23 Jan 2023 13:52:44 -0500 Subject: [PATCH 004/255] SCHEMA: Remove inconsistent name field from objects.common_principles --- src/schema/objects/common_principles.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/schema/objects/common_principles.yaml b/src/schema/objects/common_principles.yaml index 87c0d16729..ef55eeece4 100644 --- a/src/schema/objects/common_principles.yaml +++ b/src/schema/objects/common_principles.yaml @@ -4,13 +4,11 @@ # The order in which these terms are presented in the specification is defined in `rules/common_principles.yaml`, # rather than this file (`objects/common_principles.yaml`). data_acquisition: - name: Data acquisition display_name: Data acquisition description: | A continuous uninterrupted block of time during which a brain scanning instrument was acquiring data according to particular scanning sequence/protocol. data_type: - name: Data type display_name: Data type description: | A functional group of different types of data. @@ -42,13 +40,11 @@ data_type: 12. `nirs` (near infrared spectroscopy) dataset: - name: Dataset display_name: Dataset description: | A set of neuroimaging and behavioral data acquired for a purpose of a particular study. A dataset consists of data acquired from one or more subjects, possibly from multiple sessions. deprecated: - name: DEPRECATED display_name: DEPRECATED description: | A "deprecated" entity or metadata field SHOULD NOT be used in the generation of new datasets. @@ -56,7 +52,6 @@ deprecated: Validating software SHOULD warn when deprecated practices are detected and provide a suggestion for updating the dataset to preserve the curator's intent. event: - name: Event display_name: Event description: | Something that happens or may be perceived by a test subject as happening @@ -72,7 +67,6 @@ event: In BIDS, each event has an onset time and duration. Note that not all tasks will have recorded events (for example, "resting state"). extension: - name: File extension display_name: File extension description: | A portion of the file name after the left-most period (`.`) preceded by any other alphanumeric. @@ -80,13 +74,11 @@ extension: but the file extension of `test.nii.gz` is `.nii.gz`. Note that the left-most period is included in the file extension. index: - name: index display_name: index description: | A nonnegative integer, possibly prefixed with arbitrary number of 0s for consistent indentation, for example, it is `01` in `run-01` following `run-` specification. label: - name: label display_name: label description: | An alphanumeric value, possibly prefixed with arbitrary number of 0s for consistent indentation, @@ -94,7 +86,6 @@ label: Note that labels MUST not collide when casing is ignored (see [Case collision intolerance](SPEC_ROOT/common-principles.md#case-collision-intolerance)). modality: - name: Modality display_name: Modality description: | The category of brain data recorded by a file. @@ -105,7 +96,6 @@ modality: When applicable, the modality is indicated in the **suffix**. The modality may overlap with, but should not be confused with the **data type**. run: - name: Run display_name: Run description: | An uninterrupted repetition of data acquisition that has the same acquisition parameters and task @@ -119,14 +109,12 @@ run: For some types of [PET](SPEC_ROOT/modality-specific-files/positron-emission-tomography.md) acquisitions, a subject may leave and re-enter the scanner without interrupting the scan. sample: - name: Sample display_name: Sample description: | A sample pertaining to a subject such as tissue, primary cell or cell-free sample. Sample labels MUST be unique within a subject and it is RECOMMENDED that they be unique throughout the dataset. session: - name: Session display_name: Session description: | A logical grouping of neuroimaging and behavioral data consistent across subjects. @@ -144,7 +132,6 @@ session: In the [PET](SPEC_ROOT/modality-specific-files/positron-emission-tomography.md) context, a session may also indicate a group of related scans, taken in one or more visits. suffix: - name: suffix display_name: suffix description: | An alphanumeric string that forms part of a filename, located after all @@ -152,13 +139,11 @@ suffix: following a final `_`, right before the **file extension**; for example, it is `eeg` in `sub-05_task-matchingpennies_eeg.vhdr`. subject: - name: Subject display_name: Subject description: | A person or animal participating in the study. Used interchangeably with term **Participant**. task: - name: Task display_name: Task description: | A set of structured activities performed by the participant. From 00105ecaec9656edfbb5aa938491ae6752fec9a2 Mon Sep 17 00:00:00 2001 From: VisLab <1189050+VisLab@users.noreply.github.com> Date: Mon, 30 Jan 2023 08:14:18 -0600 Subject: [PATCH 005/255] Updated score library version (#1390) --- src/appendices/hed.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/appendices/hed.md b/src/appendices/hed.md index 604b445e74..abbe092d32 100644 --- a/src/appendices/hed.md +++ b/src/appendices/hed.md @@ -197,13 +197,13 @@ Example: The following `dataset_description.json` file specifies that the [HED8.1.0.xml](https://github.com/hed-standard/hed-schemas/blob/main/standard_schema/hedxml/HED8.1.0.xml) standard schema should be used along with the SCORE library for clinical neurological annotation located at -[HED_score_0.0.1.xml](https://github.com/hed-standard/hed-schemas/blob/main/library_schemas/score/hedxml/HED_score_0.0.1.xml). +[HED_score_1.0.0.xml](https://github.com/hed-standard/hed-schemas/blob/main/library_schemas/score/hedxml/HED_score_1.0.0.xml). ```JSON { "Name": "A great experiment", "BIDSVersion": "1.7.0", - "HEDVersion": ["8.1.0", "sc:score_0.0.1"] + "HEDVersion": ["8.1.0", "sc:score_1.0.0"] } ``` The `sc:` is a user-chosen prefixes used to distinguish the sources @@ -227,7 +227,7 @@ The following `dataset_description.json` indicates that only the SCORE library v { "Name": "A great experiment", "BIDSVersion": "1.7.0", - "HEDVersion": "score_0.0.1" + "HEDVersion": "score_1.0.0" } ``` From fb1806b0793988a0de6f439cdf5cb165a72d1455 Mon Sep 17 00:00:00 2001 From: Cyril Pernet Date: Wed, 1 Feb 2023 08:57:10 +0100 Subject: [PATCH 006/255] richness versus distinctness --- src/common-principles.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common-principles.md b/src/common-principles.md index 7161bb392b..00f1b9e7b0 100644 --- a/src/common-principles.md +++ b/src/common-principles.md @@ -183,6 +183,10 @@ as the labels would collide on a case-insensitive filesystem. Additionally, because the suffix `eeg` is defined, then the suffix `EEG` will not be added to future versions of the standard. +## Filesystem structure & Filenames richness versus distinctness + +BIDS provides a rich filesystem structure and rich filenames by using entites, but it is important to keep in mind that files also have to be readable. They have to be readable by machines, and this implies that filenames cannot be longer than 255 characters. They also have to be readable by humans, and this implies minimizing length. A useful way to think about filenaming is distinctness: what is the minimal information needed for distinguish files? A simple illustration is given by using the [`ses-`](https://bids-specification.readthedocs.io/en/stable/glossary.html#session-entities) and [`run-`](https://bids-specification.readthedocs.io/en/stable/appendices/entities.html#run) entities. A T1 weighted MRI image could, in principle, be called `sub-X_ses-1-run-1_T1w.nii`. When there is only 1 session, and only 1 run, this is not needed as `ses-` and `run-` do not increase distinctivness. Here, it is recommended to use the shorter version `sub-X_T1w.nii`. In some cases, this principle is enforced in the BIDS validator. + ## Source vs. raw vs. derived data BIDS was originally designed to describe and apply consistent naming conventions From 180568cab166900096031b704e3a43dcf6b566ce Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Thu, 2 Feb 2023 16:07:44 -0500 Subject: [PATCH 007/255] SCHEMA: Add rules for checking existence of referenced files, non-existence of duplicate files (#1385) Co-authored-by: Taylor Salo --- src/schema/README.md | 85 +++++++++++++---------- src/schema/meta/expression_tests.yaml | 38 ++++++++++ src/schema/rules/checks/general.yaml | 12 ++++ src/schema/rules/checks/references.yaml | 43 ++++++++++++ src/schema/rules/errors.yaml | 8 +-- src/schema/rules/files/deriv/imaging.yaml | 1 - 6 files changed, 146 insertions(+), 41 deletions(-) create mode 100644 src/schema/rules/checks/general.yaml create mode 100644 src/schema/rules/checks/references.yaml diff --git a/src/schema/README.md b/src/schema/README.md index 335c56522f..4d71f34ce2 100644 --- a/src/schema/README.md +++ b/src/schema/README.md @@ -214,30 +214,34 @@ These are to be populated by an *interpreter* of the schema, and provide the The following operators should be defined by an interpreter: -| Operator | Definition | Example | -| --------- | ------------------------------------------------------------- | --------------------------------------------- | -| `==` | equality | `suffix == "T1w"` | -| `!=` | inequality | `entities.task != "rest"` | -| `<`/`>` | less-than / greater-than | `sidecar.EchoTime < 0.5` | -| `<=`/`>=` | less-than-or-equal / greater-than-or-equal | `0 <= 4` | -| `in` | object lookup, true if RHS is a subfield of LHS | `"Units" in sidecar` | -| `!` | negation, true if the following value is false, or vice versa | `!true == false` | -| `&&` | conjunction, true if both RHS and LHS are true | `"Units" in sidecar && sidecar.Units == "mm"` | -| `\|\|` | disjunction, true if either RHS or LHS is true | `a < mn \|\| a > mx` | -| `.` | object query, returns value of subfield | `sidecar.Units` | -| `[]` | array index, returns value of Nth element (0-indexed) of list | `columns.participant_label[0]` | +| Operator | Definition | Example | +| ----------- | ------------------------------------------------------------- | --------------------------------------------- | +| `==` | equality | `suffix == "T1w"` | +| `!=` | inequality | `entities.task != "rest"` | +| `<`/`>` | less-than / greater-than | `sidecar.EchoTime < 0.5` | +| `<=`/`>=` | less-than-or-equal / greater-than-or-equal | `0 <= 4` | +| `in` | object lookup, true if RHS is a subfield of LHS | `"Units" in sidecar` | +| `!` | negation, true if the following value is false, or vice versa | `!true == false` | +| `&&` | conjunction, true if both RHS and LHS are true | `"Units" in sidecar && sidecar.Units == "mm"` | +| `\|\|` | disjunction, true if either RHS or LHS is true | `a < mn \|\| a > mx` | +| `.` | object query, returns value of subfield | `sidecar.Units` | +| `[]` | array index, returns value of Nth element (0-indexed) of list | `columns.participant_label[0]` | +| `+` | numeric addition / string concatenation | `x + 1`, `stem + "suffix"` | +| `-`/`*`/`/` | numeric operators (division coerces to float) | `length(array) - 2`, `x * 2`, `1 / 2 == 0.5` | The following functions should be defined by an interpreter: -| Function | Definition | Example | Note | -| ---------------------------------------- | ----------------------------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------ | -| `match(arg: str, pattern: str) -> bool` | `true` if `arg` matches the regular expression `pattern` (anywhere in string) | `match(extension, ".gz$")` | True if the file extension ends with `.gz` | -| `type(arg: Any) -> str` | The name of the type, including `"array"`, `"object"`, `"null"` | `type(datatypes)` | Returns `"array"` | -| `intersects(a: array, b: array) -> bool` | `true` if arguments contain any shared elements | `intersects(dataset.modalities, ["pet", "mri"])` | True if either PET or MRI data is found in dataset | -| `length(arg: array) -> int` | Number of elements in an array | `length(columns.onset) > 0` | True if there is at least one value in the onset column | -| `count(arg: array, val: any)` | Number of elements in an array equal to `val` | `count(columns.type, "EEG")` | The number of times "EEG" appears in the column "type" of the current TSV file | -| `min(arg: array)` | The smallest non-`n/a` value in an array | `min(sidecar.SliceTiming) == 0` | A check that the onset of the first slice is 0s | -| `max(arg: array)` | The largest non-`n/a` value in an array | `max(columns.onset)` | The time of the last onset in an events.tsv file | +| Function | Definition | Example | Note | +| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------ | +| `match(arg: str, pattern: str) -> bool` | `true` if `arg` matches the regular expression `pattern` (anywhere in string) | `match(extension, ".gz$")` | True if the file extension ends with `.gz` | +| `substr(arg: str, start: int, end: int) -> str` | The portion of the input string spanning from start position to end position | `substr(path, 0, length(path) - 3)` | `path` with the last three characters dropped | +| `type(arg: Any) -> str` | The name of the type, including `"array"`, `"object"`, `"null"` | `type(datatypes)` | Returns `"array"` | +| `intersects(a: array, b: array) -> bool` | `true` if arguments contain any shared elements | `intersects(dataset.modalities, ["pet", "mri"])` | True if either PET or MRI data is found in dataset | +| `length(arg: array) -> int` | Number of elements in an array | `length(columns.onset) > 0` | True if there is at least one value in the onset column | +| `count(arg: array, val: any)` | Number of elements in an array equal to `val` | `count(columns.type, "EEG")` | The number of times "EEG" appears in the column "type" of the current TSV file | +| `min(arg: array)` | The smallest non-`n/a` value in an array | `min(sidecar.SliceTiming) == 0` | A check that the onset of the first slice is 0s | +| `max(arg: array)` | The largest non-`n/a` value in an array | `max(columns.onset)` | The time of the last onset in an events.tsv file | +| `exists(arg: str \| array, rule: str) -> int` | Count of files in an array that exist in the dataset. String is array with length 1. Rules include `"bids-uri"`, `"dataset"`, `"subject"` and `"stimuli"`. | `exists(sidecar.IntendedFor, "subject")` | True if all files in `IntendedFor` exist, relative to the subject directory. | #### The special value `null` @@ -246,21 +250,30 @@ This value propagates through all of the above operations in a fully-defined, hopefully intuitive way. Most operations involving `null` simply resolve to `null`: -| Operation | Result | -| ------------------------ | ------ | -| `sidecar.MissingValue` | `null` | -| `null.anything` | `null` | -| `null[0]` | `null` | -| `null && true` | `null` | -| `null \|\| true` | `null` | -| `!null` | `null` | -| `match(null, pattern)` | `null` | -| `intersects(list, null)` | `null` | -| `length(null)` | `null` | -| `count(null, val)` | `null` | -| `count(list, null)` | `null` | -| `min(null)` | `null` | -| `max(null)` | `null` | +| Operation | Result | +| -------------------------- | ------ | +| `sidecar.MissingValue` | `null` | +| `null.anything` | `null` | +| `null[0]` | `null` | +| `null && true` | `null` | +| `null \|\| true` | `null` | +| `!null` | `null` | +| `null + 1` | `null` | +| `null - 1` | `null` | +| `null * 1` | `null` | +| `null / 1` | `null` | +| `match(null, pattern)` | `null` | +| `intersects(list, null)` | `null` | +| `substr(null, 0, 1)` | `null` | +| `substr(str, null, 1)` | `null` | +| `substr(str, 0, null)` | `null` | +| `length(null)` | `null` | +| `count(null, val)` | `null` | +| `count(list, null)` | `null` | +| `min(null)` | `null` | +| `max(null)` | `null` | +| `exists(null, "bids-uri")` | `null` | +| `exists("/path", null)` | `null` | The following operators have boolean results: diff --git a/src/schema/meta/expression_tests.yaml b/src/schema/meta/expression_tests.yaml index 0f15d03f76..e57efe498b 100644 --- a/src/schema/meta/expression_tests.yaml +++ b/src/schema/meta/expression_tests.yaml @@ -11,8 +11,36 @@ result: null - expression: null || true result: null +- expression: true && null + result: null +- expression: false && null + result: false +- expression: true || null + result: true +- expression: false || null + result: null - expression: '!null' result: null +- expression: null + 1 + result: null +- expression: null - 1 + result: null +- expression: null * 1 + result: null +- expression: null / 1 + result: null +- expression: 1 + null + result: null +- expression: 1 - null + result: null +- expression: 1 * null + result: null +- expression: 1 / null + result: null +- expression: "'str1' + null" + result: null +- expression: "null + 'str1'" + result: null - expression: intersects([], null) result: null - expression: intersects(null, []) @@ -45,9 +73,19 @@ result: false - expression: '"VolumeTiming" in null' result: false +- expression: exists(null, "bids-uri") + result: false +- expression: exists([], null) + result: false - expression: evaluate(true) result: true - expression: evaluate(false) result: false - expression: evaluate(null) result: false +- expression: 1 + 2 + result: 3 +- expression: '"cat" + "dog"' + result: 'catdog' +- expression: '1 + "cat"' + result: null diff --git a/src/schema/rules/checks/general.yaml b/src/schema/rules/checks/general.yaml new file mode 100644 index 0000000000..75f6e363d0 --- /dev/null +++ b/src/schema/rules/checks/general.yaml @@ -0,0 +1,12 @@ +--- +# BIDS Validator Original Issue Code #74 +# Was DUPLICATE_NIFTI_FILES, but applies generally +DuplicateFiles: + code: DUPLICATE_FILES + message: | + File exists with and without `.gz` extension + level: error + selectors: + - match(extension, '\.gz$') + checks: + - exists(substr(path, 0, length(path) - 3)) == 0 diff --git a/src/schema/rules/checks/references.yaml b/src/schema/rules/checks/references.yaml new file mode 100644 index 0000000000..cbbc8f5454 --- /dev/null +++ b/src/schema/rules/checks/references.yaml @@ -0,0 +1,43 @@ +--- +SubjectRelativeIntendedFor: + selectors: + - datatype != "ieeg" + - type(sidecar.IntendedFor) != "null" + checks: + - exists(sidecar.IntendedFor, "bids-uri") || exists(sidecar.IntendedFor, "subject") + +DatasetRelativeIntendedFor: + selectors: + - datatype == "ieeg" + - type(sidecar.IntendedFor) != "null" + checks: + - exists(sidecar.IntendedFor, "bids-uri") || exists(sidecar.IntendedFor, "dataset") + +AssociatedEmptyRoom: + selectors: + - suffix == "meg" + - type(sidecar.AssociatedEmptyRoom) != "null" + checks: + - exists(sidecar.AssociatedEmptyRoom, "bids-uri") || exists(sidecar.AssociatedEmptyRoom, "dataset") + +Stimuli: + selectors: + - suffix == "events" + - extension == ".tsv" + - type(columns.stim_file) != "null" + checks: + - exists(columns.stim_file, "stimuli") + +Sources: + selectors: + - dataset.dataset_description.DatasetType == "derivatives" + - type(sidecar.Sources) != "null" + checks: + - exists(sidecar.Sources, "bids-uri") || exists(sidecar.Sources, "dataset") + +SpatialReferences: + selectors: + - dataset.dataset_description.DatasetType == "derivatives" + - type(sidecar.SpatialReference.URI) != "null" + checks: + - exists(sidecar.SpatialReference.URI, "bids-uri") || exists(sidecar.SpatialReference.URI, "dataset") diff --git a/src/schema/rules/errors.yaml b/src/schema/rules/errors.yaml index 0a41808075..deb04b40fa 100644 --- a/src/schema/rules/errors.yaml +++ b/src/schema/rules/errors.yaml @@ -118,7 +118,7 @@ WrongNewLine: selectors: - extension == ".tsv" -# BIDS Validator Original Issue Code #74 +# BIDS Validator Original Issue Code #88 MalformedBvec: code: MALFORMED_BVEC message: | @@ -127,7 +127,7 @@ MalformedBvec: selectors: - extension == ".bvec" -# BIDS Validator Original Issue Code #88 +# BIDS Validator Original Issue Code #89 MalformedBval: code: MALFORMED_BVAL message: | @@ -136,7 +136,7 @@ MalformedBval: selectors: - extension == ".bval" -# BIDS Validator Original Issue Code #89 +# BIDS Validator Original Issue Code #90 SidecarWithoutDatafile: code: SIDECAR_WITHOUT_DATAFILE message: | @@ -145,7 +145,7 @@ SidecarWithoutDatafile: selectors: - extension == ".json" -# BIDS Validator Original Issue Code #90 +# BIDS Validator Original Issue Code #97 MissingSession: code: MISSING_SESSION message: | diff --git a/src/schema/rules/files/deriv/imaging.yaml b/src/schema/rules/files/deriv/imaging.yaml index f5d82d570d..e037dcc10c 100644 --- a/src/schema/rules/files/deriv/imaging.yaml +++ b/src/schema/rules/files/deriv/imaging.yaml @@ -99,7 +99,6 @@ anat_parametric_discrete_segmentation: - .json - .tsv - anat_nonparametric_discrete_segmentation: $ref: rules.files.raw.anat.nonparametric suffixes: From eb658db44deade3ef3eb0ffaa192bb5b77d78de0 Mon Sep 17 00:00:00 2001 From: Anthony Galassi <28850131+bendhouseart@users.noreply.github.com> Date: Thu, 2 Feb 2023 16:30:45 -0500 Subject: [PATCH 008/255] update workflow for schema-docs (#1394) --- .github/workflows/schema_docs.yaml | 70 +++++++++++++++++------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/.github/workflows/schema_docs.yaml b/.github/workflows/schema_docs.yaml index 70b1ac4bc9..b3fee9aca3 100644 --- a/.github/workflows/schema_docs.yaml +++ b/.github/workflows/schema_docs.yaml @@ -31,39 +31,51 @@ jobs: os: ["ubuntu-latest"] steps: - uses: actions/checkout@v3 + with: + path: source + persist-credentials: false - - name: Debug - uses: mxschmitt/action-tmate@v3 - if: ${{ inputs.debug_enabled }} - timeout-minutes: 15 + - name: Collect schema-docs repo + uses: actions/checkout@v3 with: - limit-access-to-actor: true + ref: main + repository: ${{ github.repository_owner }}/schema-docs + path: dest + token: ${{ secrets.SCHEMA_DOCS_GITHUB_API_TOKEN }} + fetch-depth: 0 + persist-credentials: true + + - name: Update schema-docs from bids-specification + run: | + rsync --recursive --copy-links source/tools/schemacode/bidsschematools dest/ + rsync --recursive --copy-links source/tools/schemacode/docs dest/ + cat source/requirements.txt | grep -v tools/schemacode >> dest/requirements.txt + sort dest/requirements.txt | uniq > dest/requirements.txt.tmp + mv dest/requirements.txt.tmp dest/requirements.txt - - name: Pull schema-docs and reqs from bids-specification + - name: Check for Updates + id: updatesmade + working-directory: dest run: | - - pushd $HOME - git config --global user.name "${{ github.actor }}" - git config --global user.email "${{ github.actor}}@noreply.com" - git clone https://${{ github.repository_owner }}:${{ secrets.SCHEMA_DOCS_GITHUB_API_TOKEN }}@github.com/${{ github.repository_owner }}/schema-docs - popd - rsync --recursive --copy-links ${{ github.workspace }}/tools/schemacode/bidsschematools $HOME/schema-docs/ - rsync --recursive --copy-links ${{ github.workspace }}/tools/schemacode/docs $HOME/schema-docs/ - cat ${{ github.workspace }}/requirements.txt | grep -v tools/schemacode >> $HOME/schema-docs/requirements.txt - sort $HOME/schema-docs/requirements.txt | uniq > $HOME/schema-docs/requirements.txt.tmp - mv $HOME/schema-docs/requirements.txt.tmp $HOME/schema-docs/requirements.txt - - - name: Push Updates to schema-docs - run: | - - pushd $HOME/schema-docs - - git diff --exit-code > $HOME/gitdiff.log - if [[ $? == 0 ]]; then + git add --all + git status --porcelain + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + echo "changesmade=true" >> $GITHUB_OUTPUT + else echo "No changes found schema code, this workflow should only run when changes are made to schema-*." echo "You must be up to no good." - else - git add --all - git commit -m "Auto update from bids-specification $GITHUB_SHA" - git push origin main + echo "changesmade=false" >> $GITHUB_OUTPUT fi + + - name: Push Updates + if: ${{ steps.updatesmade.outputs.changesmade == 'true' }} + working-directory: dest + env: + ACTOR: ${{ github.actor }} + OWNER: ${{ github.repository_owner }} + run: | + git config --global user.name "$ACTOR" + git config --global user.email "$ACTOR@noreply.com" + git add --all + git commit -m "Auto update from bids-specification $GITHUB_SHA" + git push From 896e6e3007d195f612bb0cb809c822a4caa1ce6f Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 5 Feb 2023 17:35:12 +0100 Subject: [PATCH 009/255] refactor beh metadata --- .../behavioral-experiments.md | 41 +++++++++++-------- src/schema/rules/sidecars/beh.yaml | 7 +++- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/modality-specific-files/behavioral-experiments.md b/src/modality-specific-files/behavioral-experiments.md index 181edf9dcc..dac1d1551c 100644 --- a/src/modality-specific-files/behavioral-experiments.md +++ b/src/modality-specific-files/behavioral-experiments.md @@ -7,30 +7,32 @@ The inputs for this macro can be found in the directory and a guide for using macros can be found at https://github.com/bids-standard/bids-specification/blob/master/macros_doc.md --> + {{ MACROS___make_filename_template("raw", datatypes=["beh"]) }} In addition to logs from behavioral experiments performed alongside imaging data -acquisitions, one can also include data from experiments performed with no neural -recordings. -The results of those experiments can be stored in the `beh` directory using the same -formats for event timing (`_events.tsv`), metadata (`_events.json`), -physiological (`_physio.tsv.gz`, `_physio.json`) -and other continuous recordings (`_stim.tsv.gz`, `_stim.json`) -as for tasks performed during MRI, electrophysiological or other neural recordings. -Additionally, events files that do not include the mandatory `onset` and -`duration` columns can still be included, but should be labeled `_beh.tsv` -rather than `_events.tsv`. +acquisitions, one can also include data from experiments performed with no +neural recordings. The results of those experiments can be stored in the `beh` +directory using the same formats for event timing (`_events.tsv`), metadata +(`_events.json`), physiological (`_physio.tsv.gz`, `_physio.json`) and other +continuous recordings (`_stim.tsv.gz`, `_stim.json`) as for tasks performed +during MRI, electrophysiological or other neural recordings. Additionally, +events files that do not include the mandatory `onset` and `duration` columns +can still be included, but should be labeled `_beh.tsv` rather than +`_events.tsv`. ## RECOMMENDED metadata In addition to the metadata that is either: -- RECOMMENDED for sidecar JSON files for [tabular data](../common-principles.md#tabular-data), or +- RECOMMENDED for sidecar JSON files for + [tabular data](../common-principles.md#tabular-data), or -- REQUIRED for some data that can be found in the `beh` directory - (for example `SamplingFrequency` and `StartTime` for `*_.tsv.gz` files), +- REQUIRED for some data that can be found in the `beh` directory (for example + `SamplingFrequency` and `StartTime` for `*_.tsv.gz` files), -it is RECOMMENDED to add the following metadata to the JSON files of this directory: +it is RECOMMENDED to add the following metadata to the JSON files of this +directory: -{{ MACROS___make_sidecar_table("beh.BEHTabularData") }} -Example of the content of a `_beh.tsv` and its accompanying `_beh.json` sidecar file: +{{ MACROS___make_sidecar_table("beh.BEHInstitutionInformation") }} + +{{ MACROS___make_sidecar_table("beh.BEHTaskInformation") }} + +Example of the content of a `_beh.tsv` and its accompanying `_beh.json` sidecar +file: ```Text trial response response_time stim_file @@ -50,7 +56,8 @@ congruent red 1.435 images/word-red_color-red.jpg incongruent red 1.739 images/word-red_color-blue.jpg ``` -In the accompanying JSON sidecar, the `trial` column might be documented as follows: +In the accompanying JSON sidecar, the `trial` column might be documented as +follows: ```JSON { diff --git a/src/schema/rules/sidecars/beh.yaml b/src/schema/rules/sidecars/beh.yaml index e9221986f6..1221bf84d7 100644 --- a/src/schema/rules/sidecars/beh.yaml +++ b/src/schema/rules/sidecars/beh.yaml @@ -6,7 +6,7 @@ --- # Metadata for either beh or events files -BEHTabularData: +BEHTaskInformation: selectors: - intersects([suffix], ["beh", "events"]) fields: @@ -15,6 +15,11 @@ BEHTabularData: TaskDescription: recommended CogAtlasID: recommended CogPOID: recommended + +BEHInstitutionInformation: + selectors: + - intersects([suffix], ["beh", "events"]) + fields: InstitutionName: recommended InstitutionAddress: recommended InstitutionalDepartmentName: recommended From 478688c0ed21848acbbe698e97251c0fe95d59ae Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 5 Feb 2023 17:39:22 +0100 Subject: [PATCH 010/255] refactor PET --- .../positron-emission-tomography.md | 278 ++++++++++-------- src/schema/rules/sidecars/pet.yaml | 12 +- 2 files changed, 160 insertions(+), 130 deletions(-) diff --git a/src/modality-specific-files/positron-emission-tomography.md b/src/modality-specific-files/positron-emission-tomography.md index d26ba71df6..3b801c17ef 100644 --- a/src/modality-specific-files/positron-emission-tomography.md +++ b/src/modality-specific-files/positron-emission-tomography.md @@ -1,62 +1,60 @@ # Positron Emission Tomography Support for Positron Emission Tomography (PET) was developed as a -[BIDS Extension Proposal](../extensions.md#bids-extension-proposals). -Please see [Citing BIDS](../introduction.md#citing-bids) -on how to appropriately credit this extension when referring to it in the -context of the academic literature. +[BIDS Extension Proposal](../extensions.md#bids-extension-proposals). Please see +[Citing BIDS](../introduction.md#citing-bids) on how to appropriately credit +this extension when referring to it in the context of the academic literature. -Several [example PET datasets](https://github.com/bids-standard/bids-examples#pet-datasets) -have been formatted using this specification -and can be used for practical guidance when curating a new dataset. +Several +[example PET datasets](https://github.com/bids-standard/bids-examples#pet-datasets) +have been formatted using this specification and can be used for practical +guidance when curating a new dataset. Further PET datasets are available from [OpenNeuro](https://openneuro.org). ## Terminology and conventions -PET-BIDS is fully consistent with the BIDS specification as a whole. -However, BIDS was initially developed in the context of MRI, -so some terminology may be unfamiliar to researchers from each field. -This section adds clarifications to -[Common Principles - Definitions](../common-principles.md#definitions) -for the PET context, and introduces the term "time zero" which is currently -specific to PET. +PET-BIDS is fully consistent with the BIDS specification as a whole. However, +BIDS was initially developed in the context of MRI, so some terminology may be +unfamiliar to researchers from each field. This section adds clarifications to +[Common Principles - Definitions](../common-principles.md#definitions) for the +PET context, and introduces the term "time zero" which is currently specific to +PET. 1. **Session** - In most cases, a new session with respect to PET corresponds - to a visit to the scanning site, and starts with a new injection. - In situations where different data types are obtained over several visits - (for example, FDG PET on one day followed by amyloid PET a couple days after) - these scans may be grouped into the same session. - In other datasets, a subject leaving the scanner and returning under the same - injection may be considered separate sessions. - -1. **Run** - In PET, subjects may have to leave the scanner to use the bathroom. - While leaving the scanner would interrupt an MR acquisition, in PET this - disruption is more appropriately considered missing data during a run. - -1. **Time zero** - A reference point in time, to which all timestamps pertaining - to a recording are relative. - Time zero will most commonly be the time of injection of a radioisotope, or - the time at which the first scan of an acquisition is begun. - If a pharmacological within-scan challenge is performed, - another time zero may be more convenient. - -An overview of a common PET experiment (with blood data) can be seen in Figure 1, -defined on a single time scale relative to a predefined "time zero". + to a visit to the scanning site, and starts with a new injection. In + situations where different data types are obtained over several visits (for + example, FDG PET on one day followed by amyloid PET a couple days after) + these scans may be grouped into the same session. In other datasets, a + subject leaving the scanner and returning under the same injection may be + considered separate sessions. + +1. **Run** - In PET, subjects may have to leave the scanner to use the + bathroom. While leaving the scanner would interrupt an MR acquisition, in + PET this disruption is more appropriately considered missing data during a + run. + +1. **Time zero** - A reference point in time, to which all timestamps + pertaining to a recording are relative. Time zero will most commonly be the + time of injection of a radioisotope, or the time at which the first scan of + an acquisition is begun. If a pharmacological within-scan challenge is + performed, another time zero may be more convenient. + +An overview of a common PET experiment (with blood data) can be seen in Figure +1, defined on a single time scale relative to a predefined "time zero". ![Figure 1](images/PET_scan_overview.png "Overview of PET recording data") -Figure 1: Overview of a common PET experiment, including blood measurements, -and defined on a common time scale. -Note, "time zero" is often defined as time of injection or scan start, -but if a pharmaceutical challenge is carried out during the scan, -this time point may also be chosen as time zero. -The injected dose, the PET data, and blood data should all be decay-corrected to time zero, -but because the time of injection does not always coincide with scan start, -the PET data may not always be decay-corrected to the time of injection. -If this is not the case, this may be indicated in the reconstruction section -(`ImageDecayCorrected` and `ImageDecayCorrectionTime`). -In this example, tracer injection coincides with scan start. +Figure 1: Overview of a common PET experiment, including blood measurements, and +defined on a common time scale. Note, "time zero" is often defined as time of +injection or scan start, but if a pharmaceutical challenge is carried out during +the scan, this time point may also be chosen as time zero. The injected dose, +the PET data, and blood data should all be decay-corrected to time zero, but +because the time of injection does not always coincide with scan start, the PET +data may not always be decay-corrected to the time of injection. If this is not +the case, this may be indicated in the reconstruction section +(`ImageDecayCorrected` and `ImageDecayCorrectionTime`). In this example, tracer +injection coincides with scan start. ## PET recording data @@ -67,77 +65,78 @@ The inputs for this macro can be found in the directory and a guide for using macros can be found at https://github.com/bids-standard/bids-specification/blob/master/macros_doc.md --> + {{ MACROS___make_filename_template( "raw", datatypes=["pet"], suffixes=["pet", "events", "physio", "stim"]) }} -PET data MUST be stored in the `pet` directory. -PET imaging data SHOULD be stored in 4D (or 3D, if only one volume was acquired) -NIfTI files with the `_pet` suffix. -Volumes MUST be stored in chronological order (the order they were acquired in). +PET data MUST be stored in the `pet` directory. PET imaging data SHOULD be +stored in 4D (or 3D, if only one volume was acquired) NIfTI files with the +`_pet` suffix. Volumes MUST be stored in chronological order (the order they +were acquired in). The OPTIONAL [`task-