diff --git a/docs/code/LeedsSportsPose.py b/docs/code/LeedsSportsPose.py index 67e43dca9..4bb983b83 100644 --- a/docs/code/LeedsSportsPose.py +++ b/docs/code/LeedsSportsPose.py @@ -43,6 +43,10 @@ dataset = Dataset("LeedsSportsPose", gas) """""" +"""Read Dataset / list segment names""" +dataset.keys() +"""""" + """Read Dataset / get segment""" segment = dataset[0] """""" diff --git a/docs/code/NeolixOD.py b/docs/code/NeolixOD.py index eb60f677b..5909bc30e 100644 --- a/docs/code/NeolixOD.py +++ b/docs/code/NeolixOD.py @@ -42,6 +42,10 @@ dataset = Dataset("NeolixOD", gas) """""" +"""Read Dataset / list segment names""" +dataset.keys() +"""""" + """Read Dataset / get segment""" segment = dataset[0] """""" diff --git a/docs/source/conf.py b/docs/source/conf.py index 3bb1f2b56..2fb912c03 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,9 +15,12 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # """Configuration file for the Sphinx documentation builder.""" +import os import sys from pathlib import Path +import jinja2 + sys.path.insert(0, str(Path(__file__).parents[2])) @@ -79,3 +82,39 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ["_static"] + +source_path = os.path.dirname(os.path.abspath(__file__)) +pattern = os.environ.get("SPHINX_PATTERN") +single_doc = pattern is not None and pattern not in ("-api", "whatsnew") +include_api = pattern is None or pattern == "whatsnew" +example_path = os.path.join(source_path, "quick_start", "examples") +datasets = ( + "Dogs Vs Cats", + "20 Newsgroups", + "BSTLD", + "Neolix OD", + "Leeds Sports Pose", + "THCHS-30", +) +label_types = ( + "Classification", + "Classification", + "Box2D", + "Box3D", + "Keypoints2D", + "Sentence", +) +file_names = ("DogsVsCats", "Newsgroups20", "BSTLD", "NeolixOD", "LeedsSportsPose", "THCHS30") +for dataset_name, label_type, file_name in zip(datasets, label_types, file_names): + with open(os.path.join(example_path, "test.rst.template")) as f: + t = jinja2.Template(f.read()) + with open(os.path.join(example_path, f"{file_name}.rst"), "w") as f: + f.write( + t.render( + include_api=include_api, + single_doc=(pattern if single_doc else None), + dataset_name=dataset_name, + label_type=label_type, + file_name=file_name, + ) + ) diff --git a/docs/source/quick_start/examples/BSTLD.rst b/docs/source/quick_start/examples/BSTLD.rst deleted file mode 100644 index 062b22be7..000000000 --- a/docs/source/quick_start/examples/BSTLD.rst +++ /dev/null @@ -1,182 +0,0 @@ -######## - BSTLD -######## - -This topic describes how to manage the `BSTLD Dataset `_, -which is a dataset with :ref:`reference/label_format/Box2D:Box2D` label(:numref:`Fig. %s `). - -.. _example-bstld: - -.. figure:: ../../images/example-Box2D.png - :scale: 50 % - :align: center - - The preview of a cropped image with labels from "BSTLD". - -***************************** - Authorize a Client Instance -***************************** - -An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Authorize a Client Instance""" - :end-before: """""" - -**************** - Create Dataset -**************** - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Create Dataset""" - :end-before: """""" - -****************** - Organize Dataset -****************** - -Normally, ``dataloader.py`` and ``catalog.json`` are required to organize the "BSTLD" dataset into the :class:`~tensorbay.dataset.dataset.Dataset` instance. -In this example, they are stored in the same directory like:: - - BSTLD/ - catalog.json - dataloader.py - -Step 1: Write the Catalog -========================= - -A :ref:`reference/dataset_structure:catalog` contains all label information of one dataset, which -is typically stored in a json file like ``catalog.json``. - -.. literalinclude:: ../../../../tensorbay/opendataset/BSTLD/catalog.json - :language: json - :name: BSTLD-catalog - :linenos: - -The only annotation type for "BSTLD" is :ref:`reference/label_format/Box2D:Box2D`, and there are 13 -:ref:`reference/label_format/CommonLabelProperties:category` types and one :ref:`reference/label_format/CommonLabelProperties:attributes` type. - -.. note:: - - By passing the path of the ``catalog.json``, :func:`~tensorbay.dataset.dataset.DatasetBase.load_catalog` supports loading the catalog into dataset. - -.. important:: - - See :ref:`catalog table ` for more catalogs with different label types. - -Step 2: Write the Dataloader -============================ - -A :ref:`reference/glossary:dataloader` is needed to organize the dataset into a :class:`~tensorbay.dataset.dataset.Dataset` instance. - -.. literalinclude:: ../../../../tensorbay/opendataset/BSTLD/loader.py - :language: python - :name: BSTLD-dataloader - :linenos: - -See :ref:`Box2D annotation ` for more details. - -There are already a number of dataloaders in TensorBay SDK provided by the community. -Thus, instead of writing, importing an available dataloader is also feasible. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Organize dataset / import dataloader""" - :end-before: """""" - -.. note:: - - Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. - -.. important:: - - See :ref:`dataloader table ` for dataloaders with different label types. - -******************* - Visualize Dataset -******************* - -Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. -This step can help users to check whether the dataset is correctly organized. -Please see :ref:`features/visualization:Visualization` for more details. - -**************** - Upload Dataset -**************** - -The organized "BSTLD" dataset can be uploaded to TensorBay for sharing, reuse, etc. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Upload Dataset""" - :end-before: """""" - -.. note:: - Set `skip_uploaded_files=True` to skip uploaded data. - The data will be skiped if its name and segment name is the same as remote data. - -Similar with Git, the commit step after uploading can record changes to the dataset as a version. -If needed, do the modifications and commit again. -Please see :ref:`features/version_control/index:Version Control` for more details. - -************** - Read Dataset -************** - -Now "BSTLD" dataset can be read from TensorBay. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Read Dataset / get dataset""" - :end-before: """""" - -In :ref:`reference/dataset_structure:dataset` "BSTLD", there are three -:ref:`segments `: ``train``, ``test`` and ``additional``. -Get the segment names by listing them all. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Read Dataset / list segment names""" - :end-before: """""" - -Get a segment by passing the required segment name. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Read Dataset / get segment""" - :end-before: """""" - - -In the train :ref:`reference/dataset_structure:segment`, there is a sequence of :ref:`reference/dataset_structure:data`, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Read Dataset / get data""" - :end-before: """""" - -In each :ref:`reference/dataset_structure:data`, -there is a sequence of :ref:`reference/label_format/Box2D:Box2D` annotations, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Read Dataset / get label""" - :end-before: """""" - -There is only one label type in "BSTLD" dataset, which is ``box2d``. -The information stored in :ref:`reference/label_format/CommonLabelProperties:category` is -one of the names in "categories" list of :ref:`catalog.json `. The information stored -in :ref:`reference/label_format/CommonLabelProperties:attributes` is one or several of the attributes in "attributes" list of :ref:`catalog.json `. -See :ref:`reference/label_format/Box2D:Box2D` label format for more details. - -**************** - Delete Dataset -**************** - -.. literalinclude:: ../../../../docs/code/BSTLD.py - :language: python - :start-after: """Delete Dataset""" - :end-before: """""" diff --git a/docs/source/quick_start/examples/DogsVsCats.rst b/docs/source/quick_start/examples/DogsVsCats.rst deleted file mode 100644 index c166acc7a..000000000 --- a/docs/source/quick_start/examples/DogsVsCats.rst +++ /dev/null @@ -1,169 +0,0 @@ -############## - Dogs vs Cats -############## - -This topic describes how to manage the `Dogs vs Cats Dataset `_, -which is a dataset with :ref:`reference/label_format/Classification:Classification` label. - -***************************** - Authorize a Client Instance -***************************** - -An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Authorize a Client Instance""" - :end-before: """""" - -**************** - Create Dataset -**************** - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Create Dataset""" - :end-before: """""" - -****************** - Organize Dataset -****************** - -Normally, ``dataloader.py`` and ``catalog.json`` are required to organize the "Dogs vs Cats" dataset into the :class:`~tensorbay.dataset.dataset.Dataset` instance. -In this example, they are stored in the same directory like:: - - Dogs vs Cats/ - catalog.json - dataloader.py - -Step 1: Write the Catalog -========================= - -A :ref:`reference/dataset_structure:catalog` contains all label information of one dataset, which -is typically stored in a json file like ``catalog.json``. - -.. literalinclude:: ../../../../tensorbay/opendataset/DogsVsCats/catalog.json - :language: json - :name: dogsvscats-catalog - :linenos: - -The only annotation type for "Dogs vs Cats" is :ref:`reference/label_format/Classification:Classification`, and there are 2 -:ref:`reference/label_format/CommonLabelProperties:category` types. - -.. note:: - - By passing the path of the ``catalog.json``, :func:`~tensorbay.dataset.dataset.DatasetBase.load_catalog` supports loading the catalog into dataset. - -.. important:: - - See :ref:`catalog table ` for more catalogs with different label types. - -Step 2: Write the Dataloader -============================ - -A :ref:`reference/glossary:dataloader` is needed to organize the dataset into -a :class:`~tensorbay.dataset.dataset.Dataset` instance. - -.. literalinclude:: ../../../../tensorbay/opendataset/DogsVsCats/loader.py - :language: python - :name: dogsvscats-dataloader - :linenos: - -See :ref:`Classification annotation ` for more details. - - -There are already a number of dataloaders in TensorBay SDK provided by the community. -Thus, instead of writing, importing an available dataloadert is also feasible. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Organize dataset / import dataloader""" - :end-before: """""" - -.. note:: - - Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. - -.. important:: - - See :ref:`dataloader table ` for more examples of dataloaders with different label types. - -******************* - Visualize Dataset -******************* - -Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. -This step can help users to check whether the dataset is correctly organized. -Please see :ref:`features/visualization:Visualization` for more details. - -**************** - Upload Dataset -**************** - -The organized "Dogs vs Cats" dataset can be uploaded to TensorBay for sharing, reuse, etc. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Upload Dataset""" - :end-before: """""" - -Similar with Git, the commit step after uploading can record changes to the dataset as a version. -If needed, do the modifications and commit again. -Please see :ref:`features/version_control/index:Version Control` for more details. - -************** - Read Dataset -************** - -Now "Dogs vs Cats" dataset can be read from TensorBay. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Read Dataset / get dataset""" - :end-before: """""" - -In :ref:`reference/dataset_structure:dataset` "Dogs vs Cats", there are two -:ref:`segments `: ``train`` and ``test``. -Get the segment names by listing them all. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Read Dataset / list segment names""" - :end-before: """""" - -Get a segment by passing the required segment name. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Read Dataset / get segment""" - :end-before: """""" - -In the train :ref:`reference/dataset_structure:segment`, there is a sequence of :ref:`reference/dataset_structure:data`, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Read Dataset / get data""" - :end-before: """""" - -In each :ref:`reference/dataset_structure:data`, -there is a sequence of :ref:`reference/label_format/Classification:Classification` annotations, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Read Dataset / get label""" - :end-before: """""" - -There is only one label type in "Dogs vs Cats" dataset, which is ``classification``. The information stored in :ref:`reference/label_format/CommonLabelProperties:category` is -one of the names in "categories" list of :ref:`catalog.json `. -See :ref:`reference/label_format/Classification:Classification` label format for more details. - -**************** - Delete Dataset -**************** - -.. literalinclude:: ../../../../docs/code/DogsVsCats.py - :language: python - :start-after: """Delete Dataset""" - :end-before: """""" diff --git a/docs/source/quick_start/examples/LeedsSportsPose.rst b/docs/source/quick_start/examples/LeedsSportsPose.rst deleted file mode 100644 index fa8ef1ad7..000000000 --- a/docs/source/quick_start/examples/LeedsSportsPose.rst +++ /dev/null @@ -1,169 +0,0 @@ -################### - Leeds Sports Pose -################### - -This topic describes how to manage the `Leeds Sports Pose Dataset `_, -which is a dataset with :ref:`reference/label_format/Keypoints2D:Keypoints2D` label(:numref:`Fig. %s `). - -.. _example-leedssportspose: - -.. figure:: ../../images/example-Keypoints2D.png - :scale: 80 % - :align: center - - The preview of an image with labels from "Leeds Sports Pose". - -***************************** - Authorize a Client Instance -***************************** - -An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Authorize a Client Instance""" - :end-before: """""" - -**************** - Create Dataset -**************** - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Create Dataset""" - :end-before: """""" - -****************** - Organize Dataset -****************** - -Normally, ``dataloader.py`` and ``catalog.json`` are required to organize the "Leeds Sports Pose" dataset into the :class:`~tensorbay.dataset.dataset.Dataset` instance. -In this example, they are stored in the same directory like:: - - Leeds Sports Pose/ - catalog.json - dataloader.py - -Step 1: Write the Catalog -========================= - -A :ref:`reference/dataset_structure:catalog` contains all label information of one dataset, which -is typically stored in a json file like ``catalog.json``. - -.. literalinclude:: ../../../../tensorbay/opendataset/LeedsSportsPose/catalog.json - :language: json - :name: LeedsSportsPose-catalog - :linenos: - -The only annotation type for "Leeds Sports Pose" is :ref:`reference/label_format/Keypoints2D:Keypoints2D`. - -.. note:: - - By passing the path of the ``catalog.json``, :func:`~tensorbay.dataset.dataset.DatasetBase.load_catalog` supports loading the catalog into dataset. - -.. important:: - - See :ref:`catalog table ` for more catalogs with different label types. - -Step 2: Write the Dataloader -============================ - -A :ref:`reference/glossary:dataloader` is needed to organize the dataset into -a :class:`~tensorbay.dataset.dataset.Dataset` instance. - -.. literalinclude:: ../../../../tensorbay/opendataset/LeedsSportsPose/loader.py - :language: python - :name: LeedsSportsPose-dataloader - :linenos: - -See :ref:`Keipoints2D annotation ` for more details. - - -There are already a number of dataloaders in TensorBay SDK provided by the community. -Thus, instead of writing, importing an available dataloader is also feasible. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Organize dataset / import dataloader""" - :end-before: """""" - -.. note:: - - Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. - -.. important:: - - See :ref:`dataloader table ` for dataloaders with different label types. - -******************* - Visualize Dataset -******************* - -Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. -This step can help users to check whether the dataset is correctly organized. -Please see :ref:`features/visualization:Visualization` for more details. - -**************** - Upload Dataset -**************** - -The organized "BSTLD" dataset can be uploaded to TensorBay for sharing, reuse, etc. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Upload Dataset""" - :end-before: """""" - -Similar with Git, the commit step after uploading can record changes to the dataset as a version. -If needed, do the modifications and commit again. -Please see :ref:`features/version_control/index:Version Control` for more details. - -************** - Read Dataset -************** - -Now "Leeds Sports Pose" dataset can be read from TensorBay. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Read Dataset / get dataset""" - :end-before: """""" - -In :ref:`reference/dataset_structure:dataset` "Leeds Sports Pose", there is one -:ref:`reference/dataset_structure:segment` named ``default``. Get it by passing the segment name or the index. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Read Dataset / get segment""" - :end-before: """""" - -In the default :ref:`reference/dataset_structure:segment`, there is a sequence of :ref:`reference/dataset_structure:data`, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Read Dataset / get data""" - :end-before: """""" - -In each :ref:`reference/dataset_structure:data`, -there is a sequence of :ref:`reference/label_format/Keypoints2D:Keypoints2D` annotations, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Read Dataset / get label""" - :end-before: """""" - -There is only one label type in "Leeds Sports Pose" dataset, which is ``keypoints2d``. The information stored in ``x`` (``y``) is -the x (y) coordinate of one keypoint of one keypoints list. The information stored in ``v`` is -the visible status of one keypoint of one keypoints list. See :ref:`reference/label_format/Keypoints2D:Keypoints2D` -label format for more details. - -**************** - Delete Dataset -**************** - -.. literalinclude:: ../../../../docs/code/LeedsSportsPose.py - :language: python - :start-after: """Delete Dataset""" - :end-before: """""" diff --git a/docs/source/quick_start/examples/NeolixOD.rst b/docs/source/quick_start/examples/NeolixOD.rst deleted file mode 100644 index 4107765db..000000000 --- a/docs/source/quick_start/examples/NeolixOD.rst +++ /dev/null @@ -1,177 +0,0 @@ -########### - Neolix OD -########### - -This topic describes how to manage the `Neolix OD dataset`_, -which is a dataset with :ref:`reference/label_format/Box3D:Box3D` label type -(:numref:`Fig. %s `). - -.. _Neolix OD dataset: https://gas.graviti.cn/dataset/graviti-open-dataset/NeolixOD - -.. _example-neolixod: - -.. figure:: ../../images/example-Box3D.png - :scale: 50 % - :align: center - - The preview of a point cloud from "Neolix OD" with Box3D labels. - -***************************** - Authorize a Client Instance -***************************** - -An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Authorize a Client Instance""" - :end-before: """""" - -**************** - Create Dataset -**************** - - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Create Dataset""" - :end-before: """""" - -****************** - Organize Dataset -****************** - -Normally, ``dataloader.py`` and ``catalog.json`` are required to organize the "Neolix OD" dataset into the :class:`~tensorbay.dataset.dataset.Dataset` instance. -In this example, they are stored in the same directory like:: - - Neolix OD/ - catalog.json - dataloader.py - -Step 1: Write the Catalog -========================= - -A :ref:`Catalog ` contains all label information of one dataset, -which is typically stored in a json file like ``catalog.json``. - -.. literalinclude:: ../../../../tensorbay/opendataset/NeolixOD/catalog.json - :language: json - :name: neolixod-catalog - :linenos: - -The only annotation type for "Neolix OD" is :ref:`reference/label_format/Box3D:Box3D`, and there are 15 -:ref:`reference/label_format/CommonLabelProperties:Category` types and 3 :ref:`reference/label_format/CommonLabelProperties:Attributes` types. - -.. note:: - - By passing the path of the ``catalog.json``, :func:`~tensorbay.dataset.dataset.DatasetBase.load_catalog` supports loading the catalog into dataset. - -.. important:: - - See :ref:`catalog table ` for more catalogs with different label types. - -Step 2: Write the Dataloader -============================ - -A :ref:`reference/glossary:dataloader` is needed to organize the dataset into -a :class:`~tensorbay.dataset.dataset.Dataset` instance. - -.. literalinclude:: ../../../../tensorbay/opendataset/NeolixOD/loader.py - :language: python - :name: neolixod-dataloader - :linenos: - -See :ref:`Box3D annotation ` for more details. - - -There are already a number of dataloaders in TensorBay SDK provided by the community. -Thus, instead of writing, importing an available dataloader is also feasible. - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Organize dataset / import dataloader""" - :end-before: """""" - -.. note:: - - Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. - -.. important:: - - See :ref:`dataloader table ` for dataloaders with different label types. - -******************* - Visualize Dataset -******************* - -Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. -This step can help users to check whether the dataset is correctly organized. -Please see :ref:`features/visualization:Visualization` for more details. - -**************** - Upload Dataset -**************** - -The organized "Neolix OD" dataset can be uploaded to tensorBay for sharing, reuse, etc. - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Upload Dataset""" - :end-before: """""" - -Similar with Git, the commit step after uploading can record changes to the dataset as a version. -If needed, do the modifications and commit again. -Please see :ref:`features/version_control/index:Version Control` for more details. - -************** - Read Dataset -************** - -Now "Neolix OD" dataset can be read from TensorBay. - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Read Dataset / get dataset""" - :end-before: """""" - -In :ref:`reference/dataset_structure:Dataset` "Neolix OD", there is only one -:ref:`segment `: ``default``. -Get a segment by passing the required segment name or the index. - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Read Dataset / get segment""" - :end-before: """""" - -In the default :ref:`reference/dataset_structure:Segment`, -there is a sequence of :ref:`reference/dataset_structure:Data`, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Read Dataset / get data""" - :end-before: """""" - -In each :ref:`reference/dataset_structure:Data`, -there is a sequence of :ref:`reference/label_format/Box3D:Box3D` annotations, - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Read Dataset / get label""" - :end-before: """""" - -There is only one label type in "Neolix OD" dataset, which is ``box3d``. -The information stored in :ref:`reference/label_format/CommonLabelProperties:Category` is -one of the category names in "categories" list of :ref:`catalog.json `. -The information stored in :ref:`reference/label_format/CommonLabelProperties:Attributes` -is one of the attributes in "attributes" list of :ref:`catalog.json `. -See :ref:`reference/label_format/Box3D:Box3D` label format for more details. - -**************** - Delete Dataset -**************** - -.. literalinclude:: ../../../../docs/code/NeolixOD.py - :language: python - :start-after: """Delete Dataset""" - :end-before: """""" diff --git a/docs/source/quick_start/examples/Newsgroups20.rst b/docs/source/quick_start/examples/Newsgroups20.rst deleted file mode 100644 index 02045b430..000000000 --- a/docs/source/quick_start/examples/Newsgroups20.rst +++ /dev/null @@ -1,187 +0,0 @@ -############### - 20 Newsgroups -############### - -This topic describes how to manage the `20 Newsgroups dataset`_, which is a dataset -with :ref:`reference/label_format/Classification:Classification` label type. - -.. _20 Newsgroups dataset: https://gas.graviti.cn/dataset/data-decorators/Newsgroups20 - -***************************** - Authorize a Client Instance -***************************** - -An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Authorize a Client Instance""" - :end-before: """""" - -**************** - Create Dataset -**************** - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Create Dataset""" - :end-before: """""" - -****************** - Organize Dataset -****************** - -Normally, ``dataloader.py`` and ``catalog.json`` are required to organize the "20 Newsgroups" dataset into the :class:`~tensorbay.dataset.dataset.Dataset` instance. -In this example, they are stored in the same directory like:: - - 20 Newsgroups/ - catalog.json - dataloader.py - - -It takes the following steps to organize the "20 Newsgroups" dataset by -the :class:`~tensorbay.dataset.dataset.Dataset` instance. - -Step 1: Write the Catalog -========================= - -A :ref:`Catalog ` contains all label information of one dataset, -which is typically stored in a json file like ``catalog.json``. - -.. literalinclude:: ../../../../tensorbay/opendataset/Newsgroups20/catalog.json - :language: json - :name: Newsgroups20-catalog - :linenos: - -The only annotation type for "20 Newsgroups" is :ref:`reference/label_format/Classification:Classification`, -and there are 20 :ref:`reference/label_format/CommonLabelProperties:Category` types. - -.. note:: - - * The :ref:`categories` in - :ref:`reference/dataset_structure:Dataset` "20 Newsgroups" have parent-child relationship, - and it use "." to sparate different levels. - * By passing the path of the ``catalog.json``, :func:`~tensorbay.dataset.dataset.DatasetBase.load_catalog` supports loading the catalog into dataset. - -.. important:: - - See :ref:`catalog table ` for more catalogs with different label types. - -Step 2: Write the Dataloader -============================ - -A :ref:`reference/glossary:Dataloader` is neeeded to organize the dataset into a -:class:`~tensorbay.dataset.dataset.Dataset` instance. - -.. literalinclude:: ../../../../tensorbay/opendataset/Newsgroups20/loader.py - :language: python - :name: Newsgroups20-dataloader - :linenos: - -See :ref:`Classification annotation ` for more details. - -.. note:: - - The data in "20 Newsgroups" do not have extensions - so that a "txt" extension is added to the remote path of each data file - to ensure the loaded dataset could function well on TensorBay. - - -There are already a number of dataloaders in TensorBay SDK provided by the community. -Thus, instead of writing, importing an available dataloader is also feasible. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Organize dataset / import dataloader""" - :end-before: """""" - -.. note:: - - Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. - -.. important:: - - See :ref:`dataloader table ` for dataloaders with different label types. - -******************* - Visualize Dataset -******************* - -Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. -This step can help users to check whether the dataset is correctly organized. -Please see :ref:`features/visualization:Visualization` for more details. - -**************** - Upload Dataset -**************** - -The organized "20 Newsgroups" dataset can be uploaded to TensorBay for sharing, reuse, etc. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Upload Dataset""" - :end-before: """""" - -Similar with Git, the commit step after uploading can record changes to the dataset as a version. -If needed, do the modifications and commit again. -Please see :ref:`features/version_control/index:Version Control` for more details. - -************** - Read Dataset -************** - -Now "20 Newsgroups" dataset can be read from TensorBay. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Read Dataset / get dataset""" - :end-before: """""" - -In :ref:`reference/dataset_structure:Dataset` "20 Newsgroups", there are four -:ref:`Segments `: ``20news-18828``, -``20news-bydate-test`` and ``20news-bydate-train``, ``20_newsgroups``. -Get the segment names by listing them all. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Read Dataset / list segment names""" - :end-before: """""" - -Get a segment by passing the required segment name. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Read Dataset / get segment""" - :end-before: """""" - -In the 20news-18828 :ref:`reference/dataset_structure:Segment`, there is a sequence of :ref:`reference/dataset_structure:Data`, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Read Dataset / get data""" - :end-before: """""" - -In each :ref:`reference/dataset_structure:Data`, -there is a sequence of :ref:`reference/label_format/Classification:Classification` annotations, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Read Dataset / get label""" - :end-before: """""" - -There is only one label type in "20 Newsgroups" dataset, which is ``Classification``. -The information stored in :ref:`reference/label_format/CommonLabelProperties:Category` is -one of the category names in "categories" list of :ref:`catalog.json `. -See :ref:`this page ` for more details about the -structure of Classification. - -**************** - Delete Dataset -**************** - -.. literalinclude:: ../../../../docs/code/Newsgroups20.py - :language: python - :start-after: """Delete Dataset""" - :end-before: """""" diff --git a/docs/source/quick_start/examples/THCHS30.rst b/docs/source/quick_start/examples/THCHS30.rst deleted file mode 100644 index 66020089a..000000000 --- a/docs/source/quick_start/examples/THCHS30.rst +++ /dev/null @@ -1,159 +0,0 @@ -########### - THCHS-30 -########### - -This topic describes how to manage the `THCHS-30 Dataset`_, -which is a dataset with :ref:`reference/label_format/Sentence:Sentence` label - -.. _THCHS-30 Dataset: https://www.graviti.com/open-datasets/data-decorators/THCHS30 - -***************************** - Authorize a Client Instance -***************************** - -An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Authorize a Client Instance""" - :end-before: """""" - -**************** - Create Dataset -**************** - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Create Dataset""" - :end-before: """""" - -****************** -Organize Dataset -****************** - -It takes the following steps to organize the “THCHS-30” dataset by the :class:`~tensorbay.dataset.dataset.Dataset` instance. - -Step 1: Write the Catalog -========================= - -A :ref:`Catalog ` contains all label information of one -dataset, which is typically stored in a json file. However the catalog of THCHS-30 is too -large, instead of reading it from json file, we read it by mapping from subcatalog that is -loaded by the raw file. Check the :ref:`dataloader ` below for more details. - -.. important:: - - See :ref:`catalog table ` for more catalogs with different - label types. - -Step 2: Write the Dataloader -============================ - -A :ref:`dataloader ` is needed to organize the dataset -into a :class:`~tensorbay.dataset.dataset.Dataset` instance. - -.. literalinclude:: ../../../../tensorbay/opendataset/THCHS30/loader.py - :language: python - :name: THCHS30-dataloader - :linenos: - -See :ref:`Sentence annotation ` for more details. - - -There are already a number of dataloaders in TensorBay SDK provided by the community. -Thus, instead of writing, importing an available dataloadert is also feasible. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Organize dataset / import dataloader""" - :end-before: """""" - -.. note:: - - Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. - -.. important:: - - See :ref:`dataloader table ` for dataloaders with different label types. - -******************* - Visualize Dataset -******************* - -Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. -This step can help users to check whether the dataset is correctly organized. -Please see :ref:`features/visualization:Visualization` for more details. - -**************** -Upload Dataset -**************** - -The organized "THCHS-30" dataset can be uploaded to TensorBay for sharing, reuse, etc. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Upload Dataset""" - :end-before: """""" - -Similar with Git, the commit step after uploading can record changes to the dataset as a version. -If needed, do the modifications and commit again. -Please see :ref:`features/version_control/index:Version Control` for more details. - -************** -Read Dataset -************** - -Now "THCHS-30" dataset can be read from TensorBay. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Read Dataset / get dataset""" - :end-before: """""" - -In :ref:`reference/dataset_structure:Dataset` "THCHS-30", there are three -:ref:`Segments `: -``dev``, ``train`` and ``test``. -Get the segment names by listing them all. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Read Dataset / list segment names""" - :end-before: """""" - -Get a segment by passing the required segment name. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Read Dataset / get segment""" - :end-before: """""" - -In the dev :ref:`reference/dataset_structure:Segment`, -there is a sequence of :ref:`reference/dataset_structure:Data`, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Read Dataset / get data""" - :end-before: """""" - -In each :ref:`reference/dataset_structure:Data`, -there is a sequence of :ref:`reference/label_format/Sentence:Sentence` annotations, -which can be obtained by index. - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Read Dataset / get label""" - :end-before: """""" - -There is only one label type in "THCHS-30" dataset, which is ``Sentence``. It contains -``sentence``, ``spell`` and ``phone`` information. See :ref:`Sentence ` -label format for more details. - -**************** -Delete Dataset -**************** - -.. literalinclude:: ../../../../docs/code/THCHS30.py - :language: python - :start-after: """Delete Dataset""" - :end-before: """""" diff --git a/docs/source/quick_start/examples/test.rst.template b/docs/source/quick_start/examples/test.rst.template new file mode 100644 index 000000000..073b32e54 --- /dev/null +++ b/docs/source/quick_start/examples/test.rst.template @@ -0,0 +1,215 @@ +{% set datasets_with_image=("BSTLD", "Neolix OD", "Leeds Sports Pose", ) %} + +################## + {{dataset_name}} +################## + +This topic describes how to manage the `{{dataset_name}} Dataset `_, +which is a dataset with :ref:`reference/label_format/{{label_type}}:{{label_type}}` label{% if dataset_name in datasets_with_image %} +(:numref:`Fig. %s `). + +.. _example-{{file_name | lower}}: + +.. figure:: ../../images/example-{{label_type}}.png + :scale: 50 % + :align: center + + The preview of a cropped image with labels from "{{dataset_name}}". +{% endif %} + +***************************** + Authorize a Client Instance +***************************** + +An :ref:`reference/glossary:accesskey` is needed to authenticate identity when using TensorBay. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Authorize a Client Instance""" + :end-before: """""" + +**************** + Create Dataset +**************** + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Create Dataset""" + :end-before: """""" + +****************** + Organize Dataset +****************** + +Normally, ``dataloader.py`` and ``catalog.json`` are required to organize the "{{dataset_name}}" dataset into the :class:`~tensorbay.dataset.dataset.Dataset` instance. +In this example, they are stored in the same directory like:: + + {{dataset_name}}/ + catalog.json + dataloader.py + +Step 1: Write the Catalog +========================= + +A :ref:`reference/dataset_structure:catalog` contains all label information of one dataset, which +is typically stored in a json file like ``catalog.json``. {% if dataset_name == "THCHS-30" %} +However the catalog of THCHS-30 is too +large, instead of reading it from json file, we read it by mapping from subcatalog that is +loaded by the raw file. Check the :ref:`dataloader ` below for more details. +{% endif %} + +{% if dataset_name != "THCHS-30" %} +.. literalinclude:: ../../../../tensorbay/opendataset/{{file_name}}/catalog.json + :language: json + :name: {{file_name}}-catalog + :linenos: +{% endif %} + +{% if dataset_name == "BSTLD" %} +The only annotation type for "{{dataset_name}}" is :ref:`reference/label_format/{{label_type}}:{{label_type}}`, and there are 13 +:ref:`reference/label_format/CommonLabelProperties:category` types and one :ref:`reference/label_format/CommonLabelProperties:attributes` type. +{% endif %} + +{% if dataset_name == "Dogs Vs Cats" %} +The only annotation type for "{{dataset_name}}" is :ref:`reference/label_format/{{label_type}}:{{label_type}}`, and there are 2 +:ref:`reference/label_format/CommonLabelProperties:category` types. +{% endif %} + +{% if dataset_name == "Leeds Sports Pose" %} +The only annotation type for "{{dataset_name}}" is :ref:`reference/label_format/{{label_type}}:{{label_type}}`. +{% endif %} + +{% if dataset_name == "Neolix OD" %} +The only annotation type for "{{dataset_name}}" is :ref:`reference/label_format/{{label_type}}:{{label_type}}`, and there are 15 +:ref:`reference/label_format/CommonLabelProperties:Category` types and 3 :ref:`reference/label_format/CommonLabelProperties:Attributes` types. +{% endif %} + +{% if dataset_name == "20 Newsgroups" %} +The only annotation type for "{{dataset_name}}" is :ref:`reference/label_format/{{label_type}}:{{label_type}}`, +and there are 20 :ref:`reference/label_format/CommonLabelProperties:Category` types. +{% endif %} + +.. note:: + + By passing the path of the ``catalog.json``, :func:`~tensorbay.dataset.dataset.DatasetBase.load_catalog` supports loading the catalog into dataset. + +.. important:: + + See :ref:`catalog table ` for more catalogs with different label types. + +Step 2: Write the Dataloader +============================ + +A :ref:`reference/glossary:dataloader` is needed to organize the dataset into a :class:`~tensorbay.dataset.dataset.Dataset` instance. + +.. literalinclude:: ../../../../tensorbay/opendataset/{{file_name}}/loader.py + :language: python + :name: {{file_name}}-dataloader + :linenos: + +See :ref:`{{label_type}} annotation ` for more details. + +There are already a number of dataloaders in TensorBay SDK provided by the community. +Thus, instead of writing, importing an available dataloader is also feasible. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Organize dataset / import dataloader""" + :end-before: """""" + +.. note:: + + Note that catalogs are automatically loaded in available dataloaders, users do not have to write them again. + +.. important:: + + See :ref:`dataloader table ` for dataloaders with different label types. + +******************* + Visualize Dataset +******************* + +Optionally, the organized dataset can be visualized by **Pharos**, which is a TensorBay SDK plug-in. +This step can help users to check whether the dataset is correctly organized. +Please see :ref:`features/visualization:Visualization` for more details. + +**************** + Upload Dataset +**************** + +The organized "{{dataset_name}}" dataset can be uploaded to TensorBay for sharing, reuse, etc. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Upload Dataset""" + :end-before: """""" + +.. note:: + Set ``skip_uploaded_files=True`` to skip uploaded data. + The data will be skiped if its name and segment name is the same as remote data. + +Similar with Git, the commit step after uploading can record changes to the dataset as a version. +If needed, do the modifications and commit again. +Please see :ref:`features/version_control/index:Version Control` for more details. + +************** + Read Dataset +************** + +Now "{{dataset_name}}" dataset can be read from TensorBay. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Read Dataset / get dataset""" + :end-before: """""" + +Get the segment names by listing them all. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Read Dataset / list segment names""" + :end-before: """""" + +Get a segment by passing the required segment name. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Read Dataset / get segment""" + :end-before: """""" + +In the :ref:`reference/dataset_structure:segment`, there is a sequence of :ref:`reference/dataset_structure:data`, +which can be obtained by index. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Read Dataset / get data""" + :end-before: """""" + +In each :ref:`reference/dataset_structure:data`, +there is a sequence of :ref:`reference/label_format/{{label_type}}:{{label_type}}` annotations, +which can be obtained by index. + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Read Dataset / get label""" + :end-before: """""" + +There is only one label type in "{{dataset_name}}" dataset, which is ``{{label_type | lower}}``. +{% if dataset_name != "THCHS-30" %} +The information stored in :ref:`reference/label_format/CommonLabelProperties:category` is +one of the names in "categories" list of :ref:`catalog.json <{{file_name}}-catalog>`. The information stored +in :ref:`reference/label_format/CommonLabelProperties:attributes` is one or several of the attributes in "attributes" list of :ref:`catalog.json <{{file_name}}-catalog>`. +See :ref:`reference/label_format/{{label_type}}:{{label_type}}` label format for more details. +{% else %} +It contains ``sentence``, ``spell`` and ``phone`` information. See :ref:`Sentence ` +label format for more details. +{% endif %} + +**************** + Delete Dataset +**************** + +.. literalinclude:: ../../../../docs/code/{{file_name}}.py + :language: python + :start-after: """Delete Dataset""" + :end-before: """""" diff --git a/requirements_linter.txt b/requirements_linter.txt index a5d323d66..1fdbb8223 100644 --- a/requirements_linter.txt +++ b/requirements_linter.txt @@ -14,3 +14,4 @@ xmltodict >= 0.2 pyyaml >= 5.1 h5py >= 3.2.1 pillow >= 2.0.0 +jinja2 >= 3.0.1