From 55b8878659e310f9012d5a13709b0ed316979fed Mon Sep 17 00:00:00 2001 From: Riccardo Massei Date: Fri, 11 Oct 2024 09:47:19 +0200 Subject: [PATCH] add new tools (omero get and omero filter) and updated the ci/pr pipeline to add metadat to the dummy set) --- .github/workflows/ci.yaml | 2 + .github/workflows/pr.yaml | 2 + tools/omero/.shed.yml | 6 +- tools/omero/omero_filter.py | 80 ++++++++++++ tools/omero/omero_filter.xml | 102 +++++++++++++++ tools/omero/omero_get.py | 90 +++++++++++++ tools/omero/omero_get.xml | 118 ++++++++++++++++++ .../test-data/output_filter_filename.tsv | 2 + tools/omero/test-data/output_filter_tag.tsv | 2 + tools/omero/test-data/output_ids_dataset.tsv | 2 + tools/omero/test-data/output_ids_image.tsv | 3 + tools/omero/test-data/output_ids_project.tsv | 2 + 12 files changed, 408 insertions(+), 3 deletions(-) create mode 100644 tools/omero/omero_filter.py create mode 100644 tools/omero/omero_filter.xml create mode 100644 tools/omero/omero_get.py create mode 100644 tools/omero/omero_get.xml create mode 100644 tools/omero/test-data/output_filter_filename.tsv create mode 100644 tools/omero/test-data/output_filter_tag.tsv create mode 100644 tools/omero/test-data/output_ids_dataset.tsv create mode 100644 tools/omero/test-data/output_ids_image.tsv create mode 100644 tools/omero/test-data/output_ids_project.tsv diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a140ee5..e3d139d2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -188,6 +188,8 @@ jobs: DID=$(omero obj new Dataset name='test_dts') omero obj new ProjectDatasetLink parent=$PID child=$DID omero import -d $DID .github/dummy-dts-omero + omero tag create --name test_tag --desc 'description of my_tag' + omero tag link Image:1 1 echo "Created the dummy dataset into OMERO" # download or create large test data via script diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 908d4530..40ba89f7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -329,6 +329,8 @@ jobs: DID=$(omero obj new Dataset name='test_dts') omero obj new ProjectDatasetLink parent=$PID child=$DID omero import -d $DID .github/dummy-dts-omero + omero tag create --name test_tag --desc 'description of my_tag' + omero tag link Image:1 1 echo "Created the dummy dataset into OMERO" # download or create large test data via script diff --git a/tools/omero/.shed.yml b/tools/omero/.shed.yml index 70fa0277..870afe5a 100644 --- a/tools/omero/.shed.yml +++ b/tools/omero/.shed.yml @@ -1,8 +1,8 @@ categories: - Imaging name: omero_upload -description: Import images, region of interest, metadata into an OMERO.server using omero-py -long_description: Tool to import and link different objects into OMERO +description: Interact with an OMERO.server using omero-py and ezomero. +long_description: This set of tool allows to import images and metadata into an OMERO.server. owner: ufz remote_repository_url: https://github.com/Helmholtz-UFZ/galaxy-tools/tree/main/tools/omero homepage_url: https://github.com/ome/omero-py/ @@ -13,4 +13,4 @@ suite: name: "suite_omero_py" description: "A suite of tools that brings the omero-py and ezomero project into Galaxy." long_description: | - OMERO.py provides an interface to the OMERO.blitz server. + OMERO.py and ezomero provides an interface to the OMERO.blitz server. diff --git a/tools/omero/omero_filter.py b/tools/omero/omero_filter.py new file mode 100644 index 00000000..ef8d1e83 --- /dev/null +++ b/tools/omero/omero_filter.py @@ -0,0 +1,80 @@ +import argparse +import csv +import json + +import ezomero as ez + + +def filter_ids_ezo(user, pws, host, port, filter, id, value1, value2=None, tsv_file="filter_list.tsv"): + + # Transform the id input in a list of integer + id = id.split(',') + id = list(map(int, id)) + + # Function to write tabular file from the ezomero output + def write_ids_to_tsv(data, header): + with open(tsv_file, 'a+', newline='') as f: + f.seek(0) + is_empty = f.tell() == 0 # Check if file is empty + writer = csv.writer(f, delimiter='\t') + if is_empty: + writer.writerow([header]) # Write the header + for item in data: + writer.writerow([item]) # Write each ID + + try: + with ez.connect(user, pws, "", host, port, secure=True) as conn: + if filter == "filename": + fn_ids = ez.filter_by_filename(conn, id, value1) + write_ids_to_tsv(fn_ids, [f"Images with filename {value1}"]) + return fn_ids + elif filter == "KP": + kp_ims = ez.filter_by_kv(conn, id, value1, value2) + write_ids_to_tsv(kp_ims, [f"Images with KV {value1} and {value2}"]) + return kp_ims + elif filter == "tag": + tg_dict = ez.filter_by_tag_value(conn, id, value1) + write_ids_to_tsv(tg_dict, [f"Images with tag {value1}"]) + return tg_dict + + else: + raise ValueError(f"Unsupported object type") + + except Exception as e: + print(f"Connection error: {str(e)}") + return None + + +# Argument parsing +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fetch and save data as TSV based on object type.") + parser.add_argument("--credential-file", dest="credential_file", type=str, required=True, + help="Credential file (JSON file with username and password for OMERO)") + parser.add_argument('--host', required=True, + help="Host server address.") + parser.add_argument('--port', required=True, type=int, + help='OMERO port') + parser.add_argument('--filter', required=True, + help="Filter type - Filename, Key-Value Pairs, Tag") + parser.add_argument('--id', required=True, + help="List of images IDs") + parser.add_argument('--value1', required=True, + help="First searching values - Filename, Key, Tag") + parser.add_argument('--value2', required=False, + help="Second searching values - Value (necessary just for Key-Value Pairs filter") + parser.add_argument('--tsv_file', default='filter_list.tsv', required=True, + help="Output TSV file path.") + + args = parser.parse_args() + + with open(args.credential_file, 'r') as f: + crds = json.load(f) + + # Call the main function to get the object and save it as a TSV + filter_ids_ezo(user=crds['username'], pws=crds['password'], host=args.host, + port=args.port, + filter=args.filter, + value1=args.value1, + value2=args.value2, + id=args.id, + tsv_file=args.tsv_file) diff --git a/tools/omero/omero_filter.xml b/tools/omero/omero_filter.xml new file mode 100644 index 00000000..82f721b6 --- /dev/null +++ b/tools/omero/omero_filter.xml @@ -0,0 +1,102 @@ + + with ezomero + + 5.18.0 + 0 + + + omero + + + ezomero + + openjdk + + + + + + + + ^[a-zA-Z0-9._-]*$ + '..' not in value + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Description +----------- + +Tool to filter images IDs by filename, Key-Value Pairs and Tag value. +For Key-Value Pairs search, two values are required (Value1 = Key, Value2 = Pair). +IDs are a list of image IDs which can be fetched using the omero_get tool. + + + + 10.1038/nmeth.1896 + + \ No newline at end of file diff --git a/tools/omero/omero_get.py b/tools/omero/omero_get.py new file mode 100644 index 00000000..c8cc484d --- /dev/null +++ b/tools/omero/omero_get.py @@ -0,0 +1,90 @@ +import argparse +import csv +import json + +import ezomero as ez + + +def get_object_ezo(user, pws, host, port, obj_type, id=None, tsv_file="id_list.tsv"): + # Function to write tabular file from the ezomero output + def write_ids_to_tsv(data, header): + with open(tsv_file, 'a+', newline='') as f: + f.seek(0) + is_empty = f.tell() == 0 # Check if file is empty + writer = csv.writer(f, delimiter='\t') + if is_empty: + writer.writerow([header]) # Write the header + for item in data: + writer.writerow([item]) # Write each ID + + # Function to write tabular file from a dictionary ezomero output + def write_dict_to_tsv(data, headers): + with open(tsv_file, 'a+', newline='') as f: + f.seek(0) + is_empty = f.tell() == 0 # Check if file is empty + writer = csv.writer(f, delimiter='\t') + if is_empty: + writer.writerow(headers) # Write the headers + for key, value in data.items(): + writer.writerow([key, value]) # Write each key-value pair + + try: + with ez.connect(user, pws, "", host, port, secure=True) as conn: + if obj_type == "dataset": + ds_ids = ez.get_dataset_ids(conn, project=int(id)) + write_ids_to_tsv(ds_ids, "Dataset IDs") + return ds_ids + elif obj_type == "image": + ds_ims = ez.get_image_ids(conn, dataset=int(id)) + write_ids_to_tsv(ds_ims, "Image IDs") + return ds_ims + elif obj_type == "annotation": + ma_dict = ez.get_map_annotation(conn, int(id)) + write_dict_to_tsv(ma_dict, ["Annotation ID", "Annotation Value"]) + return ma_dict + elif obj_type == "project": + proj_ids = ez.get_project_ids(conn) + write_ids_to_tsv(proj_ids, "Project IDs") + return proj_ids + elif obj_type == "roi": + roi_ids = ez.get_roi_ids(conn, int(id)) + write_ids_to_tsv(roi_ids, "ROI IDs") + return roi_ids + elif obj_type == "table": + table = ez.get_table(conn, int(id)) + write_dict_to_tsv(table, ["Table ID", "Table Value"]) + return table + else: + raise ValueError(f"Unsupported object type: {obj_type}") + + except Exception as e: + print(f"Connection error: {str(e)}") + return None + + +# Argument parsing +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fetch and save data as TSV based on object type.") + parser.add_argument("--credential-file", dest="credential_file", type=str, + required=True, help="Credential file (JSON file with username and password for OMERO)") + parser.add_argument('--host', required=True, + help="Host server address.") + parser.add_argument('--port', required=True, type=int, + help='OMERO port') + parser.add_argument('--obj_type', required=True, + help="Type of object to fetch: dataset, image, annotation, project, roi, or table.") + parser.add_argument('--id', required=False, + help="ID of the specific OMERO object.") + parser.add_argument('--tsv_file', default='id_list.tsv', required=True, + help="Output TSV file path.") + args = parser.parse_args() + + with open(args.credential_file, 'r') as f: + crds = json.load(f) + + # Call the main function to get the object and save it as a TSV + get_object_ezo(user=crds['username'], pws=crds['password'], host=args.host, + port=args.port, + obj_type=args.obj_type, + id=args.id, + tsv_file=args.tsv_file) diff --git a/tools/omero/omero_get.xml b/tools/omero/omero_get.xml new file mode 100644 index 00000000..b21a0cca --- /dev/null +++ b/tools/omero/omero_get.xml @@ -0,0 +1,118 @@ + + with ezomero + + 5.18.0 + 0 + + + omero + + + ezomero + + openjdk + + + + + + + + ^[a-zA-Z0-9._-]*$ + '..' not in value + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Description +----------- + +Tool to fetch project, dataset, images and ROIs IDs user defined OMERO.server. +Additionally, this tool fetch attachments and tables linked to an Image + +Options: +Project -> Project IDs present in the OMERO.server +Dataset -> Dataset IDs present in a specific Project +Image -> Image IDs present in a specific Dataset +Annotation -> Annotation File (Key-Value Pairs) linked to an Image +ROI -> ROI IDs linked to an Image +Table -> Table linked to an Image + + + + 10.1038/nmeth.1896 + + \ No newline at end of file diff --git a/tools/omero/test-data/output_filter_filename.tsv b/tools/omero/test-data/output_filter_filename.tsv new file mode 100644 index 00000000..8cfddd1d --- /dev/null +++ b/tools/omero/test-data/output_filter_filename.tsv @@ -0,0 +1,2 @@ +['Images with filename sample_image_2.jpg'] +2 diff --git a/tools/omero/test-data/output_filter_tag.tsv b/tools/omero/test-data/output_filter_tag.tsv new file mode 100644 index 00000000..f725cde0 --- /dev/null +++ b/tools/omero/test-data/output_filter_tag.tsv @@ -0,0 +1,2 @@ +['Images with tag test_tag'] +1 diff --git a/tools/omero/test-data/output_ids_dataset.tsv b/tools/omero/test-data/output_ids_dataset.tsv new file mode 100644 index 00000000..10d1a2e4 --- /dev/null +++ b/tools/omero/test-data/output_ids_dataset.tsv @@ -0,0 +1,2 @@ +Dataset IDs +1 diff --git a/tools/omero/test-data/output_ids_image.tsv b/tools/omero/test-data/output_ids_image.tsv new file mode 100644 index 00000000..94ec02fa --- /dev/null +++ b/tools/omero/test-data/output_ids_image.tsv @@ -0,0 +1,3 @@ +Image IDs +1 +2 diff --git a/tools/omero/test-data/output_ids_project.tsv b/tools/omero/test-data/output_ids_project.tsv new file mode 100644 index 00000000..242726b5 --- /dev/null +++ b/tools/omero/test-data/output_ids_project.tsv @@ -0,0 +1,2 @@ +Project IDs +1