moved timezone saving to UTC

MRICenterline/app/database/save_points.py

@@ -1,4 +1,5 @@
 import sqlite3
+import pytz
 from datetime import datetime, timezone
 
 from MRICenterline.app.database import name_id
@@ -15,11 +16,18 @@ def save_points(case_name: str,
                 sequence_name: str,
                 length_points: PointArray,
                 mpr_points: PointArray,
-                timer_data: Timer):
+                timer_data: Timer,
+                timestamp: datetime or None = None):
 
     case_id = name_id.get_case_id(case_name)
     seq_id = name_id.get_sequence_id(sequence_name, case_id)
-    timestamp = datetime.now(timezone.utc).astimezone().strftime(CONST.TIMESTAMP_FORMAT)
+
+    if timestamp:
+        # convert timestamp to UTC and then to string
+        timestamp_string = timestamp.astimezone(pytz.utc).strftime(CONST.TIMESTAMP_FORMAT)
+    else:
+        # use timezone now
+        timestamp_string = datetime.utcnow().strftime(CONST.TIMESTAMP_FORMAT)
     time_gap = timer_data.calculate_time_gap()
 
     con = sqlite3.connect(CFG.get_db())
@@ -69,7 +77,7 @@ def save_points(case_name: str,
         cl_id = None
 
     session_data = {
-        'timestamp': timestamp,
+        'timestamp': timestamp_string,
         'time_elapsed_seconds': time_gap,
         'lengths_id': lengths_id,
         'cl_id': cl_id,
@@ -88,5 +96,7 @@ def save_points(case_name: str,
 
     session_id = con.cursor().execute("select count(*) from 'sessions'").fetchone()[0]
     con.close()
-    logging.info(f"Saved session with id [{session_id}] successfully.")
-    MSG.msg_box_info(f"Saved session with id [{session_id}] successfully.")
+
+    session_saved_message = f"Saved session with id [{session_id}] successfully."
+    logging.info(session_saved_message)
+    MSG.msg_box_info(session_saved_message)

MRICenterline/app/file_reader/AbstractReader.py

@@ -2,8 +2,9 @@
 
 
 class AbstractReader:
-    def __init__(self, case_id: int, folder: str, is_new_case: bool = False):
+    def __init__(self, case_id: int, case_name: str, folder: str, is_new_case: bool = False):
         self.case_id = case_id
+        self.case_name = case_name
         self.folder = folder
         self.sitk_image = None
         self.is_new_case = is_new_case

MRICenterline/app/file_reader/dicom/DICOMReader.py

@@ -14,11 +14,11 @@
 
 
 class DICOMReader(AbstractReader):
-    def __init__(self, case_id, folder, is_new_case):
-        super().__init__(case_id, folder, is_new_case)
+    def __init__(self, case_id, case_name, folder, is_new_case):
+        super().__init__(case_id, case_name, folder, is_new_case)
 
         if self.is_new_case:
-            InitialDatabaseBuild.build(self.folder)
+            InitialDatabaseBuild.build(self.folder, case_name)
         self.read_from_database()
 
     def __repr__(self):

MRICenterline/app/file_reader/dicom/InitialDatabaseBuild.py

@@ -7,8 +7,8 @@
 from MRICenterline import CFG
 
 
-def build(folder):
-    case_name = os.path.relpath(folder, CFG.get_folder('raw'))
+def build(folder, case_name):
+    # case_name = os.path.relpath(folder, CFG.get_folder('raw'))
 
     # get case_id
     con = sqlite3.connect(CFG.get_db())

MRICenterline/app/file_reader/imager.py

@@ -11,9 +11,13 @@
 
 
 class Imager:
-    def __init__(self, directory):
+    def __init__(self, directory, root_folder=None):
         self.directory = directory
-        self.case_name = os.path.relpath(directory, CFG.get_folder('raw'))
+
+        if root_folder:
+            self.case_name = os.path.relpath(directory, root_folder)
+        else:
+            self.case_name = os.path.relpath(directory, CFG.get_folder('raw'))
 
         self.database_check()
         self.assign_reader()
@@ -73,7 +77,7 @@ def database_check(self):
 
     def assign_reader(self):
         if self.file_type.upper() == "DICOM":
-            self.reader = DICOMReader(self.case_id, self.directory, self.new_case_flag)
+            self.reader = DICOMReader(self.case_id, self.case_name, self.directory, self.new_case_flag)
         elif self.file_type.upper() == "NRRD":
             pass
         else:
@@ -84,7 +88,7 @@ def assign_reader(self):
     def initialize_folder(self):
         logging.debug("Checking folder type")
         self.file_type = read_folder(self.directory)
-        logging.debug(f"Folder {self.directory} is type {self.file_type}")
+        logging.debug(f"Folder {self.directory} is type {self.file_type}, adding with case name [{self.case_name}]")
 
         # add the folder to the case_list table
         con = sqlite3.connect(CFG.get_db())

MRICenterline/app/points/import_from_v3.py

@@ -0,0 +1,126 @@
+import os
+import json
+
+from MRICenterline import CFG
+from MRICenterline.app.points.point import Point
+from MRICenterline.app.points.point_array import PointArray
+from MRICenterline.app.points.status import PointStatus
+from MRICenterline.app.points.timer import Timer
+from MRICenterline.app.database.save_points import save_points
+from MRICenterline.app.database.name_id import get_case_id
+from MRICenterline.app.gui_data_handling.slice_loc_based_image_properties import SliceLocImageProperties
+from MRICenterline.app.file_reader.dicom.DICOMReader import DICOMReader
+
+import logging
+logging.getLogger(__name__)
+
+
+def timestamp_parse(original: str):
+    from datetime import datetime
+
+    split1 = original.split("T")
+    date = datetime.strptime(split1[0], "%Y-%m-%d")
+
+    split2 = split1[1].split(".")
+    time = datetime.strptime(split2[0], "%H:%M:%S")
+    tz = datetime.strptime(split2[1].lstrip("0123456789"), "%z")
+
+    return datetime(date.year, date.month, date.day, hour=time.hour, minute=time.minute, second=time.second,
+                    microsecond=0, tzinfo=tz.tzinfo)
+
+
+class Ver3AnnotationImport:
+    def __init__(self, filename, root_folder=None):
+        self.filename = filename
+
+        if root_folder:
+            self.case_name = os.path.relpath(self.filename.parents[1], root_folder)
+        else:
+            self.case_name = os.path.relpath(self.filename.parents[1], CFG.get_folder('raw'))
+
+        logging.info(f"Reading file: {self.filename}")
+
+        with open(self.filename, 'r') as f:
+            file = json.load(f)
+
+            self.sequence_name = file['SeriesDescription']
+            self.timestamp = timestamp_parse(file['annotation timestamp'])
+
+            try:
+                self.time_measured = file['Time measurement']
+            except KeyError:
+                logging.info("No time measured listed")
+                self.time_measured = None
+
+            self.mpr_point_array = PointArray(PointStatus.MPR)
+            self.length_point_array = PointArray(PointStatus.LENGTH)
+
+            try:
+                mpr_coords = file['MPR points']
+            except KeyError:
+                logging.info(f"Found no MPR points")
+            else:
+                logging.info(f"Found {len(mpr_coords)} MPR points")
+                self.parse_points(mpr_coords, self.mpr_point_array)
+
+            try:
+                length_coords = file['length points']
+            except KeyError:
+                logging.info(f"Found no length points")
+            else:
+                logging.info(f"Found {len(length_coords)} length points")
+                self.parse_points(length_coords, self.length_point_array)
+
+    def parse_points(self, points_from_json, point_array: PointArray):
+        dcm_reader = DICOMReader(case_name=self.case_name,
+                                 case_id=get_case_id(self.case_name),
+                                 folder=self.filename.parents[1],
+                                 is_new_case=False)
+
+        np_array, file_list = dcm_reader[self.sequence_name]
+
+        image_properties = SliceLocImageProperties(np_array=np_array,
+                                                   z_coords=dcm_reader.get_z_coords(self.sequence_name),
+                                                   file_list=file_list)
+
+        for pt in points_from_json:
+            parsed = Point.point_from_vtk_coords(pt, image_properties)
+            point_array.add_point(parsed)
+
+    def __repr__(self):
+        return f"""
+        Importing {self.filename} into database
+        Timestamp  {self.timestamp}
+        Time measured {self.time_measured}
+        Length Points: {len(self.length_point_array)}
+        MPR Points: {len(self.mpr_point_array)}
+        """
+
+    def commit(self):
+        if self.time_measured and len(self.time_measured.strip()) > 0:
+            timer = DummyTimer(self.time_measured.strip())
+        else:
+            timer = Timer()
+
+        save_points(case_name=self.case_name,
+                    sequence_name=self.sequence_name,
+                    length_points=self.length_point_array,
+                    mpr_points=self.mpr_point_array,
+                    timer_data=timer,
+                    timestamp=self.timestamp)
+
+
+class DummyTimer(Timer):
+    def __init__(self, gap_from_file):
+        super().__init__()
+        from datetime import datetime, timedelta
+
+        clean_gap_string = gap_from_file.split(".")[0]
+
+        t = datetime.strptime(clean_gap_string, "%H:%M:%S")
+        delta = timedelta(hours=t.hour, minutes=t.minute, seconds=t.second)
+
+        self.measured_gap = delta
+
+    def calculate_time_gap(self):
+        return int(self.measured_gap.total_seconds())

MRICenterline/app/scanner/load_v3_points.py

@@ -1,8 +1,36 @@
+from glob import glob
+from typing import List
 from pathlib import Path
 
+from MRICenterline.app.points.import_from_v3 import Ver3AnnotationImport
+
 import logging
 logging.getLogger(__name__)
 
 
-def load_v3_points(folder: str or Path):
-    pass
+def load_v3_points(folders: List[str] or List[Path]):
+    logging.info("Starting v3 point loader")
+    num_folders = len(folders)
+
+    for index, folder in enumerate(folders):
+        logging.info(f"[{1 + index} / {num_folders}] Reading {folder}")
+        root_folder = folder.parents[1]
+
+        if Path(folder).parts[-1] == 'data':
+            for annotation_file in [Path(i) for i in glob(f"{folder}/*.annotation.json")]:
+                if Path(annotation_file).name.split(".")[-3] == "centerline":
+                    pass
+                    # centerline_files.add(annotation_file)
+                else:
+                    try:
+                        importer = Ver3AnnotationImport(annotation_file, root_folder)
+                    # TODO: should not be needed when using the Rambam PC
+                    except KeyError:
+                        pass
+                    else:
+                        print(importer)
+                        importer.commit()
+
+        else:
+            pass
+

MRICenterline/app/scanner/run_metadata_sequence_scan.py

@@ -1,18 +1,43 @@
+from pathlib import Path
 from MRICenterline.app.file_reader.imager import Imager
 
+import logging
+logging.getLogger(__name__)
+
+
+def run_metadata_sequence_scan(folders, parent_widget=None, running_for_v3_scanner=False):
+    if running_for_v3_scanner:
+        root_folder = folders.pop(0)
+        logging.info(f"Using {root_folder} as folder root")
+    else:
+        root_folder = None
 
-def run_metadata_sequence_scan(folders, parent_widget=None):
     num_folders = len(folders)
     for index, folder in enumerate(folders):
+        if running_for_v3_scanner and Path(folder).parts[-1] == 'data':
+            logging.info(f"Skipping {folder}")
+            continue
+
+        reading_info = f"[{1 + index} / {num_folders}] Reading {folder}"
+        logging.info(reading_info)
         if parent_widget:
-            parent_widget.add_to_textbox(f"[{1 + index} / {num_folders}] Reading {folder}")
+            parent_widget.add_to_textbox(reading_info)
 
         try:
-            imager = Imager(folder)
+            imager = Imager(folder, root_folder=root_folder)
         except NotImplementedError:
-            parent_widget.add_to_textbox(f"{folder} is either not supported or has no MRI images.", color='red')
+            error_info = f"{folder} is either not supported or has no MRI images."
+            logging.warning(error_info)
+            if parent_widget:
+                parent_widget.add_to_textbox(f"{folder} is either not supported or has no MRI images.", color='red')
         else:
+            reading_done_info = f"Folder is {imager.file_type} | {len(imager)} sequences found"
+            logging.info(reading_done_info)
             if parent_widget:
-                parent_widget.add_to_textbox(f"Folder is {imager.file_type} | {len(imager)} sequences found")
+                parent_widget.add_to_textbox(reading_done_info)
+
+    logging.info("Metadata sequence scan complete!")
 
-    parent_widget.add_to_textbox(f"Done! You may close.")
+    if parent_widget:
+        done_message = f"Done! You may close."
+        parent_widget.add_to_textbox(done_message)

MRICenterline/cli/arg_setup.py

@@ -1,4 +1,6 @@
 import argparse
+from glob import glob
+from pathlib import Path
 
 from MRICenterline.app import scanner
 
@@ -23,10 +25,9 @@ def arg_setup():
         parser.add_argument(sh, lo, action='store_true', help=he)
 
     args = parser.parse_args()
+    directories = [Path(i) for i in glob(f"{args.folder}/**/", recursive=True)]
 
-    logging.info(f"Starting scan: {args.folder}")
-
-    # TODO: create a temporary CFG file for the scanner to use
+    logging.info(f"Starting scan: {args.folder}, found {len(directories)} folders.")
 
     if args.scan:
         logging.info("Generate sequence/metadata report")
@@ -36,5 +37,5 @@ def arg_setup():
         logging.info("Generate timing report")
     if args.ver3:
         logging.info("Load points from v3")
-        scanner.run_metadata_sequence_scan(args.folder)
-        scanner.load_v3_points(args.folder)
+        scanner.run_metadata_sequence_scan(directories, running_for_v3_scanner=True)
+        scanner.load_v3_points(directories)

requirements.txt

@@ -23,4 +23,5 @@ scipy
 SimpleITK==2.0.2
 vtk
 tqdm
-nibabel
+nibabel
+pytz