Fix #178. Ensure most acquisition panel acquisitions include metadata…

… in result.

nion/instrumentation/Acquisition.py

@@ -1881,6 +1881,10 @@ def _process(self, channel_data: ChannelData) -> typing.Sequence[ChannelData]:
         if self.__axis is not None:
             summed_xdata = xd.sum(data_and_metadata, self.__axis)
             assert summed_xdata
+            summed_xdata._set_metadata(data_and_metadata.metadata)
+            summed_xdata.timestamp = data_and_metadata.timestamp
+            summed_xdata.timezone = data_and_metadata.timezone
+            summed_xdata.timezone_offset = data_and_metadata.timezone_offset
             return [ChannelData(channel_data.channel, summed_xdata)]
         else:
             data = data_and_metadata.data

nion/instrumentation/HardwareSource.py

@@ -1997,8 +1997,7 @@ def get_calibration_descriptions(self, data_metadata: DataAndMetadata.DataMetada
             calibration_descriptions.append(CalibrationDescription("temporal", "calculated", "data", None, temporal_calibrations))
         if angular_calibrations := self.__get_angular_calibrations(dimensional_shape, dimensional_calibrations, metadata):
             calibration_descriptions.append(CalibrationDescription("angular", "calculated", "data", None, angular_calibrations))
-        calibration_descriptions.extend(
-            self.__get_intensity_e_calibration_descriptions(dimensional_shape, intensity_calibration, dimensional_calibrations, metadata))
+        calibration_descriptions.extend(self.__get_intensity_e_calibration_descriptions(data_metadata))
         return calibration_descriptions
 
     def __get_spatial_calibrations(self, dimensional_shape: DataAndMetadata.ShapeType,
@@ -2043,14 +2042,14 @@ def __get_angular_calibrations(self, dimensional_shape: DataAndMetadata.ShapeTyp
                 return [Calibration.Calibration(offset=offset, scale=angular_scale, units="rad") for _, offset in zip(dimensional_shape, offsets)]
         return None
 
-    def __get_intensity_e_calibration_descriptions(self, dimensional_shape: DataAndMetadata.ShapeType,
-                                                   intensity_calibration: Calibration.Calibration,
-                                                   dimensional_calibrations: DataAndMetadata.CalibrationListType,
-                                                   metadata: typing.Optional[DataAndMetadata.MetadataType]) -> typing.Sequence[CalibrationDescription]:
+    def __get_intensity_e_calibration_descriptions(self, data_metadata: DataAndMetadata.DataMetadata) -> typing.Sequence[CalibrationDescription]:
+        intensity_calibration = data_metadata.intensity_calibration
+        dimensional_calibrations = data_metadata.datum_dimensional_calibrations
+        metadata = data_metadata.metadata
         calibration_descriptions = list[CalibrationDescription]()
         counts_per_electron = metadata.get("hardware_source", dict()).get("counts_per_electron") if metadata else None
         exposure = metadata.get("hardware_source", dict()).get("exposure") if metadata else None
-        if counts_per_electron and intensity_calibration and intensity_calibration.units == "counts":
+        if counts_per_electron and intensity_calibration.units == "counts":
             calibration_descriptions.append(CalibrationDescription("intensity-e", "calculated", "data",
                                                                    Calibration.Calibration(
                                                                        scale=intensity_calibration.scale / counts_per_electron,

nion/instrumentation/MultiAcquire.py

@@ -423,6 +423,7 @@ def __init__(self, document_model: DocumentModel.DocumentModel, channel_name: st
         self.__current_parameters_index = current_parameters_index
         self.__cumulative_elapsed_time = cumulative_elapsed_time
         self.__stack_metadata_keys = stack_metadata_keys
+        self.__stack_values = list[typing.Any]()
         self.current_frames_index = 0
         self.__data_item = self.__create_data_item(channel_name, grab_sync_info)
         self.progress_updated_event = Event.Event()
@@ -585,39 +586,26 @@ def update(self, data_and_metadata: DataAndMetadata.DataAndMetadata, state: str,
         metadata["MultiAcquire.settings"] = self.__multi_acquire_settings.as_dict()
         metadata["MultiAcquire.parameters"] = self.__multi_acquire_parameters.as_dict()
         # This is needed for metadata that changes with each spectrum image in the stack and needs to be preserved.
-        # One usecase is the storage information that comes with virtual detector data that has the full dataset saved
-        # in the background. Currently the camera defines which metadata keys to stack and we copy that information
-        # when setting up the CameraDataChannel
+        # One usecase is the storage information that comes with virtual detector data that has the full dataset
+        # saved in the background. Currently, the camera defines which metadata keys to stack and we copy that
+        # information when setting up the CameraDataChannel
         if self.__stack_metadata_keys is not None:
             for key_path in self.__stack_metadata_keys:
-                existing_data = None
                 if isinstance(key_path, str):
                     key_path = [key_path]
-                sub_dict = dict(self.__data_item.metadata)
+                existing_data = None
+                sub_dict = dict(metadata)
                 for key in key_path:
                     sub_dict = typing.cast(typing.Dict[str, typing.Any], sub_dict.get(key))
                     if sub_dict is None:
                         break
                 else:
                     existing_data = copy.deepcopy(sub_dict)
 
-                parent = None
-                sub_dict = metadata
-                for key in key_path:
-                    parent = sub_dict
-                    sub_dict = typing.cast(typing.Dict[str, typing.Any], sub_dict.get(key))
-                    if sub_dict is None:
-                        break
-                else:
-                    if self.current_frames_index == 0 and parent is not None:
-                        parent[key_path[-1]] = [sub_dict]
-                    elif existing_data is not None and parent is not None:
-                        if isinstance(existing_data, list):
-                            if len(existing_data) <= self.current_frames_index:
-                                existing_data.append(copy.deepcopy(sub_dict))
-                            else:
-                                existing_data[self.current_frames_index] = copy.deepcopy(sub_dict)
-                            parent[key_path[-1]] = existing_data
+                if len(self.__stack_values) <= self.current_frames_index:
+                    self.__stack_values.append(copy.deepcopy(existing_data))
+
+                metadata.setdefault("MultiAcquire.stack", dict())[key_path[-1]] = copy.deepcopy(self.__stack_values)
 
         data_metadata = DataAndMetadata.DataMetadata(data_shape_and_dtype,
                                                      intensity_calibration,

nion/instrumentation/camera_base.py

@@ -943,6 +943,10 @@ def __init__(self, data_element: typing.Optional[ImportExportManager.DataElement
     def data_element(self) -> ImportExportManager.DataElementType:
         return self.__data_element
 
+    @property
+    def metadata(self) -> typing.MutableMapping[str, typing.Any]:
+        return typing.cast(typing.MutableMapping[str, typing.Any], self.__data_element.setdefault("metadata", dict()))
+
     @property
     def is_signal_calibrated(self) -> bool:
         return "spatial_calibrations" in self.__data_element
@@ -1417,7 +1421,7 @@ def modes(self) -> typing.Sequence[str]: raise NotImplementedError()
 
     def get_acquire_sequence_metrics(self, frame_parameters: CameraFrameParameters) -> typing.Mapping[str, typing.Any]: ...
     def make_live_data_element(self, data: _NDArray, properties: typing.Mapping[str, typing.Any], timestamp: datetime.datetime, frame_parameters: CameraFrameParameters, frame_count: int) -> ImportExportManager.DataElementType: ...
-    def update_camera_properties(self, properties: typing.MutableMapping[str, typing.Any], frame_parameters: CameraFrameParameters, signal_type: typing.Optional[str] = None) -> None: ...
+    def update_camera_properties(self, acquisition_data: AcquisitionData, frame_parameters: CameraFrameParameters, signal_type: typing.Optional[str] = None) -> None: ...
     def get_camera_calibrations(self, camera_frame_parameters: CameraFrameParameters) -> typing.Tuple[Calibration.Calibration, ...]: ...
     def get_camera_intensity_calibration(self, camera_frame_parameters: CameraFrameParameters) -> Calibration.Calibration: ...
     def shift_click(self, mouse_position: Geometry.FloatPoint, camera_shape: DataAndMetadata.Shape2dType, logger: logging.Logger) -> None: ...
@@ -1821,8 +1825,7 @@ def __update_data_element_for_sequence(self, data_element: ImportExportManager.D
             if "spatial_calibrations" in data_element:
                 data_element["spatial_calibrations"] = [dict(), ] + data_element["spatial_calibrations"]
         self.__update_intensity_calibration(data_element, instrument_controller, self.__camera)
-        STEMController.update_instrument_properties(data_element.setdefault("metadata", dict()).setdefault("instrument", dict()), instrument_controller, self.__camera)
-        update_camera_properties(data_element.setdefault("metadata", dict()).setdefault("hardware_source", dict()), frame_parameters, self.hardware_source_id, self.display_name, data_element.get("signal_type", self.__signal_type))
+        self.update_camera_properties(AcquisitionData(data_element), frame_parameters, data_element.get("signal_type", self.__signal_type))
 
     def make_live_data_element(self, data: _NDArray, properties: typing.Mapping[str, typing.Any], timestamp: datetime.datetime, frame_parameters: CameraFrameParameters, frame_count: int) -> ImportExportManager.DataElementType:
         data_element: ImportExportManager.DataElementType = dict()
@@ -1846,9 +1849,10 @@ def make_live_data_element(self, data: _NDArray, properties: typing.Mapping[str,
         data_element["metadata"]["hardware_source"]["integration_count"] = frame_count
         return data_element
 
-    def update_camera_properties(self, properties: typing.MutableMapping[str, typing.Any], frame_parameters: CameraFrameParameters, signal_type: typing.Optional[str] = None) -> None:
-        STEMController.update_instrument_properties(properties, self.__get_instrument_controller(), self.__camera)
-        update_camera_properties(properties, frame_parameters, self.hardware_source_id, self.display_name, signal_type or self.__signal_type)
+    def update_camera_properties(self, acquisition_data: AcquisitionData, frame_parameters: CameraFrameParameters, signal_type: typing.Optional[str] = None) -> None:
+        metadata = acquisition_data.metadata
+        STEMController.update_instrument_properties(metadata.setdefault("instrument", dict()), self.__get_instrument_controller(), self.__camera)
+        update_camera_properties(metadata.setdefault("hardware_source", dict()), frame_parameters, self.hardware_source_id, self.display_name, signal_type or self.__signal_type)
 
     def get_camera_calibrations(self, camera_frame_parameters: CameraFrameParameters) -> typing.Tuple[Calibration.Calibration, ...]:
         processing = camera_frame_parameters.processing
@@ -2322,16 +2326,9 @@ def acquire_sequence(self, n: int) -> typing.Sequence[ImportExportManager.DataEl
         return [data_element]
 
     def __update_data_element_for_sequence(self, data_element: ImportExportManager.DataElementType, frame_parameters: CameraFrameParameters) -> None:
-        acquisition_data = AcquisitionData(data_element)
         data_element["version"] = 1
         data_element["state"] = "complete"
-        instrument_controller = self.__get_instrument_controller()
-        camera_calibrations = self.get_camera_calibrations(frame_parameters)
-        acquisition_data.apply_signal_calibrations([Calibration.Calibration()] + list(camera_calibrations))
-        acquisition_data.apply_intensity_calibration(self.get_camera_intensity_calibration(frame_parameters))
-        acquisition_data.counts_per_electron = self.get_counts_per_electron()
-        STEMController.update_instrument_properties(data_element.setdefault("metadata", dict()).setdefault("instrument", dict()), instrument_controller, self.__camera)
-        update_camera_properties(data_element.setdefault("metadata", dict()).setdefault("hardware_source", dict()), frame_parameters, self.hardware_source_id, self.display_name, data_element.get("signal_type", self.__signal_type))
+        self.update_camera_properties(AcquisitionData(data_element), frame_parameters, data_element.get("signal_type", self.__signal_type))
 
     def make_live_data_element(self, data: _NDArray, properties: typing.Mapping[str, typing.Any], timestamp: datetime.datetime, frame_parameters: CameraFrameParameters, frame_count: int) -> ImportExportManager.DataElementType:
         acquisition_data = AcquisitionData()
@@ -2363,9 +2360,14 @@ def make_live_data_element(self, data: _NDArray, properties: typing.Mapping[str,
         data_element["metadata"]["hardware_source"]["integration_count"] = frame_count
         return data_element
 
-    def update_camera_properties(self, properties: typing.MutableMapping[str, typing.Any], frame_parameters: CameraFrameParameters, signal_type: typing.Optional[str] = None) -> None:
-        STEMController.update_instrument_properties(properties, self.__get_instrument_controller(), self.__camera)
-        update_camera_properties(properties, frame_parameters, self.hardware_source_id, self.display_name, signal_type or self.__signal_type)
+    def update_camera_properties(self, acquisition_data: AcquisitionData, frame_parameters: CameraFrameParameters, signal_type: typing.Optional[str] = None) -> None:
+        metadata = acquisition_data.metadata
+        camera_calibrations = self.get_camera_calibrations(frame_parameters)
+        acquisition_data.apply_signal_calibrations([Calibration.Calibration()] + list(camera_calibrations))
+        acquisition_data.apply_intensity_calibration(self.get_camera_intensity_calibration(frame_parameters))
+        acquisition_data.counts_per_electron = self.get_counts_per_electron()
+        STEMController.update_instrument_properties(metadata.setdefault("instrument", dict()), self.__get_instrument_controller(), self.__camera)
+        update_camera_properties(metadata.setdefault("hardware_source", dict()), frame_parameters, self.hardware_source_id, self.display_name, signal_type or self.__signal_type)
 
     def get_camera_calibrations(self, camera_frame_parameters: CameraFrameParameters) -> typing.Tuple[Calibration.Calibration, ...]:
         calibrator = self.__get_camera_calibrator()
@@ -2699,9 +2701,10 @@ def get_next_data(self) -> typing.Optional[CameraDeviceStreamPartialData]:
         if valid_count > 0:
             uncropped_xdata = self.__partial_data_info.xdata  # this returns the entire result data array
             is_complete = self.__partial_data_info.is_complete
-            camera_metadata: typing.Dict[str, typing.Any] = dict()
-            self.__camera_hardware_source.update_camera_properties(camera_metadata, self.__camera_frame_parameters)
-            metadata = dict(uncropped_xdata.metadata)
+            acquisition_data = AcquisitionData()
+            acquisition_data.metadata.update(uncropped_xdata.metadata)
+            self.__camera_hardware_source.update_camera_properties(acquisition_data, self.__camera_frame_parameters)
+            metadata = acquisition_data.metadata
             # this is a hack to prevent potentially misleading metadata
             # from getting saved into the synchronized data. while it is acceptable to
             # assume that the hardware_source properties will get copied to the final
@@ -2711,7 +2714,6 @@ def get_next_data(self) -> typing.Optional[CameraDeviceStreamPartialData]:
             metadata.setdefault("hardware_source", dict()).pop("frame_number", None)
             metadata.setdefault("hardware_source", dict()).pop("integration_count", None)
             metadata.setdefault("hardware_source", dict()).pop("valid_rows", None)
-            metadata.setdefault("hardware_source", dict()).update(camera_metadata)
             metadata.update(copy.deepcopy(self.__additional_metadata))
 
             # TODO: this should be tracked elsewhere than here.
@@ -2793,9 +2795,10 @@ def get_next_data(self) -> typing.Optional[CameraDeviceStreamPartialData]:
         if valid_count > 0:
             uncropped_xdata = self.__partial_data_info.xdata  # this returns the entire result data array
             is_complete = self.__partial_data_info.is_complete
-            camera_metadata: typing.Dict[str, typing.Any] = dict()
-            self.__camera_hardware_source.update_camera_properties(camera_metadata, self.__camera_frame_parameters)
-            metadata = dict(uncropped_xdata.metadata)
+            acquisition_data = AcquisitionData()
+            acquisition_data.metadata.update(uncropped_xdata.metadata)
+            self.__camera_hardware_source.update_camera_properties(acquisition_data, self.__camera_frame_parameters)
+            metadata = acquisition_data.metadata
             # this is a hack to prevent some of the potentially misleading metadata
             # from getting saved into the synchronized data. while it is acceptable to
             # assume that the hardware_source properties will get copied to the final
@@ -2805,7 +2808,6 @@ def get_next_data(self) -> typing.Optional[CameraDeviceStreamPartialData]:
             metadata.setdefault("hardware_source", dict()).pop("frame_number", None)
             metadata.setdefault("hardware_source", dict()).pop("integration_count", None)
             metadata.setdefault("hardware_source", dict()).pop("valid_rows", None)
-            metadata.setdefault("hardware_source", dict()).update(camera_metadata)
             metadata.update(copy.deepcopy(self.__additional_metadata))
             # note: collection calibrations will be added in the collections stream
             data_calibrations = self.__camera_hardware_source.get_camera_calibrations(self.__camera_frame_parameters)

nion/instrumentation/scan_base.py

@@ -1364,8 +1364,8 @@ def grab_synchronized_get_info(self, *, scan_frame_parameters: ScanFrameParamete
         data_calibrations = camera.get_camera_calibrations(camera_frame_parameters)
         data_intensity_calibration = camera.get_camera_intensity_calibration(camera_frame_parameters)
 
-        camera_metadata: typing.Dict[str, typing.Any] = dict()
-        camera.update_camera_properties(camera_metadata, camera_frame_parameters)
+        acquisition_data = camera_base.AcquisitionData()
+        camera.update_camera_properties(acquisition_data, camera_frame_parameters)
 
         scan_metadata: typing.Dict[str, typing.Any] = dict()
         update_scan_metadata(scan_metadata, self.hardware_source_id, self.display_name, copy.copy(scan_frame_parameters), None, dict())
@@ -1375,7 +1375,7 @@ def grab_synchronized_get_info(self, *, scan_frame_parameters: ScanFrameParamete
 
         return GrabSynchronizedInfo(scan_size, fractional_area, is_subscan, camera_readout_size,
                                     camera_readout_size_squeezed, scan_calibrations, data_calibrations,
-                                    data_intensity_calibration, instrument_metadata, camera_metadata,
+                                    data_intensity_calibration, instrument_metadata, acquisition_data.metadata.get("hardware_source", dict()),
                                     scan_metadata, axes_descriptor)
 
     def grab_synchronized(self, *,