From 01710ca440b3d0cb1140d76c34c8fe433c7c33a1 Mon Sep 17 00:00:00 2001 From: "Vorobiov Mihail (XC-AS/EDS3-RO)" Date: Tue, 30 Jan 2024 08:44:29 +0200 Subject: [PATCH 1/4] read nested CA structures --- src/asammdf/blocks/mdf_v4.py | 159 +++++++++++++++++++++++++------- src/asammdf/blocks/v4_blocks.py | 25 +++++ 2 files changed, 149 insertions(+), 35 deletions(-) diff --git a/src/asammdf/blocks/mdf_v4.py b/src/asammdf/blocks/mdf_v4.py index 0694cf04f..87bbf9ef6 100644 --- a/src/asammdf/blocks/mdf_v4.py +++ b/src/asammdf/blocks/mdf_v4.py @@ -992,12 +992,14 @@ def _read_channels( f"Channel component address {component_addr:X} is outside the file size {self.file_limit}" ) break + + index = ch_cntr - 1 + dependencies.append(None) + # check if it is a CABLOCK or CNBLOCK stream.seek(component_addr) blk_id = stream.read(4) if blk_id == b"##CN": - index = ch_cntr - 1 - dependencies.append(None) ( ch_cntr, ret_composition, @@ -1018,10 +1020,16 @@ def _read_channels( else: # only channel arrays with storage=CN_TEMPLATE are # supported so far + channel.dtype_fmt = dtype( + get_fmt_v4( + channel.data_type, + channel.bit_offset + channel.bit_count, + channel.channel_type, + ) + ) + first_dep = ca_block = ChannelArrayBlock(address=component_addr, stream=stream, mapped=mapped) - if ca_block.storage != v4c.CA_STORAGE_TYPE_CN_TEMPLATE: - logger.warning("Only CN template arrays are supported") - ca_list = [ca_block] + ca_list = [first_dep] while ca_block.composition_addr: stream.seek(ca_block.composition_addr) @@ -1033,19 +1041,107 @@ def _read_channels( mapped=mapped, ) ca_list.append(ca_block) - else: - logger.warning("skipping CN block; CN block within CA block" " is not implemented yet") - break - dependencies.append(ca_list) + elif channel.data_type == v4c.DATA_TYPE_BYTEARRAY: + # read CA-CN nested structure + dependencies[index] = ca_list[:] + ( + ch_cntr, + ret_composition, + ret_composition_dtype, + ) = self._read_channels( + ca_block.composition_addr, + grp, + stream, + dg_cntr, + ch_cntr, + True, + mapped=mapped, + ) - channel.dtype_fmt = dtype( - get_fmt_v4( - channel.data_type, - channel.bit_offset + channel.bit_count, - channel.channel_type, - ) - ) + if ret_composition: + dependencies[index].extend(ret_composition) + + byte_offset_factors = [] + bit_pos_inval_factors = [] + dimensions = [] + total_elem = 1 + + for ca_blck in ca_list: + # only consider CN templates + if ca_blck.ca_type != v4c.CA_STORAGE_TYPE_CN_TEMPLATE: + logger.warning("Only CN template arrays are supported") + continue + + # 1D array with dimensions + for i in range(ca_blck.dims): + dim_size = ca_blck[f"dim_size_{i}"] + dimensions.append(dim_size) + total_elem *= dim_size + + # 1D arrays for byte offset and invalidation bit pos calculations + byte_offset_factors.extend(ca_blck.get_byte_offset_factors()) + bit_pos_inval_factors.extend(ca_blck.get_bit_pos_inval_factors()) + + multipliers = [1] * len(dimensions) + for i in range(len(dimensions) - 2, -1, -1): + multipliers[i] = multipliers[i + 1] * dimensions[i + 1] + + def _get_nd_coords(index, factors: list[int]) -> list[int]: + """Convert 1D index to CA's nD coordinates""" + coords = [0] * len(factors) + for i, factor in enumerate(factors): + coords[i] = index // factor + index %= factor + return coords + + channels_to_copy = channels[index:] + for elem_id in range(total_elem): + for cn_id, cn_block in enumerate(channels_to_copy): + nd_coords = _get_nd_coords(elem_id, multipliers) + + # copy composition block + new_block = deepcopy(cn_block) + + # update byte offset & position of invalidation bit + byte_offset = bit_offset = 0 + for coord, byte_factor, bit_factor in zip( + nd_coords, byte_offset_factors, bit_pos_inval_factors + ): + byte_offset += coord * byte_factor + bit_offset += coord * bit_factor + new_block.byte_offset += byte_offset + new_block.pos_invalidation_bit += bit_offset + + # update channel name + new_block.name += "[" + "][".join(str(coord) for coord in nd_coords) + "]" + + # append to channel list + channels.append(new_block) + ch_cntr += 1 + + # update channel dependencies + if dependencies[index + cn_id] is not None: + deps = [] + for dep in dependencies[index + cn_id]: + if not isinstance(dep, ChannelArrayBlock): + dep_entry = (dep[0], dep[1] + len(channels_to_copy) * (elem_id + 1)) + deps.append(dep_entry) + dependencies.append(deps) + else: + dependencies.append(None) + + # update channels db + entry = (dg_cntr, ch_cntr) + self.channels_db.add(new_block.name, entry) + + break + + else: + logger.warning( + "skipping CN block; Nested CA structure should be contained within BYTEARRAY data type" + ) + break else: dependencies.append(None) @@ -4744,15 +4840,7 @@ def _append_structure_composition( invalidation_bytes_nr: int, inval_bits: list[NDArray[Any]], inval_cntr: int, - ) -> tuple[ - int, - int, - int, - tuple[int, int], - list[NDArray[Any]], - list[tuple[str, dtype[Any], tuple[int, ...]]], - int, - ]: + ) -> tuple[int, int, int, tuple[int, int], list[NDArray[Any]], list[tuple[str, dtype[Any], tuple[int, ...]]], int,]: si_map = self._si_map fields = [] @@ -5171,14 +5259,7 @@ def _append_structure_composition_column_oriented( dg_cntr: int, ch_cntr: int, defined_texts: dict[str, int], - ) -> tuple[ - int, - int, - int, - tuple[int, int], - list[NDArray[Any]], - list[tuple[str, dtype[Any], tuple[int, ...]]], - ]: + ) -> tuple[int, int, int, tuple[int, int], list[NDArray[Any]], list[tuple[str, dtype[Any], tuple[int, ...]]],]: si_map = self._si_map fields = [] @@ -6322,7 +6403,8 @@ def get( record_offset: int = ..., record_count: int | None = ..., skip_channel_validation: bool = ..., - ) -> Signal: ... + ) -> Signal: + ... @overload def get( @@ -6338,7 +6420,8 @@ def get( record_offset: int = ..., record_count: int | None = ..., skip_channel_validation: bool = ..., - ) -> tuple[NDArray[Any], NDArray[Any]]: ... + ) -> tuple[NDArray[Any], NDArray[Any]]: + ... def get( self, @@ -6941,6 +7024,9 @@ def _get_array( cycles_nr = len(vals) for ca_block in dependency_list[:1]: + if not isinstance(ca_block, ChannelArrayBlock): + break + dims_nr = ca_block.dims if ca_block.ca_type == v4c.CA_TYPE_SCALE_AXIS: @@ -7031,6 +7117,9 @@ def _get_array( types.append(dtype_pair) for ca_block in dependency_list[1:]: + if not isinstance(ca_block, ChannelArrayBlock): + break + dims_nr = ca_block.dims if ca_block.flags & v4c.FLAG_CA_FIXED_AXIS: diff --git a/src/asammdf/blocks/v4_blocks.py b/src/asammdf/blocks/v4_blocks.py index ecbb6abb5..b5d862570 100644 --- a/src/asammdf/blocks/v4_blocks.py +++ b/src/asammdf/blocks/v4_blocks.py @@ -1768,6 +1768,31 @@ def __bytes__(self) -> bytes: result = pack(fmt, *[getattr(self, key) for key in keys]) return result + def get_byte_offset_factors(self) -> list[int]: + """Returns list of factors f(d), used to calculate byte offset""" + return self._factors(self.byte_offset_base) + + def get_bit_pos_inval_factors(self) -> list[int]: + """Returns list of factors f(d), used to calculate invalidation bit position""" + return self._factors(self.invalidation_bit_base) + + def _factors(self, base: int) -> list[int]: + factor = base + factors = [factor] + # column oriented layout + if self.flags & v4c.FLAG_CA_INVERSE_LAYOUT: + for i in range(1, self.dims): + factor *= self[f"dim_size_{i - 1}"] + factors.append(factor) + + # row oriented layout + else: + for i in range(self.dims - 2, -1, -1): + factor *= self[f"dim_size_{i + 1}"] + factors.insert(0, factor) + + return factors + class ChannelGroup: """*ChannelGroup* has the following attributes, that are also available as From d6e00121b14d6737a087fd616d7c558df875bf5c Mon Sep 17 00:00:00 2001 From: "Vorobiov Mihail (XC-AS/EDS3-RO)" Date: Thu, 22 Feb 2024 10:21:29 +0200 Subject: [PATCH 2/4] better names for nested channels (similar to how vector does it) --- src/asammdf/blocks/mdf_v4.py | 41 ++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/asammdf/blocks/mdf_v4.py b/src/asammdf/blocks/mdf_v4.py index 87bbf9ef6..b5308e611 100644 --- a/src/asammdf/blocks/mdf_v4.py +++ b/src/asammdf/blocks/mdf_v4.py @@ -17,6 +17,7 @@ import mmap import os from pathlib import Path +import re import shutil import sys from tempfile import gettempdir, NamedTemporaryFile @@ -1095,13 +1096,25 @@ def _get_nd_coords(index, factors: list[int]) -> list[int]: index %= factor return coords - channels_to_copy = channels[index:] - for elem_id in range(total_elem): - for cn_id, cn_block in enumerate(channels_to_copy): + def _get_name_with_indices(ch_name: str, ch_parent_name: str, indices: list[int]) -> str: + coords = "[" + "][".join(str(coord) for coord in indices) + "]" + m = re.match(ch_parent_name, ch_name) + n = re.search(r"\[\d+\]", ch_name) + if m: + name = ch_name[: m.end()] + coords + ch_name[m.end() :] + elif n: + name = ch_name[: n.start()] + coords + ch_name[n.start() :] + else: + name = ch_name + coords + return name + + ch_len = len(channels) + for elem_id in range(1, total_elem): + for cn_id in range(index, ch_len): nd_coords = _get_nd_coords(elem_id, multipliers) # copy composition block - new_block = deepcopy(cn_block) + new_block = deepcopy(channels[cn_id]) # update byte offset & position of invalidation bit byte_offset = bit_offset = 0 @@ -1114,18 +1127,17 @@ def _get_nd_coords(index, factors: list[int]) -> list[int]: new_block.pos_invalidation_bit += bit_offset # update channel name - new_block.name += "[" + "][".join(str(coord) for coord in nd_coords) + "]" + new_block.name = _get_name_with_indices(new_block.name, channel.name, nd_coords) # append to channel list channels.append(new_block) - ch_cntr += 1 # update channel dependencies - if dependencies[index + cn_id] is not None: + if dependencies[cn_id] is not None: deps = [] - for dep in dependencies[index + cn_id]: + for dep in dependencies[cn_id]: if not isinstance(dep, ChannelArrayBlock): - dep_entry = (dep[0], dep[1] + len(channels_to_copy) * (elem_id + 1)) + dep_entry = (dep[0], dep[1] + (ch_len - index) * elem_id) deps.append(dep_entry) dependencies.append(deps) else: @@ -1134,6 +1146,17 @@ def _get_nd_coords(index, factors: list[int]) -> list[int]: # update channels db entry = (dg_cntr, ch_cntr) self.channels_db.add(new_block.name, entry) + ch_cntr += 1 + + # modify channels' names found recursively in-place + orig_name = channel.name + for cn_id in range(index, ch_len): + nd_coords = _get_nd_coords(0, multipliers) + name = _get_name_with_indices(channels[cn_id].name, orig_name, nd_coords) + entry = self.channels_db.pop(channels[cn_id].name) + channels[cn_id].name = name + # original channel entry will only contain single source tuple + self.channels_db.add(name, entry[0]) break From 114a683d3ccafa8d384c22db4390a45f1a143bd3 Mon Sep 17 00:00:00 2001 From: "Vorobiov Mihail (XC-AS/EDS3-RO)" Date: Tue, 27 Feb 2024 10:36:13 +0200 Subject: [PATCH 3/4] fix failing tests --- src/asammdf/blocks/mdf_v4.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/asammdf/blocks/mdf_v4.py b/src/asammdf/blocks/mdf_v4.py index b5308e611..8f004a3eb 100644 --- a/src/asammdf/blocks/mdf_v4.py +++ b/src/asammdf/blocks/mdf_v4.py @@ -1030,7 +1030,7 @@ def _read_channels( ) first_dep = ca_block = ChannelArrayBlock(address=component_addr, stream=stream, mapped=mapped) - ca_list = [first_dep] + dependencies[index] = [first_dep] while ca_block.composition_addr: stream.seek(ca_block.composition_addr) @@ -1041,11 +1041,10 @@ def _read_channels( stream=stream, mapped=mapped, ) - ca_list.append(ca_block) + dependencies[index].append(ca_block) elif channel.data_type == v4c.DATA_TYPE_BYTEARRAY: # read CA-CN nested structure - dependencies[index] = ca_list[:] ( ch_cntr, ret_composition, @@ -1060,6 +1059,7 @@ def _read_channels( mapped=mapped, ) + ca_cnt = len(dependencies[index]) if ret_composition: dependencies[index].extend(ret_composition) @@ -1068,7 +1068,7 @@ def _read_channels( dimensions = [] total_elem = 1 - for ca_blck in ca_list: + for ca_blck in dependencies[index][:ca_cnt]: # only consider CN templates if ca_blck.ca_type != v4c.CA_STORAGE_TYPE_CN_TEMPLATE: logger.warning("Only CN template arrays are supported") From 74f8a9a3b632aa1052c393e77993dd3bec4258b0 Mon Sep 17 00:00:00 2001 From: "Vorobiov Mihail (XC-AS/EDS3-RO)" Date: Thu, 29 Feb 2024 08:19:01 +0200 Subject: [PATCH 4/4] black --- src/asammdf/blocks/mdf_v4.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/asammdf/blocks/mdf_v4.py b/src/asammdf/blocks/mdf_v4.py index 8f004a3eb..01584e2dc 100644 --- a/src/asammdf/blocks/mdf_v4.py +++ b/src/asammdf/blocks/mdf_v4.py @@ -4863,7 +4863,15 @@ def _append_structure_composition( invalidation_bytes_nr: int, inval_bits: list[NDArray[Any]], inval_cntr: int, - ) -> tuple[int, int, int, tuple[int, int], list[NDArray[Any]], list[tuple[str, dtype[Any], tuple[int, ...]]], int,]: + ) -> tuple[ + int, + int, + int, + tuple[int, int], + list[NDArray[Any]], + list[tuple[str, dtype[Any], tuple[int, ...]]], + int, + ]: si_map = self._si_map fields = [] @@ -5282,7 +5290,14 @@ def _append_structure_composition_column_oriented( dg_cntr: int, ch_cntr: int, defined_texts: dict[str, int], - ) -> tuple[int, int, int, tuple[int, int], list[NDArray[Any]], list[tuple[str, dtype[Any], tuple[int, ...]]],]: + ) -> tuple[ + int, + int, + int, + tuple[int, int], + list[NDArray[Any]], + list[tuple[str, dtype[Any], tuple[int, ...]]], + ]: si_map = self._si_map fields = [] @@ -6426,8 +6441,7 @@ def get( record_offset: int = ..., record_count: int | None = ..., skip_channel_validation: bool = ..., - ) -> Signal: - ... + ) -> Signal: ... @overload def get( @@ -6443,8 +6457,7 @@ def get( record_offset: int = ..., record_count: int | None = ..., skip_channel_validation: bool = ..., - ) -> tuple[NDArray[Any], NDArray[Any]]: - ... + ) -> tuple[NDArray[Any], NDArray[Any]]: ... def get( self,