Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replaced internal encoders with message encoder calling public API #767

Merged
merged 27 commits into from
Jul 24, 2024
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
e4e8250
Serialized all scalar data types for message_serializer with all tests
LazeringDeath Jun 13, 2024
54bf7df
Added array data types to message_serializer
LazeringDeath Jun 14, 2024
e55d6d0
Added enums and sub functions to message_serializer
LazeringDeath Jun 18, 2024
3131ec3
Added messages and refactored message_serializer
LazeringDeath Jun 20, 2024
3af83c0
Merge remote-tracking branch 'origin/main' into users/tynguyen/encode…
LazeringDeath Jun 20, 2024
7770383
Modified message_serializer to pass all the tests
LazeringDeath Jun 21, 2024
ea6934d
Switched current serializer with message_serializer and removed sub f…
LazeringDeath Jun 24, 2024
dd8e33a
Replaced test_message_serialzier with test_serializer
LazeringDeath Jun 24, 2024
a936584
Fixed 'Mypy statis analysis' in message_serializer
LazeringDeath Jun 24, 2024
e24da77
Changed file names corresponding to it's functionality
LazeringDeath Jun 24, 2024
371e90d
Fixed naming issue
LazeringDeath Jun 24, 2024
96f07ce
Changed 'test_serializer' to 'test_decoder'
LazeringDeath Jun 24, 2024
ce4e3e1
Implemented encoder to reuse message types, renamed and reorder files
LazeringDeath Jul 1, 2024
7ebd83f
Fixed docstrings and reordered encoder.
LazeringDeath Jul 2, 2024
48d8803
[DRAFT] Message decoder (#780)
LazeringDeath Jul 10, 2024
239eb36
Creates 2 messages per service, renamed message/fields, and reordered…
LazeringDeath Jul 15, 2024
e8999d9
Merge remote-tracking branch 'origin' into users/tynguyen/encoder-mes…
LazeringDeath Jul 15, 2024
55726f6
Deleted serialization_strategy with default_value
LazeringDeath Jul 15, 2024
7e910f5
Deleted test_serializer
LazeringDeath Jul 15, 2024
76d4cd8
Deleted _message.py
LazeringDeath Jul 15, 2024
e68913d
Fixed type errors in test encoder/decoder and docstring
LazeringDeath Jul 15, 2024
661482d
Renamed and cleaned helper functions, added initalize() in metadata.
LazeringDeath Jul 18, 2024
019cef9
Fixed sytleguide and type assignment.
LazeringDeath Jul 18, 2024
f351927
Pass enum type in initialize() and moved it in ParameterMetadata
LazeringDeath Jul 23, 2024
ff0850c
Changed EnumType to Enum
LazeringDeath Jul 23, 2024
ce5425d
Changed isinstance() to type() in _create_enum_type_class
LazeringDeath Jul 23, 2024
e76e00a
Add correct type hint to enum_type passing in initialize() and added …
LazeringDeath Jul 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
import grpc
from google.protobuf import any_pb2

from ni_measurement_plugin_sdk_service._internal.parameter import serializer
from ni_measurement_plugin_sdk_service._internal.parameter.metadata import ParameterMetadata
from ni_measurement_plugin_sdk_service._internal.parameter import decoder, encoder
from ni_measurement_plugin_sdk_service._internal.parameter.metadata import (
ParameterMetadata,
)
from ni_measurement_plugin_sdk_service._internal.stubs.ni.measurementlink.measurement.v1 import (
measurement_service_pb2 as v1_measurement_service_pb2,
measurement_service_pb2_grpc as v1_measurement_service_pb2_grpc,
Expand All @@ -23,7 +25,10 @@
measurement_service_pb2 as v2_measurement_service_pb2,
measurement_service_pb2_grpc as v2_measurement_service_pb2_grpc,
)
from ni_measurement_plugin_sdk_service.measurement.info import MeasurementInfo
from ni_measurement_plugin_sdk_service.measurement.info import (
MeasurementInfo,
ServiceInfo,
)
from ni_measurement_plugin_sdk_service.session_management import PinMapContext


Expand Down Expand Up @@ -131,9 +136,13 @@ def _get_mapping_by_parameter_name(
return mapping_by_variable_name


def _serialize_outputs(output_metadata: Dict[int, ParameterMetadata], outputs: Any) -> any_pb2.Any:
def _serialize_outputs(
output_metadata: Dict[int, ParameterMetadata], outputs: Any, service_name: str
) -> any_pb2.Any:
if isinstance(outputs, collections.abc.Sequence):
return any_pb2.Any(value=serializer.serialize_parameters(output_metadata, outputs))
return any_pb2.Any(
value=encoder.serialize_parameters(output_metadata, outputs, service_name)
)
elif outputs is None:
raise ValueError(f"Measurement function returned None")
else:
Expand Down Expand Up @@ -161,6 +170,7 @@ def __init__(
output_parameter_list: List[ParameterMetadata],
measure_function: Callable,
owner: object,
service_info: ServiceInfo,
) -> None:
"""Initialize the measurement v1 servicer."""
super().__init__()
Expand All @@ -169,6 +179,7 @@ def __init__(
self._measurement_info = measurement_info
self._measure_function = measure_function
self._owner = weakref.ref(owner) if owner is not None else None # avoid reference cycle
self._service_info = service_info

def GetMetadata( # noqa: N802 - function name should be lowercase
self, request: v1_measurement_service_pb2.GetMetadataRequest, context: grpc.ServicerContext
Expand All @@ -193,8 +204,8 @@ def GetMetadata( # noqa: N802 - function name should be lowercase
)
measurement_signature.configuration_parameters.append(configuration_parameter)

measurement_signature.configuration_defaults.value = serializer.serialize_default_values(
self._configuration_metadata
measurement_signature.configuration_defaults.value = encoder.serialize_default_values(
self._configuration_metadata, self._get_service_name() + ".Inputs"
)

for field_number, output_metadata in self._output_metadata.items():
Expand Down Expand Up @@ -224,8 +235,10 @@ def Measure( # noqa: N802 - function name should be lowercase
self, request: v1_measurement_service_pb2.MeasureRequest, context: grpc.ServicerContext
) -> v1_measurement_service_pb2.MeasureResponse:
"""RPC API that executes the registered measurement method."""
mapping_by_id = serializer.deserialize_parameters(
self._configuration_metadata, request.configuration_parameters.value
mapping_by_id = decoder.deserialize_parameters(
self._configuration_metadata,
request.configuration_parameters.value,
self._get_service_name() + ".Inputs",
)
mapping_by_variable_name = _get_mapping_by_parameter_name(
mapping_by_id, self._measure_function
Expand Down Expand Up @@ -254,9 +267,15 @@ def Measure( # noqa: N802 - function name should be lowercase

def _serialize_response(self, outputs: Any) -> v1_measurement_service_pb2.MeasureResponse:
return v1_measurement_service_pb2.MeasureResponse(
outputs=_serialize_outputs(self._output_metadata, outputs)
outputs=_serialize_outputs(
self._output_metadata, outputs, self._get_service_name() + ".Outputs"
)
)

def _get_service_name(self) -> str:
service_name = "".join(char for char in self._service_info.service_class if char.isalpha())
return service_name


class MeasurementServiceServicerV2(v2_measurement_service_pb2_grpc.MeasurementServiceServicer):
"""Measurement v2 servicer."""
Expand All @@ -268,6 +287,7 @@ def __init__(
output_parameter_list: List[ParameterMetadata],
measure_function: Callable,
owner: object,
service_info: ServiceInfo,
) -> None:
"""Initialize the measurement v2 servicer."""
super().__init__()
Expand All @@ -276,6 +296,7 @@ def __init__(
self._measurement_info = measurement_info
self._measure_function = measure_function
self._owner = weakref.ref(owner) if owner is not None else None # avoid reference cycle
self._service_info = service_info

def GetMetadata( # noqa: N802 - function name should be lowercase
self, request: v2_measurement_service_pb2.GetMetadataRequest, context: grpc.ServicerContext
Expand All @@ -301,8 +322,8 @@ def GetMetadata( # noqa: N802 - function name should be lowercase
)
measurement_signature.configuration_parameters.append(configuration_parameter)

measurement_signature.configuration_defaults.value = serializer.serialize_default_values(
self._configuration_metadata
measurement_signature.configuration_defaults.value = encoder.serialize_default_values(
self._configuration_metadata, self._get_service_name() + ".Inputs"
)

for field_number, output_metadata in self._output_metadata.items():
Expand Down Expand Up @@ -334,8 +355,10 @@ def Measure( # noqa: N802 - function name should be lowercase
self, request: v2_measurement_service_pb2.MeasureRequest, context: grpc.ServicerContext
) -> Generator[v2_measurement_service_pb2.MeasureResponse, None, None]:
"""RPC API that executes the registered measurement method."""
mapping_by_id = serializer.deserialize_parameters(
self._configuration_metadata, request.configuration_parameters.value
mapping_by_id = decoder.deserialize_parameters(
self._configuration_metadata,
request.configuration_parameters.value,
self._get_service_name() + ".Inputs",
)
mapping_by_variable_name = _get_mapping_by_parameter_name(
mapping_by_id, self._measure_function
Expand Down Expand Up @@ -363,5 +386,11 @@ def Measure( # noqa: N802 - function name should be lowercase

def _serialize_response(self, outputs: Any) -> v2_measurement_service_pb2.MeasureResponse:
return v2_measurement_service_pb2.MeasureResponse(
outputs=_serialize_outputs(self._output_metadata, outputs)
outputs=_serialize_outputs(
self._output_metadata, outputs, self._get_service_name() + ".Outputs"
)
)

def _get_service_name(self) -> str:
LazeringDeath marked this conversation as resolved.
Show resolved Hide resolved
service_name = "".join(char for char in self._service_info.service_class if char.isalpha())
return service_name
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from google.protobuf.internal import encoder, wire_format
from google.protobuf.message import Message

from ni_measurement_plugin_sdk_service._internal.parameter._serializer_types import (
from ni_measurement_plugin_sdk_service._internal.parameter._decoder_types import (
Decoder,
Key,
NewDefault,
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Parameter Serializer."""

from json import loads
from typing import Any, Dict

from google.protobuf import descriptor_pool, message_factory
from google.protobuf.descriptor_pb2 import FieldDescriptorProto

from ni_measurement_plugin_sdk_service._annotations import ENUM_VALUES_KEY
from ni_measurement_plugin_sdk_service._internal.parameter.metadata import (
ParameterMetadata,
)
from ni_measurement_plugin_sdk_service._internal.parameter.serialization_descriptors import (
_get_enum_type,
)


def deserialize_parameters(
parameter_metadata_dict: Dict[int, ParameterMetadata],
parameter_bytes: bytes,
service_name: str,
) -> Dict[int, Any]:
"""Deserialize the bytes of the parameter based on the metadata.

Args:
parameter_metadata_dict (Dict[int, ParameterMetadata]): Parameter metadata by ID.

parameter_bytes (bytes): Byte string to deserialize.

service_name (str): Unique service name.

Returns:
Dict[int, Any]: Deserialized parameters by ID
"""
pool = descriptor_pool.Default()
message_proto = pool.FindMessageTypeByName(service_name)
message_instance = message_factory.GetMessageClass(message_proto)()
parameter_values = {}

message_instance.ParseFromString(parameter_bytes)
for i in message_proto.fields_by_number.keys():
parameter_metadata = parameter_metadata_dict[i]
field_name = parameter_metadata.sanitized_display_name()
value = getattr(message_instance, field_name)

if (
parameter_metadata.type == FieldDescriptorProto.TYPE_ENUM
and _get_enum_type(parameter_metadata) is not int
):
parameter_values[i] = _deserialize_enum_parameter(parameter_metadata, value)
elif (
parameter_metadata.type == FieldDescriptorProto.TYPE_MESSAGE
and not parameter_metadata.repeated
and value.ByteSize() == 0
):
parameter_values[i] = None
else:
parameter_values[i] = value
return parameter_values


def _deserialize_enum_parameter(parameter_metadata: ParameterMetadata, field_value: Any) -> Any:
"""Convert all enums into the user defined enum type.

Args:
parameter_metadata (ParameterMetadata): Metadata of current enum value.

field_value (Any): Value of current field.

Returns:
Any: Enum type or a list of enum types.
"""
enum_dict = loads(parameter_metadata.annotations[ENUM_VALUES_KEY])
enum_type = _get_enum_type(parameter_metadata)
if parameter_metadata.repeated:
return [_get_enum_field(enum_dict, enum_type, value) for value in field_value]
else:
return _get_enum_field(enum_dict, enum_type, field_value)


def _get_enum_field(enum_dict: Dict[Any, int], enum_type: Any, field_value: int) -> Any:
"""Get enum type and value from 'field_value'.

Args:
enum_dict (Dict[Any, int]): List enum class of 'field_value'.

enum_type (Any): 'field_value' enum class name.

field_value (int): Default value of current field.

Returns:
Any: Enum type of 'field_value' from 'enum_dict' with the enum value.
"""
for name in enum_dict.keys():
enum_value = getattr(enum_type, name)
if field_value == enum_value.value:
return enum_value
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from typing import Any

from google.protobuf import type_pb2

_type_default_mapping = {
LazeringDeath marked this conversation as resolved.
Show resolved Hide resolved
type_pb2.Field.TYPE_FLOAT: float(),
type_pb2.Field.TYPE_DOUBLE: float(),
type_pb2.Field.TYPE_INT32: int(),
type_pb2.Field.TYPE_INT64: int(),
type_pb2.Field.TYPE_UINT32: int(),
type_pb2.Field.TYPE_UINT64: int(),
type_pb2.Field.TYPE_BOOL: bool(),
type_pb2.Field.TYPE_STRING: str(),
type_pb2.Field.TYPE_ENUM: int(),
}


def get_type_default(type: type_pb2.Field.Kind.ValueType, repeated: bool) -> Any:
"""Get the default value for the give type."""
if repeated:
return list()
return _type_default_mapping.get(type)
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""Parameter Serializer."""

from enum import Enum
from typing import Any, Dict, Sequence

from google.protobuf import descriptor_pool, message_factory
from google.protobuf.descriptor_pb2 import FieldDescriptorProto

from ni_measurement_plugin_sdk_service._internal.parameter.default_value import (
get_type_default,
)
from ni_measurement_plugin_sdk_service._internal.parameter.metadata import (
ParameterMetadata,
)


def serialize_parameters(
parameter_metadata_dict: Dict[int, ParameterMetadata],
parameter_values: Sequence[Any],
service_name: str,
) -> bytes:
"""Serialize the parameter values in same order based on the metadata_dict.

Args:
parameter_metadata_dict (Dict[int, ParameterMetadata]): Parameter metadata by ID.

parameter_values (Sequence[Any]): Parameter values to serialize.

service_name (str): Unique service name.

Returns:
bytes: Serialized byte string containing parameter values.
"""
pool = descriptor_pool.Default()
message_proto = pool.FindMessageTypeByName(service_name)
message_instance = message_factory.GetMessageClass(message_proto)()

for i, parameter in enumerate(parameter_values, start=1):
parameter_metadata = parameter_metadata_dict[i]
field_name = parameter_metadata.sanitized_display_name()
parameter = _get_enum_values(param=parameter)
type_default_value = get_type_default(parameter_metadata.type, parameter_metadata.repeated)

# Doesn't assign default values or None values to fields
if parameter != type_default_value and parameter is not None:
if parameter_metadata.repeated:
getattr(message_instance, field_name).extend(parameter)
elif parameter_metadata.type == FieldDescriptorProto.TYPE_MESSAGE:
getattr(message_instance, field_name).CopyFrom(parameter)
else:
setattr(message_instance, field_name, parameter)
return message_instance.SerializeToString()


def serialize_default_values(
parameter_metadata_dict: Dict[int, ParameterMetadata], service_name: str
) -> bytes:
"""Serialize the Default values in the Metadata.

Args:
parameter_metadata_dict (Dict[int, ParameterMetadata]): Configuration metadata.

service_name (str): Unique service name.

Returns:
bytes: Serialized byte string containing default values.
"""
default_value_parameter_array = [
parameter.default_value for parameter in parameter_metadata_dict.values()
]
return serialize_parameters(
parameter_metadata_dict, default_value_parameter_array, service_name
)


def _get_enum_values(param: Any) -> Any:
"""Get's value of an enum.

Args:
param (Any): A value/parameter of parameter_values.

Returns:
Any: An enum value or a list of enums or the 'param'.
"""
if param == []:
return param
if isinstance(param, list) and isinstance(param[0], Enum):
return [x.value for x in param]
elif isinstance(param, Enum):
return param.value
return param
Loading
Loading