modem_message_body_parser.proto

/* Modem message body parsers management.
 *
 * Modem message body parsers (short: parsers) are small scripts defined using the Kaitai Struct specification
 * to parse binary data.
 * Parsers are assigned to modems to parse the body of their messages dynamically.
 *
 * Parsers can be written manually as a .ksy file and uploaded through the API, or the simplified API implementation
 * can be used to create a parser.
 *
 * For more information on Kaitai Struct, see [http://kaitai.io/](http://kaitai.io/),
 * where you can find documentation, examples and a web IDE.
 */
syntax = "proto3";

package hiber.modem.message.bodyparser;

import "base.proto";
import "field.proto";
import "google/protobuf/struct.proto";
import "modem.proto";

option java_multiple_files = false;
option java_package = "global.hiber.api.grpc.modem.message.bodyparser";
option java_outer_classname = "ModemMessageBodyParserApi";
option go_package = ".;hiber";

service ModemMessageBodyParserService {
    /* List available message body parsers (including inherited from your parent organizations). */
    rpc List (ListModemMessageBodyParsersRequest) returns (ListModemMessageBodyParsersRequest.Response);

    /* Upload a .ksy file as a message body parser. */
    rpc Upload (UploadModemMessageBodyParserRequest) returns (ModemMessageBodyParser);

    /* Re-upload a .ksy file to update a message body parser, if you are the owner. */
    rpc UpdateUploaded (UpdateUploadedModemMessageBodyParserRequest) returns (ModemMessageBodyParser);

    /* Create a simple message body parser using API elements. */
    rpc CreateSimple (CreateSimpleModemMessageBodyParserRequest) returns (ModemMessageBodyParser);

    /* Update a simple message body parser created using API elements, if you are the owner. */
    rpc UpdateSimple (UpdateSimpleModemMessageBodyParserRequest) returns (ModemMessageBodyParser);

    /* Rename a message body parser, if you are the owner. */
    rpc Rename (RenameModemMessageBodyParserRequest) returns (ModemMessageBodyParser);

    /* Make a message body parser (un)available to (a selection of) child organizations. */
    rpc UpdateChildOrganizationAvailability (UpdateChildOrganizationAvailabilityRequest) returns (ModemMessageBodyParser);

    /* Make a message body parser available to a child organization. */
    rpc MakeAvailableToChildOrganization (MakeModemMessageBodyParserAvailableToChildOrganizationRequest) returns (ModemMessageBodyParser);

    /* Make a message body parser unavailable to a child organization. */
    rpc MakeUnavailableToChildOrganization (MakeModemMessageBodyParserUnavailableToChildOrganizationRequest) returns (ModemMessageBodyParser);

    /* Delete a message body parser, if you are the owner. */
    rpc Delete (DeleteModemMessageBodyParserRequest) returns (DeleteModemMessageBodyParserRequest.Response);

    /* Test a message body parser with an existing message or provided binary message body.
     * This can also be used without an existing parser by uploading a .ksy file or specifying a simple parser.
     */
    rpc Test (TestModemMessageBodyParserRequest) returns (TestModemMessageBodyParserRequest.Response);

    /* Retry parsing messages with a message body parser (for example after updating the parser). */
    rpc Retry (RetryModemMessageBodyParsing.Request) returns (RetryModemMessageBodyParsing.Response);

    /* Assign a parser to a modem. */
    rpc Assign (AssignModemMessageBodyParsers.Request) returns (AssignModemMessageBodyParsers.Response);

    /* Unassign a parser from a modem. */
    rpc Unassign (UnassignModemMessageBodyParsers.Request) returns (UnassignModemMessageBodyParsers.Response);
}

/* Modem message body parser. This defines a parsing strategy for the body of a message.
 * A parser can be defined in two ways: using a .ksy (Kaitai struct https://kaitai.io/) file or using the simple parser.
 */
message ModemMessageBodyParser {
    /* A modem message body parser with this object is available to child organizations.
     * This means the child organization can use it, but not update or delete it.
     */
    message AvailableToChildOrganizations {
        Filter.ChildOrganizations child_organizations = 1;
    }

    /* Fields in the parsed result that match common things that can be processed by the system,
     * like a location or battery percentage. */
    message MetadataFields {
        message LocationFields {
            string latitude = 1;
            string longitude = 2;
        }

        /* Specify a pair of fields in the message body that contain location data,
         * which will update the modem location and the map.
         */
        LocationFields location_fields = 2;

        /* Custom metadata fields, which will be added to the message metadata json. */
        repeated string message_metadata_fields = 1;

        /* Custom metadata fields, which will be added to the modem metadata json. */
        repeated string modem_metadata_fields = 3;

        oneof measured_at_field {
            /* Field that contains the time (epoch seconds) to use for the values extracted from the message.
             * If not set, sent time is used.
             */
            string measured_at_time_field = 4;

            /* Field that contains the time offset (second before sent time) to use for the values extracted
             * from the message. If not set, sent time is used.
             */
            string measured_at_offset_field = 5;
        }

        /* In order to use this parser on a message,
         * the metadata on the message must match the given requirement here.
         * The key of the map is the json-path to look for in the message metadata,
         * the value of the map is the json to expect at that json-path. */
        map<string, google.protobuf.Value> require_message_metadata = 6;
    }

    /* The type of post-processing to be applied to the result of this parser. */
    enum PostProcessing {
        NOTHING = 0;

        reserved 1;
    }

    /* The globally (across organizations) unique identifier for this parser. */
    string identifier = 7;

    /* The organization that created the parser. */
    string organization = 8;

    /* The name for this parser. */
    string name = 2;

    /* The content of this parsers script.
     * If simple_parser is set, this content is generated from that definition.
     * This field may be omitted by the list call to save data.
     */
    string content_ksy = 3;

    /* The simple parser this .ksy was generated from, if it was generated from a simple parser.
     * This field may be omitted on demand to save data in the list call.
     */
    SimpleModemMessageBodyParser simple_parser = 4;

    /* Fields in the parsed result that contain data.
     * Data fields are cached for efficient retrieval and allow all kinds of processing.
     */
    repeated field.Field data_fields = 11;

    /* Fields in the parsed result that contain metadata, and special things like a location. */
    MetadataFields metadata_fields = 10;

    /* If set, this parser is available to your child organizations, as a Provided parser. */
    AvailableToChildOrganizations available_to_child_organizations = 6;

    /* The list of post-processing steps applied to the result of this parser. */
    repeated PostProcessing post_processing = 13 [deprecated = true];

    /* In order to use this parser on a message,
     * the metadata on the message must match the given requirement here.
     * The key of the map is the json-path to look for in the message metadata,
     * the value of the map is the json to expect at that json-path.
     * Deprecated: use metadata_fields.require_message_metadata
     */
    map<string, google.protobuf.Value> require_message_metadata = 14 [deprecated = true];

    reserved 1, 5, 9, 12;
}

/* Simple modem message body parser. This can be used a convenient way to create a .ksy parser. */
message SimpleModemMessageBodyParser {
    message Field {
        /* Field name. This is a reduced to a value containing only lowercase letters, numbers and underscores. */
        string name = 1;

        /* A short, single line description of the field. */
        string documentation = 2;

        oneof type {
            Integer integer = 3;
            Float float = 4;
            Bytes bytes = 5;
            String string = 6;

            /* Specify an expected magic sequence (some bytes that are always the same). */
            bytes magic = 7;

            /* Specify a custom type name. */
            string custom_type = 8;
        }

        message Integer {
            /* Specifies the size of this integer in bytes. */
            int32 size_bytes = 3;

            /* Specifies whether this integer is signed or unsigned. */
            bool signed = 4;

            /* Specifies endian encoding for this field. This can be omitted if default endianness specified. */
            Endian endian = 5;
        }

        message Float {
            /* Specifies the size of this floating point number in bytes. */
            int32 size_bytes = 3;

            /* Specifies endian encoding for this field. This can be omitted if default endianness specified. */
            Endian endian = 5;
        }

        message Bytes {
            DelimitedSize size = 3;
        }

        message String {
            DelimitedSize size = 3;

            /* Specifies encoding for the string. This can be omitted if default encoding specified. */
            string encoding = 4;
        }
    }

    /* Delimited size definition, used for Bytes and Text. */
    message DelimitedSize {
        oneof size {
            /* Specifies the number of bytes in this field. */
            int32 size_bytes = 1;

            /* Specifies that this field parses all bytes until the end of the stream. */
            bool end_of_stream = 2;

            /* Specifies the number of bytes in this field using the value of an Integer field.
              * This integer field must be parsed before this field is parsed.
              */
            string field_name = 3;
        }

        /* Specify a terminator that marks the point the field terminates, optionally,
         * and we should continue to the next field. This is typically a single byte (i.e. 0x00).
         * This can be combined with a size to limit the amounts of bytes that are parsed.
         */
        bytes terminator = 4;
    }

    enum Endian {
        DEFAULT = 0;
        LITTLE_ENDIAN = 1;
        BIG_ENDIAN = 2;
    }

    /* Parser name.
     * When this structure is used as a custom type inside a parser, this name is a reduced to
     * a value containing only lowercase letters, numbers and underscores, like field names.
     */
    string name = 1;

    /* A short, single line description of the field. */
    string documentation = 2;

    /* The fields in this object. */
    repeated Field fields = 3;

    /* Sets a default endianness for this parser's fields and any fields in custom types. */
    Endian default_endian = 4;

    /* Sets a default string encoding for this parser's fields and any fields in custom types. */
    string default_text_encoding = 5;

    /* A list of custom types used for custom types fields.
     * Each is effectively a simple parser, and can have its own custom types again, which can only be used within
     * that parser structure.
     * The parser name is used for custom type fields.
     */
    repeated SimpleModemMessageBodyParser custom_types = 6;
}

/* Selection object for parsers. Used to select which parser to list, assign, etc. */
message ModemMessageBodyParserSelection {
    /* Filter parsers by their identifiers. */
    repeated string identifiers = 8;

    /* Find parsers by searching for this text in the name, content_ksy, data fields or simple parser definition. */
    optional string search = 2;

    /* Only return simple parsers, created using SimpleModemMessageBodyParser. */
    optional bool only_simple = 3;

    /* Only return parsers created from an uploaded .ksy file. */
    optional bool only_ksy_file = 4;

    /* Exclude parsers that are not owned by your organization (those that you did not create yourself). */
    optional bool only_owned_parsers = 5;

    /* Only return parsers that were created by the given parent organizations. */
    repeated string owner_organizations = 6;

    /* Only return parsers that have one of the given data fields
     * This only works when the fields are marked explicitly using the data fields option.
     */
    repeated string has_data_fields = 9;

    reserved 1, 7;
}

/* List the parser your organizations has access to. This may include inherited parsers.
 *
 * If include_content is not set, this call may omit parser specifications, like the ksy, since those are
 * typically larger multi-line strings.
 */
message ListModemMessageBodyParsersRequest {
    message Response {
        repeated ModemMessageBodyParser parsers = 1;
        Pagination.Result pagination = 2;
        ListModemMessageBodyParsersRequest request = 3;
    }

    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* Select the parsers to list. Optional, when omitted or empty everything is included. */
    optional ModemMessageBodyParserSelection selection = 2;
    optional Pagination pagination = 3;

    /* Whether to omit the content in the resulting ModemMessageBodyParsers. */
    optional bool exclude_content = 4;
}

/* Upload a new body parser from a .ksy file. */
message UploadModemMessageBodyParserRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* A descriptive name for this parser. */
    string name = 2;

    /* The ksy definition for this parser. */
    string content_ksy = 3;

    /* Fields in the parsed result that contain data.
     * This can be useful to track which fields could be plotted, etc.
     */
    repeated field.Field data_fields = 5;

    /* Fields in the parsed result that match special things that can be processed by the system, like a location. */
    optional ModemMessageBodyParser.MetadataFields metadata_fields = 10;

    repeated ModemMessageBodyParser.PostProcessing post_processing = 6;

    /* In order to use this parser on a message,
     * the metadata on the message must match the given requirement here.
     * The key of the map is the json-path to look for in the message metadata,
     * the value of the map is the json to expect at that json-path. */
    map<string, google.protobuf.Value> require_message_metadata = 7;

    reserved 4;
}

/* Upload an updated body parser from a .ksy file, replacing the previous file. */
message UpdateUploadedModemMessageBodyParserRequest {
    message MetadataFields {
        /* Update the location fields. */
        optional ModemMessageBodyParser.MetadataFields.LocationFields update_location_fields = 3;

        /* Add fields to the message metadata fields list. */
        repeated string add_message_metadata_fields = 1;

        /* Remove fields from the message metadata fields list. */
        repeated string remove_message_metadata_fields = 2;

        /* Replace the message metadata fields list. */
        repeated string replace_message_metadata_fields = 4;

        /* Add fields to the modem metadata fields list. */
        repeated string add_modem_metadata_fields = 5;

        /* Remove fields from the modem metadata fields list. */
        repeated string remove_modem_metadata_fields = 6;

        /* Replace the modem metadata fields list. */
        repeated string replace_modem_metadata_fields = 7;

        oneof measured_at_field {
            /* Update the custom field to extract to measured_at time. */
            string measured_at_time_field = 13;
            UpdateClearableString deprecated_measured_at_time_field = 8 [deprecated = true];

            /* Update the custom field to extract to measured_at offset from the sent_at time in seconds. */
            string measured_at_offset_field = 14;
            UpdateClearableString deprecated_measured_at_offset_field = 9 [deprecated = true];
        }

        /* In order to use this parser on a message,
         * the metadata on the message must match the given requirement here.
         * The key of the map is the json-path to look for in the message metadata,
         * the value of the map is the json to expect at that json-path. */
        map<string, google.protobuf.Value> add_require_message_metadata = 10;
        /* Remove a requirement for the metadata. Remove by listing the json-path here. */
        repeated string remove_require_message_metadata = 11;
        /* Replaces the entire configuration for required message metadata. */
        map<string, google.protobuf.Value> replace_require_message_metadata = 12;

    }

    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* The identifier of the parser that should be updated. */
    string identifier = 4;

    /* If set, changes the name of the parser. */
    optional string name = 13;

    /* The new ksy definition for this parser. */
    optional string content_ksy = 3;

    /* Add fields to the data fields list. */
    repeated field.Field add_data_fields = 7;

    /* Remove fields from the data fields list. */
    repeated string remove_data_fields = 6;

    /* Fields in the parsed result that match special things that can be processed by the system, like a location. */
    optional MetadataFields metadata_fields = 10;

    /* Add a post-processing step to the result of this parser. */
    repeated ModemMessageBodyParser.PostProcessing add_post_processing = 8 [deprecated = true];
    /* Remove a post-processing step to the result of this parser. */
    repeated ModemMessageBodyParser.PostProcessing remove_post_processing = 9 [deprecated = true];

    reserved 2, 5, 11, 12;
}

/* Create a simple modem message parser, which generates a .ksy specification. */
message CreateSimpleModemMessageBodyParserRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;
    SimpleModemMessageBodyParser parser = 2;
}

/* Update a simple modem message parser, updating the generated .ksy specification. */
message UpdateSimpleModemMessageBodyParserRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* The identifier of the parser that should be updated. */
    string identifier = 4;

    SimpleModemMessageBodyParser parser = 3;

    reserved 2;
}

message RenameModemMessageBodyParserRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* The identifier of the parser that should be updated. */
    string identifier = 4;

    /* The new name for this parser. */
    string name = 3;

    reserved 2;
}

/* Update the child availability for a parser.
 * Parsers can be made available to child organizations, which means that they can be viewed in,
 * and assigned to modems in, child organizations.
 */
message UpdateChildOrganizationAvailabilityRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* The identifier of the parser that should be updated. */
    string identifier = 4;

    /* The new child organization availability filter for this parser. */
    Filter.ChildOrganizations available_to_child_organizations = 3;

    reserved 2;
}

/* Make this parser available to a specific child organizations.
 *
 * This will
 * - when the organization is on the exclude list, remove it
 * - when the includeAll flag is false, add it to the include list (if not already present)
 */
message MakeModemMessageBodyParserAvailableToChildOrganizationRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* The identifier of the parser that should be updated. */
    string identifier = 2;

    /* The child organization(s) that the parser should be available to. */
    repeated string available_to = 3;
}

/* Make this parser unavailable to a specific child organizations.
 *
 * This will
 * - when the organization is on the include list, remove it
 * - when the includeAll flag is true, add it to the exclude list (if not already present)
 */
message MakeModemMessageBodyParserUnavailableToChildOrganizationRequest {
    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* The identifier of the parser that should be updated. */
    string identifier = 2;

    /* The child organization(s) that the parser should be unavailable to. */
    repeated string unavailable_to = 3;
}

message DeleteModemMessageBodyParserRequest {
    message Response {
    }

    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* A selection of parsers to be deleted. */
    ModemMessageBodyParserSelection selection = 2;
}

/* Test a parsers on a message.
 *
 * This method provides a number of options and can be used in multiple ways:
 * - Test a real message or an uploaded message body.
 * - Test using an existing parser, an ksy definition or a simple parser definition.
 *
 * This can be used to test a parser with a modem's past messages, before assigning it,
 * or while developing a parser to validate it with an expected body.
 */
message TestModemMessageBodyParserRequest {
    message Response {
        oneof result {
            google.protobuf.Struct parsed = 1;
            string error = 2;
        }
    }

    /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
    optional string organization = 1;

    /* Either an existing parser, an uploaded ksy definition or a simple parser definition to test. */
    oneof parser {
        /* The identifier of the parser you want to test. */
        string identifier = 7;

        /* A ksy definition you want to test. */
        string content_ksy = 2;

        /* A simple parser definition you want to test. */
        SimpleModemMessageBodyParser simple_parser = 3;
    }

    /* Either a real message or a given message body. */
    oneof data {
        /* A byte array (message body) to parse. */
        BytesOrHex parse_bytes = 4;

        /* The id of an existing message, testing the parser on its body. */
        uint64 modem_message_id = 5;
    }

    reserved 6;
}

/* Retry a message, parsing it with all the currently assigned parsers that are in the given selection.
 * If the message was previously parsed by any of those parsers, the previous result is replaced.
 */
message RetryModemMessageBodyParsing {
    message Request {
        /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
        optional string organization = 1;

        /* The messages to parse. */
        repeated uint64 modem_message_ids = 2;

        /* A selection of parsers to apply, if they are assigned. */
        ModemMessageBodyParserSelection parser_selection = 3;

        /* Re-parsing messages causes new events to be sent out. Set suppress_events to true to prevent that. */
        bool suppress_events = 4;
    }

    message Response {
        repeated ModemMessage.ParsedBody parsed_messages = 1;
    }
}

/* Directly assigned modem message parser to a modem. */
message AssignedModemMessageBodyParser {
    string parser_identifier = 1;
    string modem_number = 2;
}

/* Add direct assignments.
 * If an overlapping assignment using a rule exists, it is shadowed by the direct assignment.
 *
 * Simplified version of assign.AssignDirectly.
 */
message AssignModemMessageBodyParsers {
    message Request {
        /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
        optional string organization = 1;
        ModemMessageBodyParserSelection assign_parsers = 2;
        modem.ModemSelection to_modems = 3;
    }
    message Response {
        repeated AssignedModemMessageBodyParser created = 1;
        Request request = 2;
    }
}

/* Remove a direct assignment.
 * If an overlapping assignment using a rule exists, it is not affected, except that it is longer shadowed.
 *
 * Simplified version of assign.UnassignDirectly.
 */
message UnassignModemMessageBodyParsers {
    message Request {
        /* Pick the organization to use (/impersonate). If unset, your default organization is used. */
        optional string organization = 1;
        ModemMessageBodyParserSelection unassign_parsers = 2;
        modem.ModemSelection from_modems = 3;
    }
    message Response {
        repeated AssignedModemMessageBodyParser removed_direct_assignments = 1;
        Request request = 2;
    }
}