From d55a8dede62190f28a07e4a40f28176bae8e9755 Mon Sep 17 00:00:00 2001 From: S-Tim Date: Thu, 10 Oct 2024 13:53:38 +0200 Subject: [PATCH] =?UTF-8?q?feat(#1015):=20added=20property=20to=20enable?= =?UTF-8?q?=20payload=20attribute=20length=20valida=E2=80=A6=20(#1042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(#1015): added property to enable payload attribute length validation - functionality to automatically trim payload attribute values if they exceed the column length in the database * feat(#1015): added property to enable payload attribute length validation - functionality to automatically trim payload attribute values if they exceed the column length in the database * chore(#1015): add test for payload length trim --------- Co-authored-by: Michael von Bargen --- docs/reference-guide/components/view-jpa.md | 5 +++++ .../src/main/kotlin/VariableSerializer.kt | 21 ++++++++++++------- .../src/test/kotlin/JsonPathWithValueTest.kt | 13 ++++++++++++ .../jpa/JpaPolyflowViewDataEntryService.kt | 3 ++- .../view/jpa/JpaPolyflowViewTaskService.kt | 3 ++- .../view/jpa/PolyflowJpaViewProperties.kt | 8 +++++++ .../polyflow/view/jpa/data/ConverterExt.kt | 12 ++++++++--- .../polyflow/view/jpa/task/ConverterExt.kt | 5 +++-- 8 files changed, 56 insertions(+), 14 deletions(-) diff --git a/docs/reference-guide/components/view-jpa.md b/docs/reference-guide/components/view-jpa.md index 26c7e4acb..811b7f386 100644 --- a/docs/reference-guide/components/view-jpa.md +++ b/docs/reference-guide/components/view-jpa.md @@ -75,6 +75,11 @@ In addition, the `stored-items` property is holding a set of items to be persist stored items are: `task`, `data-entry`, `process-instance` and `process-definition`. By setting this property, you can disable storage of items not required by your application and save space consumption of your database. The property defaults to `data-entry`. +With the `payload-attribute-column-length` property one can specify a maximum length for payload attribute values if they are strings. Values that exceed +this length will automatically be trimmed to the max length in order to prevent exceptions when handling the event. This is especially necessary because +relational databases have limits on the length of composite primary keys. Since the combination of (id, path, value) for tasks or (id, type, path, value) for +data entries must be unique, the primary key is very large, which limits the amount of space available for the value. + The `include-correlated-data-entries-in-data-entry-queries` flag controls whether a data entry query (`DataEntriesForUserQuery` or `DataEntriesQuery`) considers the payload of correlated data entries. The data entry attributes (such as `entry_type`, `state.state`, ...) of correlated data entries are not considered. *Note:* Only one level of correlation depth is considered here and there is no option yet to change the depth. diff --git a/integration/common/variable-serializer/src/main/kotlin/VariableSerializer.kt b/integration/common/variable-serializer/src/main/kotlin/VariableSerializer.kt index 86f0bca3c..edbaf5619 100644 --- a/integration/common/variable-serializer/src/main/kotlin/VariableSerializer.kt +++ b/integration/common/variable-serializer/src/main/kotlin/VariableSerializer.kt @@ -48,17 +48,18 @@ typealias JsonPathFilterFunction = (path: String) -> Boolean * @param limit limit of levels to convert. Defaults to -1 meaning there is no limit. * @param filters filter object to identify properties to include into the result. */ -fun VariableMap.toJsonPathsWithValues(limit: Int = -1, filters: List> = emptyList()): Set> { +fun VariableMap.toJsonPathsWithValues(limit: Int = -1, filters: List> = emptyList(), valueLengthLimit: Int? = null): Set> { return this.entries .map { it.toPair() } - .map { it.toJsonPathWithValue(prefix = "", limit = limit, filter = filters) } + .map { it.toJsonPathWithValue(prefix = "", limit = limit, filter = filters, valueLengthLimit) } .flatten().toSet() } internal fun Pair.toJsonPathWithValue( prefix: String = "", limit: Int = -1, - filter: List> + filter: List>, + valueLengthLimit: Int? ): List> { // level limit check val currentLevel = prefix.count { ".".contains(it) } @@ -72,9 +73,15 @@ internal fun Pair.toJsonPathWithValue( "$prefix.${this.first}" } - val value = this.second + var value = this.second return if (value != null && value.isPrimitiveType()) { + // trim strings to the value length limit if provided + if(valueLengthLimit != null && value is String && value.length > valueLengthLimit) { + // TODO: logging + value = value.substring(0 until valueLengthLimit) + } + // check the filters if (!filter.filter { (_, type) -> type == FilterType.EXCLUDE }.all { (filter, _) -> filter.invoke(key).not() @@ -91,10 +98,10 @@ internal fun Pair.toJsonPathWithValue( @Suppress("UNCHECKED_CAST") (value as Map).entries .map { it.toPair() } - .map { it.toJsonPathWithValue(key, limit, filter) } + .map { it.toJsonPathWithValue(key, limit, filter, valueLengthLimit) } .flatten() } else if (value is List<*>) { - value.map { (key to it).toJsonPathWithValue(prefix, limit, filter) }.flatten() + value.map { (key to it).toJsonPathWithValue(prefix, limit, filter, valueLengthLimit) }.flatten() } else { // ignore complex objects, in default scenarios, complex objects got already deserialized by the sender in ProjectingCommandAccumulator.serializePayloadIfNeeded listOf() @@ -104,7 +111,7 @@ internal fun Pair.toJsonPathWithValue( internal fun Any.isPrimitiveType(): Boolean { return when (this) { // TODO: ask Jackson for the supported list of types - is String, is Boolean, is Number, is Int, is Long, is Float, is Date, is Instant -> true + is String, is Boolean, is Number, is Date, is Instant -> true else -> false } } diff --git a/integration/common/variable-serializer/src/test/kotlin/JsonPathWithValueTest.kt b/integration/common/variable-serializer/src/test/kotlin/JsonPathWithValueTest.kt index d7a5e94fd..cf518f4b3 100644 --- a/integration/common/variable-serializer/src/test/kotlin/JsonPathWithValueTest.kt +++ b/integration/common/variable-serializer/src/test/kotlin/JsonPathWithValueTest.kt @@ -249,6 +249,19 @@ internal class JsonPathWithValueTest { assertThat(result).contains("multiple.deepKey1" to "value-1") assertThat(result).contains("multiple.deepKey2" to "value-2") } + + @Test + fun `should trim strings`() { + val payload = createVariables().apply { + put("key", "String with long content. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata") + } + + val result = payload.toJsonPathsWithValues(valueLengthLimit = 25) + + val trimmedStringContent = result.find { it.first == "key" }?.second.toString() + assertThat(trimmedStringContent).hasSize(25) + .isEqualTo("String with long content.") + } } internal fun Set>.keys(): List { diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewDataEntryService.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewDataEntryService.kt index 90bd23d22..572f21b37 100644 --- a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewDataEntryService.kt +++ b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewDataEntryService.kt @@ -113,7 +113,8 @@ class JpaPolyflowViewDataEntryService( objectMapper = objectMapper, revisionValue = RevisionValue.fromMetaData(metaData), limit = polyflowJpaViewProperties.payloadAttributeLevelLimit, - filters = polyflowJpaViewProperties.dataEntryJsonPathFilters() + filters = polyflowJpaViewProperties.dataEntryJsonPathFilters(), + payLoadAttributeColumnLength = polyflowJpaViewProperties.payloadAttributeColumnLength ) ).apply { logger.debug { "JPA-VIEW-41: Business data entry created $event." } diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewTaskService.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewTaskService.kt index d7d249a2f..116bd50e1 100644 --- a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewTaskService.kt +++ b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewTaskService.kt @@ -324,7 +324,8 @@ class JpaPolyflowViewTaskService( event.toEntity( objectMapper, polyflowJpaViewProperties.payloadAttributeLevelLimit, - polyflowJpaViewProperties.taskJsonPathFilters() + polyflowJpaViewProperties.taskJsonPathFilters(), + polyflowJpaViewProperties.payloadAttributeColumnLength ) ) emitTaskUpdate(updated) diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/PolyflowJpaViewProperties.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/PolyflowJpaViewProperties.kt index c735de875..1a7a0ec3b 100644 --- a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/PolyflowJpaViewProperties.kt +++ b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/PolyflowJpaViewProperties.kt @@ -16,6 +16,14 @@ data class PolyflowJpaViewProperties( * Leg level during the path building for indexing the custom attributes. Set to -1 (no limit) as default. */ val payloadAttributeLevelLimit: Int = -1, + + /** + * Allows to specify the column length of the payload attribute values in order to trim values that are too long. This prevents exceptions when handling + * the events, which can cause the application to infinitely retry. + * When set to null, values will not be trimmed or validated. + */ + val payloadAttributeColumnLength: Int?, + /** * List of items to store in projection. Defaults to "DATA_ENTRY" */ diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/data/ConverterExt.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/data/ConverterExt.kt index ea54a9ec7..9538cc498 100644 --- a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/data/ConverterExt.kt +++ b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/data/ConverterExt.kt @@ -69,10 +69,16 @@ fun DataEntryState.toState() = DataEntryStateEmbeddable(processingType = this.pr /** * Event to entity. */ -fun DataEntryCreatedEvent.toEntity(objectMapper: ObjectMapper, revisionValue: RevisionValue, limit: Int, filters: List>) = DataEntryEntity( +fun DataEntryCreatedEvent.toEntity( + objectMapper: ObjectMapper, + revisionValue: RevisionValue, + limit: Int, + filters: List>, + payLoadAttributeColumnLength: Int? = null +) = DataEntryEntity( dataEntryId = DataEntryId(entryType = this.entryType, entryId = this.entryId), payload = this.payload.toPayloadJson(objectMapper), - payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters).map { attr -> PayloadAttribute(attr) }.toMutableSet(), + payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters, payLoadAttributeColumnLength).map { attr -> PayloadAttribute(attr) }.toMutableSet(), name = this.name, applicationName = this.applicationName, type = this.type, @@ -88,7 +94,7 @@ fun DataEntryCreatedEvent.toEntity(objectMapper: ObjectMapper, revisionValue: Re }, authorizedPrincipals = AuthorizationChange.applyUserAuthorization(mutableSetOf(), this.authorizations).map { user(it).toString() } .plus(AuthorizationChange.applyGroupAuthorization(mutableSetOf(), this.authorizations).map { group(it).toString() }).toMutableSet(), - correlations = this.correlations.toMutableMap().map { entry -> DataEntryId(entryType = entry.key, entryId = entry.value.toString()) }.toMutableSet() + correlations = this.correlations.toMutableMap().map { entry -> DataEntryId(entryType = entry.key, entryId = entry.value.toString()) }.toMutableSet() ).apply { this.protocol = this.protocol.addModification(this, this@toEntity.createModification, this@toEntity.state) } diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/task/ConverterExt.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/task/ConverterExt.kt index 6a60ef5b5..f121fabdd 100644 --- a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/task/ConverterExt.kt +++ b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/task/ConverterExt.kt @@ -23,7 +23,8 @@ import java.time.Instant fun TaskCreatedEngineEvent.toEntity( objectMapper: ObjectMapper, limit: Int, - filters: List> + filters: List>, + payLoadAttributeColumnLength: Int? ) = TaskEntity( taskId = this.id, taskDefinitionKey = this.taskDefinitionKey, @@ -35,7 +36,7 @@ fun TaskCreatedEngineEvent.toEntity( .toMutableSet(), correlations = this.correlations.map { entry -> DataEntryId(entryType = entry.key, entryId = "${entry.value}") }.toMutableSet(), payload = this.payload.toPayloadJson(objectMapper), - payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters).map { attr -> PayloadAttribute(attr) }.toMutableSet(), + payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters, payLoadAttributeColumnLength).map { attr -> PayloadAttribute(attr) }.toMutableSet(), assignee = this.assignee, businessKey = this.businessKey, description = this.description,