feat(#1015): added property to enable payload attribute length valida… (

#1042) * feat(#1015): added property to enable payload attribute length validation - functionality to automatically trim payload attribute values if they exceed the column length in the database * feat(#1015): added property to enable payload attribute length validation - functionality to automatically trim payload attribute values if they exceed the column length in the database * chore(#1015): add test for payload length trim --------- Co-authored-by: Michael von Bargen <[email protected]>
holunda-io · Oct 10, 2024 · d55a8de · d55a8de
1 parent 662baed
commit d55a8de
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 14 deletions.
diff --git a/docs/reference-guide/components/view-jpa.md b/docs/reference-guide/components/view-jpa.md
@@ -75,6 +75,11 @@ In addition, the `stored-items` property is holding a set of items to be persist
 stored items are: `task`, `data-entry`, `process-instance` and `process-definition`. By setting this property, you can disable
 storage of items not required by your application and save space consumption of your database. The property defaults to `data-entry`.
 
+With the `payload-attribute-column-length` property one can specify a maximum length for payload attribute values if they are strings. Values that exceed
+this length will automatically be trimmed to the max length in order to prevent exceptions when handling the event. This is especially necessary because
+relational databases have limits on the length of composite primary keys. Since the combination of (id, path, value) for tasks or (id, type, path, value) for
+data entries must be unique, the primary key is very large, which limits the amount of space available for the value.
+
 The `include-correlated-data-entries-in-data-entry-queries` flag controls whether a data entry query (`DataEntriesForUserQuery` or `DataEntriesQuery`) considers
 the payload of correlated data entries. The data entry attributes (such as `entry_type`, `state.state`, ...) of correlated data entries are not considered.
 *Note:* Only one level of correlation depth is considered here and there is no option yet to change the depth.

diff --git a/integration/common/variable-serializer/src/main/kotlin/VariableSerializer.kt b/integration/common/variable-serializer/src/main/kotlin/VariableSerializer.kt
@@ -48,17 +48,18 @@ typealias JsonPathFilterFunction = (path: String) -> Boolean
  * @param limit limit of levels to convert. Defaults to -1 meaning there is no limit.
  * @param filters filter object to identify properties to include into the result.
  */
-fun VariableMap.toJsonPathsWithValues(limit: Int = -1, filters: List<Pair<JsonPathFilterFunction, FilterType>> = emptyList()): Set<Pair<String, Any>> {
+fun VariableMap.toJsonPathsWithValues(limit: Int = -1, filters: List<Pair<JsonPathFilterFunction, FilterType>> = emptyList(), valueLengthLimit: Int? = null): Set<Pair<String, Any>> {
     return this.entries
         .map { it.toPair() }
-        .map { it.toJsonPathWithValue(prefix = "", limit = limit, filter = filters) }
+        .map { it.toJsonPathWithValue(prefix = "", limit = limit, filter = filters, valueLengthLimit) }
         .flatten().toSet()
 }
 
 internal fun Pair<String, Any?>.toJsonPathWithValue(
   prefix: String = "",
   limit: Int = -1,
-  filter: List<Pair<JsonPathFilterFunction, FilterType>>
+  filter: List<Pair<JsonPathFilterFunction, FilterType>>,
+  valueLengthLimit: Int?
 ): List<Pair<String, Any>> {
   // level limit check
   val currentLevel = prefix.count { ".".contains(it) }
@@ -72,9 +73,15 @@ internal fun Pair<String, Any?>.toJsonPathWithValue(
     "$prefix.${this.first}"
   }
 
-  val value = this.second
+  var value = this.second
   return if (value != null && value.isPrimitiveType()) {
 
+    // trim strings to the value length limit if provided
+    if(valueLengthLimit != null && value is String && value.length > valueLengthLimit) {
+      // TODO: logging
+      value = value.substring(0 until valueLengthLimit)
+    }
+
     // check the filters
     if (!filter.filter { (_, type) -> type == FilterType.EXCLUDE }.all { (filter, _) ->
         filter.invoke(key).not()
@@ -91,10 +98,10 @@ internal fun Pair<String, Any?>.toJsonPathWithValue(
     @Suppress("UNCHECKED_CAST")
     (value as Map<String, Any?>).entries
         .map { it.toPair() }
-        .map { it.toJsonPathWithValue(key, limit, filter) }
+        .map { it.toJsonPathWithValue(key, limit, filter, valueLengthLimit) }
         .flatten()
   } else if (value is List<*>) {
-    value.map { (key to it).toJsonPathWithValue(prefix, limit, filter) }.flatten()
+    value.map { (key to it).toJsonPathWithValue(prefix, limit, filter, valueLengthLimit) }.flatten()
   } else {
     // ignore complex objects, in default scenarios, complex objects got already deserialized by the sender in ProjectingCommandAccumulator.serializePayloadIfNeeded
     listOf()
@@ -104,7 +111,7 @@ internal fun Pair<String, Any?>.toJsonPathWithValue(
 internal fun Any.isPrimitiveType(): Boolean {
   return when (this) {
     // TODO: ask Jackson for the supported list of types
-    is String, is Boolean, is Number, is Int, is Long, is Float, is Date, is Instant -> true
+    is String, is Boolean, is Number, is Date, is Instant -> true
     else -> false
   }
 }

diff --git a/integration/common/variable-serializer/src/test/kotlin/JsonPathWithValueTest.kt b/integration/common/variable-serializer/src/test/kotlin/JsonPathWithValueTest.kt
@@ -249,6 +249,19 @@ internal class JsonPathWithValueTest {
     assertThat(result).contains("multiple.deepKey1" to "value-1")
     assertThat(result).contains("multiple.deepKey2" to "value-2")
   }
+
+  @Test
+  fun `should trim strings`() {
+    val payload = createVariables().apply {
+      put("key", "String with long content. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata")
+    }
+
+    val result = payload.toJsonPathsWithValues(valueLengthLimit = 25)
+
+    val trimmedStringContent = result.find { it.first == "key" }?.second.toString()
+    assertThat(trimmedStringContent).hasSize(25)
+      .isEqualTo("String with long content.")
+  }
 }
 
 internal fun  Set<Pair<String, Any>>.keys(): List<String> {

diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewDataEntryService.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewDataEntryService.kt
@@ -113,7 +113,8 @@ class JpaPolyflowViewDataEntryService(
           objectMapper = objectMapper,
           revisionValue = RevisionValue.fromMetaData(metaData),
           limit = polyflowJpaViewProperties.payloadAttributeLevelLimit,
-          filters = polyflowJpaViewProperties.dataEntryJsonPathFilters()
+          filters = polyflowJpaViewProperties.dataEntryJsonPathFilters(),
+          payLoadAttributeColumnLength = polyflowJpaViewProperties.payloadAttributeColumnLength
         )
       ).apply {
         logger.debug { "JPA-VIEW-41: Business data entry created $event." }

diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewTaskService.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/JpaPolyflowViewTaskService.kt
@@ -324,7 +324,8 @@ class JpaPolyflowViewTaskService(
           event.toEntity(
             objectMapper,
             polyflowJpaViewProperties.payloadAttributeLevelLimit,
-            polyflowJpaViewProperties.taskJsonPathFilters()
+            polyflowJpaViewProperties.taskJsonPathFilters(),
+            polyflowJpaViewProperties.payloadAttributeColumnLength
           )
         )
         emitTaskUpdate(updated)

diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/PolyflowJpaViewProperties.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/PolyflowJpaViewProperties.kt
@@ -16,6 +16,14 @@ data class PolyflowJpaViewProperties(
    * Leg level during the path building for indexing the custom attributes. Set to -1 (no limit) as default.
    */
   val payloadAttributeLevelLimit: Int = -1,
+
+  /**
+   * Allows to specify the column length of the payload attribute values in order to trim values that are too long. This prevents exceptions when handling
+   * the events, which can cause the application to infinitely retry.
+   * When set to null, values will not be trimmed or validated.
+   */
+  val payloadAttributeColumnLength: Int?,
+
   /**
    * List of items to store in projection. Defaults to "DATA_ENTRY"
    */

diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/data/ConverterExt.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/data/ConverterExt.kt
@@ -69,10 +69,16 @@ fun DataEntryState.toState() = DataEntryStateEmbeddable(processingType = this.pr
 /**
  * Event to entity.
  */
-fun DataEntryCreatedEvent.toEntity(objectMapper: ObjectMapper, revisionValue: RevisionValue, limit: Int, filters: List<Pair<JsonPathFilterFunction, FilterType>>) = DataEntryEntity(
+fun DataEntryCreatedEvent.toEntity(
+  objectMapper: ObjectMapper,
+  revisionValue: RevisionValue,
+  limit: Int,
+  filters: List<Pair<JsonPathFilterFunction, FilterType>>,
+  payLoadAttributeColumnLength: Int? = null
+) = DataEntryEntity(
   dataEntryId = DataEntryId(entryType = this.entryType, entryId = this.entryId),
   payload = this.payload.toPayloadJson(objectMapper),
-  payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters).map { attr -> PayloadAttribute(attr) }.toMutableSet(),
+  payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters, payLoadAttributeColumnLength).map { attr -> PayloadAttribute(attr) }.toMutableSet(),
   name = this.name,
   applicationName = this.applicationName,
   type = this.type,
@@ -88,7 +94,7 @@ fun DataEntryCreatedEvent.toEntity(objectMapper: ObjectMapper, revisionValue: Re
   },
   authorizedPrincipals = AuthorizationChange.applyUserAuthorization(mutableSetOf(), this.authorizations).map { user(it).toString() }
     .plus(AuthorizationChange.applyGroupAuthorization(mutableSetOf(), this.authorizations).map { group(it).toString() }).toMutableSet(),
-  correlations = this.correlations.toMutableMap().map { entry -> DataEntryId(entryType = entry.key, entryId =  entry.value.toString()) }.toMutableSet()
+  correlations = this.correlations.toMutableMap().map { entry -> DataEntryId(entryType = entry.key, entryId = entry.value.toString()) }.toMutableSet()
 ).apply {
   this.protocol = this.protocol.addModification(this, this@toEntity.createModification, this@toEntity.state)
 }

diff --git a/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/task/ConverterExt.kt b/view/jpa/src/main/kotlin/io/holunda/polyflow/view/jpa/task/ConverterExt.kt
@@ -23,7 +23,8 @@ import java.time.Instant
 fun TaskCreatedEngineEvent.toEntity(
   objectMapper: ObjectMapper,
   limit: Int,
-  filters: List<Pair<JsonPathFilterFunction, FilterType>>
+  filters: List<Pair<JsonPathFilterFunction, FilterType>>,
+  payLoadAttributeColumnLength: Int?
 ) = TaskEntity(
   taskId = this.id,
   taskDefinitionKey = this.taskDefinitionKey,
@@ -35,7 +36,7 @@ fun TaskCreatedEngineEvent.toEntity(
     .toMutableSet(),
   correlations = this.correlations.map { entry -> DataEntryId(entryType = entry.key, entryId = "${entry.value}") }.toMutableSet(),
   payload = this.payload.toPayloadJson(objectMapper),
-  payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters).map { attr -> PayloadAttribute(attr) }.toMutableSet(),
+  payloadAttributes = this.payload.toJsonPathsWithValues(limit, filters, payLoadAttributeColumnLength).map { attr -> PayloadAttribute(attr) }.toMutableSet(),
   assignee = this.assignee,
   businessKey = this.businessKey,
   description = this.description,