Upgrade vector (#269)

CHANGELOG.md

@@ -8,8 +8,14 @@ All notable changes to this project will be documented in this file.
 
 - Default stackableVersion to operator version. It is recommended to remove `spec.image.stackableVersion` from your custom resources ([#267], [#268]).
 
+### Changed
+
+- `vector` `0.26.0` -> `0.31.0` ([#269]).
+- `operator-rs` `0.44.0` -> `0.45.1` ([#267]).
+
 [#267]: https://github.com/stackabletech/spark-k8s-operator/pull/267
 [#268]: https://github.com/stackabletech/spark-k8s-operator/pull/268
+[#269]: https://github.com/stackabletech/spark-k8s-operator/pull/269
 
 ## [23.7.0] - 2023-07-14
 

tests/kuttl-test.yaml.jinja2

@@ -9,3 +9,19 @@ testDirs:
 startKIND: false
 suppress: ["events"]
 parallel: 2
+
+# The timeout (in seconds) is used when namespaces are created or
+# deleted, and, if not overridden, in TestSteps, TestAsserts, and
+# Commands. If not set, the timeout is 30 seconds by default.
+#
+# The deletion of a namespace can take a while until all resources are
+# gracefully shut down. If the timeout is reached in the meantime, even
+# a successful test case is considered a failure.
+#
+# For instance, the termination grace period of the Vector aggregator in
+# the logging tests is set to 60 seconds. If there are logs entries
+# which could not be forwarded yet to the external aggregator defined in
+# the VECTOR_AGGREGATOR environment variable, then the test aggregator
+# uses this period of time by trying to forward the events. In this
+# case, deleting a namespace with several Pods takes about 90 seconds.
+timeout: 120

tests/templates/kuttl/logging/01-install-spark-vector-aggregator.yaml

@@ -5,7 +5,7 @@ commands:
   - script: >-
       helm install spark-vector-aggregator vector
       --namespace $NAMESPACE
-      --version 0.20.1
+      --version 0.23.0
       --repo https://helm.vector.dev
       --values spark-vector-aggregator-values.yaml
 ---

tests/templates/kuttl/logging/spark-vector-aggregator-values.yaml.j2

@@ -275,13 +275,26 @@ customConfig:
         .cluster == "pyspark-custom-log-config" &&
         ends_with(string!(.pod), "-exec-1") &&
         .container == "vector"
+
+    # Invalid events
+
+    filteredInvalidEvents:
+      type: filter
+      inputs: [vector]
+      condition: |-
+        .timestamp == from_unix_timestamp!(0) ||
+        is_null(.level) ||
+        is_null(.logger) ||
+        is_null(.message)
   sinks:
     out:
       inputs: [filtered*]
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
       type: vector
       address: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
       buffer:
+        # Avoid back pressure from VECTOR_AGGREGATOR. The test should
+        # not fail if the aggregator is not available.
         when_full: drop_newest
 {% else %}
       type: blackhole

tests/templates/kuttl/logging/test_log_aggregation.py

@@ -2,7 +2,7 @@
 import requests
 
 
-def check_processed_events():
+def check_sent_events():
     response = requests.post(
         'http://spark-vector-aggregator:8686/graphql',
         json={
@@ -12,8 +12,8 @@ def check_processed_events():
                         nodes {
                             componentId
                             metrics {
-                                processedEventsTotal {
-                                    processedEventsTotal
+                                sentEventsTotal {
+                                    sentEventsTotal
                                 }
                             }
                         }
@@ -30,12 +30,19 @@ def check_processed_events():
 
     transforms = result['data']['transforms']['nodes']
     for transform in transforms:
-        processedEvents = transform['metrics']['processedEventsTotal']['processedEventsTotal']
+        sentEvents = transform['metrics']['sentEventsTotal']
         componentId = transform['componentId']
-        assert processedEvents > 0, \
-            f'No events were processed in "{componentId}".'
+
+        if componentId == 'filteredInvalidEvents':
+            assert sentEvents is None or \
+                sentEvents['sentEventsTotal'] == 0, \
+                'Invalid log events were sent.'
+        else:
+            assert sentEvents is not None and \
+                sentEvents['sentEventsTotal'] > 0, \
+                f'No events were sent in "{componentId}".'
 
 
 if __name__ == '__main__':
-    check_processed_events()
+    check_sent_events()
     print('Test successful!')