From a2fbb012766bc16f6379cf0b12496f61c5159301 Mon Sep 17 00:00:00 2001 From: jshlbrd Date: Wed, 2 Oct 2024 23:12:04 -0700 Subject: [PATCH] docs(examples): Add Unit Tests --- examples/condition/meta/config.jsonnet | 17 +- examples/condition/meta/data.json | 1 - examples/condition/meta/stdout.txt | 2 - examples/condition/number/config.jsonnet | 15 +- examples/condition/number/data.json | 1 - examples/condition/number/stdout.txt | 13 -- examples/condition/string/config.jsonnet | 17 +- examples/condition/string/data.json | 1 - examples/condition/string/stdout.txt | 12 -- .../transform/aggregate/sample/config.jsonnet | 35 +++- .../transform/aggregate/sample/data.jsonl | 13 -- .../transform/aggregate/sample/stdout.txt | 1 - .../aggregate/summarize/config.jsonnet | 32 ++++ .../transform/aggregate/summarize/data.jsonl | 19 -- .../transform/aggregate/summarize/stdout.txt | 5 - .../transform/array/extend/config.jsonnet | 12 +- examples/transform/array/extend/data.json | 1 - examples/transform/array/extend/stdout.txt | 4 - .../transform/array/flatten/config.jsonnet | 12 +- examples/transform/array/flatten/data.json | 1 - examples/transform/array/flatten/stdout.txt | 3 - .../array/flatten_deep/config.jsonnet | 12 +- .../transform/array/flatten_deep/data.json | 1 - .../transform/array/flatten_deep/stdout.txt | 3 - examples/transform/array/group/config.jsonnet | 27 ++- examples/transform/array/group/data.json | 1 - examples/transform/array/group/stdout.txt | 15 -- .../enrich/kvstore_csv/config.jsonnet | 12 +- .../transform/enrich/kvstore_csv/data.jsonl | 1 - .../transform/enrich/kvstore_csv/stdout.txt | 7 - .../enrich/kvstore_json/config.jsonnet | 12 +- .../transform/enrich/kvstore_json/data.jsonl | 1 - .../transform/enrich/kvstore_json/kv.json | 1 + .../transform/enrich/kvstore_json/stdout.txt | 4 - .../enrich/kvstore_set_add/config.jsonnet | 17 +- .../enrich/kvstore_set_add/data.jsonl | 6 - .../enrich/kvstore_set_add/stdout.txt | 41 ----- examples/transform/enrich/mmdb/config.jsonnet | 21 ++- examples/transform/enrich/mmdb/data.jsonl | 3 - examples/transform/enrich/mmdb/stdout.txt | 172 ------------------ examples/transform/format/zip/config.jsonnet | 12 ++ examples/transform/format/zip/data.csv | 3 - examples/transform/format/zip/data.jsonl | 3 - examples/transform/format/zip/stdout.txt | 6 - .../meta/default_value/config.jsonnet | 13 +- .../transform/meta/default_value/data.json | 1 - .../meta/each_in_array/config.jsonnet | 20 +- .../transform/meta/each_in_array/data.json | 1 - .../transform/meta/each_in_array/stdout.txt | 12 -- .../meta/exactly_once_consumer/config.jsonnet | 18 +- .../meta/exactly_once_consumer/data.jsonl | 8 - .../meta/exactly_once_consumer/stdout.txt | 11 -- .../meta/exactly_once_producer/config.jsonnet | 18 +- .../meta/exactly_once_producer/data.jsonl | 8 - .../meta/exactly_once_producer/stdout.txt | 11 -- .../meta/exactly_once_system/config.jsonnet | 18 +- .../meta/exactly_once_system/data.jsonl | 8 - .../meta/exactly_once_system/stdout.txt | 14 -- .../meta/execution_time/config.jsonnet | 10 + .../transform/meta/execution_time/data.json | 1 - .../transform/meta/execution_time/stdout.txt | 3 - .../meta/retry_with_backoff/config.jsonnet | 11 +- .../meta/retry_with_backoff/data.json | 1 - .../meta/retry_with_backoff/stdout.txt | 4 - .../transform/number/clamp/config.jsonnet | 17 +- examples/transform/number/clamp/data.txt | 3 - examples/transform/number/clamp/stdout.txt | 3 - examples/transform/number/max/config.jsonnet | 15 +- examples/transform/number/max/data.txt | 4 - examples/transform/number/max/stdout.txt | 4 - examples/transform/number/min/config.jsonnet | 15 +- examples/transform/number/min/data.txt | 4 - examples/transform/number/min/stdout.txt | 4 - .../send/aux_transforms/config.jsonnet | 1 - .../send/aws_s3_glacier/config.jsonnet | 1 - examples/transform/send/batch/config.jsonnet | 1 - .../transform/send/datadog/config.jsonnet | 1 - examples/transform/send/splunk/config.jsonnet | 1 - .../transform/send/sumologic/config.jsonnet | 1 - .../time/str_conversion/config.jsonnet | 13 +- .../transform/time/str_conversion/data.json | 1 - .../transform/time/str_conversion/stdout.txt | 3 - .../utility/generate_ctrl/config.jsonnet | 25 ++- .../utility/generate_ctrl/data.jsonl | 13 -- .../utility/generate_ctrl/stdout.txt | 59 ------ .../utility/message_bytes/config.jsonnet | 22 +++ .../utility/message_bytes/data.jsonl | 13 -- .../utility/message_bytes/stdout.txt | 2 - .../utility/message_count/config.jsonnet | 22 +++ .../utility/message_count/data.jsonl | 13 -- .../utility/message_count/stdout.txt | 2 - .../utility/message_freshness/config.jsonnet | 10 + .../utility/message_freshness/data.jsonl | 1 - .../utility/message_freshness/stdout.txt | 2 - 94 files changed, 464 insertions(+), 601 deletions(-) delete mode 100644 examples/condition/meta/data.json delete mode 100644 examples/condition/meta/stdout.txt delete mode 100644 examples/condition/number/data.json delete mode 100644 examples/condition/number/stdout.txt delete mode 100644 examples/condition/string/data.json delete mode 100644 examples/condition/string/stdout.txt delete mode 100644 examples/transform/aggregate/sample/data.jsonl delete mode 100644 examples/transform/aggregate/sample/stdout.txt delete mode 100644 examples/transform/aggregate/summarize/data.jsonl delete mode 100644 examples/transform/aggregate/summarize/stdout.txt delete mode 100644 examples/transform/array/extend/data.json delete mode 100644 examples/transform/array/extend/stdout.txt delete mode 100644 examples/transform/array/flatten/data.json delete mode 100644 examples/transform/array/flatten/stdout.txt delete mode 100644 examples/transform/array/flatten_deep/data.json delete mode 100644 examples/transform/array/flatten_deep/stdout.txt delete mode 100644 examples/transform/array/group/data.json delete mode 100644 examples/transform/array/group/stdout.txt delete mode 100644 examples/transform/enrich/kvstore_csv/data.jsonl delete mode 100644 examples/transform/enrich/kvstore_csv/stdout.txt delete mode 100644 examples/transform/enrich/kvstore_json/data.jsonl create mode 100644 examples/transform/enrich/kvstore_json/kv.json delete mode 100644 examples/transform/enrich/kvstore_json/stdout.txt delete mode 100644 examples/transform/enrich/kvstore_set_add/data.jsonl delete mode 100644 examples/transform/enrich/kvstore_set_add/stdout.txt delete mode 100644 examples/transform/enrich/mmdb/data.jsonl delete mode 100644 examples/transform/enrich/mmdb/stdout.txt delete mode 100644 examples/transform/format/zip/data.csv delete mode 100644 examples/transform/format/zip/data.jsonl delete mode 100644 examples/transform/format/zip/stdout.txt delete mode 100644 examples/transform/meta/default_value/data.json delete mode 100644 examples/transform/meta/each_in_array/data.json delete mode 100644 examples/transform/meta/each_in_array/stdout.txt delete mode 100644 examples/transform/meta/exactly_once_consumer/data.jsonl delete mode 100644 examples/transform/meta/exactly_once_consumer/stdout.txt delete mode 100644 examples/transform/meta/exactly_once_producer/data.jsonl delete mode 100644 examples/transform/meta/exactly_once_producer/stdout.txt delete mode 100644 examples/transform/meta/exactly_once_system/data.jsonl delete mode 100644 examples/transform/meta/exactly_once_system/stdout.txt delete mode 100644 examples/transform/meta/execution_time/data.json delete mode 100644 examples/transform/meta/execution_time/stdout.txt delete mode 100644 examples/transform/meta/retry_with_backoff/data.json delete mode 100644 examples/transform/meta/retry_with_backoff/stdout.txt delete mode 100644 examples/transform/number/clamp/data.txt delete mode 100644 examples/transform/number/clamp/stdout.txt delete mode 100644 examples/transform/number/max/data.txt delete mode 100644 examples/transform/number/max/stdout.txt delete mode 100644 examples/transform/number/min/data.txt delete mode 100644 examples/transform/number/min/stdout.txt delete mode 100644 examples/transform/time/str_conversion/data.json delete mode 100644 examples/transform/time/str_conversion/stdout.txt delete mode 100644 examples/transform/utility/generate_ctrl/data.jsonl delete mode 100644 examples/transform/utility/generate_ctrl/stdout.txt delete mode 100644 examples/transform/utility/message_bytes/data.jsonl delete mode 100644 examples/transform/utility/message_bytes/stdout.txt delete mode 100644 examples/transform/utility/message_count/data.jsonl delete mode 100644 examples/transform/utility/message_count/stdout.txt delete mode 100644 examples/transform/utility/message_freshness/data.jsonl delete mode 100644 examples/transform/utility/message_freshness/stdout.txt diff --git a/examples/condition/meta/config.jsonnet b/examples/condition/meta/config.jsonnet index 3716b042..3401049a 100644 --- a/examples/condition/meta/config.jsonnet +++ b/examples/condition/meta/config.jsonnet @@ -4,9 +4,18 @@ local sub = import '../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'meta', + transforms: [ + sub.tf.test.message({ value: ["alice@brex.com","bob@brex.com"] }), + sub.tf.send.stdout(), + ], + // Asserts that the message is equal to 'true'. + condition: sub.cnd.str.eq({ value: 'true' }), + } + ], transforms: [ - sub.tf.send.stdout(), // In real-world deployments, the match decision is typically used // to summarize an array of values. For this example, the decision // is represented as a boolean value and printed to stdout. @@ -15,7 +24,9 @@ local sub = import '../../../substation.libsonnet'; { condition: sub.cnd.meta.any({ object: { source_key: '@this' }, // Required to interpret the input as a JSON array. - inspectors: [sub.cnd.str.ends_with(settings={ value: '@brex.com' })], + conditions: [ + sub.cnd.str.ends_with({ value: 'brex.com' }), + ], }), transforms: [ sub.tf.obj.insert({ object: { target_key: 'meta result' }, value: true }), diff --git a/examples/condition/meta/data.json b/examples/condition/meta/data.json deleted file mode 100644 index 3fa4dcfc..00000000 --- a/examples/condition/meta/data.json +++ /dev/null @@ -1 +0,0 @@ -["alice@brex.com","bob@brex.com"] diff --git a/examples/condition/meta/stdout.txt b/examples/condition/meta/stdout.txt deleted file mode 100644 index 4b122616..00000000 --- a/examples/condition/meta/stdout.txt +++ /dev/null @@ -1,2 +0,0 @@ -["alice@brex.com","bob@brex.com"] -true diff --git a/examples/condition/number/config.jsonnet b/examples/condition/number/config.jsonnet index ca5e369a..656276a4 100644 --- a/examples/condition/number/config.jsonnet +++ b/examples/condition/number/config.jsonnet @@ -2,7 +2,20 @@ local sub = import '../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'number', + transforms: [ + sub.tf.test.message({ value: {"sourcePort":22,"bytes":20000} }), + sub.tf.send.stdout(), + ], + // Asserts that the conditional transforms were applied. + condition: sub.cnd.all([ + sub.cnd.str.eq({ obj: {src: 'service'}, value: 'SSH' }), + sub.cnd.str.eq({ obj: {src: 'severity'}, value: 'high' }), + ]) + } + ], transforms: [ sub.tf.meta.switch({ cases: [ { diff --git a/examples/condition/number/data.json b/examples/condition/number/data.json deleted file mode 100644 index edfed892..00000000 --- a/examples/condition/number/data.json +++ /dev/null @@ -1 +0,0 @@ -{"eventId":"123461","timestamp":"2024-07-29T10:00:00Z","sourceIP":"192.168.1.6","destinationIP":"172.16.0.7","sourcePort":"22","destinationPort":"22","protocol":"TCP","action":"ACCEPT","bytes":"20000"} diff --git a/examples/condition/number/stdout.txt b/examples/condition/number/stdout.txt deleted file mode 100644 index 96117951..00000000 --- a/examples/condition/number/stdout.txt +++ /dev/null @@ -1,13 +0,0 @@ -{ - "eventId": "123461", - "timestamp": "2024-07-29T10:00:00Z", - "sourceIP": "192.168.1.6", - "destinationIP": "172.16.0.7", - "sourcePort": "22", - "destinationPort": "22", - "protocol": "TCP", - "action": "ACCEPT", - "bytes": "20000", - "service": "SSH", - "severity": "high" -} diff --git a/examples/condition/string/config.jsonnet b/examples/condition/string/config.jsonnet index e2b449ee..f8eefac1 100644 --- a/examples/condition/string/config.jsonnet +++ b/examples/condition/string/config.jsonnet @@ -2,14 +2,27 @@ local sub = import '../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'string', + transforms: [ + sub.tf.test.message({ value: {"action":"ACCEPT","vpcId":"vpc-2b3c4d5e"} }), + sub.tf.send.stdout(), + ], + // Asserts that the conditional transforms were applied. + condition: sub.cnd.all([ + sub.cnd.str.eq({ obj: {src: 'outcome'}, value: 'Allow' }), + sub.cnd.str.eq({ obj: {src: 'priority'}, value: 'high' }), + ]) + } + ], transforms: [ sub.tf.meta.switch({ cases: [ { condition: sub.cnd.str.eq({ obj: { src: 'action' }, value: 'ACCEPT' }), transforms: [ // This overwrites the value of the 'action' key. - sub.tf.obj.insert({ obj: { trg: 'action' }, value: 'Allow' }), + sub.tf.obj.insert({ obj: { trg: 'outcome' }, value: 'Allow' }), ], }, ] }), diff --git a/examples/condition/string/data.json b/examples/condition/string/data.json deleted file mode 100644 index 3c5c76f3..00000000 --- a/examples/condition/string/data.json +++ /dev/null @@ -1 +0,0 @@ -{"eventId":"123461","timestamp":"2024-07-29T10:00:00Z","sourceIP":"192.168.1.6","destinationIP":"172.16.0.7","sourcePort":"80","destinationPort":"443","protocol":"TCP","action":"ACCEPT","vpcId":"vpc-2b3c4d5e"} diff --git a/examples/condition/string/stdout.txt b/examples/condition/string/stdout.txt deleted file mode 100644 index 9af356d8..00000000 --- a/examples/condition/string/stdout.txt +++ /dev/null @@ -1,12 +0,0 @@ -{ - "eventId": "123461", - "timestamp": "2024-07-29T10:00:00Z", - "sourceIP": "192.168.1.6", - "destinationIP": "172.16.0.7", - "sourcePort": "80", - "destinationPort": "443", - "protocol": "TCP", - "action": "Allow", - "vpcId": "vpc-2b3c4d5e", - "priority": "high" -} diff --git a/examples/transform/aggregate/sample/config.jsonnet b/examples/transform/aggregate/sample/config.jsonnet index d728682d..a909965a 100644 --- a/examples/transform/aggregate/sample/config.jsonnet +++ b/examples/transform/aggregate/sample/config.jsonnet @@ -4,17 +4,40 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'sample', + transforms: [ + sub.tf.test.message({ value: {"a":"b"}}), + sub.tf.test.message({ value: {"c":"d"}}), + sub.tf.test.message({ value: {"e":"f"}}), + sub.tf.test.message({ value: {"g":"h"}}), + sub.tf.test.message({ value: {"i":"j"}}), + sub.tf.test.message({ value: {"k":"l"}}), + sub.tf.test.message({ value: {"m":"n"}}), + sub.tf.test.message({ value: {"o":"p"}}), + sub.tf.test.message({ value: {"q":"r"}}), + sub.tf.test.message({ value: {"s":"t"}}), + sub.tf.test.message({ value: {"u":"v"}}), + sub.tf.test.message({ value: {"w":"x"}}), + sub.tf.test.message({ value: {"y":"z"}}), + sub.tf.test.message({ value: " "}), + sub.tf.send.stdout(), + ], + // Asserts that the message is '{"c":"d"}'. + condition: sub.cnd.num.len.greater_than({ value: 0 }), + } + ], transforms: [ // Events are aggregated into an array. This example has a sample - // rate of up to 1/10. By default, the sample rate will be lower if - // fewer than 10 events are processed by Substation. - sub.tf.aggregate.to.array({ object: { target_key: 'meta sample' }, batch: { count: 10 } }), + // rate of up to 1/5. By default, the sample rate will be lower if + // fewer than 5 events are processed by Substation. + sub.tf.aggregate.to.array({ object: { target_key: 'meta sample' }, batch: { count: 5 } }), // A strict sample rate can be enforced by dropping any events that - // contain the `sample` key, but do not have a length of 10. + // contain the `sample` key, but do not have a length of 5. sub.tf.meta.switch(settings={ cases: [ { - condition: sub.cnd.num.len.eq({ object: { source_key: 'meta sample' }, value: 10 }), + condition: sub.cnd.num.len.eq({ object: { source_key: 'meta sample' }, value: 5 }), transforms: [ sub.tf.object.copy({ object: { source_key: 'meta sample.0' } }), ], diff --git a/examples/transform/aggregate/sample/data.jsonl b/examples/transform/aggregate/sample/data.jsonl deleted file mode 100644 index d101df01..00000000 --- a/examples/transform/aggregate/sample/data.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"a":"b"} -{"c":"d"} -{"e":"f"} -{"g":"h"} -{"i":"j"} -{"k":"l"} -{"m":"n"} -{"o":"p"} -{"q":"r"} -{"s":"t"} -{"u":"v"} -{"w":"x"} -{"y":"z"} diff --git a/examples/transform/aggregate/sample/stdout.txt b/examples/transform/aggregate/sample/stdout.txt deleted file mode 100644 index 6c03b134..00000000 --- a/examples/transform/aggregate/sample/stdout.txt +++ /dev/null @@ -1 +0,0 @@ -{"c":"d"} diff --git a/examples/transform/aggregate/summarize/config.jsonnet b/examples/transform/aggregate/summarize/config.jsonnet index 7c9031e4..762e2cf6 100644 --- a/examples/transform/aggregate/summarize/config.jsonnet +++ b/examples/transform/aggregate/summarize/config.jsonnet @@ -4,6 +4,38 @@ local sub = import '../../../../substation.libsonnet'; { + tests: [ + { + name: 'summarize', + transforms: [ + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.8.8","bytes":11,"timestamp":1674429049} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.4.4","bytes":20,"timestamp":1674429050} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.4.4","bytes":15,"timestamp":1674429051} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.8.8","bytes":8,"timestamp":1674429052} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.8.8","bytes":25,"timestamp":1674429053} }), + sub.tf.test.message({ value: {"client":"10.1.1.4","server":"1.2.3.4","bytes":2400,"timestamp":1674429054} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.4.4","bytes":23,"timestamp":1674429055} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.8.8","bytes":12,"timestamp":1674429056} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.4.4","bytes":18,"timestamp":1674429057} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.8.8","bytes":6,"timestamp":1674429058} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.4.4","bytes":23,"timestamp":1674429059} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.8.8","bytes":12,"timestamp":1674429060} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.4.4","bytes":18,"timestamp":1674429061} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.8.8","bytes":6,"timestamp":1674429062} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.8.8","bytes":11,"timestamp":1674429063} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.4.4","bytes":20,"timestamp":1674429064} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.4.4","bytes":15,"timestamp":1674429065} }), + sub.tf.test.message({ value: {"client":"10.1.1.3","server":"8.8.8.8","bytes":8,"timestamp":1674429066} }), + sub.tf.test.message({ value: {"client":"10.1.1.2","server":"8.8.8.8","bytes":25,"timestamp":1674429067} }), + sub.tf.send.stdout(), + ], + // Asserts that each message has the 'count' and 'bytes_total' fields. + condition: sub.cnd.all([ + sub.cnd.num.len.greater_than({ obj: {src: 'count'}, value: 0 }), + sub.cnd.num.len.greater_than({ obj: {src: 'bytes_total'}, value: 0 }), + ]) + } + ], transforms: [ // Events are aggregated into arrays based on their client and server fields. // The resulting array is put into a new field named "reduce". diff --git a/examples/transform/aggregate/summarize/data.jsonl b/examples/transform/aggregate/summarize/data.jsonl deleted file mode 100644 index 9684c261..00000000 --- a/examples/transform/aggregate/summarize/data.jsonl +++ /dev/null @@ -1,19 +0,0 @@ -{"client":"10.1.1.2","server":"8.8.8.8","bytes":11,"timestamp":1674429049} -{"client":"10.1.1.3","server":"8.8.4.4","bytes":20,"timestamp":1674429050} -{"client":"10.1.1.2","server":"8.8.4.4","bytes":15,"timestamp":1674429051} -{"client":"10.1.1.3","server":"8.8.8.8","bytes":8,"timestamp":1674429052} -{"client":"10.1.1.2","server":"8.8.8.8","bytes":25,"timestamp":1674429053} -{"client":"10.1.1.4","server":"1.2.3.4","bytes":2400,"timestamp":1674429054} -{"client":"10.1.1.2","server":"8.8.4.4","bytes":23,"timestamp":1674429055} -{"client":"10.1.1.3","server":"8.8.8.8","bytes":12,"timestamp":1674429056} -{"client":"10.1.1.2","server":"8.8.4.4","bytes":18,"timestamp":1674429057} -{"client":"10.1.1.3","server":"8.8.8.8","bytes":6,"timestamp":1674429058} -{"client":"10.1.1.2","server":"8.8.4.4","bytes":23,"timestamp":1674429059} -{"client":"10.1.1.3","server":"8.8.8.8","bytes":12,"timestamp":1674429060} -{"client":"10.1.1.2","server":"8.8.4.4","bytes":18,"timestamp":1674429061} -{"client":"10.1.1.3","server":"8.8.8.8","bytes":6,"timestamp":1674429062} -{"client":"10.1.1.2","server":"8.8.8.8","bytes":11,"timestamp":1674429063} -{"client":"10.1.1.3","server":"8.8.4.4","bytes":20,"timestamp":1674429064} -{"client":"10.1.1.2","server":"8.8.4.4","bytes":15,"timestamp":1674429065} -{"client":"10.1.1.3","server":"8.8.8.8","bytes":8,"timestamp":1674429066} -{"client":"10.1.1.2","server":"8.8.8.8","bytes":25,"timestamp":1674429067} diff --git a/examples/transform/aggregate/summarize/stdout.txt b/examples/transform/aggregate/summarize/stdout.txt deleted file mode 100644 index 06a2f1f1..00000000 --- a/examples/transform/aggregate/summarize/stdout.txt +++ /dev/null @@ -1,5 +0,0 @@ -{"client":"10.1.1.2","server":"8.8.8.8","timestamp":1674429067,"count":4,"bytes_total":72} -{"client":"10.1.1.3","server":"8.8.8.8","timestamp":1674429066,"count":6,"bytes_total":52} -{"client":"10.1.1.2","server":"8.8.4.4","timestamp":1674429065,"count":6,"bytes_total":112} -{"client":"10.1.1.4","server":"1.2.3.4","timestamp":1674429054,"count":1,"bytes_total":2400} -{"client":"10.1.1.3","server":"8.8.4.4","timestamp":1674429064,"count":2,"bytes_total":40} diff --git a/examples/transform/array/extend/config.jsonnet b/examples/transform/array/extend/config.jsonnet index c0a28134..1c3d8a53 100644 --- a/examples/transform/array/extend/config.jsonnet +++ b/examples/transform/array/extend/config.jsonnet @@ -2,7 +2,17 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'extend', + transforms: [ + sub.tf.test.message({ value: {"a":[1,2],"z":[3,4]} }), + sub.tf.send.stdout(), + ], + // Asserts that 'a' contains 4 elements. + condition: sub.cnd.num.len.eq({ obj: {src: 'a'}, value: 4 }), + } + ], transforms: [ // Append the value of `z` to `a` (using the `-1` array index). sub.tf.object.copy({ object: { source_key: 'z', target_key: 'a.-1' } }), diff --git a/examples/transform/array/extend/data.json b/examples/transform/array/extend/data.json deleted file mode 100644 index 9915a5f3..00000000 --- a/examples/transform/array/extend/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":[1,2],"z":[3,4]} diff --git a/examples/transform/array/extend/stdout.txt b/examples/transform/array/extend/stdout.txt deleted file mode 100644 index f8cc26de..00000000 --- a/examples/transform/array/extend/stdout.txt +++ /dev/null @@ -1,4 +0,0 @@ -{ - "a": [1, 2, 3, 4], - "z": [3, 4] -} diff --git a/examples/transform/array/flatten/config.jsonnet b/examples/transform/array/flatten/config.jsonnet index 2eac7700..b2d3a954 100644 --- a/examples/transform/array/flatten/config.jsonnet +++ b/examples/transform/array/flatten/config.jsonnet @@ -2,7 +2,17 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'extend', + transforms: [ + sub.tf.test.message({ value: {"a":[1,2,[3,4]]} }), + sub.tf.send.stdout(), + ], + // Asserts that 'a' contains 4 elements. + condition: sub.cnd.num.len.eq({ obj: {src: 'a'}, value: 4 }), + } + ], transforms: [ // Flatten by copying the value and chaining GJSON's `@flatten` operator. sub.tf.obj.cp({ object: { source_key: 'a|@flatten', target_key: 'a' } }), diff --git a/examples/transform/array/flatten/data.json b/examples/transform/array/flatten/data.json deleted file mode 100644 index 667836a7..00000000 --- a/examples/transform/array/flatten/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":[1,2,[3,4]]} diff --git a/examples/transform/array/flatten/stdout.txt b/examples/transform/array/flatten/stdout.txt deleted file mode 100644 index 41096f70..00000000 --- a/examples/transform/array/flatten/stdout.txt +++ /dev/null @@ -1,3 +0,0 @@ -{ - "a": [1, 2, 3, 4] -} diff --git a/examples/transform/array/flatten_deep/config.jsonnet b/examples/transform/array/flatten_deep/config.jsonnet index ac726628..7ff36dc9 100644 --- a/examples/transform/array/flatten_deep/config.jsonnet +++ b/examples/transform/array/flatten_deep/config.jsonnet @@ -2,7 +2,17 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'extend', + transforms: [ + sub.tf.test.message({ value: {"a":[1,2,[3,4,[5,6]]]} }), + sub.tf.send.stdout(), + ], + // Asserts that 'a' contains 6 elements. + condition: sub.cnd.num.len.eq({ obj: {src: 'a'}, value: 6 }), + } + ], transforms: [ // Flatten by copying the value and chaining GJSON's `@flatten` operator // with the `deep` option. diff --git a/examples/transform/array/flatten_deep/data.json b/examples/transform/array/flatten_deep/data.json deleted file mode 100644 index 852ff7c7..00000000 --- a/examples/transform/array/flatten_deep/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":[1,2,[3,4,[5,6]]]} diff --git a/examples/transform/array/flatten_deep/stdout.txt b/examples/transform/array/flatten_deep/stdout.txt deleted file mode 100644 index 542900c5..00000000 --- a/examples/transform/array/flatten_deep/stdout.txt +++ /dev/null @@ -1,3 +0,0 @@ -{ - "a": [1, 2, 3, 4, 5, 6] -} diff --git a/examples/transform/array/group/config.jsonnet b/examples/transform/array/group/config.jsonnet index 9edbf886..65426ea1 100644 --- a/examples/transform/array/group/config.jsonnet +++ b/examples/transform/array/group/config.jsonnet @@ -5,11 +5,30 @@ local sub = import '../../../../substation.libsonnet'; local files_key = 'meta files'; { - concurrency: 1, + tests: [ + { + name: 'group', + transforms: [ + sub.tf.test.message({ value: {"file_name":["foo.txt","bar.html"],"file_type":["text/plain","text/html"],"file_size":[100,500]} }), + sub.tf.send.stdout(), + ], + // Asserts that each element in the array contains these keys: + // - file_name + // - file_type + // - file_size + // - file_extension + condition: sub.cnd.meta.all({ + object: { source_key: '@this' }, + conditions: [ + sub.cnd.num.len.greater_than({ obj: { src: 'file_type' }, value: 0 }), + sub.cnd.num.len.greater_than({ obj: { src: 'file_size' }, value: 0 }), + sub.cnd.num.len.greater_than({ obj: { src: 'file_name' }, value: 0 }), + sub.cnd.num.len.greater_than({ obj: { src: 'file_extension' }, value: 0 }), + ], + }) + } + ], transforms: [ - // This example sends data to stdout at each step to iteratively show - // how the data is transformed. - sub.tf.send.stdout(), // Copy the object to metadata, where it is grouped. sub.tf.obj.cp({ object: { target_key: files_key } }), // Elements from the file_name array are transformed and derived file extensions diff --git a/examples/transform/array/group/data.json b/examples/transform/array/group/data.json deleted file mode 100644 index a0e2819a..00000000 --- a/examples/transform/array/group/data.json +++ /dev/null @@ -1 +0,0 @@ -{"file_name":["foo.txt","bar.html"],"file_type":["text/plain","text/html"],"file_size":[100,500]} diff --git a/examples/transform/array/group/stdout.txt b/examples/transform/array/group/stdout.txt deleted file mode 100644 index 8c9b507a..00000000 --- a/examples/transform/array/group/stdout.txt +++ /dev/null @@ -1,15 +0,0 @@ -{"file_name":["foo.txt","bar.html"],"file_type":["text/plain","text/html"],"file_size":[100,500]} -[ - { - "file_name": "foo.txt", - "file_type": "text/plain", - "file_size": 100, - "file_extension": "txt" - }, - { - "file_name": "bar.html", - "file_type": "text/html", - "file_size": 500, - "file_extension": "html" - } -] diff --git a/examples/transform/enrich/kvstore_csv/config.jsonnet b/examples/transform/enrich/kvstore_csv/config.jsonnet index 1b5fa569..87811726 100644 --- a/examples/transform/enrich/kvstore_csv/config.jsonnet +++ b/examples/transform/enrich/kvstore_csv/config.jsonnet @@ -10,6 +10,17 @@ local sub = import '../../../../substation.libsonnet'; local kv = sub.kv_store.csv_file({ file: 'kv.csv', column: 'product' }); { + tests: [ + { + name: 'kvstore_csv', + transforms: [ + sub.tf.test.message({ value: {"product":"churro"} }), + sub.tf.send.stdout(), + ], + // Asserts that the message contains product info. + condition: sub.cnd.num.len.gt({ object: { source_key: 'info' }, value: 0 }), + } + ], transforms: [ // The CSV file KV store returns the entire row minus the key column. // For example, this returns {"price":"9.99","calories":"500"} for "churro". @@ -17,7 +28,6 @@ local kv = sub.kv_store.csv_file({ file: 'kv.csv', column: 'product' }); object: { source_key: 'product', target_key: 'info' }, kv_store: kv, }), - sub.tf.obj.cp({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/enrich/kvstore_csv/data.jsonl b/examples/transform/enrich/kvstore_csv/data.jsonl deleted file mode 100644 index 19a662e7..00000000 --- a/examples/transform/enrich/kvstore_csv/data.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"product":"churro"} diff --git a/examples/transform/enrich/kvstore_csv/stdout.txt b/examples/transform/enrich/kvstore_csv/stdout.txt deleted file mode 100644 index 88d0c953..00000000 --- a/examples/transform/enrich/kvstore_csv/stdout.txt +++ /dev/null @@ -1,7 +0,0 @@ -{ - "product": "churro", - "info": { - "calories": "500", - "price": "9.99" - } -} diff --git a/examples/transform/enrich/kvstore_json/config.jsonnet b/examples/transform/enrich/kvstore_json/config.jsonnet index 723b5902..0cc5a1ec 100644 --- a/examples/transform/enrich/kvstore_json/config.jsonnet +++ b/examples/transform/enrich/kvstore_json/config.jsonnet @@ -7,12 +7,22 @@ local sub = import '../../../../substation.libsonnet'; local kv = sub.kv_store.json_file({ file: 'kv.json' }); { + tests: [ + { + name: 'kvstore_json', + transforms: [ + sub.tf.test.message({ value: {"product":"churro"} }), + sub.tf.send.stdout(), + ], + // Asserts that the message contains product info. + condition: sub.cnd.num.len.gt({ object: { source_key: 'price' }, value: 0 }), + } + ], transforms: [ sub.tf.enrich.kv_store.item.get({ object: { source_key: 'product', target_key: 'price' }, kv_store: kv, }), - sub.tf.obj.cp({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/enrich/kvstore_json/data.jsonl b/examples/transform/enrich/kvstore_json/data.jsonl deleted file mode 100644 index 19a662e7..00000000 --- a/examples/transform/enrich/kvstore_json/data.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"product":"churro"} diff --git a/examples/transform/enrich/kvstore_json/kv.json b/examples/transform/enrich/kvstore_json/kv.json new file mode 100644 index 00000000..7d22510e --- /dev/null +++ b/examples/transform/enrich/kvstore_json/kv.json @@ -0,0 +1 @@ +{"churro":9.99} diff --git a/examples/transform/enrich/kvstore_json/stdout.txt b/examples/transform/enrich/kvstore_json/stdout.txt deleted file mode 100644 index 46724035..00000000 --- a/examples/transform/enrich/kvstore_json/stdout.txt +++ /dev/null @@ -1,4 +0,0 @@ -{ - "product": "churro", - "price": 9.99 -} diff --git a/examples/transform/enrich/kvstore_set_add/config.jsonnet b/examples/transform/enrich/kvstore_set_add/config.jsonnet index 1ca44d5a..fe70453b 100644 --- a/examples/transform/enrich/kvstore_set_add/config.jsonnet +++ b/examples/transform/enrich/kvstore_set_add/config.jsonnet @@ -7,6 +7,22 @@ local sub = import '../../../../substation.libsonnet'; local mem = sub.kv_store.memory(); { + tests: [ + { + name: 'kvstore_set_add', + transforms: [ + sub.tf.test.message({ value: {"date": "2021-01-01","customer":"alice@brex.com","order":"pizza"} }), + sub.tf.test.message({ value: {"date": "2021-01-01","customer":"bob@brex.com","order":"burger"} }), + sub.tf.test.message({ value: {"date": "2021-01-03","customer":"bob@brex.com","order":"pizza"} }), + sub.tf.test.message({ value: {"date": "2021-01-07","customer":"alice@brex.com","order":"pizza"} }), + sub.tf.test.message({ value: {"date": "2021-01-07","customer":"bob@brex.com","order":"burger"} }), + sub.tf.test.message({ value: {"date": "2021-01-13","customer":"alice@brex.com","order":"pizza"} }), + sub.tf.send.stdout(), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // Each order is stored in memory indexed by the customer's email // address and printed to stdout. Only unique orders are stored. @@ -21,7 +37,6 @@ local mem = sub.kv_store.memory(); object: { source_key: 'customer', target_key: 'kv_store' }, kv_store: mem, }), - sub.tf.obj.cp({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/enrich/kvstore_set_add/data.jsonl b/examples/transform/enrich/kvstore_set_add/data.jsonl deleted file mode 100644 index 5bf38854..00000000 --- a/examples/transform/enrich/kvstore_set_add/data.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"date": "2021-01-01","customer":"alice@brex.com","order":"pizza"} -{"date": "2021-01-01","customer":"bob@brex.com","order":"burger"} -{"date": "2021-01-03","customer":"bob@brex.com","order":"pizza"} -{"date": "2021-01-07","customer":"alice@brex.com","order":"pizza"} -{"date": "2021-01-07","customer":"bob@brex.com","order":"burger"} -{"date": "2021-01-13","customer":"alice@brex.com","order":"pizza"} diff --git a/examples/transform/enrich/kvstore_set_add/stdout.txt b/examples/transform/enrich/kvstore_set_add/stdout.txt deleted file mode 100644 index 76487eca..00000000 --- a/examples/transform/enrich/kvstore_set_add/stdout.txt +++ /dev/null @@ -1,41 +0,0 @@ -{ - "date": "2021-01-07", - "customer": "alice@brex.com", - "order": "pizza", - "kv_store": ["pizza"] -} - -{ - "date": "2021-01-01", - "customer": "alice@brex.com", - "order": "pizza", - "kv_store": ["pizza"] -} - -{ - "date": "2021-01-03", - "customer": "bob@brex.com", - "order": "pizza", - "kv_store": ["burger", "pizza"] -} - -{ - "date": "2021-01-07", - "customer": "bob@brex.com", - "order": "burger", - "kv_store": ["burger", "pizza"] -} - -{ - "date": "2021-01-01", - "customer": "bob@brex.com", - "order": "burger", - "kv_store": ["burger", "pizza"] -} - -{ - "date": "2021-01-13", - "customer": "alice@brex.com", - "order": "pizza", - "kv_store": ["pizza"] -} diff --git a/examples/transform/enrich/mmdb/config.jsonnet b/examples/transform/enrich/mmdb/config.jsonnet index 9097fedc..5a717502 100644 --- a/examples/transform/enrich/mmdb/config.jsonnet +++ b/examples/transform/enrich/mmdb/config.jsonnet @@ -1,12 +1,27 @@ local sub = import '../../../../substation.libsonnet'; -local city = sub.kv_store.mmdb({ file: 'https://gist.github.com/jshlbrd/59641ccc71ba2873fb204ac44d101640/raw/3ad0e8c09563c614c50de4671caef8c1983cbb4d/GeoLite2-City.mmdb' }); - local asn = sub.kv_store.mmdb({ file: 'https://gist.github.com/jshlbrd/59641ccc71ba2873fb204ac44d101640/raw/3ad0e8c09563c614c50de4671caef8c1983cbb4d/GeoLite2-ASN.mmdb' }); { + tests: [ + { + name: 'mmdb-cloudflare', + transforms: [ + sub.tf.test.message({ value: {"ip":"1.1.1.1"} }), + ], + // Asserts that the message contains ASN info. + condition: sub.cnd.str.eq({ obj: {src: 'asn.autonomous_system_organization'}, value: 'CLOUDFLARENET' }), + }, + { + name: 'mmdb-google', + transforms: [ + sub.tf.test.message({ value: {"ip":"8.8.8.8"} }), + ], + // Asserts that the message contains ASN info. + condition: sub.cnd.str.eq({ obj: {src: 'asn.autonomous_system_organization'}, value: 'GOOGLE' }), + } + ], transforms: [ - sub.tf.enrich.kv_store.item.get({ object: { source_key: 'ip', target_key: 'city' }, kv_store: city }), sub.tf.enrich.kv_store.item.get({ object: { source_key: 'ip', target_key: 'asn' }, kv_store: asn }), sub.tf.obj.cp({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), diff --git a/examples/transform/enrich/mmdb/data.jsonl b/examples/transform/enrich/mmdb/data.jsonl deleted file mode 100644 index 6ea857b7..00000000 --- a/examples/transform/enrich/mmdb/data.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"ip":"1.1.1.1"} -{"ip":"8.8.8.8"} -{"ip":"9.9.9.9"} diff --git a/examples/transform/enrich/mmdb/stdout.txt b/examples/transform/enrich/mmdb/stdout.txt deleted file mode 100644 index c14c9930..00000000 --- a/examples/transform/enrich/mmdb/stdout.txt +++ /dev/null @@ -1,172 +0,0 @@ -{ - "ip": "8.8.8.8", - "city": { - "continent": { - "code": "NA", - "geoname_id": 6255149, - "names": { - "de": "Nordamerika", - "en": "North America", - "es": "Norteamérica", - "fr": "Amérique du Nord", - "ja": "北アメリカ", - "pt-BR": "América do Norte", - "ru": "Северная Америка", - "zh-CN": "北美洲" - } - }, - "country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "Vereinigte Staaten", - "en": "United States", - "es": "Estados Unidos", - "fr": "États Unis", - "ja": "アメリカ", - "pt-BR": "EUA", - "ru": "США", - "zh-CN": "美国" - } - }, - "location": { - "accuracy_radius": 1000, - "latitude": 37.751, - "longitude": -97.822, - "time_zone": "America/Chicago" - }, - "registered_country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "Vereinigte Staaten", - "en": "United States", - "es": "Estados Unidos", - "fr": "États Unis", - "ja": "アメリカ", - "pt-BR": "EUA", - "ru": "США", - "zh-CN": "美国" - } - } - }, - "asn": { - "autonomous_system_number": 15169, - "autonomous_system_organization": "GOOGLE" - } -} - -{ - "ip": "9.9.9.9", - "city": { - "city": { - "geoname_id": 5327684, - "names": { - "de": "Berkeley", - "en": "Berkeley", - "es": "Berkeley", - "fr": "Berkeley", - "ja": "バークリー", - "pt-BR": "Berkeley", - "ru": "Беркли", - "zh-CN": "伯克利" - } - }, - "continent": { - "code": "NA", - "geoname_id": 6255149, - "names": { - "de": "Nordamerika", - "en": "North America", - "es": "Norteamérica", - "fr": "Amérique du Nord", - "ja": "北アメリカ", - "pt-BR": "América do Norte", - "ru": "Северная Америка", - "zh-CN": "北美洲" - } - }, - "country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "Vereinigte Staaten", - "en": "United States", - "es": "Estados Unidos", - "fr": "États Unis", - "ja": "アメリカ", - "pt-BR": "EUA", - "ru": "США", - "zh-CN": "美国" - } - }, - "location": { - "accuracy_radius": 20, - "latitude": 37.8767, - "longitude": -122.2676, - "metro_code": 807, - "time_zone": "America/Los_Angeles" - }, - "postal": { - "code": "94709" - }, - "registered_country": { - "geoname_id": 2658434, - "iso_code": "CH", - "names": { - "de": "Schweiz", - "en": "Switzerland", - "es": "Suiza", - "fr": "Suisse", - "ja": "スイス連邦", - "pt-BR": "Suíça", - "ru": "Швейцария", - "zh-CN": "瑞士" - } - }, - "subdivisions": [ - { - "geoname_id": 5332921, - "iso_code": "CA", - "names": { - "de": "Kalifornien", - "en": "California", - "es": "California", - "fr": "Californie", - "ja": "カリフォルニア州", - "pt-BR": "Califórnia", - "ru": "Калифорния", - "zh-CN": "加州" - } - } - ] - }, - "asn": { - "autonomous_system_number": 19281, - "autonomous_system_organization": "QUAD9-AS-1" - } -} - -{ - "ip": "1.1.1.1", - "city": { - "registered_country": { - "geoname_id": 2077456, - "iso_code": "AU", - "names": { - "de": "Australien", - "en": "Australia", - "es": "Australia", - "fr": "Australie", - "ja": "オーストラリア", - "pt-BR": "Austrália", - "ru": "Австралия", - "zh-CN": "澳大利亚" - } - } - }, - "asn": { - "autonomous_system_number": 13335, - "autonomous_system_organization": "CLOUDFLARENET" - } -} diff --git a/examples/transform/format/zip/config.jsonnet b/examples/transform/format/zip/config.jsonnet index 5a379018..a88a365f 100644 --- a/examples/transform/format/zip/config.jsonnet +++ b/examples/transform/format/zip/config.jsonnet @@ -5,6 +5,18 @@ local sub = import '../../../../substation.libsonnet'; { + tests: [ + { + name: 'zip', + transforms: [ + // This is a Zip file containing a CSV file and a JSONL file. + sub.tf.test.message({ value: 'UEsDBAoAAAAAAMeuFFmAS9oWGgAAABoAAAAIABwAZGF0YS5jc3ZVVAkAA/VyxWb1csVmdXgLAAEE9gEAAAQUAAAAZm9vLGJhcgpiYXoscXV4CnF1dXgsY29yZ2VQSwMEFAAAAAgAx64UWYViom8nAAAALAAAAAoAHABkYXRhLmpzb25sVVQJAAP1csVm9XLFZnV4CwABBPYBAAAEFAAAAKtWSsvPV7JSSkosUqrlqgbSVUBeYWkFmFdYCmRYKSXnF6WnKtUCAFBLAQIeAwoAAAAAAMeuFFmAS9oWGgAAABoAAAAIABgAAAAAAAEAAACkgQAAAABkYXRhLmNzdlVUBQAD9XLFZnV4CwABBPYBAAAEFAAAAFBLAQIeAxQAAAAIAMeuFFmFYqJvJwAAACwAAAAKABgAAAAAAAEAAACkgVwAAABkYXRhLmpzb25sVVQFAAP1csVmdXgLAAEE9gEAAAQUAAAAUEsFBgAAAAACAAIAngAAAMcAAAAAAA==' }), + sub.tf.fmt.from.b64(), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // Unzip the file. The contents of each file in the Zip file are // now messages in the pipeline (including EOL characters, if any). diff --git a/examples/transform/format/zip/data.csv b/examples/transform/format/zip/data.csv deleted file mode 100644 index d6e746e3..00000000 --- a/examples/transform/format/zip/data.csv +++ /dev/null @@ -1,3 +0,0 @@ -foo,bar -baz,qux -quux,corge \ No newline at end of file diff --git a/examples/transform/format/zip/data.jsonl b/examples/transform/format/zip/data.jsonl deleted file mode 100644 index b7519d55..00000000 --- a/examples/transform/format/zip/data.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"foo":"bar"} -{"baz":"qux"} -{"quux":"corge"} \ No newline at end of file diff --git a/examples/transform/format/zip/stdout.txt b/examples/transform/format/zip/stdout.txt deleted file mode 100644 index 5f8d1265..00000000 --- a/examples/transform/format/zip/stdout.txt +++ /dev/null @@ -1,6 +0,0 @@ -foo,bar -baz,qux -quux,corge -{"foo":"bar"} -{"baz":"qux"} -{"quux":"corge"} diff --git a/examples/transform/meta/default_value/config.jsonnet b/examples/transform/meta/default_value/config.jsonnet index 8ae2e064..77719789 100644 --- a/examples/transform/meta/default_value/config.jsonnet +++ b/examples/transform/meta/default_value/config.jsonnet @@ -1,8 +1,20 @@ local sub = import '../../../../substation.libsonnet'; { + tests: [ + { + name: 'default_value', + transforms: [ + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.send.stdout(), + ], + // Asserts that 'z' exists. + condition: sub.cnd.num.len.greater_than({ obj: {src: 'z'}, value: 0 }), + } + ], transforms: [ sub.tf.object.insert({ object: { target_key: 'z' }, value: true }), + sub.tf.send.stdout(), // This simulates a transform that may not produce an output. sub.tf.meta.switch({ cases: [ { @@ -12,7 +24,6 @@ local sub = import '../../../../substation.libsonnet'; ], }, ] }), - sub.tf.object.copy({ source_key: '@pretty' }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/meta/default_value/data.json b/examples/transform/meta/default_value/data.json deleted file mode 100644 index b6e81411..00000000 --- a/examples/transform/meta/default_value/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"} diff --git a/examples/transform/meta/each_in_array/config.jsonnet b/examples/transform/meta/each_in_array/config.jsonnet index ef596e4a..3de8710f 100644 --- a/examples/transform/meta/each_in_array/config.jsonnet +++ b/examples/transform/meta/each_in_array/config.jsonnet @@ -4,6 +4,25 @@ local sub = import '../../../../substation.libsonnet'; { + tests: [ + { + name: 'each_in_array', + transforms: [ + sub.tf.test.message({ value: {"a":[{"b":1,"c":2},{"b":3,"c":4}]} }), + sub.tf.send.stdout(), + ], + // Asserts that each object in the array 'a' has: + // - No key 'b' + // - Key 'z' with value 'true' + condition: sub.cnd.meta.all({ + object: { src: 'a' }, + conditions: [ + sub.cnd.num.len.eq({ obj: {src: 'b'}, value: 0 }), + sub.cnd.str.eq({ obj: {src: 'z'}, value: 'true' }), + ], + }), + } + ], transforms: [ sub.tf.meta.for_each({ object: { source_key: 'a', target_key: 'a' }, @@ -15,7 +34,6 @@ local sub = import '../../../../substation.libsonnet'; sub.tf.object.insert({ object: { target_key: 'z' }, value: true }), ], }), - sub.tf.object.copy({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/meta/each_in_array/data.json b/examples/transform/meta/each_in_array/data.json deleted file mode 100644 index b3c96393..00000000 --- a/examples/transform/meta/each_in_array/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":[{"b":1,"c":2},{"b":3,"c":4}]} diff --git a/examples/transform/meta/each_in_array/stdout.txt b/examples/transform/meta/each_in_array/stdout.txt deleted file mode 100644 index 600c8c15..00000000 --- a/examples/transform/meta/each_in_array/stdout.txt +++ /dev/null @@ -1,12 +0,0 @@ -{ - "a": [ - { - "c": 2, - "z": true - }, - { - "c": 4, - "z": true - } - ] -} diff --git a/examples/transform/meta/exactly_once_consumer/config.jsonnet b/examples/transform/meta/exactly_once_consumer/config.jsonnet index 67e58ec5..d2e52210 100644 --- a/examples/transform/meta/exactly_once_consumer/config.jsonnet +++ b/examples/transform/meta/exactly_once_consumer/config.jsonnet @@ -6,6 +6,23 @@ local sub = import '../../../../substation.libsonnet'; local kv = sub.kv_store.memory(); { + tests: [ + { + name: 'exactly_once_consumer', + transforms: [ + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"e":"f"} }), + sub.tf.test.message({ value: {"a":"b"} }), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // If a message acquires a lock, then it is tagged for inspection. sub.tf.meta.kv_store.lock(settings={ @@ -28,7 +45,6 @@ local kv = sub.kv_store.memory(); }, ] }), // At this point only locked messages exist in the pipeline. - sub.tf.object.copy({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/meta/exactly_once_consumer/data.jsonl b/examples/transform/meta/exactly_once_consumer/data.jsonl deleted file mode 100644 index 864e8bfd..00000000 --- a/examples/transform/meta/exactly_once_consumer/data.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"a":"b"} -{"a":"b"} -{"c":"d"} -{"a":"b"} -{"c":"d"} -{"c":"d"} -{"e":"f"} -{"a":"b"} diff --git a/examples/transform/meta/exactly_once_consumer/stdout.txt b/examples/transform/meta/exactly_once_consumer/stdout.txt deleted file mode 100644 index deb69e5f..00000000 --- a/examples/transform/meta/exactly_once_consumer/stdout.txt +++ /dev/null @@ -1,11 +0,0 @@ -{ - "a": "b" -} - -{ - "c": "d" -} - -{ - "e": "f" -} diff --git a/examples/transform/meta/exactly_once_producer/config.jsonnet b/examples/transform/meta/exactly_once_producer/config.jsonnet index 1e440fdc..f2660925 100644 --- a/examples/transform/meta/exactly_once_producer/config.jsonnet +++ b/examples/transform/meta/exactly_once_producer/config.jsonnet @@ -6,6 +6,23 @@ local sub = import '../../../../substation.libsonnet'; local kv = sub.kv_store.memory(); { + tests: [ + { + name: 'exactly_once_producer', + transforms: [ + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"e":"f"} }), + sub.tf.test.message({ value: {"a":"b"} }), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // This only prints messages that acquire a lock. Any message // that fails to acquire a lock will be skipped. An error in the @@ -17,7 +34,6 @@ local kv = sub.kv_store.memory(); prefix: 'eo_producer', ttl_offset: '1m', transforms: [ - sub.tf.object.copy({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], }), diff --git a/examples/transform/meta/exactly_once_producer/data.jsonl b/examples/transform/meta/exactly_once_producer/data.jsonl deleted file mode 100644 index 864e8bfd..00000000 --- a/examples/transform/meta/exactly_once_producer/data.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"a":"b"} -{"a":"b"} -{"c":"d"} -{"a":"b"} -{"c":"d"} -{"c":"d"} -{"e":"f"} -{"a":"b"} diff --git a/examples/transform/meta/exactly_once_producer/stdout.txt b/examples/transform/meta/exactly_once_producer/stdout.txt deleted file mode 100644 index deb69e5f..00000000 --- a/examples/transform/meta/exactly_once_producer/stdout.txt +++ /dev/null @@ -1,11 +0,0 @@ -{ - "a": "b" -} - -{ - "c": "d" -} - -{ - "e": "f" -} diff --git a/examples/transform/meta/exactly_once_system/config.jsonnet b/examples/transform/meta/exactly_once_system/config.jsonnet index b5a718ea..16529c09 100644 --- a/examples/transform/meta/exactly_once_system/config.jsonnet +++ b/examples/transform/meta/exactly_once_system/config.jsonnet @@ -6,6 +6,23 @@ local sub = import '../../../../substation.libsonnet'; local kv = sub.kv_store.memory(); { + tests: [ + { + name: 'exactly_once_producer', + transforms: [ + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"a":"b"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"c":"d"} }), + sub.tf.test.message({ value: {"e":"f"} }), + sub.tf.test.message({ value: {"a":"b"} }), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // All messages are locked before being sent through other transform // functions, ensuring that the message is processed only once. @@ -18,7 +35,6 @@ local kv = sub.kv_store.memory(); ttl_offset: '1m', transforms: [ sub.tf.obj.insert({ object: { target_key: 'processed' }, value: true }), - sub.tf.object.copy({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], }), diff --git a/examples/transform/meta/exactly_once_system/data.jsonl b/examples/transform/meta/exactly_once_system/data.jsonl deleted file mode 100644 index 864e8bfd..00000000 --- a/examples/transform/meta/exactly_once_system/data.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"a":"b"} -{"a":"b"} -{"c":"d"} -{"a":"b"} -{"c":"d"} -{"c":"d"} -{"e":"f"} -{"a":"b"} diff --git a/examples/transform/meta/exactly_once_system/stdout.txt b/examples/transform/meta/exactly_once_system/stdout.txt deleted file mode 100644 index 468b2b2c..00000000 --- a/examples/transform/meta/exactly_once_system/stdout.txt +++ /dev/null @@ -1,14 +0,0 @@ -{ - "a": "b", - "processed": true -} - -{ - "c": "d", - "processed": true -} - -{ - "e": "f", - "processed": true -} diff --git a/examples/transform/meta/execution_time/config.jsonnet b/examples/transform/meta/execution_time/config.jsonnet index 0984356d..c49ea3b1 100644 --- a/examples/transform/meta/execution_time/config.jsonnet +++ b/examples/transform/meta/execution_time/config.jsonnet @@ -6,6 +6,16 @@ local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; { + tests: [ + { + name: 'execution_time', + transforms: [ + sub.tf.test.message({ value: {"a":"b"} }), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // The `meta_metric_duration` transform measures the execution time of // the transform that it wraps. diff --git a/examples/transform/meta/execution_time/data.json b/examples/transform/meta/execution_time/data.json deleted file mode 100644 index b6e81411..00000000 --- a/examples/transform/meta/execution_time/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"} diff --git a/examples/transform/meta/execution_time/stdout.txt b/examples/transform/meta/execution_time/stdout.txt deleted file mode 100644 index d8b7594c..00000000 --- a/examples/transform/meta/execution_time/stdout.txt +++ /dev/null @@ -1,3 +0,0 @@ -{"_aws":{"Timestamp":1724300928469,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"ObjectCopyDuration"}]}]},"AppName":"example","ObjectCopyDuration":28209} -{"_aws":{"Timestamp":1724300928469,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"UtilityDelayDuration"}]}]},"AppName":"example","UtilityDelayDuration":101036000} -{"_aws":{"Timestamp":1724300928470,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"UtilityMultiDuration"}]}]},"AppName":"example","UtilityMultiDuration":303265541} diff --git a/examples/transform/meta/retry_with_backoff/config.jsonnet b/examples/transform/meta/retry_with_backoff/config.jsonnet index a2e52066..c35b3c18 100644 --- a/examples/transform/meta/retry_with_backoff/config.jsonnet +++ b/examples/transform/meta/retry_with_backoff/config.jsonnet @@ -13,6 +13,16 @@ local cnd = sub.cnd.all([ ]); { + tests: [ + { + name: 'retry_with_backoff', + transforms: [ + sub.tf.test.message({ value: {"a":"b"} }), + ], + // Asserts that the target key 'c' exists. + condition: sub.cnd.num.len.greater_than({ object: { source_key: key }, value: 1 }), + } + ], transforms: [ sub.tf.meta.retry({ transforms: [ @@ -21,7 +31,6 @@ local cnd = sub.cnd.all([ condition: cnd, // If this returns false, then the transforms are retried. retry: { delay: '1s', count: 4 }, // Retry up to 4 times with a 1 second backoff (1s, 1s, 1s, 1s). }), - sub.tf.object.copy({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/meta/retry_with_backoff/data.json b/examples/transform/meta/retry_with_backoff/data.json deleted file mode 100644 index b6e81411..00000000 --- a/examples/transform/meta/retry_with_backoff/data.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"} diff --git a/examples/transform/meta/retry_with_backoff/stdout.txt b/examples/transform/meta/retry_with_backoff/stdout.txt deleted file mode 100644 index 567a78fe..00000000 --- a/examples/transform/meta/retry_with_backoff/stdout.txt +++ /dev/null @@ -1,4 +0,0 @@ -{ - "a": "b", - "c": true -} diff --git a/examples/transform/number/clamp/config.jsonnet b/examples/transform/number/clamp/config.jsonnet index 5067dd2e..d9209bf5 100644 --- a/examples/transform/number/clamp/config.jsonnet +++ b/examples/transform/number/clamp/config.jsonnet @@ -2,7 +2,22 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'clamp', + transforms: [ + sub.tf.test.message({ value: -1 }), + sub.tf.test.message({ value: 101 }), + sub.tf.test.message({ value: 50 }), + sub.tf.send.stdout(), + ], + // Asserts that each number is within the range [0, 100]. + condition: sub.cnd.all([ + sub.cnd.num.greater_than({ value: -1 }), + sub.cnd.num.less_than({ value: 101 }), + ]), + } + ], transforms: [ sub.tf.number.maximum({ value: 0 }), sub.tf.number.minimum({ value: 100 }), diff --git a/examples/transform/number/clamp/data.txt b/examples/transform/number/clamp/data.txt deleted file mode 100644 index 7779e6cb..00000000 --- a/examples/transform/number/clamp/data.txt +++ /dev/null @@ -1,3 +0,0 @@ --1 -101 -50 diff --git a/examples/transform/number/clamp/stdout.txt b/examples/transform/number/clamp/stdout.txt deleted file mode 100644 index e7e321ee..00000000 --- a/examples/transform/number/clamp/stdout.txt +++ /dev/null @@ -1,3 +0,0 @@ -100 -0 -50 diff --git a/examples/transform/number/max/config.jsonnet b/examples/transform/number/max/config.jsonnet index 74709096..ed1f7647 100644 --- a/examples/transform/number/max/config.jsonnet +++ b/examples/transform/number/max/config.jsonnet @@ -3,7 +3,20 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'max', + transforms: [ + sub.tf.test.message({ value: 0 }), + sub.tf.test.message({ value: 10 }), + sub.tf.test.message({ value: -1 }), + sub.tf.test.message({ value: -1.1 }), + sub.tf.send.stdout(), + ], + // Asserts that each number is greater than -1. + condition: sub.cnd.num.greater_than({ value: -1 }), + } + ], transforms: [ sub.tf.num.max({ value: 0 }), sub.tf.send.stdout(), diff --git a/examples/transform/number/max/data.txt b/examples/transform/number/max/data.txt deleted file mode 100644 index a5c891fc..00000000 --- a/examples/transform/number/max/data.txt +++ /dev/null @@ -1,4 +0,0 @@ -0 --1 --1.1 -10 diff --git a/examples/transform/number/max/stdout.txt b/examples/transform/number/max/stdout.txt deleted file mode 100644 index 2360ec2f..00000000 --- a/examples/transform/number/max/stdout.txt +++ /dev/null @@ -1,4 +0,0 @@ -0 -0 -0 -10 diff --git a/examples/transform/number/min/config.jsonnet b/examples/transform/number/min/config.jsonnet index 9a70e26f..c117ce68 100644 --- a/examples/transform/number/min/config.jsonnet +++ b/examples/transform/number/min/config.jsonnet @@ -3,7 +3,20 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'min', + transforms: [ + sub.tf.test.message({ value: 0 }), + sub.tf.test.message({ value: 10 }), + sub.tf.test.message({ value: -1 }), + sub.tf.test.message({ value: -1.1 }), + sub.tf.send.stdout(), + ], + // Asserts that each number is less than 1. + condition: sub.cnd.num.less_than({ value: 1 }), + } + ], transforms: [ sub.tf.num.min({ value: 0 }), sub.tf.send.stdout(), diff --git a/examples/transform/number/min/data.txt b/examples/transform/number/min/data.txt deleted file mode 100644 index a5c891fc..00000000 --- a/examples/transform/number/min/data.txt +++ /dev/null @@ -1,4 +0,0 @@ -0 --1 --1.1 -10 diff --git a/examples/transform/number/min/stdout.txt b/examples/transform/number/min/stdout.txt deleted file mode 100644 index f27880a1..00000000 --- a/examples/transform/number/min/stdout.txt +++ /dev/null @@ -1,4 +0,0 @@ -0 --1 --1.1 -0 diff --git a/examples/transform/send/aux_transforms/config.jsonnet b/examples/transform/send/aux_transforms/config.jsonnet index 3d726095..eae9a327 100644 --- a/examples/transform/send/aux_transforms/config.jsonnet +++ b/examples/transform/send/aux_transforms/config.jsonnet @@ -5,7 +5,6 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, transforms: [ // By default all data is buffered before it is sent. sub.tf.send.stdout({ diff --git a/examples/transform/send/aws_s3_glacier/config.jsonnet b/examples/transform/send/aws_s3_glacier/config.jsonnet index b9784e6f..1f5c3750 100644 --- a/examples/transform/send/aws_s3_glacier/config.jsonnet +++ b/examples/transform/send/aws_s3_glacier/config.jsonnet @@ -5,7 +5,6 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, transforms: [ sub.tf.send.aws.s3({ // Glacier Instant Retrieval charges a minimum of 128KB per object, otherwise diff --git a/examples/transform/send/batch/config.jsonnet b/examples/transform/send/batch/config.jsonnet index 00477a44..f2f32c57 100644 --- a/examples/transform/send/batch/config.jsonnet +++ b/examples/transform/send/batch/config.jsonnet @@ -4,7 +4,6 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, transforms: [ sub.tf.object.copy({ object: { source_key: '@pretty' } }), // Each object is organized by the value retrieved from the `group_id` key. diff --git a/examples/transform/send/datadog/config.jsonnet b/examples/transform/send/datadog/config.jsonnet index ca6b8906..cd956ec1 100644 --- a/examples/transform/send/datadog/config.jsonnet +++ b/examples/transform/send/datadog/config.jsonnet @@ -13,7 +13,6 @@ local max_size = 1000 * 1000 * 5; local max_count = 1000; { - concurrency: 1, transforms: [ // Connections to the Datadog Logs API are authenticated using an API key. sub.transform.utility.secret({ secret: sub.secrets.environment_variable({ id: 'DD', name: 'DATADOG_API_KEY' }) }), diff --git a/examples/transform/send/splunk/config.jsonnet b/examples/transform/send/splunk/config.jsonnet index 9b2e60c0..5bb7bf65 100644 --- a/examples/transform/send/splunk/config.jsonnet +++ b/examples/transform/send/splunk/config.jsonnet @@ -11,7 +11,6 @@ local sub = import '../../../../substation.libsonnet'; local max_size = 1000 * 1000; { - concurrency: 1, transforms: [ // Connections to the Splunk HEC are authenticated using a token. sub.transform.utility.secret({ secret: sub.secrets.environment_variable({ id: 'SPLUNK', name: 'SPLUNK_TOKEN_ID' }) }), diff --git a/examples/transform/send/sumologic/config.jsonnet b/examples/transform/send/sumologic/config.jsonnet index 90252a03..2e030a9c 100644 --- a/examples/transform/send/sumologic/config.jsonnet +++ b/examples/transform/send/sumologic/config.jsonnet @@ -9,7 +9,6 @@ local sub = import '../../../../substation.libsonnet'; local max_size = 1000 * 1000; { - concurrency: 1, transforms: [ sub.tf.send.http.post({ batch: { size: max_size }, diff --git a/examples/transform/time/str_conversion/config.jsonnet b/examples/transform/time/str_conversion/config.jsonnet index ea16d4e5..3db00c3a 100644 --- a/examples/transform/time/str_conversion/config.jsonnet +++ b/examples/transform/time/str_conversion/config.jsonnet @@ -2,7 +2,17 @@ local sub = import '../../../../substation.libsonnet'; { - concurrency: 1, + tests: [ + { + name: 'str_conversion', + transforms: [ + sub.tf.test.message({ value: {"time":"2024-01-01T01:02:03.123Z"}}), + sub.tf.send.stdout(), + ], + // Asserts that the time value is equal to the expected value. + condition: sub.cnd.str.eq({ obj: {src: 'time'}, value: '2024-01-01T01:02:03' }), + } + ], // Substation uses Go's pattern-based time format (https://gobyexample.com/time-formatting-parsing) // to convert time values to and from strings. All time values in the system are in epoch / Unix format // with nanosecond precision. @@ -11,7 +21,6 @@ local sub = import '../../../../substation.libsonnet'; sub.tf.time.from.string({ obj: { source_key: 'time', target_key: 'time' }, format: '2006-01-02T15:04:05.000Z' }), // This converts the Unix time back to a string. sub.tf.time.to.string({ obj: { source_key: 'time', target_key: 'time' }, format: '2006-01-02T15:04:05' }), - sub.tf.obj.cp({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], } diff --git a/examples/transform/time/str_conversion/data.json b/examples/transform/time/str_conversion/data.json deleted file mode 100644 index 86e52d7c..00000000 --- a/examples/transform/time/str_conversion/data.json +++ /dev/null @@ -1 +0,0 @@ -{"time":"2024-01-01T01:02:03.123Z"} diff --git a/examples/transform/time/str_conversion/stdout.txt b/examples/transform/time/str_conversion/stdout.txt deleted file mode 100644 index b05f7192..00000000 --- a/examples/transform/time/str_conversion/stdout.txt +++ /dev/null @@ -1,3 +0,0 @@ -{ - "time": "2024-01-01T01:02:03" -} diff --git a/examples/transform/utility/generate_ctrl/config.jsonnet b/examples/transform/utility/generate_ctrl/config.jsonnet index be754bdf..63f453df 100644 --- a/examples/transform/utility/generate_ctrl/config.jsonnet +++ b/examples/transform/utility/generate_ctrl/config.jsonnet @@ -5,9 +5,32 @@ local sub = import '../../../../substation.libsonnet'; { + tests: [ + { + name: 'generate_ctrl', + transforms: [ + sub.tf.test.message({ value: {"a":"b"}}), + sub.tf.test.message({ value: {"c":"d"}}), + sub.tf.test.message({ value: {"e":"f"}}), + sub.tf.test.message({ value: {"g":"h"}}), + sub.tf.test.message({ value: {"i":"j"}}), + sub.tf.test.message({ value: {"k":"l"}}), + sub.tf.test.message({ value: {"m":"n"}}), + sub.tf.test.message({ value: {"o":"p"}}), + sub.tf.test.message({ value: {"q":"r"}}), + sub.tf.test.message({ value: {"s":"t"}}), + sub.tf.test.message({ value: {"u":"v"}}), + sub.tf.test.message({ value: {"w":"x"}}), + sub.tf.test.message({ value: {"y":"z"}}), + sub.tf.send.stdout(), + ], + // Asserts that the number of objects in each array is less than 3. + condition: sub.cnd.num.len.less_than({ obj: {src: '@this'}, value: 3 }), + } + ], transforms: [ sub.tf.utility.control({ batch: { count: 2 } }), - sub.tf.aggregate.to.array({ batch: { count: 10000 } }), + sub.tf.aggregate.to.array(), sub.tf.obj.cp({ object: { source_key: '@pretty' } }), sub.tf.send.stdout(), ], diff --git a/examples/transform/utility/generate_ctrl/data.jsonl b/examples/transform/utility/generate_ctrl/data.jsonl deleted file mode 100644 index d101df01..00000000 --- a/examples/transform/utility/generate_ctrl/data.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"a":"b"} -{"c":"d"} -{"e":"f"} -{"g":"h"} -{"i":"j"} -{"k":"l"} -{"m":"n"} -{"o":"p"} -{"q":"r"} -{"s":"t"} -{"u":"v"} -{"w":"x"} -{"y":"z"} diff --git a/examples/transform/utility/generate_ctrl/stdout.txt b/examples/transform/utility/generate_ctrl/stdout.txt deleted file mode 100644 index 71a916df..00000000 --- a/examples/transform/utility/generate_ctrl/stdout.txt +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "a": "b" - }, - { - "c": "d" - } -] - -[ - { - "e": "f" - }, - { - "g": "h" - } -] - -[ - { - "i": "j" - }, - { - "k": "l" - } -] - -[ - { - "m": "n" - }, - { - "o": "p" - } -] - -[ - { - "q": "r" - }, - { - "s": "t" - } -] - -[ - { - "u": "v" - }, - { - "w": "x" - } -] - -[ - { - "y": "z" - } -] diff --git a/examples/transform/utility/message_bytes/config.jsonnet b/examples/transform/utility/message_bytes/config.jsonnet index 8fbdc123..e3e9d462 100644 --- a/examples/transform/utility/message_bytes/config.jsonnet +++ b/examples/transform/utility/message_bytes/config.jsonnet @@ -6,6 +6,28 @@ local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; { + tests: [ + { + name: 'message_bytes', + transforms: [ + sub.tf.test.message({ value: {"a":"b"}}), + sub.tf.test.message({ value: {"c":"d"}}), + sub.tf.test.message({ value: {"e":"f"}}), + sub.tf.test.message({ value: {"g":"h"}}), + sub.tf.test.message({ value: {"i":"j"}}), + sub.tf.test.message({ value: {"k":"l"}}), + sub.tf.test.message({ value: {"m":"n"}}), + sub.tf.test.message({ value: {"o":"p"}}), + sub.tf.test.message({ value: {"q":"r"}}), + sub.tf.test.message({ value: {"s":"t"}}), + sub.tf.test.message({ value: {"u":"v"}}), + sub.tf.test.message({ value: {"w":"x"}}), + sub.tf.test.message({ value: {"y":"z"}}), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // If the transform is configured first, then the metric reflects // the sum of bytes received by Substation. diff --git a/examples/transform/utility/message_bytes/data.jsonl b/examples/transform/utility/message_bytes/data.jsonl deleted file mode 100644 index d101df01..00000000 --- a/examples/transform/utility/message_bytes/data.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"a":"b"} -{"c":"d"} -{"e":"f"} -{"g":"h"} -{"i":"j"} -{"k":"l"} -{"m":"n"} -{"o":"p"} -{"q":"r"} -{"s":"t"} -{"u":"v"} -{"w":"x"} -{"y":"z"} diff --git a/examples/transform/utility/message_bytes/stdout.txt b/examples/transform/utility/message_bytes/stdout.txt deleted file mode 100644 index 4141fade..00000000 --- a/examples/transform/utility/message_bytes/stdout.txt +++ /dev/null @@ -1,2 +0,0 @@ -{"_aws":{"Timestamp":1724299043477,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"BytesReceived"}]}]},"AppName":"example","BytesReceived":117} -{"_aws":{"Timestamp":1724299043477,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"BytesTransformed"}]}]},"AppName":"example","BytesTransformed":195} diff --git a/examples/transform/utility/message_count/config.jsonnet b/examples/transform/utility/message_count/config.jsonnet index 9443e128..f12c6cf9 100644 --- a/examples/transform/utility/message_count/config.jsonnet +++ b/examples/transform/utility/message_count/config.jsonnet @@ -6,6 +6,28 @@ local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; { + tests: [ + { + name: 'message_count', + transforms: [ + sub.tf.test.message({ value: {"a":"b"}}), + sub.tf.test.message({ value: {"c":"d"}}), + sub.tf.test.message({ value: {"e":"f"}}), + sub.tf.test.message({ value: {"g":"h"}}), + sub.tf.test.message({ value: {"i":"j"}}), + sub.tf.test.message({ value: {"k":"l"}}), + sub.tf.test.message({ value: {"m":"n"}}), + sub.tf.test.message({ value: {"o":"p"}}), + sub.tf.test.message({ value: {"q":"r"}}), + sub.tf.test.message({ value: {"s":"t"}}), + sub.tf.test.message({ value: {"u":"v"}}), + sub.tf.test.message({ value: {"w":"x"}}), + sub.tf.test.message({ value: {"y":"z"}}), + ], + // Asserts that each message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ // If the transform is configured first, then the count reflects // the number of messages received by Substation. diff --git a/examples/transform/utility/message_count/data.jsonl b/examples/transform/utility/message_count/data.jsonl deleted file mode 100644 index d101df01..00000000 --- a/examples/transform/utility/message_count/data.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"a":"b"} -{"c":"d"} -{"e":"f"} -{"g":"h"} -{"i":"j"} -{"k":"l"} -{"m":"n"} -{"o":"p"} -{"q":"r"} -{"s":"t"} -{"u":"v"} -{"w":"x"} -{"y":"z"} diff --git a/examples/transform/utility/message_count/stdout.txt b/examples/transform/utility/message_count/stdout.txt deleted file mode 100644 index 5589b801..00000000 --- a/examples/transform/utility/message_count/stdout.txt +++ /dev/null @@ -1,2 +0,0 @@ -{"_aws":{"Timestamp":1724299095089,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"MessagesReceived"}]}]},"AppName":"example","MessagesReceived":13} -{"_aws":{"Timestamp":1724299095089,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName"]],"Metrics":[{"Name":"MessagesTransformed"}]}]},"AppName":"example","MessagesTransformed":0} diff --git a/examples/transform/utility/message_freshness/config.jsonnet b/examples/transform/utility/message_freshness/config.jsonnet index 2bbfef4e..473075b1 100644 --- a/examples/transform/utility/message_freshness/config.jsonnet +++ b/examples/transform/utility/message_freshness/config.jsonnet @@ -14,6 +14,16 @@ local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; { + tests: [ + { + name: 'message_freshness', + transforms: [ + sub.tf.test.message({ value: {"timestamp":1724299266000000000}}), + ], + // Asserts that the message is not empty. + condition: sub.cnd.num.len.gt({ value: 0 }), + } + ], transforms: [ sub.transform.utility.metric.freshness({ threshold: '5s', // Amount of time spent in the system before considered stale. diff --git a/examples/transform/utility/message_freshness/data.jsonl b/examples/transform/utility/message_freshness/data.jsonl deleted file mode 100644 index 7ba48645..00000000 --- a/examples/transform/utility/message_freshness/data.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"timestamp":1724299266000000000} diff --git a/examples/transform/utility/message_freshness/stdout.txt b/examples/transform/utility/message_freshness/stdout.txt deleted file mode 100644 index db57f7be..00000000 --- a/examples/transform/utility/message_freshness/stdout.txt +++ /dev/null @@ -1,2 +0,0 @@ -{"_aws":{"Timestamp":1724299370516,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["AppName","FreshnessType"]],"Metrics":[{"Name":"MessageFreshness"}]}]},"AppName":"example","FreshnessType":"Success","MessageFreshness":0} -{"_aws":{"Timestamp":1724299370516,"CloudWatchMetrics":[{"Namespace":"Substation","Dimensions":[["FreshnessType","AppName"]],"Metrics":[{"Name":"MessageFreshness"}]}]},"FreshnessType":"Failure","AppName":"example","MessageFreshness":1}