diff --git a/examples/condition/meta/config.jsonnet b/examples/condition/meta/config.jsonnet index 19028494..f6d8179f 100644 --- a/examples/condition/meta/config.jsonnet +++ b/examples/condition/meta/config.jsonnet @@ -1,7 +1,7 @@ // This example determines if all values in an array are email addresses // that have the DNS domain "brex.com". This technique can be used to // validate or summarize values in an array. -local sub = import '../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/condition/number/config.jsonnet b/examples/condition/number/config.jsonnet index cbbff57a..f3618437 100644 --- a/examples/condition/number/config.jsonnet +++ b/examples/condition/number/config.jsonnet @@ -1,5 +1,5 @@ // This example shows usage of the 'number.equal_to' and 'number.greater_than' conditions. -local sub = import '../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/condition/string/config.jsonnet b/examples/condition/string/config.jsonnet index 6b7785da..a1237316 100644 --- a/examples/condition/string/config.jsonnet +++ b/examples/condition/string/config.jsonnet @@ -1,5 +1,5 @@ // This example shows usage of the 'string.equal_to' and 'string.greater_than' conditions. -local sub = import '../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/aggregate/sample/config.jsonnet b/examples/transform/aggregate/sample/config.jsonnet index 0904be2e..e1a652cb 100644 --- a/examples/transform/aggregate/sample/config.jsonnet +++ b/examples/transform/aggregate/sample/config.jsonnet @@ -1,7 +1,7 @@ // This example samples data by aggregating events into an array, then // selecting the first event in the array as a sample. The sampling rate // is 1/N, where N is the count of events in the buffer. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/aggregate/summarize/config.jsonnet b/examples/transform/aggregate/summarize/config.jsonnet index 21c03040..8274ad99 100644 --- a/examples/transform/aggregate/summarize/config.jsonnet +++ b/examples/transform/aggregate/summarize/config.jsonnet @@ -1,7 +1,7 @@ // This example reduces data by summarizing multiple network events into a single event, // simulating the behavior of flow records. This technique can be used to reduce // any JSON data that contains common fields, not just network events. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/array/extend/config.jsonnet b/examples/transform/array/extend/config.jsonnet index 22f65db4..a93a3adc 100644 --- a/examples/transform/array/extend/config.jsonnet +++ b/examples/transform/array/extend/config.jsonnet @@ -1,5 +1,5 @@ // This example extends an array by appending and flattening values. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/array/flatten/config.jsonnet b/examples/transform/array/flatten/config.jsonnet index b7ffb6e6..7a1c8fe4 100644 --- a/examples/transform/array/flatten/config.jsonnet +++ b/examples/transform/array/flatten/config.jsonnet @@ -1,5 +1,5 @@ // This example flattens an array of arrays. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/array/flatten_deep/config.jsonnet b/examples/transform/array/flatten_deep/config.jsonnet index 82bfe7d2..c5e3398f 100644 --- a/examples/transform/array/flatten_deep/config.jsonnet +++ b/examples/transform/array/flatten_deep/config.jsonnet @@ -1,5 +1,5 @@ // This example flattens an array of arrays. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/array/group/config.jsonnet b/examples/transform/array/group/config.jsonnet index 007684d3..4b09a8ae 100644 --- a/examples/transform/array/group/config.jsonnet +++ b/examples/transform/array/group/config.jsonnet @@ -1,6 +1,6 @@ // This example groups an array of arrays into an array of objects // based on index and configured keys. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local files_key = 'meta files'; diff --git a/examples/transform/enrich/http_secret/config.jsonnet b/examples/transform/enrich/http_secret/config.jsonnet index f8995e9b..dc4815d4 100644 --- a/examples/transform/enrich/http_secret/config.jsonnet +++ b/examples/transform/enrich/http_secret/config.jsonnet @@ -3,7 +3,7 @@ // // Test this example using the substation CLI: // SUBSTATION_EXAMPLE_URL=https://www.gutenberg.org/files/2701/old/moby10b.txt substation test config.jsonnet -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // The secret is retrieved from the environment variable named // `SUBSTATION_EXAMPLE_URL` and referenced in subsequent transforms using diff --git a/examples/transform/enrich/kvstore_csv/config.jsonnet b/examples/transform/enrich/kvstore_csv/config.jsonnet index 47153d46..275ce5cd 100644 --- a/examples/transform/enrich/kvstore_csv/config.jsonnet +++ b/examples/transform/enrich/kvstore_csv/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `enrich_kv_store_item_get` transform // to lookup data in a KV store backed by a CSV file. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // This CSV file must be local to the Substation app. Absolute paths are // recommended. Files accessible over HTTPS and hosted in AWS S3 also work. diff --git a/examples/transform/enrich/kvstore_json/config.jsonnet b/examples/transform/enrich/kvstore_json/config.jsonnet index bb56d207..d7225715 100644 --- a/examples/transform/enrich/kvstore_json/config.jsonnet +++ b/examples/transform/enrich/kvstore_json/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `enrich_kv_store_item_get` transform // to lookup data in a KV store backed by a JSON file. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // This JSON file must be local to the Substation app. Absolute paths are // recommended. Files accessible over HTTPS and hosted in AWS S3 also work. diff --git a/examples/transform/enrich/kvstore_set_add/config.jsonnet b/examples/transform/enrich/kvstore_set_add/config.jsonnet index 9d7452fb..2592b9e0 100644 --- a/examples/transform/enrich/kvstore_set_add/config.jsonnet +++ b/examples/transform/enrich/kvstore_set_add/config.jsonnet @@ -1,7 +1,7 @@ // This example shows how to use the `enrich_kv_store_set_add` transform // to track data over time in a KV store. The sample data contains food // orders and is indexed by each customer's email address. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // Default Memory store is used. local mem = sub.kv_store.memory(); diff --git a/examples/transform/enrich/mmdb/config.jsonnet b/examples/transform/enrich/mmdb/config.jsonnet index d3ce76b6..4ce6ee3a 100644 --- a/examples/transform/enrich/mmdb/config.jsonnet +++ b/examples/transform/enrich/mmdb/config.jsonnet @@ -1,4 +1,4 @@ -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local asn = sub.kv_store.mmdb({ file: 'https://gist.github.com/jshlbrd/59641ccc71ba2873fb204ac44d101640/raw/3ad0e8c09563c614c50de4671caef8c1983cbb4d/GeoLite2-ASN.mmdb' }); diff --git a/examples/transform/enrich/urlscan/config.jsonnet b/examples/transform/enrich/urlscan/config.jsonnet index 0aa8bd9b..42760744 100644 --- a/examples/transform/enrich/urlscan/config.jsonnet +++ b/examples/transform/enrich/urlscan/config.jsonnet @@ -3,7 +3,7 @@ // // Test this example using the substation CLI: // URLSCAN_API_KEY=xx substation test config.jsonnet -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local headers = { 'API-Key': '${SECRET:URLSCAN}', 'Content-Type': 'application/json' }; diff --git a/examples/transform/format/zip/config.jsonnet b/examples/transform/format/zip/config.jsonnet index 172549db..2d5590c3 100644 --- a/examples/transform/format/zip/config.jsonnet +++ b/examples/transform/format/zip/config.jsonnet @@ -2,7 +2,7 @@ // Add the two data files in this directory to a Zip file and send it to // Substation. You can use this command to create the Zip file: // zip data.zip data.jsonl data.csv -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/meta/crash_program/config.jsonnet b/examples/transform/meta/crash_program/config.jsonnet index 8df3a7f7..a3011794 100644 --- a/examples/transform/meta/crash_program/config.jsonnet +++ b/examples/transform/meta/crash_program/config.jsonnet @@ -1,7 +1,7 @@ // This example shows how to intentionally crash a program if a transform // does not produce an output. This technique can be used to provide strict // guarantees about the result of data transformations. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // `key` is the target of the transform that may not produce an output and is // checked to determine if the transform was successful. diff --git a/examples/transform/meta/default_value/config.jsonnet b/examples/transform/meta/default_value/config.jsonnet index 4eeda314..3a7998c2 100644 --- a/examples/transform/meta/default_value/config.jsonnet +++ b/examples/transform/meta/default_value/config.jsonnet @@ -1,4 +1,4 @@ -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/meta/each_in_array/config.jsonnet b/examples/transform/meta/each_in_array/config.jsonnet index db4333b0..acf93916 100644 --- a/examples/transform/meta/each_in_array/config.jsonnet +++ b/examples/transform/meta/each_in_array/config.jsonnet @@ -1,7 +1,7 @@ // This example shows how to use the `meta.for_each` transform to // modify objects in an array. In this example, keys are removed // and added to each object in the array. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/meta/exactly_once_consumer/config.jsonnet b/examples/transform/meta/exactly_once_consumer/config.jsonnet index 6e2f279b..c46be7a9 100644 --- a/examples/transform/meta/exactly_once_consumer/config.jsonnet +++ b/examples/transform/meta/exactly_once_consumer/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `meta_kv_store_lock` transform to // create an "exactly once" semantic for a pipeline consumer. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // In production environments a distributed KV store should be used. local kv = sub.kv_store.memory(); diff --git a/examples/transform/meta/exactly_once_producer/config.jsonnet b/examples/transform/meta/exactly_once_producer/config.jsonnet index 01051b0f..dc0c3086 100644 --- a/examples/transform/meta/exactly_once_producer/config.jsonnet +++ b/examples/transform/meta/exactly_once_producer/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `meta_kv_store_lock` transform to // create an "exactly once" semantic for a pipeline producer. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // In production environments a distributed KV store should be used. local kv = sub.kv_store.memory(); diff --git a/examples/transform/meta/exactly_once_system/config.jsonnet b/examples/transform/meta/exactly_once_system/config.jsonnet index 2757aa3c..3594a38d 100644 --- a/examples/transform/meta/exactly_once_system/config.jsonnet +++ b/examples/transform/meta/exactly_once_system/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `meta_kv_store_lock` transform to // create an "exactly once" semantic for an entire pipeline system. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // In production environments a distributed KV store should be used. local kv = sub.kv_store.memory(); diff --git a/examples/transform/meta/execution_time/config.jsonnet b/examples/transform/meta/execution_time/config.jsonnet index 69f6c67f..50b467db 100644 --- a/examples/transform/meta/execution_time/config.jsonnet +++ b/examples/transform/meta/execution_time/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `meta_metric_duration` transform to // measure the execution time of other transforms. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; diff --git a/examples/transform/meta/retry_with_backoff/config.jsonnet b/examples/transform/meta/retry_with_backoff/config.jsonnet index d9635cb7..e8c09034 100644 --- a/examples/transform/meta/retry_with_backoff/config.jsonnet +++ b/examples/transform/meta/retry_with_backoff/config.jsonnet @@ -1,7 +1,7 @@ // This example shows how to implement retry with backoff behavior for any // transform that does not produce an output. This technique may be useful // when enriching data with external services or asynchronous data pipelines. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // `key` is the target of the transform that may not produce an output and is // checked to determine if the transform was successful. diff --git a/examples/transform/number/clamp/config.jsonnet b/examples/transform/number/clamp/config.jsonnet index 72956274..66068335 100644 --- a/examples/transform/number/clamp/config.jsonnet +++ b/examples/transform/number/clamp/config.jsonnet @@ -1,5 +1,5 @@ // This example shows how to clamp a number to a range. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/number/max/config.jsonnet b/examples/transform/number/max/config.jsonnet index 6e906b54..026484db 100644 --- a/examples/transform/number/max/config.jsonnet +++ b/examples/transform/number/max/config.jsonnet @@ -1,6 +1,6 @@ // This example uses the `number_maximum` transform to return the larger // of two values, where one value is a constant and the other is a message. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/number/min/config.jsonnet b/examples/transform/number/min/config.jsonnet index b5ae1c22..e9ccb353 100644 --- a/examples/transform/number/min/config.jsonnet +++ b/examples/transform/number/min/config.jsonnet @@ -1,6 +1,6 @@ // This example uses the `number_minimum` transform to return the smaller // of two values, where one value is a constant and the other is a message. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/send/aux_transforms/config.jsonnet b/examples/transform/send/aux_transforms/config.jsonnet index bc642b6d..943231f9 100644 --- a/examples/transform/send/aux_transforms/config.jsonnet +++ b/examples/transform/send/aux_transforms/config.jsonnet @@ -2,7 +2,7 @@ // are executed after the data is buffered and before it is sent. The // transforms applied inside of the send transform do not affect the data // sent through the main pipeline. All send transforms use this behavior. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/send/aws_s3_glacier/config.jsonnet b/examples/transform/send/aws_s3_glacier/config.jsonnet index 1f5c3750..0a8ddaf7 100644 --- a/examples/transform/send/aws_s3_glacier/config.jsonnet +++ b/examples/transform/send/aws_s3_glacier/config.jsonnet @@ -2,7 +2,7 @@ // The Glacier Instant Retrieval class is recommended for archival data that is // compatible with Substation's serverless architecture; this class can be read // directly by a Lambda function triggered by an SNS notification. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { transforms: [ @@ -11,7 +11,7 @@ local sub = import '../../../../substation.libsonnet'; // the other values are set to impossibly high values to ensure all events are // written to the same file. batch: { size: 128 * 1000, count: 1000 * 1000, duration: '60m' }, - bucket_name: 'substation', + aws: { arn: 'arn:aws:s3:::substation-bucket' }, // Replace with your S3 bucket ARN. storage_class: 'GLACIER_IR', // Glacier Instant Retrieval. // S3 objects are organized by time to the nearest hour and have a UUID filename. file_path: { time_format: '2006/01/02/15', uuid: true, suffix: '.jsonl.gz' }, diff --git a/examples/transform/send/batch/config.jsonnet b/examples/transform/send/batch/config.jsonnet index ec6a5583..54a09094 100644 --- a/examples/transform/send/batch/config.jsonnet +++ b/examples/transform/send/batch/config.jsonnet @@ -1,7 +1,7 @@ // This example configures send transforms with batch keys to organize // data before it is sent externally. Every send transform supports batching // and optionally grouping JSON objects by a value derived from the object. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/send/datadog/config.jsonnet b/examples/transform/send/datadog/config.jsonnet index cd956ec1..73630ffc 100644 --- a/examples/transform/send/datadog/config.jsonnet +++ b/examples/transform/send/datadog/config.jsonnet @@ -3,7 +3,7 @@ // // More information about the Datadog Logs API can be found here: // https://docs.datadoghq.com/api/latest/logs/#send-logs -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // Datadog has a strict limit of 5MB per payload. Any individual event // larger than 1MB will be truncated on ingest. @@ -22,20 +22,11 @@ local max_count = 1000; sub.tf.agg.to.array({ object: { target_key: 'message' } }), ], url: 'https://http-intake.logs.datadoghq.com/api/v2/logs', - headers: [ - { - key: 'DD-API-KEY', - value: '${SECRET:DD}', - }, - { - key: 'ddsource', - value: 'my-source', - }, - { - key: 'service', - value: 'my-service', - }, - ], + headers: { + 'DD-API-KEY': '${SECRET:DD}', + ddsource: 'my-source', + service: 'my-service', + }, }), ], } diff --git a/examples/transform/send/splunk/config.jsonnet b/examples/transform/send/splunk/config.jsonnet index 5bb7bf65..d2460190 100644 --- a/examples/transform/send/splunk/config.jsonnet +++ b/examples/transform/send/splunk/config.jsonnet @@ -5,7 +5,7 @@ // // More information about the Splunk HEC can be found here: // https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/HECExamples -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // By default the Splunk HEC limits the size of each request to 1MB. local max_size = 1000 * 1000; @@ -21,10 +21,9 @@ local max_size = 1000 * 1000; sub.tf.array.join({ separator: '' }), ], url: 'https://my-instance.cloud.splunk.com:8088/services/collector', - headers: [{ - key: 'Authorization', - value: 'Splunk ${SECRET:SPLUNK}', - }], + headers: { + Authorization: 'Splunk ${SECRET:SPLUNK}', + }, }), ], } diff --git a/examples/transform/send/sumologic/config.jsonnet b/examples/transform/send/sumologic/config.jsonnet index 2e030a9c..03b9fcc7 100644 --- a/examples/transform/send/sumologic/config.jsonnet +++ b/examples/transform/send/sumologic/config.jsonnet @@ -3,7 +3,7 @@ // // More information about Sumo Logic HTTP upload can be found here: // https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/logs-metrics/upload-logs/ -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); // Sumo Logic has a strict limit of 1MB per request. local max_size = 1000 * 1000; @@ -19,7 +19,9 @@ local max_size = 1000 * 1000; // There is no authentication, so the URL should be treated like a secret. url: 'https://endpoint6.collection.us2.sumologic.com/receiver/v1/http/xxxxxxxxxx', // You can override the default source category associated with the URL. - // headers: [{key: 'X-Sumo-Category', value: 'testing/substation'}] + headers: { + 'X-Sumo-Category': 'testing/substation', + }, }), ], } diff --git a/examples/transform/test/config_test/config.jsonnet b/examples/transform/test/config_test/config.jsonnet index 7fa7eb7d..1f5d024d 100644 --- a/examples/transform/test/config_test/config.jsonnet +++ b/examples/transform/test/config_test/config.jsonnet @@ -1,4 +1,4 @@ -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/time/str_conversion/config.jsonnet b/examples/transform/time/str_conversion/config.jsonnet index 5ab554d1..0741ed3a 100644 --- a/examples/transform/time/str_conversion/config.jsonnet +++ b/examples/transform/time/str_conversion/config.jsonnet @@ -1,5 +1,5 @@ // This example shows how to convert time values between string formats. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/utility/generate_ctrl/config.jsonnet b/examples/transform/utility/generate_ctrl/config.jsonnet index 74be6e73..e4c35541 100644 --- a/examples/transform/utility/generate_ctrl/config.jsonnet +++ b/examples/transform/utility/generate_ctrl/config.jsonnet @@ -2,7 +2,7 @@ // generate a control (ctrl) Message based on the amount of data Messages // received by the system. ctrl Messages overrides the settings of the // `aggregate_to_array` transform (and any other transform that supports). -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); { tests: [ diff --git a/examples/transform/utility/message_bytes/config.jsonnet b/examples/transform/utility/message_bytes/config.jsonnet index fe586351..95485645 100644 --- a/examples/transform/utility/message_bytes/config.jsonnet +++ b/examples/transform/utility/message_bytes/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `utility_metric_bytes` transform to // sum the amount of data received and transformed by Substation. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; diff --git a/examples/transform/utility/message_count/config.jsonnet b/examples/transform/utility/message_count/config.jsonnet index 1bc6fc04..6287eb26 100644 --- a/examples/transform/utility/message_count/config.jsonnet +++ b/examples/transform/utility/message_count/config.jsonnet @@ -1,6 +1,6 @@ // This example shows how to use the `utility_metric_count` transform to // count the number of messages received and transformed by Substation. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; diff --git a/examples/transform/utility/message_freshness/config.jsonnet b/examples/transform/utility/message_freshness/config.jsonnet index ca1a0c00..0d4642e3 100644 --- a/examples/transform/utility/message_freshness/config.jsonnet +++ b/examples/transform/utility/message_freshness/config.jsonnet @@ -8,7 +8,7 @@ // // The transform emits two metrics that describe success and failure, annotated // in the `FreshnessType` attribute. -local sub = import '../../../../substation.libsonnet'; +local sub = std.extVar('sub'); local attr = { AppName: 'example' }; local dest = { type: 'aws_cloudwatch_embedded_metrics' }; diff --git a/substation_test.jsonnet b/substation_test.jsonnet index a13766e4..eb8dba1c 100644 --- a/substation_test.jsonnet +++ b/substation_test.jsonnet @@ -1,7 +1,6 @@ -local sub = import 'substation.libsonnet'; +local sub = std.extVar('sub'); local src = 'source'; -local trg = 'target'; { condition: {