Skip to content

Commit

Permalink
docs: Update Jsonnet Imports
Browse files Browse the repository at this point in the history
  • Loading branch information
jshlbrd committed Oct 27, 2024
1 parent df35d4c commit c681674
Show file tree
Hide file tree
Showing 40 changed files with 52 additions and 61 deletions.
2 changes: 1 addition & 1 deletion examples/condition/meta/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example determines if all values in an array are email addresses
// that have the DNS domain "brex.com". This technique can be used to
// validate or summarize values in an array.
local sub = import '../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/condition/number/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example shows usage of the 'number.equal_to' and 'number.greater_than' conditions.
local sub = import '../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/condition/string/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example shows usage of the 'string.equal_to' and 'string.greater_than' conditions.
local sub = import '../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/aggregate/sample/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example samples data by aggregating events into an array, then
// selecting the first event in the array as a sample. The sampling rate
// is 1/N, where N is the count of events in the buffer.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/aggregate/summarize/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example reduces data by summarizing multiple network events into a single event,
// simulating the behavior of flow records. This technique can be used to reduce
// any JSON data that contains common fields, not just network events.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/array/extend/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example extends an array by appending and flattening values.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/array/flatten/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example flattens an array of arrays.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/array/flatten_deep/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example flattens an array of arrays.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/array/group/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example groups an array of arrays into an array of objects
// based on index and configured keys.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local files_key = 'meta files';

Expand Down
2 changes: 1 addition & 1 deletion examples/transform/enrich/http_secret/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//
// Test this example using the substation CLI:
// SUBSTATION_EXAMPLE_URL=https://www.gutenberg.org/files/2701/old/moby10b.txt substation test config.jsonnet
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// The secret is retrieved from the environment variable named
// `SUBSTATION_EXAMPLE_URL` and referenced in subsequent transforms using
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/enrich/kvstore_csv/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `enrich_kv_store_item_get` transform
// to lookup data in a KV store backed by a CSV file.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// This CSV file must be local to the Substation app. Absolute paths are
// recommended. Files accessible over HTTPS and hosted in AWS S3 also work.
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/enrich/kvstore_json/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `enrich_kv_store_item_get` transform
// to lookup data in a KV store backed by a JSON file.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// This JSON file must be local to the Substation app. Absolute paths are
// recommended. Files accessible over HTTPS and hosted in AWS S3 also work.
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/enrich/kvstore_set_add/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example shows how to use the `enrich_kv_store_set_add` transform
// to track data over time in a KV store. The sample data contains food
// orders and is indexed by each customer's email address.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// Default Memory store is used.
local mem = sub.kv_store.memory();
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/enrich/mmdb/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local asn = sub.kv_store.mmdb({ file: 'https://gist.github.com/jshlbrd/59641ccc71ba2873fb204ac44d101640/raw/3ad0e8c09563c614c50de4671caef8c1983cbb4d/GeoLite2-ASN.mmdb' });

Expand Down
2 changes: 1 addition & 1 deletion examples/transform/enrich/urlscan/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//
// Test this example using the substation CLI:
// URLSCAN_API_KEY=xx substation test config.jsonnet
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local headers = { 'API-Key': '${SECRET:URLSCAN}', 'Content-Type': 'application/json' };

Expand Down
2 changes: 1 addition & 1 deletion examples/transform/format/zip/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// Add the two data files in this directory to a Zip file and send it to
// Substation. You can use this command to create the Zip file:
// zip data.zip data.jsonl data.csv
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/meta/crash_program/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example shows how to intentionally crash a program if a transform
// does not produce an output. This technique can be used to provide strict
// guarantees about the result of data transformations.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// `key` is the target of the transform that may not produce an output and is
// checked to determine if the transform was successful.
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/meta/default_value/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/meta/each_in_array/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example shows how to use the `meta.for_each` transform to
// modify objects in an array. In this example, keys are removed
// and added to each object in the array.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `meta_kv_store_lock` transform to
// create an "exactly once" semantic for a pipeline consumer.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// In production environments a distributed KV store should be used.
local kv = sub.kv_store.memory();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `meta_kv_store_lock` transform to
// create an "exactly once" semantic for a pipeline producer.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// In production environments a distributed KV store should be used.
local kv = sub.kv_store.memory();
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/meta/exactly_once_system/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `meta_kv_store_lock` transform to
// create an "exactly once" semantic for an entire pipeline system.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// In production environments a distributed KV store should be used.
local kv = sub.kv_store.memory();
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/meta/execution_time/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `meta_metric_duration` transform to
// measure the execution time of other transforms.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local attr = { AppName: 'example' };
local dest = { type: 'aws_cloudwatch_embedded_metrics' };
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/meta/retry_with_backoff/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example shows how to implement retry with backoff behavior for any
// transform that does not produce an output. This technique may be useful
// when enriching data with external services or asynchronous data pipelines.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// `key` is the target of the transform that may not produce an output and is
// checked to determine if the transform was successful.
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/number/clamp/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example shows how to clamp a number to a range.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/number/max/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example uses the `number_maximum` transform to return the larger
// of two values, where one value is a constant and the other is a message.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/number/min/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example uses the `number_minimum` transform to return the smaller
// of two values, where one value is a constant and the other is a message.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/send/aux_transforms/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// are executed after the data is buffered and before it is sent. The
// transforms applied inside of the send transform do not affect the data
// sent through the main pipeline. All send transforms use this behavior.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
4 changes: 2 additions & 2 deletions examples/transform/send/aws_s3_glacier/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// The Glacier Instant Retrieval class is recommended for archival data that is
// compatible with Substation's serverless architecture; this class can be read
// directly by a Lambda function triggered by an SNS notification.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
transforms: [
Expand All @@ -11,7 +11,7 @@ local sub = import '../../../../substation.libsonnet';
// the other values are set to impossibly high values to ensure all events are
// written to the same file.
batch: { size: 128 * 1000, count: 1000 * 1000, duration: '60m' },
bucket_name: 'substation',
aws: { arn: 'arn:aws:s3:::substation-bucket' }, // Replace with your S3 bucket ARN.
storage_class: 'GLACIER_IR', // Glacier Instant Retrieval.
// S3 objects are organized by time to the nearest hour and have a UUID filename.
file_path: { time_format: '2006/01/02/15', uuid: true, suffix: '.jsonl.gz' },
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/send/batch/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// This example configures send transforms with batch keys to organize
// data before it is sent externally. Every send transform supports batching
// and optionally grouping JSON objects by a value derived from the object.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
21 changes: 6 additions & 15 deletions examples/transform/send/datadog/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//
// More information about the Datadog Logs API can be found here:
// https://docs.datadoghq.com/api/latest/logs/#send-logs
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// Datadog has a strict limit of 5MB per payload. Any individual event
// larger than 1MB will be truncated on ingest.
Expand All @@ -22,20 +22,11 @@ local max_count = 1000;
sub.tf.agg.to.array({ object: { target_key: 'message' } }),
],
url: 'https://http-intake.logs.datadoghq.com/api/v2/logs',
headers: [
{
key: 'DD-API-KEY',
value: '${SECRET:DD}',
},
{
key: 'ddsource',
value: 'my-source',
},
{
key: 'service',
value: 'my-service',
},
],
headers: {
'DD-API-KEY': '${SECRET:DD}',
ddsource: 'my-source',
service: 'my-service',
},
}),
],
}
9 changes: 4 additions & 5 deletions examples/transform/send/splunk/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
//
// More information about the Splunk HEC can be found here:
// https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/HECExamples
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// By default the Splunk HEC limits the size of each request to 1MB.
local max_size = 1000 * 1000;
Expand All @@ -21,10 +21,9 @@ local max_size = 1000 * 1000;
sub.tf.array.join({ separator: '' }),
],
url: 'https://my-instance.cloud.splunk.com:8088/services/collector',
headers: [{
key: 'Authorization',
value: 'Splunk ${SECRET:SPLUNK}',
}],
headers: {
Authorization: 'Splunk ${SECRET:SPLUNK}',
},
}),
],
}
6 changes: 4 additions & 2 deletions examples/transform/send/sumologic/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//
// More information about Sumo Logic HTTP upload can be found here:
// https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/logs-metrics/upload-logs/
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

// Sumo Logic has a strict limit of 1MB per request.
local max_size = 1000 * 1000;
Expand All @@ -19,7 +19,9 @@ local max_size = 1000 * 1000;
// There is no authentication, so the URL should be treated like a secret.
url: 'https://endpoint6.collection.us2.sumologic.com/receiver/v1/http/xxxxxxxxxx',
// You can override the default source category associated with the URL.
// headers: [{key: 'X-Sumo-Category', value: 'testing/substation'}]
headers: {
'X-Sumo-Category': 'testing/substation',
},
}),
],
}
2 changes: 1 addition & 1 deletion examples/transform/test/config_test/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/time/str_conversion/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// This example shows how to convert time values between string formats.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/utility/generate_ctrl/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// generate a control (ctrl) Message based on the amount of data Messages
// received by the system. ctrl Messages overrides the settings of the
// `aggregate_to_array` transform (and any other transform that supports).
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

{
tests: [
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/utility/message_bytes/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `utility_metric_bytes` transform to
// sum the amount of data received and transformed by Substation.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local attr = { AppName: 'example' };
local dest = { type: 'aws_cloudwatch_embedded_metrics' };
Expand Down
2 changes: 1 addition & 1 deletion examples/transform/utility/message_count/config.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// This example shows how to use the `utility_metric_count` transform to
// count the number of messages received and transformed by Substation.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local attr = { AppName: 'example' };
local dest = { type: 'aws_cloudwatch_embedded_metrics' };
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
//
// The transform emits two metrics that describe success and failure, annotated
// in the `FreshnessType` attribute.
local sub = import '../../../../substation.libsonnet';
local sub = std.extVar('sub');

local attr = { AppName: 'example' };
local dest = { type: 'aws_cloudwatch_embedded_metrics' };
Expand Down
3 changes: 1 addition & 2 deletions substation_test.jsonnet
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
local sub = import 'substation.libsonnet';
local sub = std.extVar('sub');

local src = 'source';
local trg = 'target';

{
condition: {
Expand Down

0 comments on commit c681674

Please sign in to comment.