From e0b7345e447ad64f6ca46ffcb2555f1ca5a5278d Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 19 Nov 2024 12:35:15 -0800 Subject: [PATCH 01/18] Re-format for hugo docs --- docs/{README.md => _index.md} | 7 +++--- docs/commands/_category_.yaml | 2 -- docs/commands/{README.md => _index.md} | 5 ++++- docs/commands/super-db.md | 6 ++--- docs/commands/super.md | 8 +++---- docs/formats/{README.md => _index.md} | 11 +++++++--- docs/formats/bsup.md | 9 ++++---- docs/formats/compression.md | 7 +++--- docs/formats/csup.md | 11 +++++----- docs/formats/jsup.md | 7 +++--- docs/formats/zed.md | 10 ++++----- docs/formats/zjson.md | 7 +++--- docs/install.md | 6 ++--- docs/integrations/_category_.yaml | 2 -- docs/integrations/_index.md | 4 ++++ docs/integrations/amazon-s3.md | 6 ++--- docs/integrations/fluentd.md | 6 ++--- docs/integrations/grafana.md | 7 +++--- docs/integrations/zed-lake-auth.md | 9 ++++---- docs/integrations/zeek/_category_.yaml | 2 -- .../zeek/{README.md => _index.md} | 5 ++++- .../zeek/data-type-compatibility.md | 6 ++--- .../zeek/reading-zeek-log-formats.md | 6 ++--- docs/integrations/zeek/shaping-zeek-json.md | 6 ++--- docs/lake/_category_.yaml | 2 -- docs/lake/_index.md | 4 ++++ docs/lake/api.md | 7 +++--- docs/lake/format.md | 9 ++++---- docs/language/{README.md => _index.md} | 6 ++++- .../aggregates/{README.md => _index.md} | 5 +++-- docs/language/aggregates/and.md | 8 +++---- docs/language/aggregates/any.md | 8 +++---- docs/language/aggregates/avg.md | 8 +++---- docs/language/aggregates/collect.md | 8 +++---- docs/language/aggregates/collect_map.md | 6 ++--- docs/language/aggregates/count.md | 14 ++++++------ docs/language/aggregates/dcount.md | 10 ++++----- docs/language/aggregates/fuse.md | 6 ++--- docs/language/aggregates/max.md | 8 +++---- docs/language/aggregates/min.md | 8 +++---- docs/language/aggregates/or.md | 8 +++---- docs/language/aggregates/sum.md | 8 +++---- docs/language/aggregates/union.md | 8 +++---- docs/language/conventions.md | 7 +++--- docs/language/data-types.md | 8 +++---- docs/language/expressions.md | 8 +++---- .../functions/{README.md => _index.md} | 4 ++-- docs/language/functions/abs.md | 17 +++++--------- docs/language/functions/base64.md | 8 +++---- docs/language/functions/bucket.md | 2 +- docs/language/functions/ceil.md | 2 +- docs/language/functions/cidr_match.md | 6 ++--- docs/language/functions/coalesce.md | 4 ++-- docs/language/functions/compare.md | 2 +- docs/language/functions/error.md | 8 +++---- docs/language/functions/fields.md | 6 ++--- docs/language/functions/flatten.md | 2 +- docs/language/functions/floor.md | 2 +- docs/language/functions/grep.md | 10 ++++----- docs/language/functions/grok.md | 8 +++---- docs/language/functions/has.md | 2 +- docs/language/functions/has_error.md | 2 +- docs/language/functions/hex.md | 8 +++---- docs/language/functions/is.md | 8 +++---- docs/language/functions/is_error.md | 6 ++--- docs/language/functions/join.md | 4 ++-- docs/language/functions/kind.md | 8 +++---- docs/language/functions/ksuid.md | 2 +- docs/language/functions/len.md | 2 +- docs/language/functions/levenshtein.md | 2 +- docs/language/functions/log.md | 4 ++-- docs/language/functions/lower.md | 2 +- docs/language/functions/map.md | 4 ++-- docs/language/functions/missing.md | 2 +- docs/language/functions/nameof.md | 4 ++-- docs/language/functions/nest_dotted.md | 2 +- docs/language/functions/network_of.md | 10 ++++----- docs/language/functions/parse_uri.md | 2 +- docs/language/functions/parse_zson.md | 4 ++-- docs/language/functions/pow.md | 2 +- docs/language/functions/quiet.md | 6 ++--- docs/language/functions/regexp.md | 4 ++-- docs/language/functions/regexp_replace.md | 8 +++---- docs/language/functions/replace.md | 2 +- docs/language/functions/round.md | 2 +- docs/language/functions/rune_len.md | 4 ++-- docs/language/functions/split.md | 4 ++-- docs/language/functions/sqrt.md | 2 +- docs/language/functions/strftime.md | 4 ++-- docs/language/functions/trim.md | 2 +- docs/language/functions/typename.md | 6 ++--- docs/language/functions/typeof.md | 4 ++-- docs/language/functions/typeunder.md | 2 +- docs/language/functions/under.md | 8 +++---- docs/language/functions/unflatten.md | 4 ++-- docs/language/functions/upper.md | 4 ++-- docs/language/lateral-subqueries.md | 6 ++--- .../operators/{README.md => _index.md} | 4 ++-- docs/language/operators/assert.md | 4 ++-- docs/language/operators/combine.md | 2 +- docs/language/operators/cut.md | 10 ++++----- docs/language/operators/drop.md | 4 ++-- docs/language/operators/fork.md | 2 +- docs/language/operators/from.md | 16 +++++++------- docs/language/operators/fuse.md | 8 +++---- docs/language/operators/head.md | 6 ++--- docs/language/operators/load.md | 4 ++-- docs/language/operators/merge.md | 2 +- docs/language/operators/over.md | 18 +++++++-------- docs/language/operators/pass.md | 4 ++-- docs/language/operators/put.md | 10 ++++----- docs/language/operators/rename.md | 12 +++++----- docs/language/operators/sample.md | 4 ++-- docs/language/operators/search.md | 18 +++++++-------- docs/language/operators/sort.md | 22 +++++++++---------- docs/language/operators/summarize.md | 20 ++++++++--------- docs/language/operators/switch.md | 4 ++-- docs/language/operators/tail.md | 6 ++--- docs/language/operators/top.md | 4 ++-- docs/language/operators/uniq.md | 10 ++++----- docs/language/operators/where.md | 14 ++++++------ docs/language/operators/yield.md | 8 +++---- docs/language/overview.md | 9 +++----- docs/language/pipe-ambiguity.md | 4 +++- docs/language/pipeline-model.md | 6 ++--- docs/language/search-expressions.md | 6 ++--- docs/language/shaping.md | 6 ++--- docs/language/statements.md | 7 +++--- docs/libraries/{README.md => _index.md} | 5 ++++- docs/libraries/go.md | 6 ++--- docs/libraries/javascript.md | 6 ++--- docs/libraries/python.md | 6 ++--- docs/tutorials/{README.md => _index.md} | 5 ++++- docs/tutorials/join.md | 6 ++--- docs/tutorials/zed.md | 7 +++--- docs/tutorials/zq.md | 9 ++++---- 136 files changed, 410 insertions(+), 446 deletions(-) rename docs/{README.md => _index.md} (98%) delete mode 100644 docs/commands/_category_.yaml rename docs/commands/{README.md => _index.md} (94%) rename docs/formats/{README.md => _index.md} (99%) delete mode 100644 docs/integrations/_category_.yaml create mode 100644 docs/integrations/_index.md delete mode 100644 docs/integrations/zeek/_category_.yaml rename docs/integrations/zeek/{README.md => _index.md} (86%) delete mode 100644 docs/lake/_category_.yaml create mode 100644 docs/lake/_index.md rename docs/language/{README.md => _index.md} (91%) rename docs/language/aggregates/{README.md => _index.md} (94%) rename docs/language/functions/{README.md => _index.md} (99%) rename docs/language/operators/{README.md => _index.md} (98%) rename docs/libraries/{README.md => _index.md} (93%) rename docs/tutorials/{README.md => _index.md} (79%) diff --git a/docs/README.md b/docs/_index.md similarity index 98% rename from docs/README.md rename to docs/_index.md index 88447c6b0c..ff45bd9c84 100644 --- a/docs/README.md +++ b/docs/_index.md @@ -1,10 +1,9 @@ --- -sidebar_position: 1 -sidebar_label: Introduction +weight: 1 +title: Docs +heading: SuperDB --- -# SuperDB - SuperDB offers a new approach that makes it easier to manipulate and manage your data. With its [super-structured data model](formats/README.md#2-a-super-structured-pattern), messy JSON data can easily be given the fully-typed precision of relational tables diff --git a/docs/commands/_category_.yaml b/docs/commands/_category_.yaml deleted file mode 100644 index e5c770bade..0000000000 --- a/docs/commands/_category_.yaml +++ /dev/null @@ -1,2 +0,0 @@ -position: 3 -label: Commands diff --git a/docs/commands/README.md b/docs/commands/_index.md similarity index 94% rename from docs/commands/README.md rename to docs/commands/_index.md index c4eaa41682..979c5678ca 100644 --- a/docs/commands/README.md +++ b/docs/commands/_index.md @@ -1,4 +1,7 @@ -# Command Tooling +--- +title: Commands +weight: 2 +--- The [`super` command](super.md) is used to execute command-line queries on inputs from files, HTTP URLs, or [S3](../integrations/amazon-s3.md). diff --git a/docs/commands/super-db.md b/docs/commands/super-db.md index a2237260ab..a3bf860615 100644 --- a/docs/commands/super-db.md +++ b/docs/commands/super-db.md @@ -1,10 +1,8 @@ --- -sidebar_position: 2 -sidebar_label: super db +weight: 2 +title: super db --- -# `super db` - > **TL;DR** `super db` is a sub-command of `super` to manage and query SuperDB data lakes. > You can import data from a variety of formats and it will automatically > be committed in [super-structured](../formats/README.md) diff --git a/docs/commands/super.md b/docs/commands/super.md index 1faec39c6a..9fb7ad80bf 100644 --- a/docs/commands/super.md +++ b/docs/commands/super.md @@ -1,10 +1,8 @@ --- -sidebar_position: 1 -sidebar_label: super +weight: 1 +title: super --- -# `super` - > **TL;DR** `super` is a command-line tool that uses [SuperSQL](../language/README.md) > to query a variety of data formats in files, over HTTP, or in [S3](../integrations/amazon-s3.md) > storage. It is particularly fast when operating on data in binary formats such as @@ -43,7 +41,7 @@ but its comprehensive [type system](../formats/zed.md) obviates the need for schema specification or registries. Also, the [Super JSON](../formats/jsup.md) format is human-readable and entirely one-to-one with Super Binary so there is no need to represent non-readable formats like Avro or Protocol Buffers -in a clunky JSON encapsulated form. +in a clunky JSON encapsulated form. `super` typically operates on Super Binary-encoded data and when you want to inspect human-readable bits of output, you merely format it as Super JSON, which is the diff --git a/docs/formats/README.md b/docs/formats/_index.md similarity index 99% rename from docs/formats/README.md rename to docs/formats/_index.md index 98395229ee..e7fa09c7bf 100644 --- a/docs/formats/README.md +++ b/docs/formats/_index.md @@ -1,4 +1,7 @@ -# Formats +--- +title: Formats +weight: 5 +--- > **TL;DR** The super data model defines a new and easy way to manage, store, > and process data utilizing an emerging concept called @@ -228,11 +231,13 @@ anywhere that a value can appear. In particular, types can be aggregation keys. This is very powerful for data discovery and introspection. For example, to count the different shapes of data, you might have a SuperSQL query, operating on each input value as `this`, that has the form: -``` + +```sql SELECT count(), typeof(this) AS shape GROUP BY shape, count ``` Likewise, you could select a sample value of each shape like this: -``` + +```sql SELECT shape FROM ( SELECT any(this) AS sample, typeof(this) AS shape GROUP BY shape,sample ) diff --git a/docs/formats/bsup.md b/docs/formats/bsup.md index 4982f6f295..333f245d24 100644 --- a/docs/formats/bsup.md +++ b/docs/formats/bsup.md @@ -1,10 +1,9 @@ --- -sidebar_position: 2 -sidebar_label: Super Binary +weight: 2 +title: Super Binary +heading: Super Binary Specification --- -# Super Binary Specification - ## 1. Introduction Super Binary is an efficient, sequence-oriented serialization format for any data @@ -296,7 +295,7 @@ indicated by ``. #### 2.1.8 Named Type Typedef -A named type defines a new type ID that binds a name to a previously existing type ID. +A named type defines a new type ID that binds a name to a previously existing type ID. A named type is encoded as follows: ``` diff --git a/docs/formats/compression.md b/docs/formats/compression.md index 5934ad09b6..4b3ed3e40a 100644 --- a/docs/formats/compression.md +++ b/docs/formats/compression.md @@ -1,10 +1,9 @@ --- -sidebar_position: 6 -sidebar_label: Compression +weight: 6 +title: Compression +heading: ZNG Compression Types --- -# ZNG Compression Types - This document specifies values for the `` byte of a [Super Binary compressed value message block](bsup.md#2-the-super-binary-format) and the corresponding algorithms for the `` byte sequence. diff --git a/docs/formats/csup.md b/docs/formats/csup.md index 55377db18d..2630b09a2c 100644 --- a/docs/formats/csup.md +++ b/docs/formats/csup.md @@ -1,10 +1,9 @@ --- -sidebar_position: 4 -sidebar_label: Super Columnar +weight: 4 +title: Super Columnar +heading: Super Columnar Specification --- -# Super Columnar Specification - Super Columnar is a file format based on the [super data model](zed.md) where data is stacked to form columns. Its purpose is to provide for efficient analytics and search over @@ -245,7 +244,7 @@ and has the form: :{column:,presence:}, ... :{column:,presence:} -} +} ``` where * `` through `` are the names of the top-level fields of the @@ -268,7 +267,7 @@ An `` has the form: {values:,lengths:} ``` where -* `values` represents a continuous sequence of values of the array elements +* `values` represents a continuous sequence of values of the array elements that are sliced into array values based on the length information, and * `lengths` encodes an `int32` sequence of values that represent the length of each array value. diff --git a/docs/formats/jsup.md b/docs/formats/jsup.md index 601ee9df48..44d24e8257 100644 --- a/docs/formats/jsup.md +++ b/docs/formats/jsup.md @@ -1,10 +1,9 @@ --- -sidebar_position: 3 -sidebar_label: Super JSON +weight: 3 +title: Super JSON +heading: Super JSON Specification --- -# Super JSON Specification - ## 1. Introduction Super JSON is the human-readable, text-based serialization format of diff --git a/docs/formats/zed.md b/docs/formats/zed.md index 6018704201..17e5e6b12e 100644 --- a/docs/formats/zed.md +++ b/docs/formats/zed.md @@ -1,10 +1,8 @@ --- -sidebar_position: 1 -sidebar_label: Data Model +weight: 1 +title: Data Model --- -# Zed Data Model - Zed data is defined as an ordered sequence of one or more typed data values. Each value's type is either a "primitive type", a "complex type", the "type type", a "named type", or the "null type". @@ -154,7 +152,7 @@ have a common Zed type and the values have a common Zed type. Each key across an instance of a map value must be a unique value. -A map value may be empty. +A map value may be empty. A map type is uniquely defined by its key type and value type. @@ -196,7 +194,7 @@ the type order of the constituent types in left to right order. ### 2.7 Error -An error represents any value designated as an error. +An error represents any value designated as an error. The type order of an error is the type order of the type of its contained value. diff --git a/docs/formats/zjson.md b/docs/formats/zjson.md index 9d677be7c9..4deb1df9ce 100644 --- a/docs/formats/zjson.md +++ b/docs/formats/zjson.md @@ -1,10 +1,9 @@ --- -sidebar_position: 5 -sidebar_label: ZJSON +weight: 5 +title: ZJSON +heading: ZJSON Specification --- -# ZJSON Specification - ## 1. Introduction The [super data model](zed.md) diff --git a/docs/install.md b/docs/install.md index b643127e72..fe1e8db596 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,10 +1,8 @@ --- -sidebar_position: 2 -sidebar_label: Installation +weight: 2 +title: Installation --- -# Installation - Several options for installing `super` are available: * [Homebrew](#homebrew) for Mac or Linux, * [Binary download](#binary-download), or diff --git a/docs/integrations/_category_.yaml b/docs/integrations/_category_.yaml deleted file mode 100644 index 034b94791f..0000000000 --- a/docs/integrations/_category_.yaml +++ /dev/null @@ -1,2 +0,0 @@ -position: 9 -label: Integrations diff --git a/docs/integrations/_index.md b/docs/integrations/_index.md new file mode 100644 index 0000000000..f5b53b81af --- /dev/null +++ b/docs/integrations/_index.md @@ -0,0 +1,4 @@ +--- +weight: 8 +title: Integrations +--- diff --git a/docs/integrations/amazon-s3.md b/docs/integrations/amazon-s3.md index 54cf0b57d8..f377ce1453 100644 --- a/docs/integrations/amazon-s3.md +++ b/docs/integrations/amazon-s3.md @@ -1,10 +1,8 @@ --- -sidebar_position: 1 -sidebar_label: Amazon S3 +weight: 1 +title: Amazon S3 --- -# Amazon S3 - Zed tools can access [Amazon S3](https://aws.amazon.com/s3/) and S3-compatible storage via `s3://` URIs. Details are described below. diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index 7bea84d9f8..492a4ee262 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -1,10 +1,8 @@ --- -sidebar_position: 3 -sidebar_label: Fluentd +weight: 3 +title: Fluentd --- -# Fluentd - The [Fluentd](https://www.fluentd.org/) open source data collector can be used to push log data to a [SuperDB data lake](../commands/super-db.md) in a continuous manner. This allows for querying near-"live" event data to enable use cases such as diff --git a/docs/integrations/grafana.md b/docs/integrations/grafana.md index 230bebdae4..b1bad65e6a 100644 --- a/docs/integrations/grafana.md +++ b/docs/integrations/grafana.md @@ -1,10 +1,9 @@ --- -sidebar_position: 4 -sidebar_label: Grafana +weight: 4 +title: Grafana +heading: Grafana Data Source Plugin --- -# Grafana Data Source Plugin - A [data source plugin](https://grafana.com/grafana/plugins/?type=datasource) for [Grafana](https://grafana.com/) is available that enables plotting of time-series data that's stored in [SuperDB data lakes](../commands/super-db.md). See the diff --git a/docs/integrations/zed-lake-auth.md b/docs/integrations/zed-lake-auth.md index e1c432e499..79efc5f92b 100644 --- a/docs/integrations/zed-lake-auth.md +++ b/docs/integrations/zed-lake-auth.md @@ -1,10 +1,9 @@ --- -sidebar_position: 2 -sidebar_label: Authentication Configuration +weight: 2 +title: Authentication Configuration +heading: Configuring Authentication for a Zed Lake Service --- -# Configuring Authentication for a Zed Lake Service - A [SuperDB data lake service](../commands/super-db.md#serve) may be configured to require user authentication to be accessed from clients such as the [Zui](https://zui.brimdata.io/) application, the @@ -136,7 +135,7 @@ authentication configuration along with a directory name for lake storage. -auth.jwkspath=jwks.json \ -auth.audience=$auth0_api_identifier \ -lake=lake - + {"level":"info","ts":1678909988.9797907,"logger":"core","msg":"Started"} {"level":"info","ts":1678909988.9804773,"logger":"httpd","msg":"Listening","addr":"[::]:9867"} ... diff --git a/docs/integrations/zeek/_category_.yaml b/docs/integrations/zeek/_category_.yaml deleted file mode 100644 index 71bc2e94e2..0000000000 --- a/docs/integrations/zeek/_category_.yaml +++ /dev/null @@ -1,2 +0,0 @@ -position: 4 -label: Zeek diff --git a/docs/integrations/zeek/README.md b/docs/integrations/zeek/_index.md similarity index 86% rename from docs/integrations/zeek/README.md rename to docs/integrations/zeek/_index.md index 3a7c17b28e..1298a50099 100644 --- a/docs/integrations/zeek/README.md +++ b/docs/integrations/zeek/_index.md @@ -1,4 +1,7 @@ -# Zed Interoperability with Zeek Logs +--- +title: Zeek +heading: Zed Interoperability with Zeek Logs +--- Zed includes functionality and reference configurations specific to working with logs from the [Zeek](https://zeek.org/) open source network security diff --git a/docs/integrations/zeek/data-type-compatibility.md b/docs/integrations/zeek/data-type-compatibility.md index 1d0b109828..6c050b1b50 100644 --- a/docs/integrations/zeek/data-type-compatibility.md +++ b/docs/integrations/zeek/data-type-compatibility.md @@ -1,10 +1,8 @@ --- -sidebar_position: 2 -sidebar_label: Zed/Zeek Data Type Compatibility +weight: 2 +title: Zed/Zeek Data Type Compatibility --- -# Zed/Zeek Data Type Compatibility - As the [super data model](../../formats/zed.md) was in many ways inspired by the [Zeek TSV log format](https://docs.zeek.org/en/master/log-formats.html#zeek-tsv-format-logs), SuperDB's rich storage formats ([Super JSON](../../formats/jsup.md), diff --git a/docs/integrations/zeek/reading-zeek-log-formats.md b/docs/integrations/zeek/reading-zeek-log-formats.md index fdc191e62b..cdbac244cf 100644 --- a/docs/integrations/zeek/reading-zeek-log-formats.md +++ b/docs/integrations/zeek/reading-zeek-log-formats.md @@ -1,10 +1,8 @@ --- -sidebar_position: 1 -sidebar_label: Reading Zeek Log Formats +weight: 1 +title: Reading Zeek Log Formats --- -# Reading Zeek Log Formats - Zed is capable of reading both common Zeek log formats. This document provides guidance for what to expect when reading logs of these formats using the Zed [command line tools](../../commands/README.md). diff --git a/docs/integrations/zeek/shaping-zeek-json.md b/docs/integrations/zeek/shaping-zeek-json.md index 9ff9c13d0f..b574028ae7 100644 --- a/docs/integrations/zeek/shaping-zeek-json.md +++ b/docs/integrations/zeek/shaping-zeek-json.md @@ -1,10 +1,8 @@ --- -sidebar_position: 3 -sidebar_label: Shaping Zeek JSON +weight: 3 +title: Shaping Zeek JSON --- -# Shaping Zeek JSON - When [reading Zeek JSON format logs](reading-zeek-log-formats.md#zeek-json), much of the rich data typing that was originally present inside Zeek is at risk of being lost. This detail can be restored using a Zed diff --git a/docs/lake/_category_.yaml b/docs/lake/_category_.yaml deleted file mode 100644 index 5beb193f7a..0000000000 --- a/docs/lake/_category_.yaml +++ /dev/null @@ -1,2 +0,0 @@ -position: 7 -label: Lake diff --git a/docs/lake/_index.md b/docs/lake/_index.md new file mode 100644 index 0000000000..5e175901aa --- /dev/null +++ b/docs/lake/_index.md @@ -0,0 +1,4 @@ +--- +title: Lake +weight: 6 +--- diff --git a/docs/lake/api.md b/docs/lake/api.md index cfefb8164b..ab8448c07a 100644 --- a/docs/lake/api.md +++ b/docs/lake/api.md @@ -1,10 +1,9 @@ --- -sidebar_position: 1 -sidebar_label: API +weight: 1 +title: API +heading: Zed lake API --- -# Zed lake API - ## _Status_ > This is a brief sketch of the functionality exposed in the diff --git a/docs/lake/format.md b/docs/lake/format.md index dcc7a93921..3d30c67064 100644 --- a/docs/lake/format.md +++ b/docs/lake/format.md @@ -1,10 +1,9 @@ --- -sidebar_position: 2 -sidebar_label: Format +weight: 2 +title: Format +heading: Zed Lake Format --- -# Zed Lake Format - ## _Status_ >This document is a rough draft and work in progress. We plan to @@ -69,7 +68,7 @@ such an assumption). #### Data Objects A data object is created by a single writer using a globally unique name -with an embedded KSUID. +with an embedded KSUID. New objects are written in their entirety. No updates, appends, or modifications may be made once an object exists. Given these semantics, any such object may be diff --git a/docs/language/README.md b/docs/language/_index.md similarity index 91% rename from docs/language/README.md rename to docs/language/_index.md index 2bcd9ab64d..cb7d545616 100644 --- a/docs/language/README.md +++ b/docs/language/_index.md @@ -1,4 +1,8 @@ -# The Zed Language +--- +title: Language +weight: 4 +heading: The Zed Language +--- The language documents: * provide an [overview](overview.md) of the Zed language, diff --git a/docs/language/aggregates/README.md b/docs/language/aggregates/_index.md similarity index 94% rename from docs/language/aggregates/README.md rename to docs/language/aggregates/_index.md index 86a5f8488c..2f28163798 100644 --- a/docs/language/aggregates/README.md +++ b/docs/language/aggregates/_index.md @@ -1,5 +1,6 @@ -# Aggregate Functions - +--- +title: Aggregates +heading: Aggregate Functions --- Aggregate functions appear in either [summarization](../operators/summarize.md) diff --git a/docs/language/aggregates/and.md b/docs/language/aggregates/and.md index 1de11e15b6..44b90d88cc 100644 --- a/docs/language/aggregates/and.md +++ b/docs/language/aggregates/and.md @@ -17,7 +17,7 @@ Anded value of simple sequence: ```mdtest-command echo 'true false true' | super -z -c 'and(this)' - ``` -=> + ```mdtest-output false ``` @@ -26,7 +26,7 @@ Continuous AND of simple sequence: ```mdtest-command echo 'true false true' | super -z -c 'yield and(this)' - ``` -=> + ```mdtest-output true false @@ -37,7 +37,7 @@ Unrecognized types are ignored and not coerced for truthiness: ```mdtest-command echo 'true "foo" 0 false true' | super -z -c 'yield and(this)' - ``` -=> + ```mdtest-output true true @@ -51,7 +51,7 @@ AND of values grouped by key: echo '{a:true,k:1} {a:true,k:1} {a:true,k:2} {a:false,k:2}' | super -z -c 'and(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,and:true} {k:2,and:false} diff --git a/docs/language/aggregates/any.md b/docs/language/aggregates/any.md index 5c866ac46c..f7f081aae2 100644 --- a/docs/language/aggregates/any.md +++ b/docs/language/aggregates/any.md @@ -18,7 +18,7 @@ Any picks the first one in this scenario but this behavior is undefined: ```mdtest-command echo '1 2 3 4' | super -z -c 'any(this)' - ``` -=> + ```mdtest-output 1 ``` @@ -27,7 +27,7 @@ Continuous any over a simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'yield any(this)' - ``` -=> + ```mdtest-output 1 1 @@ -39,7 +39,7 @@ Any is not sensitive to mixed types as it just picks one: ```mdtest-command echo '"foo" 1 2 3 ' | super -z -c 'any(this)' - ``` -=> + ```mdtest-output "foo" ``` @@ -49,7 +49,7 @@ Pick from groups bucketed by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'any(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,any:1} {k:2,any:3} diff --git a/docs/language/aggregates/avg.md b/docs/language/aggregates/avg.md index 368cccbd10..c342394f92 100644 --- a/docs/language/aggregates/avg.md +++ b/docs/language/aggregates/avg.md @@ -17,7 +17,7 @@ Average value of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'avg(this)' - ``` -=> + ```mdtest-output 2.5 ``` @@ -26,7 +26,7 @@ Continuous average of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'yield avg(this)' - ``` -=> + ```mdtest-output 1. 1.5 @@ -38,7 +38,7 @@ Unrecognized types are ignored: ```mdtest-command echo '1 2 3 4 "foo"' | super -z -c 'avg(this)' - ``` -=> + ```mdtest-output 2.5 ``` @@ -48,7 +48,7 @@ Average of values bucketed by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'avg(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,avg:1.5} {k:2,avg:3.5} diff --git a/docs/language/aggregates/collect.md b/docs/language/aggregates/collect.md index 74c4cd3e6d..b6bc14dc4f 100644 --- a/docs/language/aggregates/collect.md +++ b/docs/language/aggregates/collect.md @@ -19,7 +19,7 @@ Simple sequence collected into an array: ```mdtest-command echo '1 2 3 4' | super -z -c 'collect(this)' - ``` -=> + ```mdtest-output [1,2,3,4] ``` @@ -28,7 +28,7 @@ Continuous collection over a simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'yield collect(this)' - ``` -=> + ```mdtest-output [1] [1,2] @@ -40,7 +40,7 @@ Mixed types create a union type for the array elements: ```mdtest-command echo '1 2 3 4 "foo"' | super -z -c 'collect(this)' - ``` -=> + ```mdtest-output [1,2,3,4,"foo"] ``` @@ -50,7 +50,7 @@ Create arrays of values bucketed by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'collect(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,collect:[1,2]} {k:2,collect:[3,4]} diff --git a/docs/language/aggregates/collect_map.md b/docs/language/aggregates/collect_map.md index 3032ac13b0..e421ad60d1 100644 --- a/docs/language/aggregates/collect_map.md +++ b/docs/language/aggregates/collect_map.md @@ -21,7 +21,7 @@ Combine a sequence of records into a map: echo '{stock:"APPL",price:145.03} {stock:"GOOG",price:87.07}' | super -z -c 'collect_map(|{stock:price}|)' - ``` -=> + ```mdtest-output |{"APPL":145.03,"GOOG":87.07}| ``` @@ -31,7 +31,7 @@ Continuous collection over a simple sequence: echo '|{"APPL":145.03}| |{"GOOG":87.07}| |{"APPL":150.13}|' | super -z -c 'yield collect_map(this)' - ``` -=> + ```mdtest-output |{"APPL":145.03}| |{"APPL":145.03,"GOOG":87.07}| @@ -46,7 +46,7 @@ echo '{stock:"APPL",price:145.03,day:0} {stock:"GOOG",price:89.15,day:1}' | super -z -c 'collect_map(|{stock:price}|) by day |> sort' - ``` -=> + ```mdtest-output {day:0,collect_map:|{"APPL":145.03,"GOOG":87.07}|} {day:1,collect_map:|{"APPL":150.13,"GOOG":89.15}|} diff --git a/docs/language/aggregates/count.md b/docs/language/aggregates/count.md index 22a25b27ee..bc2a7a9b21 100644 --- a/docs/language/aggregates/count.md +++ b/docs/language/aggregates/count.md @@ -17,7 +17,7 @@ Count of values in a simple sequence: ```mdtest-command echo '1 2 3' | super -z -c 'count()' - ``` -=> + ```mdtest-output 3(uint64) ``` @@ -26,7 +26,7 @@ Continuous count of simple sequence: ```mdtest-command echo '1 2 3' | super -z -c 'yield count()' - ``` -=> + ```mdtest-output 1(uint64) 2(uint64) @@ -37,7 +37,7 @@ Mixed types are handled: ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'yield count()' - ``` -=> + ```mdtest-output 1(uint64) 2(uint64) @@ -48,7 +48,7 @@ Count of values in buckets grouped by key: ```mdtest-command echo '{a:1,k:1} {a:2,k:1} {a:3,k:2}' | super -z -c 'count() by k |> sort' - ``` -=> + ```mdtest-output {k:1,count:2(uint64)} {k:2,count:1(uint64)} @@ -58,7 +58,7 @@ A simple count with no input values returns no output: ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'where grep("bar") |> count()' - ``` -=> + ```mdtest-output ``` @@ -66,7 +66,7 @@ Count can return an explicit zero when using a `where` clause in the aggregation ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'count() where grep("bar")' - ``` -=> + ```mdtest-output 0(uint64) ``` @@ -75,7 +75,7 @@ Note that the number of input values are counted, unlike the [`len` function](.. ```mdtest-command echo '[1,2,3]' | super -z -c 'count()' - ``` -=> + ```mdtest-output 1(uint64) ``` diff --git a/docs/language/aggregates/dcount.md b/docs/language/aggregates/dcount.md index 52357046bd..656151147a 100644 --- a/docs/language/aggregates/dcount.md +++ b/docs/language/aggregates/dcount.md @@ -18,7 +18,7 @@ Count of values in a simple sequence: ```mdtest-command echo '1 2 2 3' | super -z -c 'dcount(this)' - ``` -=> + ```mdtest-output 3(uint64) ``` @@ -27,7 +27,7 @@ Continuous count of simple sequence: ```mdtest-command echo '1 2 2 3' | super -z -c 'yield dcount(this)' - ``` -=> + ```mdtest-output 1(uint64) 2(uint64) @@ -39,7 +39,7 @@ Mixed types are handled: ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'yield dcount(this)' - ``` -=> + ```mdtest-output 1(uint64) 2(uint64) @@ -50,7 +50,7 @@ The estimated result may become less accurate with more unique input values: ```mdtest-command seq 10000 | super -z -c 'dcount(this)' - ``` -=> + ```mdtest-output 9987(uint64) ``` @@ -59,7 +59,7 @@ Count of values in buckets grouped by key: ```mdtest-command echo '{a:1,k:1} {a:2,k:1} {a:3,k:2}' | super -z -c 'dcount(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,dcount:2(uint64)} {k:2,dcount:1(uint64)} diff --git a/docs/language/aggregates/fuse.md b/docs/language/aggregates/fuse.md index 498c856bcf..1d010d7a13 100644 --- a/docs/language/aggregates/fuse.md +++ b/docs/language/aggregates/fuse.md @@ -12,7 +12,7 @@ fuse(any) -> type The _fuse_ aggregate function applies [type fusion](../shaping.md#type-fusion) to its input and returns the fused type. -This aggregation is useful with group-by for data exploration and discovery +This aggregation is useful with group-by for data exploration and discovery when searching for shaping rules to cluster a large number of varied input types to a smaller number of fused types each from a set of interrelated types. @@ -22,7 +22,7 @@ Fuse two records: ```mdtest-command echo '{a:1,b:2}{a:2,b:"foo"}' | super -z -c 'fuse(this)' - ``` -=> + ```mdtest-output <{a:int64,b:(int64,string)}> ``` @@ -31,7 +31,7 @@ Fuse records with a group-by key: echo '{a:1,b:"bar"}{a:2.1,b:"foo"}{a:3,b:"bar"}' | super -z -c 'fuse(this) by b |> sort' - ``` -=> + ```mdtest-output {b:"bar",fuse:<{a:int64,b:string}>} {b:"foo",fuse:<{a:float64,b:string}>} diff --git a/docs/language/aggregates/max.md b/docs/language/aggregates/max.md index 4b672e3ffe..0c377eaf1c 100644 --- a/docs/language/aggregates/max.md +++ b/docs/language/aggregates/max.md @@ -17,7 +17,7 @@ Maximum value of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'max(this)' - ``` -=> + ```mdtest-output 4 ``` @@ -26,7 +26,7 @@ Continuous maximum of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'yield max(this)' - ``` -=> + ```mdtest-output 1 2 @@ -38,7 +38,7 @@ Unrecognized types are ignored: ```mdtest-command echo '1 2 3 4 "foo"' | super -z -c 'max(this)' - ``` -=> + ```mdtest-output 4 ``` @@ -48,7 +48,7 @@ Maximum value within buckets grouped by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'max(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,max:2} {k:2,max:4} diff --git a/docs/language/aggregates/min.md b/docs/language/aggregates/min.md index 06a8786494..3a61bad03d 100644 --- a/docs/language/aggregates/min.md +++ b/docs/language/aggregates/min.md @@ -17,7 +17,7 @@ Minimum value of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'min(this)' - ``` -=> + ```mdtest-output 1 ``` @@ -26,7 +26,7 @@ Continuous minimum of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'yield min(this)' - ``` -=> + ```mdtest-output 1 1 @@ -38,7 +38,7 @@ Unrecognized types are ignored: ```mdtest-command echo '1 2 3 4 "foo"' | super -z -c 'min(this)' - ``` -=> + ```mdtest-output 1 ``` @@ -48,7 +48,7 @@ Minimum value within buckets grouped by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'min(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,min:1} {k:2,min:3} diff --git a/docs/language/aggregates/or.md b/docs/language/aggregates/or.md index 884cca4e41..78c53b097d 100644 --- a/docs/language/aggregates/or.md +++ b/docs/language/aggregates/or.md @@ -17,7 +17,7 @@ Ored value of simple sequence: ```mdtest-command echo 'false true false' | super -z -c 'or(this)' - ``` -=> + ```mdtest-output true ``` @@ -26,7 +26,7 @@ Continuous OR of simple sequence: ```mdtest-command echo 'false true false' | super -z -c 'yield or(this)' - ``` -=> + ```mdtest-output false true @@ -37,7 +37,7 @@ Unrecognized types are ignored and not coerced for truthiness: ```mdtest-command echo 'false "foo" 1 true false' | super -z -c 'yield or(this)' - ``` -=> + ```mdtest-output false false @@ -51,7 +51,7 @@ OR of values grouped by key: echo '{a:true,k:1} {a:false,k:1} {a:false,k:2} {a:false,k:2}' | super -z -c 'or(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,or:true} {k:2,or:false} diff --git a/docs/language/aggregates/sum.md b/docs/language/aggregates/sum.md index 00dfd41e0f..a4fd68f084 100644 --- a/docs/language/aggregates/sum.md +++ b/docs/language/aggregates/sum.md @@ -17,7 +17,7 @@ Sum of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'sum(this)' - ``` -=> + ```mdtest-output 10 ``` @@ -26,7 +26,7 @@ Continuous sum of simple sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'yield sum(this)' - ``` -=> + ```mdtest-output 1 3 @@ -38,7 +38,7 @@ Unrecognized types are ignored: ```mdtest-command echo '1 2 3 4 "foo"' | super -z -c 'sum(this)' - ``` -=> + ```mdtest-output 10 ``` @@ -48,7 +48,7 @@ Sum of values bucketed by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'sum(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,sum:3} {k:2,sum:7} diff --git a/docs/language/aggregates/union.md b/docs/language/aggregates/union.md index 4265fa1da9..ff38c3d6d2 100644 --- a/docs/language/aggregates/union.md +++ b/docs/language/aggregates/union.md @@ -20,7 +20,7 @@ Create a set of values from a simple sequence: ```mdtest-command echo '1 2 3 3' | super -z -c 'union(this)' - ``` -=> + ```mdtest-output |[1,2,3]| ``` @@ -29,7 +29,7 @@ Create sets continuously from values in a simple sequence: ```mdtest-command echo '1 2 3 3' | super -z -c 'yield union(this)' - ``` -=> + ```mdtest-output |[1]| |[1,2]| @@ -41,7 +41,7 @@ Mixed types create a union type for the set elements: ```mdtest-command echo '1 2 3 "foo"' | super -z -c 'set:=union(this) |> yield this,typeof(set)' - ``` -=> + ```mdtest-output {set:|[1,2,3,"foo"]|} <|[(int64,string)]|> @@ -52,7 +52,7 @@ Create sets of values bucketed by key: echo '{a:1,k:1} {a:2,k:1} {a:3,k:2} {a:4,k:2}' | super -z -c 'union(a) by k |> sort' - ``` -=> + ```mdtest-output {k:1,union:|[1,2]|} {k:2,union:|[3,4]|} diff --git a/docs/language/conventions.md b/docs/language/conventions.md index 39d5743eca..0359014178 100644 --- a/docs/language/conventions.md +++ b/docs/language/conventions.md @@ -1,10 +1,9 @@ --- -sidebar_position: 9 -sidebar_label: Conventions +weight: 9 +title: Conventions +heading: Type Conventions --- -# Type Conventions - [Function](functions/README.md) arguments and [operator](operators/README.md) input values are all dynamically typed, yet certain functions expect certain specific [data types](data-types.md) or classes of data types. To this end, the function and operator prototypes diff --git a/docs/language/data-types.md b/docs/language/data-types.md index 44d4969562..9283a74b47 100644 --- a/docs/language/data-types.md +++ b/docs/language/data-types.md @@ -1,10 +1,8 @@ --- -sidebar_position: 3 -sidebar_label: Data Types +weight: 3 +title: Data Types --- -# Data Types - The SuperPipe language includes most data types of a typical programming language as defined in the [super data model](../formats/zed.md). @@ -315,7 +313,7 @@ results from accessing a field that is not present. Thus, `x==NULL` and `x==MISSING` could disambiguate the two cases above. SuperPipe, instead, recognizes that the SQL value `MISSING` is a paradox: -I'm here but I'm not. +I'm here but I'm not. In reality, a `MISSING` value is not a value. It's an error condition that resulted from trying to reference something that didn't exist. diff --git a/docs/language/expressions.md b/docs/language/expressions.md index b2257b9eab..c1059a44e7 100644 --- a/docs/language/expressions.md +++ b/docs/language/expressions.md @@ -1,10 +1,8 @@ --- -sidebar_position: 5 -sidebar_label: Expressions +weight: 5 +title: Expressions --- -# Expressions - Zed expressions follow the typical patterns in programming languages. Expressions are typically used within pipeline operators to perform computations on input values and are typically evaluated once per each @@ -243,7 +241,7 @@ produces ``` Zed includes many [built-in functions](functions/README.md), some of which take -a variable number of arguments. +a variable number of arguments. Zed also allows you to create [user-defined functions](statements.md#func-statements). diff --git a/docs/language/functions/README.md b/docs/language/functions/_index.md similarity index 99% rename from docs/language/functions/README.md rename to docs/language/functions/_index.md index 65280e0848..27d2c54652 100644 --- a/docs/language/functions/README.md +++ b/docs/language/functions/_index.md @@ -1,5 +1,5 @@ -# Functions - +--- +title: Functions --- Functions appear in [expression](../expressions.md) context and diff --git a/docs/language/functions/abs.md b/docs/language/functions/abs.md index 7bef9ce3bb..c5d047e188 100644 --- a/docs/language/functions/abs.md +++ b/docs/language/functions/abs.md @@ -15,17 +15,12 @@ must be a numeric type. ### Examples -Absolute value of a various numbers: -```mdtest-command -echo '1 -1 0 -1.0 -1(int8) 1(uint8) "foo"' | super -z -c 'yield abs(this)' - -``` -=> -```mdtest-output -1 +{{< super-playground "yield abs(this)" >}} 1 +-1 0 -1. -1(int8) +-1.0 +-1(int8) 1(uint8) -error({message:"abs: not a number",on:"foo"}) -``` +"foo" +{{}} diff --git a/docs/language/functions/base64.md b/docs/language/functions/base64.md index 01443ee3ac..a62a58432f 100644 --- a/docs/language/functions/base64.md +++ b/docs/language/functions/base64.md @@ -21,7 +21,7 @@ Encode byte sequence `0x010203` into its Base64 string: ```mdtest-command echo '0x010203' | super -z -c 'yield base64(this)' - ``` -=> + ```mdtest-output "AQID" ``` @@ -29,7 +29,7 @@ Decode "AQID" into byte sequence `0x010203`: ```mdtest-command echo '"AQID"' | super -z -c 'yield base64(this)' - ``` -=> + ```mdtest-output 0x010203 ``` @@ -37,7 +37,7 @@ Encode ASCII string into Base64-encoded string: ```mdtest-command echo '"hello, world"' | super -z -c 'yield base64(bytes(this))' - ``` -=> + ```mdtest-output "aGVsbG8sIHdvcmxk" ``` @@ -45,7 +45,7 @@ Decode a Base64 string and cast the decoded bytes to a string: ```mdtest-command echo '"aGVsbG8gd29ybGQ="' | super -z -c 'yield string(base64(this))' - ``` -=> + ```mdtest-output "hello world" ``` diff --git a/docs/language/functions/bucket.md b/docs/language/functions/bucket.md index ad3dd90fa7..be3c8c2afa 100644 --- a/docs/language/functions/bucket.md +++ b/docs/language/functions/bucket.md @@ -23,7 +23,7 @@ Bucket a couple times to hour intervals: echo '2020-05-26T15:27:47Z "5/26/2020 3:27pm"' | super -z -c 'yield bucket(time(this), 1h)' - ``` -=> + ```mdtest-output 2020-05-26T15:00:00Z 2020-05-26T15:00:00Z diff --git a/docs/language/functions/ceil.md b/docs/language/functions/ceil.md index 5f0425a93a..d5e2e3c6a9 100644 --- a/docs/language/functions/ceil.md +++ b/docs/language/functions/ceil.md @@ -19,7 +19,7 @@ The ceiling of a various numbers: ```mdtest-command echo '1.5 -1.5 1(uint8) 1.5(float32)' | super -z -c 'yield ceil(this)' - ``` -=> + ```mdtest-output 2. -1. diff --git a/docs/language/functions/cidr_match.md b/docs/language/functions/cidr_match.md index 81c6f9fd4b..74a5e4265d 100644 --- a/docs/language/functions/cidr_match.md +++ b/docs/language/functions/cidr_match.md @@ -22,7 +22,7 @@ Test whether values are IP addresses in a network: echo '10.1.2.129 11.1.2.129 10 "foo"' | super -z -c 'yield cidr_match(10.0.0.0/8, this)' - ``` -=> + ```mdtest-output true false @@ -35,7 +35,7 @@ It also works for IPs in complex values: echo '[10.1.2.129,11.1.2.129] {a:10.0.0.1} {a:11.0.0.1}' | super -z -c 'yield cidr_match(10.0.0.0/8, this)' - ``` -=> + ```mdtest-output true true @@ -46,7 +46,7 @@ The first argument must be a network: ```mdtest-command echo '10.0.0.1' | super -z -c 'yield cidr_match([1,2,3], this)' - ``` -=> + ```mdtest-output error({message:"cidr_match: not a net",on:[1,2,3]}) ``` diff --git a/docs/language/functions/coalesce.md b/docs/language/functions/coalesce.md index 4cf16c8950..f25789782a 100644 --- a/docs/language/functions/coalesce.md +++ b/docs/language/functions/coalesce.md @@ -19,7 +19,7 @@ are null, `error("missing")`, or `error("quiet")`. ```mdtest-command super -z -c 'yield coalesce(null, error("missing"), error("quiet"), 1)' ``` -=> + ```mdtest-output 1 ``` @@ -27,7 +27,7 @@ super -z -c 'yield coalesce(null, error("missing"), error("quiet"), 1)' ```mdtest-command super -z -c 'yield coalesce(null, error("missing"), error("quiet"))' ``` -=> + ```mdtest-output null ``` diff --git a/docs/language/functions/compare.md b/docs/language/functions/compare.md index a0f157feb2..eb08111d74 100644 --- a/docs/language/functions/compare.md +++ b/docs/language/functions/compare.md @@ -23,7 +23,7 @@ is treated as the minimum or maximum value. ```mdtest-command echo '{a: 2, b: "1"}' | super -z -c 'yield compare(a, b)' - ``` -=> + ```mdtest-output -1 ``` diff --git a/docs/language/functions/error.md b/docs/language/functions/error.md index b67c34ce8c..1fe24c1582 100644 --- a/docs/language/functions/error.md +++ b/docs/language/functions/error.md @@ -21,7 +21,7 @@ Wrap a record as a structured error: echo '{foo:"foo"}' | super -z -c 'yield error({message:"bad value", value:this})' - ``` -=> + ```mdtest-output error({message:"bad value",value:{foo:"foo"}}) ``` @@ -30,7 +30,7 @@ Wrap any value as an error: ```mdtest-command echo '1 "foo" [1,2,3]' | super -z -c 'yield error(this)' - ``` -=> + ```mdtest-output error(1) error("foo") @@ -42,7 +42,7 @@ Test if a value is an error and show its type "kind": echo 'error("exception") "exception"' | super -Z -c 'yield {this,err:is_error(this),kind:kind(this)}' - ``` -=> + ```mdtest-output { this: error("exception"), @@ -62,7 +62,7 @@ missing fields to succeed: ```mdtest-command echo '{}' | super -z -c 'badfield:=x |> yield badfield==error("missing")' - ``` -=> + ```mdtest-output error("missing") ``` diff --git a/docs/language/functions/fields.md b/docs/language/functions/fields.md index ebce2916c6..35e3052597 100644 --- a/docs/language/functions/fields.md +++ b/docs/language/functions/fields.md @@ -23,7 +23,7 @@ Extract the fields of a nested record: ```mdtest-command echo '{a:1,b:2,c:{d:3,e:4}}' | super -z -c 'yield fields(this)' - ``` -=> + ```mdtest-output [["a"],["b"],["c","d"],["c","e"]] ``` @@ -32,7 +32,7 @@ Easily convert to dotted names if you prefer: echo '{a:1,b:2,c:{d:3,e:4}}' | super -z -c 'over fields(this) |> yield join(this,".")' - ``` -=> + ```mdtest-output "a" "b" @@ -43,7 +43,7 @@ A record is expected: ```mdtest-command echo 1 | super -z -c 'yield {f:fields(this)}' - ``` -=> + ```mdtest-output {f:error("missing")} ``` diff --git a/docs/language/functions/flatten.md b/docs/language/functions/flatten.md index b5d69da1d4..4514ac9743 100644 --- a/docs/language/functions/flatten.md +++ b/docs/language/functions/flatten.md @@ -20,7 +20,7 @@ inner type is a union of the record types present. ```mdtest-command echo '{a:1,b:{c:"foo"}}' | super -z -c 'yield flatten(this)' - ``` -=> + ```mdtest-output [{key:["a"],value:1},{key:["b","c"],value:"foo"}] ``` diff --git a/docs/language/functions/floor.md b/docs/language/functions/floor.md index bcaff652a8..7fb08cf33e 100644 --- a/docs/language/functions/floor.md +++ b/docs/language/functions/floor.md @@ -19,7 +19,7 @@ The floor of a various numbers: ```mdtest-command echo '1.5 -1.5 1(uint8) 1.5(float32)' | super -z -c 'yield floor(this)' - ``` -=> + ```mdtest-output 1. -2. diff --git a/docs/language/functions/grep.md b/docs/language/functions/grep.md index 904bdde6d8..1cf0ccce91 100644 --- a/docs/language/functions/grep.md +++ b/docs/language/functions/grep.md @@ -33,7 +33,7 @@ _Reach into nested records_ ```mdtest-command echo '{foo:10}{bar:{s:"baz"}}' | super -z -c 'grep("baz")' - ``` -=> + ```mdtest-output {bar:{s:"baz"}} ``` @@ -41,14 +41,14 @@ _It only matches string fields_ ```mdtest-command echo '{foo:10}{bar:{s:"baz"}}' | super -z -c 'grep("10")' - ``` -=> + ```mdtest-output ``` _Match a field name_ ```mdtest-command echo '{foo:10}{bar:{s:"baz"}}' | super -z -c 'grep("foo")' - ``` -=> + ```mdtest-output {foo:10} ``` @@ -56,7 +56,7 @@ _Regular expression_ ```mdtest-command echo '{foo:10}{bar:{s:"baz"}}' | super -z -c 'grep(/foo|baz/)' - ``` -=> + ```mdtest-output {foo:10} {bar:{s:"baz"}} @@ -66,7 +66,7 @@ _Glob with a second argument_ ```mdtest-command echo '{s:"bar"}{s:"foo"}{s:"baz"}{t:"baz"}' | super -z -c 'grep(b*, s)' - ``` -=> + ```mdtest-output {s:"bar"} {s:"baz"} diff --git a/docs/language/functions/grok.md b/docs/language/functions/grok.md index ab6ae22341..56cf02e474 100644 --- a/docs/language/functions/grok.md +++ b/docs/language/functions/grok.md @@ -155,7 +155,7 @@ echo '"2020-09-16T04:20:42.45+01:00 DEBUG This is a sample debug log message"' | super -Z -c 'yield grok("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}", this)' - ``` -=> + ```mdtest-output { timestamp: "2020-09-16T04:20:42.45+01:00", @@ -175,7 +175,7 @@ echo '"+7000"' | this, "MY_NUMTZ [+-]\\d{4}")' - ``` -=> + ```mdtest-output {tz:"+7000"} ``` @@ -187,11 +187,11 @@ readability. ```mdtest-command echo '"(555)-1212"' | super -z -c 'yield grok("\\(%{PH_PREFIX:prefix}\\)-%{PH_LINE_NUM:line_number}", - this, + this, "PH_PREFIX \\d{3}\n" + "PH_LINE_NUM \\d{4}")' - ``` -=> + ```mdtest-output {prefix:"555",line_number:"1212"} ``` diff --git a/docs/language/functions/has.md b/docs/language/functions/has.md index 3a7459f660..8d296fe37a 100644 --- a/docs/language/functions/has.md +++ b/docs/language/functions/has.md @@ -39,7 +39,7 @@ echo '{foo:10}' | super -z -c 'yield {yes:has(foo+1),no:has(bar+1)}' - echo 1 | super -z -c 'yield has(bar)' - echo '{x:error("missing")}' | super -z -c 'yield has(x)' - ``` -=> + ```mdtest-output {yes:true,no:false} {yes:true,no:false} diff --git a/docs/language/functions/has_error.md b/docs/language/functions/has_error.md index d7af8be356..8e9b1c997c 100644 --- a/docs/language/functions/has_error.md +++ b/docs/language/functions/has_error.md @@ -20,7 +20,7 @@ into value's leaves to determine if there is an error in the value. echo '{a:{b:"foo"}}' | super -z -c 'yield has_error(this)' - echo '{a:{b:"foo"}}' | super -z -c 'a.x := a.y + 1 |> yield has_error(this)' - ``` -=> + ```mdtest-output false true diff --git a/docs/language/functions/hex.md b/docs/language/functions/hex.md index a9c5df9ab3..7c7dfeab18 100644 --- a/docs/language/functions/hex.md +++ b/docs/language/functions/hex.md @@ -20,7 +20,7 @@ Encode a simple bytes sequence as a hexadecimal string: ```mdtest-command echo '0x0102ff' | super -z -c 'yield hex(this)' - ``` -=> + ```mdtest-output "0102ff" ``` @@ -28,7 +28,7 @@ Decode a simple hex string: ```mdtest-command echo '"0102ff"' | super -z -c 'yield hex(this)' - ``` -=> + ```mdtest-output 0x0102ff ``` @@ -36,7 +36,7 @@ Encode the bytes of an ASCII string as a hexadecimal string: ```mdtest-command echo '"hello, world"' | super -z -c 'yield hex(bytes(this))' - ``` -=> + ```mdtest-output "68656c6c6f2c20776f726c64" ``` @@ -44,7 +44,7 @@ Decode hex string representing ASCII into its string form: ```mdtest-command echo '"68656c6c6f20776f726c64"' | super -z -c 'yield string(hex(this))' - ``` -=> + ```mdtest-output "hello world" ``` diff --git a/docs/language/functions/is.md b/docs/language/functions/is.md index 765aa4f13a..5ff78a388d 100644 --- a/docs/language/functions/is.md +++ b/docs/language/functions/is.md @@ -19,7 +19,7 @@ Test simple types: ```mdtest-command echo '1.' | super -z -c 'yield {yes:is(),no:is()}' - ``` -=> + ```mdtest-output {yes:true,no:false} ``` @@ -28,7 +28,7 @@ Test for a given input's record type or "shape": ```mdtest-command echo '{s:"hello"}' | super -z -c 'yield is(<{s:string}>)' - ``` -=> + ```mdtest-output true ``` @@ -38,7 +38,7 @@ but if you use the type name or typeunder function, there is a match: echo '{s:"hello"}(=foo)' | super -z -c 'yield is(<{s:string}>)' - echo '{s:"hello"}(=foo)' | super -z -c 'yield is()' - ``` -=> + ```mdtest-output false true @@ -48,7 +48,7 @@ To test the underlying type, just use `==`: ```mdtest-command echo '{s:"hello"}(=foo)' | super -z -c 'yield typeunder(this)==<{s:string}>' - ``` -=> + ```mdtest-output true ``` diff --git a/docs/language/functions/is_error.md b/docs/language/functions/is_error.md index 6b9552e250..6da2e0d78d 100644 --- a/docs/language/functions/is_error.md +++ b/docs/language/functions/is_error.md @@ -19,7 +19,7 @@ A simple value is not an error: ```mdtest-command echo 1 | super -z -c 'yield is_error(this)' - ``` -=> + ```mdtest-output false ``` @@ -28,7 +28,7 @@ An error value is an error: ```mdtest-command echo "error(1)" | super -z -c 'yield is_error(this)' - ``` -=> + ```mdtest-output true ``` @@ -38,7 +38,7 @@ Convert an error string into a record with an indicator and a message: echo '"not an error" error("an error")' | super -z -c 'yield {err:is_error(this),message:under(this)}' - ``` -=> + ```mdtest-output {err:false,message:"not an error"} {err:true,message:"an error"} diff --git a/docs/language/functions/join.md b/docs/language/functions/join.md index 297e04b4e0..9e62c55c26 100644 --- a/docs/language/functions/join.md +++ b/docs/language/functions/join.md @@ -19,7 +19,7 @@ Join a symbol array of strings: ```mdtest-command echo '["a","b","c"]' | super -z -c 'yield join(this, ",")' - ``` -=> + ```mdtest-output "a,b,c" ``` @@ -29,7 +29,7 @@ Join non-string arrays by first casting: echo '[1,2,3] [10.0.0.1,10.0.0.2]' | super -z -c 'yield join(cast(this, <[string]>), "...")' - ``` -=> + ```mdtest-output "1...2...3" "10.0.0.1...10.0.0.2" diff --git a/docs/language/functions/kind.md b/docs/language/functions/kind.md index 3c876a052c..461631c4ec 100644 --- a/docs/language/functions/kind.md +++ b/docs/language/functions/kind.md @@ -20,7 +20,7 @@ A primitive value's kind is "primitive": ```mdtest-command echo '1 "a" 10.0.0.1' | super -z -c 'yield kind(this)' - ``` -=> + ```mdtest-output "primitive" "primitive" @@ -32,7 +32,7 @@ these empty values of various complex types: ```mdtest-command echo '{} [] |[]| |{}| 1((int64,string))' | super -z -c 'yield kind(this)' - ``` -=> + ```mdtest-output "record" "array" @@ -45,7 +45,7 @@ A Zed error has kind "error": ```mdtest-command echo null | super -z -c 'yield kind(1/0)' - ``` -=> + ```mdtest-output "error" ``` @@ -54,7 +54,7 @@ A Zed type's kind is the kind of the type: ```mdtest-command echo '<{s:string}>' | super -z -c 'yield kind(this)' - ``` -=> + ```mdtest-output "record" ``` diff --git a/docs/language/functions/ksuid.md b/docs/language/functions/ksuid.md index a6359c77b6..b078832885 100644 --- a/docs/language/functions/ksuid.md +++ b/docs/language/functions/ksuid.md @@ -25,7 +25,7 @@ returned as a bytes value. echo '{id:0x0dfc90519b60f362e84a3fdddd9b9e63e1fb90d1}' | super -z -c 'id := ksuid(id)' - ``` -=> + ```mdtest-output {id:"1zjJzTWWCJNVrGwqB8kZwhTM2fR"} ``` diff --git a/docs/language/functions/len.md b/docs/language/functions/len.md index 7c918f1de4..988870a7e2 100644 --- a/docs/language/functions/len.md +++ b/docs/language/functions/len.md @@ -33,7 +33,7 @@ Take the length of various types: echo '[1,2,3] |["hello"]| {a:1,b:2} "hello" 10.0.0.1 1' | super -z -c 'yield {this,len:len(this)}' - ``` -=> + ```mdtest-output {this:[1,2,3],len:3} {this:|["hello"]|,len:1} diff --git a/docs/language/functions/levenshtein.md b/docs/language/functions/levenshtein.md index 3b518159ce..e1380b467b 100644 --- a/docs/language/functions/levenshtein.md +++ b/docs/language/functions/levenshtein.md @@ -19,7 +19,7 @@ distance](https://en.wikipedia.org/wiki/Levenshtein_distance) between strings ```mdtest-command echo '{a:"kitten",b:"sitting"}' | super -z -c 'yield levenshtein(a, b)' - ``` -=> + ```mdtest-output 3 ``` diff --git a/docs/language/functions/log.md b/docs/language/functions/log.md index ea5e4949f8..7b6294ceb2 100644 --- a/docs/language/functions/log.md +++ b/docs/language/functions/log.md @@ -19,7 +19,7 @@ The logarithm of various numbers: ```mdtest-command echo '4 4.0 2.718 -1' | super -z -c 'yield log(this)' - ``` -=> + ```mdtest-output 1.3862943611198906 1.3862943611198906 @@ -31,7 +31,7 @@ The largest power of 10 smaller than the input: ```mdtest-command echo '9 10 20 1000 1100 30000' | super -z -c 'yield int64(log(this)/log(10))' - ``` -=> + ```mdtest-output 0 1 diff --git a/docs/language/functions/lower.md b/docs/language/functions/lower.md index a8d0de6b16..81a2074064 100644 --- a/docs/language/functions/lower.md +++ b/docs/language/functions/lower.md @@ -18,7 +18,7 @@ to lower case and returns the result. ```mdtest-command echo '"Zed"' | super -z -c 'yield lower(this)' - ``` -=> + ```mdtest-output "zed" ``` diff --git a/docs/language/functions/map.md b/docs/language/functions/map.md index c37b7ef261..d73ce659ad 100644 --- a/docs/language/functions/map.md +++ b/docs/language/functions/map.md @@ -21,7 +21,7 @@ Upper case each element of an array: ```mdtest-command echo '["foo","bar","baz"]' | super -z -c 'yield map(this, upper)' - ``` -=> + ```mdtest-output ["FOO","BAR","BAZ"] ``` @@ -37,7 +37,7 @@ echo '[1697151533.41415,1697151540.716529]' | yield map(this, floatToTime) ' - ``` -=> + ```mdtest-output [2023-10-12T22:58:53.414149888Z,2023-10-12T22:59:00.716528896Z] ``` diff --git a/docs/language/functions/missing.md b/docs/language/functions/missing.md index 83381fb4fe..ade2c86381 100644 --- a/docs/language/functions/missing.md +++ b/docs/language/functions/missing.md @@ -38,7 +38,7 @@ echo '{foo:10}' | super -z -c 'yield {yes:missing(bar+1),no:missing(foo+1)}' - echo 1 | super -z -c 'yield missing(bar)' - echo '{x:error("missing")}' | super -z -c 'yield missing(x)' - ``` -=> + ```mdtest-output {yes:true,no:false} {yes:false,no:true} diff --git a/docs/language/functions/nameof.md b/docs/language/functions/nameof.md index 55890c7c96..3b395d933b 100644 --- a/docs/language/functions/nameof.md +++ b/docs/language/functions/nameof.md @@ -19,7 +19,7 @@ A named type yields its name and unnamed types yield a missing error: ```mdtest-command echo '80(port=int16) 80' | super -z -c 'yield nameof(this)' - ``` -=> + ```mdtest-output "port" error("missing") @@ -29,7 +29,7 @@ The missing value can be ignored with quiet: ```mdtest-command echo '80(port=int16) 80' | super -z -c 'yield quiet(nameof(this))' - ``` -=> + ```mdtest-output "port" ``` diff --git a/docs/language/functions/nest_dotted.md b/docs/language/functions/nest_dotted.md index 3f3d9ae9b9..c6cfb8aab9 100644 --- a/docs/language/functions/nest_dotted.md +++ b/docs/language/functions/nest_dotted.md @@ -19,7 +19,7 @@ converted into nested records. If no argument is supplied to `nest_dotted`, ```mdtest-command echo '{"a.b.c":"foo"}' | super -z -c 'yield nest_dotted()' - ``` -=> + ```mdtest-output {a:{b:{c:"foo"}}} ``` diff --git a/docs/language/functions/network_of.md b/docs/language/functions/network_of.md index 1a2de4f6c8..42f7a2da61 100644 --- a/docs/language/functions/network_of.md +++ b/docs/language/functions/network_of.md @@ -22,7 +22,7 @@ Compute the network address of an IP using an `ip` mask argument: ```mdtest-command echo '10.1.2.129' | super -z -c 'yield network_of(this, 255.255.255.128)' - ``` -=> + ```mdtest-output 10.1.2.128/25 ``` @@ -31,7 +31,7 @@ Compute the network address of an IP given an integer prefix argument: ```mdtest-command echo '10.1.2.129' | super -z -c 'yield network_of(this, 25)' - ``` -=> + ```mdtest-output 10.1.2.128/25 ``` @@ -40,7 +40,7 @@ Compute the network address implied by IP classful addressing: ```mdtest-command echo '10.1.2.129' | super -z -c 'yield network_of(this)' - ``` -=> + ```mdtest-output 10.0.0.0/8 ``` @@ -49,7 +49,7 @@ The network of a value that is not an IP is an error: ```mdtest-command echo 1 | super -z -c 'yield network_of(this)' - ``` -=> + ```mdtest-output error({message:"network_of: not an IP",on:1}) ``` @@ -58,7 +58,7 @@ Network masks must be contiguous: ```mdtest-command echo '10.1.2.129' | super -z -c 'yield network_of(this, 255.255.128.255)' - ``` -=> + ```mdtest-output error({message:"network_of: mask is non-contiguous",on:255.255.128.255}) ``` diff --git a/docs/language/functions/parse_uri.md b/docs/language/functions/parse_uri.md index 33ecdc0419..ecf977c7a1 100644 --- a/docs/language/functions/parse_uri.md +++ b/docs/language/functions/parse_uri.md @@ -34,7 +34,7 @@ with the following type signature: echo '"scheme://user:password@host:12345/path?a=1&a=2&b=3&c=#fragment"' | super -Z -c 'yield parse_uri(this)' - ``` -=> + ```mdtest-output { scheme: "scheme", diff --git a/docs/language/functions/parse_zson.md b/docs/language/functions/parse_zson.md index 816bbfb0b1..0a4c7efa73 100644 --- a/docs/language/functions/parse_zson.md +++ b/docs/language/functions/parse_zson.md @@ -23,7 +23,7 @@ _Parse Super JSON text_ ```mdtest-command echo '{foo:"{a:\"1\",b:2}"}' | super -z -c 'foo := parse_zson(foo)' - ``` -=> + ```mdtest-output {foo:{a:"1",b:2}} ``` @@ -33,7 +33,7 @@ _Parse JSON text_ echo '{"foo": "{\"a\": \"1\", \"b\": 2}"}' | super -z -c 'foo := parse_zson(foo)' - ``` -=> + ```mdtest-output {foo:{a:"1",b:2}} ``` diff --git a/docs/language/functions/pow.md b/docs/language/functions/pow.md index 722c3be873..57983fbc09 100644 --- a/docs/language/functions/pow.md +++ b/docs/language/functions/pow.md @@ -18,7 +18,7 @@ The return value is a float64 or an error. ```mdtest-command echo '2' | super -z -c 'yield pow(this, 5)' - ``` -=> + ```mdtest-output 32. ``` diff --git a/docs/language/functions/quiet.md b/docs/language/functions/quiet.md index 3e8b18729e..698053e640 100644 --- a/docs/language/functions/quiet.md +++ b/docs/language/functions/quiet.md @@ -21,7 +21,7 @@ Yield processes a quiet error and thus no output: ```mdtest-command echo 'error("missing")' | super -z -c 'yield quiet(this)' - ``` -=> + ```mdtest-output ``` @@ -29,7 +29,7 @@ Without quiet, yield produces the missing error: ```mdtest-command echo 'error("missing")' | super -z -c 'yield this' - ``` -=> + ```mdtest-output error("missing") ``` @@ -38,7 +38,7 @@ The `cut` operator drops quiet errors but retains missing errors: ```mdtest-command echo '{a:1}' | super -z -c 'cut b:=x+1,c:=quiet(x+1),d:=quiet(a+1)' - ``` -=> + ```mdtest-output {b:error("missing"),d:2} ``` diff --git a/docs/language/functions/regexp.md b/docs/language/functions/regexp.md index 77ee0f9920..b736dc5db2 100644 --- a/docs/language/functions/regexp.md +++ b/docs/language/functions/regexp.md @@ -22,7 +22,7 @@ Regexp returns an array of the match and its subexpressions: echo '"seafood fool friend"' | super -z -c 'yield regexp(/foo(.?) (\w+) fr.*/, this)' - ``` -=> + ```mdtest-output ["food fool friend","d","fool"] ``` @@ -31,7 +31,7 @@ A null is returned if there is no match: ```mdtest-command echo '"foo"' | super -z -c 'yield regexp("bar", this)' - ``` -=> + ```mdtest-output null([string]) ``` diff --git a/docs/language/functions/regexp_replace.md b/docs/language/functions/regexp_replace.md index 4fb1ff96b5..8bfc3fc43d 100644 --- a/docs/language/functions/regexp_replace.md +++ b/docs/language/functions/regexp_replace.md @@ -34,7 +34,7 @@ Replace regular expression matches with a letter: ```mdtest-command echo '"-ab-axxb-"' | super -z -c 'yield regexp_replace(this, /ax*b/, "T")' - ``` -=> + ```mdtest-output "-T-T-" ``` @@ -45,7 +45,7 @@ Replace regular expression matches using numeric references to submatches: echo '"option: value"' | super -z -c 'yield regexp_replace(this,/(\w+):\s+(\w+)$/,"$1=$2")' - ``` -=> + ```mdtest-output "option=value" ``` @@ -60,7 +60,7 @@ echo '"option: value"' | "$key=$value") ' - ``` -=> + ```mdtest-output "option=value" ``` @@ -75,7 +75,7 @@ echo '"option: value"' | "$key=${value}AppendedText") ' - ``` -=> + ```mdtest-output "option=valueAppendedText" ``` diff --git a/docs/language/functions/replace.md b/docs/language/functions/replace.md index d8727b0da7..b25dbdd8e0 100644 --- a/docs/language/functions/replace.md +++ b/docs/language/functions/replace.md @@ -18,7 +18,7 @@ that occur in string `s` with the string `new`. ```mdtest-command echo '"oink oink oink"' | super -z -c 'yield replace(this, "oink", "moo")' - ``` -=> + ```mdtest-output "moo moo moo" ``` diff --git a/docs/language/functions/round.md b/docs/language/functions/round.md index 1e9242790b..bdc09ad976 100644 --- a/docs/language/functions/round.md +++ b/docs/language/functions/round.md @@ -18,7 +18,7 @@ which must be a numeric type. The return type retains the type of the argument. ```mdtest-command echo '3.14 -1.5 0 1' | super -z -c 'yield round(this)' - ``` -=> + ```mdtest-output 3. -2. diff --git a/docs/language/functions/rune_len.md b/docs/language/functions/rune_len.md index 87020d3f0e..7c5fdd845f 100644 --- a/docs/language/functions/rune_len.md +++ b/docs/language/functions/rune_len.md @@ -20,7 +20,7 @@ The length in UTF-8 characters of a smiley is 1: ```mdtest-command echo '"hello" "😎"' | super -z -c 'yield rune_len(this)' - ``` -=> + ```mdtest-output 5 1 @@ -30,7 +30,7 @@ The length in bytes of a smiley is 4: ```mdtest-command echo '"hello" "😎"' | super -z -c 'yield len(bytes(this))' - ``` -=> + ```mdtest-output 5 4 diff --git a/docs/language/functions/split.md b/docs/language/functions/split.md index e811a5541c..a037edc886 100644 --- a/docs/language/functions/split.md +++ b/docs/language/functions/split.md @@ -20,7 +20,7 @@ Split a semi-colon delimited list of fruits: ```mdtest-command echo '"apple;banana;pear;peach"' | super -z -c 'yield split(this,";")' - ``` -=> + ```mdtest-output ["apple","banana","pear","peach"] ``` @@ -31,7 +31,7 @@ array of IPs: echo '"10.0.0.1,10.0.0.2,10.0.0.3"' | super -z -c 'yield cast(split(this,","),<[ip]>)' - ``` -=> + ```mdtest-output [10.0.0.1,10.0.0.2,10.0.0.3] ``` diff --git a/docs/language/functions/sqrt.md b/docs/language/functions/sqrt.md index a1583ffb62..4676bf0ab2 100644 --- a/docs/language/functions/sqrt.md +++ b/docs/language/functions/sqrt.md @@ -18,7 +18,7 @@ The logarithm of a various numbers: ```mdtest-command echo '4 2. 1e10 -1' | super -z -c 'yield sqrt(this)' - ``` -=> + ```mdtest-output 2. 1.4142135623730951 diff --git a/docs/language/functions/strftime.md b/docs/language/functions/strftime.md index 63eaf43fee..af27520eef 100644 --- a/docs/language/functions/strftime.md +++ b/docs/language/functions/strftime.md @@ -61,7 +61,7 @@ Print the year number as a string ```mdtest-command echo 2024-07-30T20:05:15.118252Z | super -z -c 'strftime("%Y", this)' - ``` -=> + ```mdtest-output "2024" ``` @@ -70,7 +70,7 @@ Print a date in European format with slashes ```mdtest-command echo 2024-07-30T20:05:15.118252Z | super -z -c 'strftime("%d/%m/%Y", this)' - ``` -=> + ```mdtest-output "30/07/2024" ``` diff --git a/docs/language/functions/trim.md b/docs/language/functions/trim.md index be0996c53c..759bf9533a 100644 --- a/docs/language/functions/trim.md +++ b/docs/language/functions/trim.md @@ -18,7 +18,7 @@ from string argument `s` and returns the result. ```mdtest-command echo '" = Zed = "' | super -z -c 'yield trim(this)' - ``` -=> + ```mdtest-output "= Zed =" ``` diff --git a/docs/language/functions/typename.md b/docs/language/functions/typename.md index 8e68df0fab..21c043405c 100644 --- a/docs/language/functions/typename.md +++ b/docs/language/functions/typename.md @@ -19,7 +19,7 @@ Return a simple named type with a string constant argument: ```mdtest-command echo '80(port=int16)' | super -z -c 'yield typename("port")' - ``` -=> + ```mdtest-output ``` @@ -27,7 +27,7 @@ Return a named type using an expression: ```mdtest-command echo '{name:"port",p:80(port=int16)}' | super -z -c 'yield typename(name)' - ``` -=> + ```mdtest-output ``` @@ -35,7 +35,7 @@ The result is `error("missing")` if the type name does not exist: ```mdtest-command echo '80' | super -z -c 'yield typename("port")' - ``` -=> + ```mdtest-output error("missing") ``` diff --git a/docs/language/functions/typeof.md b/docs/language/functions/typeof.md index c8499c6456..c8924e8c56 100644 --- a/docs/language/functions/typeof.md +++ b/docs/language/functions/typeof.md @@ -22,7 +22,7 @@ The types of various values: echo '1 "foo" 10.0.0.1 [1,2,3] {s:"foo"} null error("missing")' | super -z -c 'yield typeof(this)' - ``` -=> + ```mdtest-output @@ -36,7 +36,7 @@ The type of a type is type `type`: ```mdtest-command echo null | super -z -c 'yield typeof(typeof(this))' - ``` -=> + ```mdtest-output ``` diff --git a/docs/language/functions/typeunder.md b/docs/language/functions/typeunder.md index 7b8b094f09..89f59f0b71 100644 --- a/docs/language/functions/typeunder.md +++ b/docs/language/functions/typeunder.md @@ -20,7 +20,7 @@ returned instead of the named type. echo '{which:"chocolate"}(=flavor)' | super -z -c 'yield {typeof:typeof(this),typeunder:typeunder(this)}' - ``` -=> + ```mdtest-output {typeof:,typeunder:<{which:string}>} ``` diff --git a/docs/language/functions/under.md b/docs/language/functions/under.md index 4363e70831..540c2f312b 100644 --- a/docs/language/functions/under.md +++ b/docs/language/functions/under.md @@ -26,7 +26,7 @@ echo '1((int64,string)) "foo"((int64,string))' | echo '1((int64,string)) "foo"((int64,string))' | super -z -c 'yield under(this)' - ``` -=> + ```mdtest-output 1((int64,string)) "foo"((int64,string)) @@ -39,7 +39,7 @@ Errors are unwrapped: echo 'error("foo") error({err:"message"})' | super -z -c 'yield this' - echo 'error("foo") error({err:"message"})' | super -z -c 'yield under(this)' - ``` -=> + ```mdtest-output error("foo") error({err:"message"}) @@ -52,7 +52,7 @@ Values of named types are unwrapped: echo '80(port=uint16)' | super -z -c 'yield this' - echo '80(port=uint16)' | super -z -c 'yield under(this)' - ``` -=> + ```mdtest-output 80(port=uint16) 80(uint16) @@ -61,7 +61,7 @@ Values that are not wrapped are unmodified: ```mdtest-command echo '1 "foo" {x:1}' | super -z -c 'yield under(this)' - ``` -=> + ```mdtest-output 1 "foo" diff --git a/docs/language/functions/unflatten.md b/docs/language/functions/unflatten.md index ffad06a3b1..4015037eb8 100644 --- a/docs/language/functions/unflatten.md +++ b/docs/language/functions/unflatten.md @@ -20,7 +20,7 @@ Simple: echo '[{key:"a",value:1},{key:["b"],value:2}]' | super -z -c 'yield unflatten(this)' - ``` -=> + ```mdtest-output {a:1,b:2} ``` @@ -35,7 +35,7 @@ echo '{a:1,rm:2}' | |> yield unflatten(this) ' - ``` -=> + ```mdtest-output {a:1} ``` diff --git a/docs/language/functions/upper.md b/docs/language/functions/upper.md index 50f2385323..f3c3f44223 100644 --- a/docs/language/functions/upper.md +++ b/docs/language/functions/upper.md @@ -18,7 +18,7 @@ to upper case and returns the result. ```mdtest-command echo '"Super JSON"' | super -z -c 'yield upper(this)' - ``` -=> + ```mdtest-output "SUPER JSON" ``` @@ -33,7 +33,7 @@ echo '"super JSON"' | yield capitalize(this) ' - ``` -=> + ```mdtest-output "Super JSON" ``` diff --git a/docs/language/lateral-subqueries.md b/docs/language/lateral-subqueries.md index 615943662f..1b7cdd2a7f 100644 --- a/docs/language/lateral-subqueries.md +++ b/docs/language/lateral-subqueries.md @@ -1,10 +1,8 @@ --- -sidebar_position: 7 -sidebar_label: Lateral Subqueries +weight: 7 +title: Lateral Subqueries --- -# Lateral Subqueries - Lateral subqueries provide a powerful means to apply a Zed query to each subsequence of values generated from an outer sequence of values. The inner query may be _any_ pipeline operator sequence (excluding diff --git a/docs/language/operators/README.md b/docs/language/operators/_index.md similarity index 98% rename from docs/language/operators/README.md rename to docs/language/operators/_index.md index 2d1eb93043..e6192a512a 100644 --- a/docs/language/operators/README.md +++ b/docs/language/operators/_index.md @@ -1,5 +1,5 @@ -# Operators - +--- +title: Operators --- Operators process a sequence of input values to create an output sequence diff --git a/docs/language/operators/assert.md b/docs/language/operators/assert.md index e88120e1e2..1984855b8c 100644 --- a/docs/language/operators/assert.md +++ b/docs/language/operators/assert.md @@ -18,7 +18,7 @@ structured error if it does not. ```mdtest-command echo {a:1} | super -z -c 'assert a > 0' - ``` -=> + ```mdtest-output {a:1} ``` @@ -26,7 +26,7 @@ echo {a:1} | super -z -c 'assert a > 0' - ```mdtest-command echo {a:-1} | super -z -c 'assert a > 0' - ``` -=> + ```mdtest-output error({message:"assertion failed",expr:"a > 0",on:{a:-1}}) ``` diff --git a/docs/language/operators/combine.md b/docs/language/operators/combine.md index 7219b65476..037304acb7 100644 --- a/docs/language/operators/combine.md +++ b/docs/language/operators/combine.md @@ -23,7 +23,7 @@ _Copy input to two pipeline branches and combine with the implied operator_ ```mdtest-command echo '1 2' | super -z -c 'fork (=>pass =>pass) |> sort this' - ``` -=> + ```mdtest-output 1 1 diff --git a/docs/language/operators/cut.md b/docs/language/operators/cut.md index fc58461308..6c05d82b77 100644 --- a/docs/language/operators/cut.md +++ b/docs/language/operators/cut.md @@ -45,7 +45,7 @@ _A simple Unix-like cut_ ```mdtest-command echo '{a:1,b:2,c:3}' | super -z -c 'cut a,c' - ``` -=> + ```mdtest-output {a:1,c:3} ``` @@ -53,7 +53,7 @@ _Missing fields show up as missing errors_ ```mdtest-command echo '{a:1,b:2,c:3}' | super -z -c 'cut a,d' - ``` -=> + ```mdtest-output {a:1,d:error("missing")} ``` @@ -61,7 +61,7 @@ _The missing fields can be ignored with quiet_ ```mdtest-command echo '{a:1,b:2,c:3}' | super -z -c 'cut a:=quiet(a),d:=quiet(d)' - ``` -=> + ```mdtest-output {a:1} ``` @@ -69,7 +69,7 @@ _Non-record values generate missing errors for fields not present in a non-recor ```mdtest-command echo '1 {a:1,b:2,c:3}' | super -z -c 'cut a,b' - ``` -=> + ```mdtest-output {a:error("missing"),b:error("missing")} {a:1,b:2} @@ -85,7 +85,7 @@ the output will be exported in formats such as `csv` or `parquet` (see also: ```mdtest-command echo '{a:1,b:null}{a:1,b:2}' | super -z -c 'cut a,b:=coalesce(b, 0)' - ``` -=> + ```mdtest-output {a:1,b:0} {a:1,b:2} diff --git a/docs/language/operators/drop.md b/docs/language/operators/drop.md index c27bd7b150..5f9ab69dee 100644 --- a/docs/language/operators/drop.md +++ b/docs/language/operators/drop.md @@ -20,7 +20,7 @@ _Drop of a field_ ```mdtest-command echo '{a:1,b:2,c:3}' | super -z -c 'drop b' - ``` -=> + ```mdtest-output {a:1,c:3} ``` @@ -28,7 +28,7 @@ _Non-record values are copied to output_ ```mdtest-command echo '1 {a:1,b:2,c:3}' | super -z -c 'drop a,b' - ``` -=> + ```mdtest-output 1 {c:3} diff --git a/docs/language/operators/fork.md b/docs/language/operators/fork.md index 8761cf8cf7..d57c6a1fe7 100644 --- a/docs/language/operators/fork.md +++ b/docs/language/operators/fork.md @@ -26,7 +26,7 @@ _Copy input to two pipeline branches and merge_ ```mdtest-command echo '1 2' | super -z -c 'fork (=>pass =>pass) |> sort this' - ``` -=> + ```mdtest-output 1 1 diff --git a/docs/language/operators/from.md b/docs/language/operators/from.md index 2b4a808771..961d20e865 100644 --- a/docs/language/operators/from.md +++ b/docs/language/operators/from.md @@ -82,7 +82,7 @@ super db -q init super db -q create -orderby flip:desc coinflips echo '{flip:1,result:"heads"} {flip:2,result:"tails"}' | super db load -q -use coinflips - -super db branch -q -use coinflips trial +super db branch -q -use coinflips trial echo '{flip:3,result:"heads"}' | super db load -q -use coinflips@trial - super db -q create numbers echo '{number:1,word:"one"} {number:2,word:"two"} {number:3,word:"three"}' | @@ -114,7 +114,7 @@ _Source structured data from a local file_ ```mdtest-command super -z -c 'file hello.jsup |> yield greeting' ``` -=> + ```mdtest-output "hello world!" ``` @@ -123,7 +123,7 @@ _Source data from a local file, but in line format_ ```mdtest-command super -z -c 'file hello.jsup format line' ``` -=> + ```mdtest-output "{greeting:\"hello world!\"}" ``` @@ -142,7 +142,7 @@ _Source data from the `main` branch of a pool_ ```mdtest-command super db -lake example query -z 'from coinflips' ``` -=> + ```mdtest-output {flip:2,result:"tails"} {flip:1,result:"heads"} @@ -152,7 +152,7 @@ _Source data from a specific branch of a pool_ ```mdtest-command super db -lake example query -z 'from coinflips@trial' ``` -=> + ```mdtest-output {flip:3,result:"heads"} {flip:2,result:"tails"} @@ -163,7 +163,7 @@ _Count the number of values in the `main` branch of all pools_ ```mdtest-command super db -lake example query -f text 'from * |> count()' ``` -=> + ```mdtest-output 5 ``` @@ -175,7 +175,7 @@ super db -lake example query -z ' from numbers |> sort number ) on flip=number word' ``` -=> + ```mdtest-output {flip:1,result:"heads",word:"one"} {flip:2,result:"tails",word:"two"} @@ -195,7 +195,7 @@ super db -lake example query -z ' |> yield f"There were {int64(c)} flips" ) |> sort this' ``` -=> + ```mdtest-output "There were 3 flips" {flip:1,result:"heads",word:"one"} diff --git a/docs/language/operators/fuse.md b/docs/language/operators/fuse.md index 66b724b358..f7142265f1 100644 --- a/docs/language/operators/fuse.md +++ b/docs/language/operators/fuse.md @@ -47,7 +47,7 @@ _Fuse two records_ ```mdtest-command echo '{a:1}{b:2}' | super -z -c fuse - ``` -=> + ```mdtest-output {a:1,b:null(int64)} {a:null(int64),b:2} @@ -56,7 +56,7 @@ _Fuse records with type variation_ ```mdtest-command echo '{a:1}{a:"foo"}' | super -z -c fuse - ``` -=> + ```mdtest-output {a:1((int64,string))} {a:"foo"((int64,string))} @@ -65,7 +65,7 @@ _Fuse records with complex type variation_ ```mdtest-command echo '{a:[1,2]}{a:["foo","bar"],b:10.0.0.1}' | super -z -c fuse - ``` -=> + ```mdtest-output {a:[1,2]([(int64,string)]),b:null(ip)} {a:["foo","bar"]([(int64,string)]),b:10.0.0.1} @@ -74,7 +74,7 @@ _The table format clarifies what fuse does_ ```mdtest-command echo '{a:1}{b:2}{a:3}' | super -f table -c fuse - ``` -=> + ```mdtest-output a b 1 - diff --git a/docs/language/operators/head.md b/docs/language/operators/head.md index eb95268382..319c2fc03f 100644 --- a/docs/language/operators/head.md +++ b/docs/language/operators/head.md @@ -20,7 +20,7 @@ _Grab first two values of arbitrary sequence_ ```mdtest-command echo '1 "foo" [1,2,3]' | super -z -c 'head 2' - ``` -=> + ```mdtest-output 1 "foo" @@ -30,7 +30,7 @@ _Grab first two values of arbitrary sequence, using a different representation o ```mdtest-command echo '1 "foo" [1,2,3]' | super -z -c 'head 1+1' - ``` -=> + ```mdtest-output 1 "foo" @@ -40,7 +40,7 @@ _Grab the first record of a record sequence_ ```mdtest-command echo '{a:"hello"}{b:"world"}' | super -z -c head - ``` -=> + ```mdtest-output {a:"hello"} ``` diff --git a/docs/language/operators/load.md b/docs/language/operators/load.md index 1f223edd53..4bf2a6c4d5 100644 --- a/docs/language/operators/load.md +++ b/docs/language/operators/load.md @@ -66,7 +66,7 @@ super db -lake example query ' super db -lake example query -z 'from bigflips' ``` -=> + ```mdtest-output {flip:1,result:"HEADS"} {flip:2,result:"TAILS"} @@ -85,7 +85,7 @@ super db -lake example query ' super db -lake example query -z 'from coinflips@onlytails' ``` -=> + ```mdtest-output {flip:2,result:"tails"} ``` diff --git a/docs/language/operators/merge.md b/docs/language/operators/merge.md index d85ab581cb..da0e6944ea 100644 --- a/docs/language/operators/merge.md +++ b/docs/language/operators/merge.md @@ -20,7 +20,7 @@ _Copy input to two pipeline branches and merge_ ```mdtest-command echo '1 2' | super -z -c 'fork (=>pass =>pass) |> merge this' - ``` -=> + ```mdtest-output 1 1 diff --git a/docs/language/operators/over.md b/docs/language/operators/over.md index 237a66c79c..f16781d8cb 100644 --- a/docs/language/operators/over.md +++ b/docs/language/operators/over.md @@ -34,7 +34,7 @@ _Over evaluates each expression and emits it_ ```mdtest-command echo null | super -z -c 'over 1,2,"foo"' - ``` -=> + ```mdtest-output 1 2 @@ -44,7 +44,7 @@ _The over clause is evaluated once per each input value_ ```mdtest-command echo "null null" | super -z -c 'over 1,2' - ``` -=> + ```mdtest-output 1 2 @@ -55,7 +55,7 @@ _Array elements are enumerated_ ```mdtest-command echo null | super -z -c 'over [1,2],[3,4,5]' - ``` -=> + ```mdtest-output 1 2 @@ -67,7 +67,7 @@ _Over traversing an array_ ```mdtest-command echo '{a:[1,2,3]}' | super -z -c 'over a' - ``` -=> + ```mdtest-output 1 2 @@ -78,7 +78,7 @@ _Filter the traversed values_ ```mdtest-command echo '{a:[6,5,4]} {a:[3,2,1]}' | super -z -c 'over a |> this % 2 == 0' - ``` -=> + ```mdtest-output 6 4 @@ -89,7 +89,7 @@ _Aggregate the traversed values_ ```mdtest-command echo '{a:[1,2]} {a:[3,4,5]}' | super -z -c 'over a |> sum(this)' - ``` -=> + ```mdtest-output 15 ``` @@ -97,7 +97,7 @@ _Aggregate the traversed values in a lateral query_ ```mdtest-command echo '{a:[1,2]} {a:[3,4,5]}' | super -z -c 'over a => ( sum(this) )' - ``` -=> + ```mdtest-output 3 12 @@ -107,7 +107,7 @@ _Access the outer values in a lateral query_ echo '{a:[1,2],s:"foo"} {a:[3,4,5],s:"bar"}' | super -z -c 'over a with s => (sum(this) |> yield {s,sum:this})' - ``` -=> + ```mdtest-output {s:"foo",sum:3} {s:"bar",sum:12} @@ -117,7 +117,7 @@ _Traverse a record by flattening it_ echo '{s:"foo",r:{a:1,b:2}} {s:"bar",r:{a:3,b:4}} ' | super -z -c 'over flatten(r) with s => (yield {s,key:key[0],value})' - ``` -=> + ```mdtest-output {s:"foo",key:"a",value:1} {s:"foo",key:"b",value:2} diff --git a/docs/language/operators/pass.md b/docs/language/operators/pass.md index 77860c874a..38f0ef582a 100644 --- a/docs/language/operators/pass.md +++ b/docs/language/operators/pass.md @@ -19,7 +19,7 @@ _Copy input to output_ ```mdtest-command echo '1 2 3' | super -z -c pass - ``` -=> + ```mdtest-output 1 2 @@ -35,7 +35,7 @@ echo '"HeLlo, WoRlD!"' | super -z -c ' => lower(this) ) |> sort' - ``` -=> + ```mdtest-output "HELLO, WORLD!" "HeLlo, WoRlD!" diff --git a/docs/language/operators/put.md b/docs/language/operators/put.md index df9b83f77c..2426e3387c 100644 --- a/docs/language/operators/put.md +++ b/docs/language/operators/put.md @@ -47,7 +47,7 @@ _A simple put_ ```mdtest-command echo '{a:1,b:2}' | super -z -c 'put c:=3' - ``` -=> + ```mdtest-output {a:1,b:2,c:3} ``` @@ -55,7 +55,7 @@ _The `put` keyword may be omitted_ ```mdtest-command echo '{a:1,b:2}' | super -z -c 'c:=3' - ``` -=> + ```mdtest-output {a:1,b:2,c:3} ``` @@ -63,7 +63,7 @@ _A `put` operation can also be done with a record literal_ ```mdtest-command echo '{a:1,b:2}' | super -z -c 'yield {...this, c:3}' - ``` -=> + ```mdtest-output {a:1,b:2,c:3} ``` @@ -71,7 +71,7 @@ _Missing fields show up as missing errors_ ```mdtest-command echo '{a:1,b:2,c:3}' | super -z -c 'put d:=e' - ``` -=> + ```mdtest-output {a:1,b:2,c:3,d:error("missing")} ``` @@ -79,7 +79,7 @@ _Non-record input values generate errors_ ```mdtest-command echo '{a:1} 1' | super -z -c 'b:=2' - ``` -=> + ```mdtest-output {a:1,b:2} error({message:"put: not a record",on:1}) diff --git a/docs/language/operators/rename.md b/docs/language/operators/rename.md index 5995e7ff98..f595b5dc3b 100644 --- a/docs/language/operators/rename.md +++ b/docs/language/operators/rename.md @@ -31,7 +31,7 @@ _A simple rename_ ```mdtest-command echo '{a:1,b:2}' | super -z -c 'rename c:=b' - ``` -=> + ```mdtest-output {a:1,c:2} ``` @@ -39,7 +39,7 @@ _Nested rename_ ```mdtest-command echo '{a:1,r:{b:2,c:3}}' | super -z -c 'rename r.a:=r.b' - ``` -=> + ```mdtest-output {a:1,r:{a:2,c:3}} ``` @@ -47,7 +47,7 @@ _Trying to mutate records with rename produces a compile-time error_ ```mdtest-command fails echo '{a:1,r:{b:2,c:3}}' | super -z -c 'rename w:=r.b' - ``` -=> + ```mdtest-output left-hand side and right-hand side must have the same depth (w vs r.b) at line 1, column 8: rename w:=r.b @@ -57,7 +57,7 @@ _Record literals can be used instead of rename for mutation_ ```mdtest-command echo '{a:1,r:{b:2,c:3}}' | super -z -c 'yield {a,r:{c:r.c},w:r.b}' - ``` -=> + ```mdtest-output {a:1,r:{c:3},w:2} ``` @@ -65,7 +65,7 @@ _Alternatively, mutations can be more generic and use drop_ ```mdtest-command echo '{a:1,r:{b:2,c:3}}' | super -z -c 'yield {a,r,w:r.b} |> drop r.b' - ``` -=> + ```mdtest-output {a:1,r:{c:3},w:2} ``` @@ -73,7 +73,7 @@ _Duplicate fields create structured errors_ ```mdtest-command echo '{b:1} {a:1,b:1} {c:1}' | super -z -c 'rename a:=b' - ``` -=> + ```mdtest-output {a:1} error({message:"rename: duplicate field: \"a\"",on:{a:1,b:1}}) diff --git a/docs/language/operators/sample.md b/docs/language/operators/sample.md index b7530a6d55..0dad1aee94 100644 --- a/docs/language/operators/sample.md +++ b/docs/language/operators/sample.md @@ -25,7 +25,7 @@ _A simple sample_ ```mdtest-command echo '1 2 3 "foo" "bar" 10.0.0.1 10.0.0.2' | super -z -c 'sample |> sort this' - ``` -=> + ```mdtest-output 1 "foo" @@ -37,7 +37,7 @@ _Sampling record shapes_ echo '{a:1}{a:2}{s:"foo"}{s:"bar"}{a:3,s:"baz"}' | super -z -c 'sample |> sort a' - ``` -=> + ```mdtest-output {a:1} {a:3,s:"baz"} diff --git a/docs/language/operators/search.md b/docs/language/operators/search.md index bc75526da1..69aeef6b66 100644 --- a/docs/language/operators/search.md +++ b/docs/language/operators/search.md @@ -25,7 +25,7 @@ _A simple keyword search for "world"_ ```mdtest-command echo '"hello, world" "say hello" "goodbye, world"' | super -z -c 'search world' - ``` -=> + ```mdtest-output "hello, world" "goodbye, world" @@ -34,7 +34,7 @@ Search can utilize _arithmetic comparisons_ ```mdtest-command echo '1 2 3' | super -z -c 'search this >= 2' - ``` -=> + ```mdtest-output 2 3 @@ -43,7 +43,7 @@ _The "search" keyword may be dropped_ ```mdtest-command echo '1 2 3' | super -z -c '? 2 or 3' - ``` -=> + ```mdtest-output 2 3 @@ -52,7 +52,7 @@ _A search with [Boolean logic](../search-expressions.md#boolean-logic)_ ```mdtest-command echo '1 2 3' | super -z -c 'search this >= 2 AND this <= 2' - ``` -=> + ```mdtest-output 2 ``` @@ -60,7 +60,7 @@ _The AND operator may be omitted through predicate concatenation_ ```mdtest-command echo '1 2 3' | super -z -c 'search this >= 2 this <= 2' - ``` -=> + ```mdtest-output 2 ``` @@ -68,7 +68,7 @@ _Concatenation for keyword search_ ```mdtest-command echo '"foo" "foo bar" "foo bar baz" "baz"' | super -z -c '? foo bar' - ``` -=> + ```mdtest-output "foo bar" "foo bar baz" @@ -77,7 +77,7 @@ _Search expressions match fields names too_ ```mdtest-command echo '{foo:1} {bar:2} {foo:3}' | super -z -c '? foo' - ``` -=> + ```mdtest-output {foo:1} {foo:3} @@ -86,7 +86,7 @@ _Boolean functions may be called_ ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'search is()' - ``` -=> + ```mdtest-output 1 ``` @@ -94,7 +94,7 @@ _Boolean functions with Boolean logic_ ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'search is() or is()' - ``` -=> + ```mdtest-output 1 10.0.0.1 diff --git a/docs/language/operators/sort.md b/docs/language/operators/sort.md index 515a547fad..0e98672818 100644 --- a/docs/language/operators/sort.md +++ b/docs/language/operators/sort.md @@ -59,7 +59,7 @@ _A simple sort with a null_ ```mdtest-command echo '2 null 1 3' | super -z -c 'sort this' - ``` -=> + ```mdtest-output 1 2 @@ -70,7 +70,7 @@ _With no sort expression, sort will sort by [`this`](../pipeline-model.md#the-sp ```mdtest-command echo '2 null 1 3' | super -z -c sort - ``` -=> + ```mdtest-output 1 2 @@ -81,7 +81,7 @@ _The "nulls last" default may be overridden_ ```mdtest-command echo '2 null 1 3' | super -z -c 'sort -nulls first' - ``` -=> + ```mdtest-output null 1 @@ -92,7 +92,7 @@ _With no sort expression, sort's heuristics will find a numeric key_ ```mdtest-command echo '{s:"bar",k:2}{s:"bar",k:3}{s:"foo",k:1}' | super -z -c sort - ``` -=> + ```mdtest-output {s:"foo",k:1} {s:"bar",k:2} @@ -102,7 +102,7 @@ _It's best practice to provide the sort key_ ```mdtest-command echo '{s:"bar",k:2}{s:"bar",k:3}{s:"foo",k:1}' | super -z -c 'sort k' - ``` -=> + ```mdtest-output {s:"foo",k:1} {s:"bar",k:2} @@ -112,7 +112,7 @@ _Sort with a secondary key_ ```mdtest-command echo '{s:"bar",k:2}{s:"bar",k:3}{s:"foo",k:2}' | super -z -c 'sort k,s' - ``` -=> + ```mdtest-output {s:"bar",k:2} {s:"foo",k:2} @@ -122,7 +122,7 @@ _Sort by secondary key in reverse order when the primary keys are identical_ ```mdtest-command echo '{s:"bar",k:2}{s:"bar",k:3}{s:"foo",k:2}' | super -z -c 'sort k,s desc' - ``` -=> + ```mdtest-output {s:"foo",k:2} {s:"bar",k:2} @@ -133,7 +133,7 @@ _Sort with a numeric expression_ echo '{s:"sum 2",x:2,y:0}{s:"sum 3",x:1,y:2}{s:"sum 0",x:-1,y:-1}' | super -z -c 'sort x+y' - ``` -=> + ```mdtest-output {s:"sum 0",x:-1,y:-1} {s:"sum 2",x:2,y:0} @@ -144,7 +144,7 @@ _Case sensitivity affects sorting "lowest value to highest" in string values_ echo '{word:"hello"}{word:"Hi"}{word:"WORLD"}' | super -z -c 'sort' - ``` -=> + ```mdtest-output {word:"Hi"} {word:"WORLD"} @@ -155,7 +155,7 @@ _Case-insensitive sort by using a string expression_ echo '{word:"hello"}{word:"Hi"}{word:"WORLD"}' | super -z -c 'sort lower(word)' - ``` -=> + ```mdtest-output {word:"hello"} {word:"Hi"} @@ -165,7 +165,7 @@ _Shorthand to reverse the sort order for each key_ ```mdtest-command echo '2 null 1 3' | super -z -c 'sort -r' - ``` -=> + ```mdtest-output 3 2 diff --git a/docs/language/operators/summarize.md b/docs/language/operators/summarize.md index 45a7e59ac4..6d7f4863fc 100644 --- a/docs/language/operators/summarize.md +++ b/docs/language/operators/summarize.md @@ -56,7 +56,7 @@ Average the input sequence: ```mdtest-command echo '1 2 3 4' | super -z -c 'summarize avg(this)' - ``` -=> + ```mdtest-output 2.5 ``` @@ -66,7 +66,7 @@ an explicit field for the output: ```mdtest-command echo '1 2 3 4' | super -z -c 'summarize mean:=avg(this)' - ``` -=> + ```mdtest-output {mean:2.5} ``` @@ -76,7 +76,7 @@ a record result is generated with field names implied by the functions: ```mdtest-command echo '1 2 3 4' | super -z -c 'summarize avg(this),sum(this),count()' - ``` -=> + ```mdtest-output {avg:2.5,sum:10,count:4(uint64)} ``` @@ -85,7 +85,7 @@ Sum the input sequence, leaving out the `summarize` keyword: ```mdtest-command echo '1 2 3 4' | super -z -c 'sum(this)' - ``` -=> + ```mdtest-output 10 ``` @@ -95,7 +95,7 @@ Create integer sets by key and sort the output to get a deterministic order: echo '{k:"foo",v:1}{k:"bar",v:2}{k:"foo",v:3}{k:"baz",v:4}' | super -z -c 'set:=union(v) by key:=k |> sort' - ``` -=> + ```mdtest-output {key:"bar",set:|[2]|} {key:"baz",set:|[4]|} @@ -107,7 +107,7 @@ Use a `where` clause: echo '{k:"foo",v:1}{k:"bar",v:2}{k:"foo",v:3}{k:"baz",v:4}' | super -z -c 'set:=union(v) where v > 1 by key:=k |> sort' - ``` -=> + ```mdtest-output {key:"bar",set:|[2]|} {key:"baz",set:|[4]|} @@ -121,7 +121,7 @@ echo '{k:"foo",v:1}{k:"bar",v:2}{k:"foo",v:3}{k:"baz",v:4}' | array:=collect(v) where k=="foo" by key:=k |> sort' - ``` -=> + ```mdtest-output {key:"bar",set:|[2]|,array:null} {key:"baz",set:|[4]|,array:null} @@ -134,7 +134,7 @@ clauses are used inside `summarize`: echo '{k:"foo",v:1}{k:"bar",v:2}{k:"foo",v:3}{k:"baz",v:4}' | super -z -c 'sum(v) where k=="bar" by key:=k |> sort' - ``` -=> + ```mdtest-output {key:"bar",sum:2} {key:"baz",sum:null} @@ -146,7 +146,7 @@ To avoid null results for `by` groupings a just shown, filter before `summarize` echo '{k:"foo",v:1}{k:"bar",v:2}{k:"foo",v:3}{k:"baz",v:4}' | super -z -c 'k=="bar" |> sum(v) by key:=k |> sort' - ``` -=> + ```mdtest-output {key:"bar",sum:2} ``` @@ -156,7 +156,7 @@ Output just the unique key values: echo '{k:"foo",v:1}{k:"bar",v:2}{k:"foo",v:3}{k:"baz",v:4}' | super -z -c 'by k |> sort' - ``` -=> + ```mdtest-output {k:"bar"} {k:"baz"} diff --git a/docs/language/operators/switch.md b/docs/language/operators/switch.md index a70047d1ac..8e65955fee 100644 --- a/docs/language/operators/switch.md +++ b/docs/language/operators/switch.md @@ -57,7 +57,7 @@ echo '1 2 3 4' | |> sort odd,even ' - ``` -=> + ```mdtest-output {odd:1} {odd:3} @@ -75,7 +75,7 @@ echo '1 2 3 4' | |> sort ' - ``` -=> + ```mdtest-output "1!" "2" diff --git a/docs/language/operators/tail.md b/docs/language/operators/tail.md index 78921c79e5..f4f2356a5e 100644 --- a/docs/language/operators/tail.md +++ b/docs/language/operators/tail.md @@ -20,7 +20,7 @@ _Grab last two values of arbitrary sequence_ ```mdtest-command echo '1 "foo" [1,2,3]' | super -z -c 'tail 2' - ``` -=> + ```mdtest-output "foo" [1,2,3] @@ -30,7 +30,7 @@ _Grab last two values of arbitrary sequence, using a different representation of ```mdtest-command echo '1 "foo" [1,2,3]' | super -z -c 'tail 1+1' - ``` -=> + ```mdtest-output "foo" [1,2,3] @@ -40,7 +40,7 @@ _Grab the last record of a record sequence_ ```mdtest-command echo '{a:"hello"}{b:"world"}' | super -z -c tail - ``` -=> + ```mdtest-output {b:"world"} ``` diff --git a/docs/language/operators/top.md b/docs/language/operators/top.md index 7dfc68872b..559fc964cb 100644 --- a/docs/language/operators/top.md +++ b/docs/language/operators/top.md @@ -23,7 +23,7 @@ _Grab the top two values from a sequence of integers_ ```mdtest-command echo '1 5 3 9 23 7' | super -z -c 'top 2 this' - ``` -=> + ```mdtest-output 23 9 @@ -35,7 +35,7 @@ echo '{name:"joe", age:22} {name:"bob", age:37} {name:"liz", age:25} {name:"ray", age:44} {name:"sue", age:41} {name:"liz", age:60}' | super -z -c 'count() by name |> top 2 count' - ``` -=> + ```mdtest-output {name:"liz",count:3(uint64)} {name:"bob",count:2(uint64)} diff --git a/docs/language/operators/uniq.md b/docs/language/operators/uniq.md index 0425cfbb7a..ca78243080 100644 --- a/docs/language/operators/uniq.md +++ b/docs/language/operators/uniq.md @@ -11,7 +11,7 @@ uniq [-c] Inspired by the traditional Unix shell command of the same name, the `uniq` operator copies its input to its output but removes duplicate values -that are adjacent to one another. +that are adjacent to one another. This operator is most often used with `cut` and `sort` to find and eliminate duplicate values. @@ -27,7 +27,7 @@ _Simple deduplication_ ```mdtest-command echo '1 2 2 3' | super -z -c uniq - ``` -=> + ```mdtest-output 1 2 @@ -38,7 +38,7 @@ _Simple deduplication with -c_ ```mdtest-command echo '1 2 2 3' | super -z -c 'uniq -c' - ``` -=> + ```mdtest-output {value:1,count:1(uint64)} {value:2,count:2(uint64)} @@ -50,7 +50,7 @@ _Use sort to deduplicate non-adjacent values_ echo '"hello" "world" "goodbye" "world" "hello" "again"' | super -z -c 'sort |> uniq' - ``` -=> + ```mdtest-output "again" "goodbye" @@ -67,7 +67,7 @@ echo '{ts:2024-09-10T21:12:33Z, action:"start"} {ts:2024-09-10T21:12:36Z, action:"stop"}' | super -z -c 'uniq' - ``` -=> + ```mdtest-output {ts:2024-09-10T21:12:33Z,action:"start"} {ts:2024-09-10T21:12:34Z,action:"running"} diff --git a/docs/language/operators/where.md b/docs/language/operators/where.md index 26918a398e..2f26062486 100644 --- a/docs/language/operators/where.md +++ b/docs/language/operators/where.md @@ -29,7 +29,7 @@ _An arithmetic comparison_ ```mdtest-command echo '1 2 3' | super -z -c 'where this >= 2' - ``` -=> + ```mdtest-output 2 3 @@ -38,7 +38,7 @@ _The "where" keyword may be dropped_ ```mdtest-command echo '1 2 3' | super -z -c 'this >= 2' - ``` -=> + ```mdtest-output 2 3 @@ -47,7 +47,7 @@ _A filter with Boolean logic_ ```mdtest-command echo '1 2 3' | super -z -c 'where this >= 2 AND this <= 2' - ``` -=> + ```mdtest-output 2 ``` @@ -55,7 +55,7 @@ _A filter with array containment logic_ ```mdtest-command echo '1 2 3 4' | super -z -c 'where this in [1,4]' - ``` -=> + ```mdtest-output 1 4 @@ -64,7 +64,7 @@ _A filter with inverse containment logic_ ```mdtest-command echo '1 2 3 4' | super -z -c 'where ! (this in [1,4])' - ``` -=> + ```mdtest-output 2 3 @@ -73,7 +73,7 @@ _Boolean functions may be called_ ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'where is()' - ``` -=> + ```mdtest-output 1 ``` @@ -81,7 +81,7 @@ _Boolean functions with Boolean logic_ ```mdtest-command echo '1 "foo" 10.0.0.1' | super -z -c 'where is() or is()' - ``` -=> + ```mdtest-output 1 10.0.0.1 diff --git a/docs/language/operators/yield.md b/docs/language/operators/yield.md index f5aedca706..fa08062018 100644 --- a/docs/language/operators/yield.md +++ b/docs/language/operators/yield.md @@ -23,7 +23,7 @@ _Hello, world_ ```mdtest-command echo null | super -z -c 'yield "hello, world"' - ``` -=> + ```mdtest-output "hello, world" ``` @@ -31,7 +31,7 @@ _Yield evaluates each expression for every input value_ ```mdtest-command echo 'null null null' | super -z -c 'yield 1,2' - ``` -=> + ```mdtest-output 1 2 @@ -44,7 +44,7 @@ _Yield typically operates on its input_ ```mdtest-command echo '1 2 3' | super -z -c 'yield this*2+1' - ``` -=> + ```mdtest-output 3 5 @@ -54,7 +54,7 @@ _Yield is often used to transform records_ ```mdtest-command echo '{a:1,b:2}{a:3,b:4}' | super -z -c 'yield [a,b],[b,a] |> collect(this)' - ``` -=> + ```mdtest-output [[1,2],[2,1],[3,4],[4,3]] ``` diff --git a/docs/language/overview.md b/docs/language/overview.md index d8f12ac78e..1a9daa3b1a 100644 --- a/docs/language/overview.md +++ b/docs/language/overview.md @@ -1,10 +1,7 @@ --- -sidebar_position: 1 -sidebar_label: Overview ---- - -# Zed Language Overview - +weight: 1 +title: Overview +heading: Zed Language Overview --- The Zed language is a query language for search, analytics, diff --git a/docs/language/pipe-ambiguity.md b/docs/language/pipe-ambiguity.md index 03fb19e440..adc6c17c1d 100644 --- a/docs/language/pipe-ambiguity.md +++ b/docs/language/pipe-ambiguity.md @@ -1,4 +1,6 @@ -# The Pipeline Symbol +--- +title: The Pipeline Symbol +--- In SuperSQL, you can use one of two symbols to separate pipeline operators: `|` or `|>`. diff --git a/docs/language/pipeline-model.md b/docs/language/pipeline-model.md index 4108f2a063..5bd3c25c9e 100644 --- a/docs/language/pipeline-model.md +++ b/docs/language/pipeline-model.md @@ -1,10 +1,8 @@ --- -sidebar_position: 2 -sidebar_label: Pipeline Model +weight: 2 +title: Pipeline Model --- -# The Pipeline Model - In SuperPipe, each operator takes its input from the output of its upstream operator beginning either with a data source or with an implied source. diff --git a/docs/language/search-expressions.md b/docs/language/search-expressions.md index f6a2c99abe..810f79feee 100644 --- a/docs/language/search-expressions.md +++ b/docs/language/search-expressions.md @@ -1,10 +1,8 @@ --- -sidebar_position: 6 -sidebar_label: Search Expressions +weight: 6 +title: Search Expressions --- -# Search Expressions - Search expressions provide a hybrid syntax between keyword search and boolean expressions. In this way, a search is a shorthand for a "lean forward" style activity where one is interactively exploring diff --git a/docs/language/shaping.md b/docs/language/shaping.md index 98e659da31..f29fb882a7 100644 --- a/docs/language/shaping.md +++ b/docs/language/shaping.md @@ -1,10 +1,8 @@ --- -sidebar_position: 8 -sidebar_label: Shaping and Type Fusion +weight: 8 +title: Shaping and Type Fusion --- -# Shaping and Type Fusion - ## Shaping Data that originates from heterogeneous sources typically has diff --git a/docs/language/statements.md b/docs/language/statements.md index 2a006a9f91..4852647ef7 100644 --- a/docs/language/statements.md +++ b/docs/language/statements.md @@ -1,10 +1,9 @@ --- -sidebar_position: 4 -sidebar_label: Const, Func, Operator, and Type Statements +weight: 4 +title: Const, Func, Operator, and Type Statements +heading: Statements --- -# Statements - ## Const Statements Constants may be defined and assigned to a symbolic name with the syntax diff --git a/docs/libraries/README.md b/docs/libraries/_index.md similarity index 93% rename from docs/libraries/README.md rename to docs/libraries/_index.md index afdb909fc0..a3c7ad189b 100644 --- a/docs/libraries/README.md +++ b/docs/libraries/_index.md @@ -1,4 +1,7 @@ -# Libraries +--- +title: Libraries +weight: 7 +--- Zed currently supports a small number of languages with client libraries for manipulating Zed data and interacting diff --git a/docs/libraries/go.md b/docs/libraries/go.md index fed92aeb93..0cc0ae43d6 100644 --- a/docs/libraries/go.md +++ b/docs/libraries/go.md @@ -1,10 +1,8 @@ --- -sidebar_position: 1 -sidebar_label: Go +weight: 1 +title: Go --- -# Go - The Zed system was developed in Go so support for Go clients is fairly comprehensive. That said, the code-embedded documentation of exported package functions is scant and we are actively working to document diff --git a/docs/libraries/javascript.md b/docs/libraries/javascript.md index 2d11791c74..14f9fff927 100644 --- a/docs/libraries/javascript.md +++ b/docs/libraries/javascript.md @@ -1,10 +1,8 @@ --- -sidebar_position: 2 -sidebar_label: JavaScript +weight: 2 +title: JavaScript --- -# JavaScript - The [zed-js library](https://github.com/brimdata/zui/tree/main/packages/zed-js) provides support for the Zed data model from within JavaScript as well as methods for communicating with a Zed lake. diff --git a/docs/libraries/python.md b/docs/libraries/python.md index 9ffa763d8d..7c8feb0e75 100644 --- a/docs/libraries/python.md +++ b/docs/libraries/python.md @@ -1,10 +1,8 @@ --- -sidebar_position: 3 -sidebar_label: Python +weight: 3 +title: Python --- -# Python - Zed includes preliminary support for Python-based interaction with a Zed lake. The Zed Python package supports loading data into a Zed lake as well as diff --git a/docs/tutorials/README.md b/docs/tutorials/_index.md similarity index 79% rename from docs/tutorials/README.md rename to docs/tutorials/_index.md index 62d75c14d0..985c92df37 100644 --- a/docs/tutorials/README.md +++ b/docs/tutorials/_index.md @@ -1,4 +1,7 @@ -# Tutorials +--- +title: Tutorials +weight: 3 +--- The tutorials contain a number of overviews of different aspects of the Zed system walking through examples with various sets diff --git a/docs/tutorials/join.md b/docs/tutorials/join.md index 4e06e371c4..7badba8cad 100644 --- a/docs/tutorials/join.md +++ b/docs/tutorials/join.md @@ -1,10 +1,8 @@ --- -sidebar_position: 3 -sidebar_label: Join +weight: 3 +title: Join Overview --- -# Join Overview - This is a brief primer on the SuperPipe [`join` operator](../language/operators/join.md). Currently, `join` is limited in that only equi-join (i.e., a join predicate diff --git a/docs/tutorials/zed.md b/docs/tutorials/zed.md index 3c21db432a..bb6151f667 100644 --- a/docs/tutorials/zed.md +++ b/docs/tutorials/zed.md @@ -1,10 +1,9 @@ --- -sidebar_position: 2 -sidebar_label: zed +weight: 2 +title: zed +heading: zed Tutorial --- -# zed Tutorial - `zq` is great, but what if we have a lot of data on which we want to perform search and analytics? This is where the `zed` command comes in. `zed` builds on the type system and language found in `zq` and adds a high performance data lake on top. diff --git a/docs/tutorials/zq.md b/docs/tutorials/zq.md index cf9d208073..f7b030491c 100644 --- a/docs/tutorials/zq.md +++ b/docs/tutorials/zq.md @@ -1,10 +1,9 @@ --- -sidebar_position: 1 -sidebar_label: super +weight: 1 +title: super +heading: super Tutorial --- -# super Tutorial - This tour provides new users of `super` an overview of the tool and the [SuperPipe language](../language/README.md) by walking through a number of examples on the command-line. @@ -624,7 +623,7 @@ produces ```mdtest-output "array" ``` -Ok got it. But, how many items are in the array? +Ok got it. But, how many items are in the array? ```mdtest-command dir=docs/tutorials super -z -c 'len(this)' prs.json ``` From 309851dbfc06d0f239fa1523c4670c37fadcc9c3 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Sat, 23 Nov 2024 17:49:22 -0800 Subject: [PATCH 02/18] Send a dispatch event to the superdb-website repo when docs change on this hugo-docs branch --- .github/workflows/notify-docs-update.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/notify-docs-update.yaml b/.github/workflows/notify-docs-update.yaml index 261c2d02d1..131cd478de 100644 --- a/.github/workflows/notify-docs-update.yaml +++ b/.github/workflows/notify-docs-update.yaml @@ -3,7 +3,7 @@ name: Notify docs update on: push: branches: - - main + - hugo-docs paths: - 'docs/**.md' @@ -13,4 +13,4 @@ jobs: steps: - name: Send dispatch event run: | - curl -XPOST -u "${{ secrets.PAT_USERNAME }}:${{ secrets.PAT_TOKEN }}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/brimdata/zed-docs-site/dispatches --data '{"event_type":"zed-docs-update"}' + curl -XPOST -u "${{ secrets.PAT_USERNAME }}:${{ secrets.PAT_TOKEN }}" -H "Accept: application/vnd.github.v3+json" -H "Content-Type: application/json" https://api.github.com/repos/brimdata/superdb-website/dispatches --data '{"event_type":"super-docs-update", "client_payload": {"commit_sha": "${{ github.sha }}"}}' From fbf68a888d7d9b242de654fd8f758d7a91eaae43 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Sat, 23 Nov 2024 17:54:36 -0800 Subject: [PATCH 03/18] Fix merge conflict --- docs/commands/super.md | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/docs/commands/super.md b/docs/commands/super.md index 75e76a9d8c..bf6790716a 100644 --- a/docs/commands/super.md +++ b/docs/commands/super.md @@ -24,36 +24,9 @@ to filter, transform, and/or analyze input data. Super's SQL pipes dialect is extensive, so much so that it can resemble a log-search experience despite its SQL foundation. -<<<<<<< HEAD -Each `input` argument must be a file path, an HTTP or HTTPS URL, -an S3 URL, or standard input specified with `-`. - -For built-in command help and a listing of all available options, -simply run `super` with no arguments. - -`super` supports a number of [input](#input-formats) and [output](#output-formats) formats, but [Super Binary](../formats/bsup.md) -tends to be the most space-efficient and most performant. Super Binary has efficiency similar to -[Avro](https://avro.apache.org) -and [Protocol Buffers](https://developers.google.com/protocol-buffers) -but its comprehensive [type system](../formats/zed.md) obviates -the need for schema specification or registries. -Also, the [Super JSON](../formats/jsup.md) format is human-readable and entirely one-to-one with Super Binary -so there is no need to represent non-readable formats like Avro or Protocol Buffers -in a clunky JSON encapsulated form. - -`super` typically operates on Super Binary-encoded data and when you want to inspect -human-readable bits of output, you merely format it as Super JSON, which is the -default format when output is directed to the terminal. Super Binary is the default -when redirecting to a non-terminal output like a file or pipe. - -When run with input arguments, each input's format is [automatically inferred](#auto-detection) -and each input is scanned -in the order appearing on the command line forming the input stream. -======= The `super` command works with data from ephemeral sources like files and URLs. If you want to persist your data into a data lake for persistent storage, check out the [`super db`](super-db.md) set of commands. ->>>>>>> origin/main By invoking the `-c` option, a query expressed in the [SuperSQL language](../language/README.md) may be specified and applied to the input stream. From 6b125c08d87d79ff7a1208a51d9f6b6bf36e95ba Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Mon, 25 Nov 2024 11:58:28 -0800 Subject: [PATCH 04/18] Rename README.md->_index.md to fix broken hyperlinks --- CHANGELOG.md | 34 +++++++++---------- docs/_index.md | 10 +++--- docs/commands/_index.md | 4 +-- docs/commands/super-db.md | 6 ++-- docs/commands/super.md | 18 +++++----- docs/formats/csup.md | 2 +- docs/formats/zjson.md | 2 +- docs/integrations/fluentd.md | 4 +-- .../zeek/data-type-compatibility.md | 2 +- .../zeek/reading-zeek-log-formats.md | 2 +- docs/lake/format.md | 2 +- docs/language/_index.md | 4 +-- docs/language/conventions.md | 2 +- docs/language/data-types.md | 2 +- docs/language/expressions.md | 4 +-- docs/language/lateral-subqueries.md | 2 +- docs/language/operators/summarize.md | 2 +- docs/language/overview.md | 4 +-- docs/language/pipeline-model.md | 6 ++-- docs/language/search-expressions.md | 2 +- docs/language/shaping.md | 2 +- docs/tutorials/zed.md | 2 +- docs/tutorials/zq.md | 8 ++--- 23 files changed, 63 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 120fc2e3a4..a95b819aed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,10 @@ * Improve the performance of the [`load` operator](docs/language/operators/load.md) by removing an unnecessary/inefficient merge (#5200) * Improve the [`sort` operator](docs/language/operators/sort.md) to allow different ordering for each key (#5203, #5262) * Update the [Zeek reference shaper docs](docs/integrations/zeek/shaping-zeek-json.md#reference-shaper-contents) to incorporate changes for [Zeek v7.0.0](https://github.com/zeek/zeek/releases/tag/v7.0.0) logs (#5212) -* Update the [`summarize` operator docs](docs/language/operators/summarize.md) to show the use of `by` without an [aggregate function](docs/language/aggregates/README.md) (#5216) +* Update the [`summarize` operator docs](docs/language/operators/summarize.md) to show the use of `by` without an [aggregate function](docs/language/aggregates/_index.md) (#5216) * Update the [`grok` function docs](docs/language/functions/grok.md) with additional examples and guidance (#5243) * Update the [Lateral Subquery docs](docs/language/lateral-subqueries.md) with an emphasis on when primitive values or arrays are returned by [Lateral Expressions](docs/language/lateral-subqueries.md#lateral-expressions) (#5264) -* The terms "pipeline" and "branch" are now used throughout the [Zed docs](docs/README.md) instead of "dataflow" and "leg" (#5272) +* The terms "pipeline" and "branch" are now used throughout the [Zed docs](docs/_index.md) instead of "dataflow" and "leg" (#5272) * Add docs for [`lake` output format](docs/commands/super.md#superdb-data-lake-metadata-output) and [`zed ls`](docs/commands/super-db.md#ls) (#5187) * Add docs for the [`top` operator](docs/language/operators/top.md) (#5276) * Add [`fluentd` integration docs](docs/integrations/fluentd.md) (#5190, #5195) @@ -22,13 +22,13 @@ * Add the `-pool` flag to [`zed manage`](docs/commands/super-db.md#manage) (#5164) * Fix an issue where the lake API was not providing query descriptions for Zed programs that contain scopes (#5152) * Fix an issue where attempts to use the [`load` operator](docs/language/operators/load.md) in `zq` caused a panic (#5162) -* Fix a parser issue with collisions between the names of [user-defined operators](docs/language/statements.md#operator-statements) and [functions](docs/language/statements.md#func-statements) and some built-in [operators](docs/language/operators/README.md) (#5161) +* Fix a parser issue with collisions between the names of [user-defined operators](docs/language/statements.md#operator-statements) and [functions](docs/language/statements.md#func-statements) and some built-in [operators](docs/language/operators/_index.md) (#5161) * Fix an issue where using `null` values in math caused a panic (#5163) ## v1.16.0 * Improve ZNG scanning performance (#5101, #5103) * Improve the error message shown when `zq` is invoked with a single argument that's not a valid query and doesn't contain a source (#5119) -* Update [Zeek integration docs](docs/integrations/zeek/README.md), including [reference shaper](docs/integrations/zeek/shaping-zeek-json.md) changes for [Zeek v6.2.0](https://github.com/zeek/zeek/releases/tag/v6.2.0) data (#5106) +* Update [Zeek integration docs](docs/integrations/zeek/_index.md), including [reference shaper](docs/integrations/zeek/shaping-zeek-json.md) changes for [Zeek v6.2.0](https://github.com/zeek/zeek/releases/tag/v6.2.0) data (#5106) * [String literals](docs/language/expressions.md#formatted-string-literals) now use the "f-string" format `f"{ }"` instead of the previous `${ }` (#5123) * Prototype SQL support has been dropped from the Zed language (full SQL support is expected at a later date) (#5109) * Empty objects and arrays in JSON output are now consistently printed on a single line (#5127) @@ -241,8 +241,8 @@ ## v1.0.0 -* Comprehensive [documentation](docs/README.md) -* Substantial improvements to the [Zed language](docs/language/README.md) +* Comprehensive [documentation](docs/_index.md) +* Substantial improvements to the [Zed language](docs/language/_index.md) * Revamped [`zed` command](docs/commands/super-db.md) * New Zed lake format (see #3634 for a migration script) * New version of the [ZNG format](docs/formats/bsup.md) (with read-only support for the previous version) @@ -264,7 +264,7 @@ * `zapi`, `zed lake serve`: Add authentication with Auth0 (#3266) * Fix an issue preventing casting from `ip` to `ip` (#3259) * `zed lake serve`: Respect the Accept request header for `GET /events` (#3246) -* Add [function documentation](docs/language/functions/README.md) (#3215) +* Add [function documentation](docs/language/functions/_index.md) (#3215) * `zed lake serve`: Change the default response content encoding to ZSON (#3242) * `zapi load`, `zed lake load`: Add the `-meta` flag to embed custom metadata in commits (#3237) @@ -280,7 +280,7 @@ * Fix a panic when compiling `SELECT ... GROUP BY ...` (#3193) * Fix a bug in which data loaded through the Zed lake service was stored uncompressed (#3198) * Add all lake index commands to Zed lake service (#3181) -* Reorganize [language documentation](docs/language/README.md) (#3187) +* Reorganize [language documentation](docs/language/_index.md) (#3187) * Make `fuse()` output deterministic (#3190) * Use lake indexes to speed up queries (#3158) * Fix bug where constants blocked `from` operator wiring logic (#3185) @@ -329,7 +329,7 @@ As you can see below, there's been many changes since the last Zed GA release! * Enhancements to the Zed language to unify search and expression syntax, introduce new operators and functions for data exploration and shaping, and more! Review the - [Zed language docs](docs/language/README.md) + [Zed language docs](docs/language/_index.md) for details. The exhaustive set of changes is listed below. Come talk to us on @@ -454,7 +454,7 @@ questions. * Fix an issue where `len()` of a `null` array was evaluating to something greater than zero (#2761) * Fix an issue where `sort` with no fields was ignoring alias types and nested fields when picking a sort field (#2762) * Fix an issue where unexpected `cut: no record found` warnings were returned by `zed lake query` but not when the same data was queried via `zq` (#2764) -* Move and extend the [Zeek interoperability docs](docs/integrations/zeek/README.md) (#2770, #2782, #2830) +* Move and extend the [Zeek interoperability docs](docs/integrations/zeek/_index.md) (#2770, #2782, #2830) * Create endpoints in the Zed lake service API that correspond to underlying Zed lake operations, and expose them via `zapi` commands (#2741, #2774, #2786, #2775, #2794, #2795, #2796, #2920, #2925, #2928) * Fix an issue where `zq` would surface a syntax error when reading ZSON it had sent as output (#2792) * Add an `/events` endpoint to the API, which can be used by clients such as the Brim app to be notified of pool updates (#2791) @@ -469,8 +469,8 @@ questions. * Have the [Python client](python) use the `/query` endpoint for the Zed lake (#2869) * Minimize the amount of surrounding context shown when reporting parse errors (#2864) * Field assignments in [`join`](docs/language/operators/join.md) now behave like [`cut`](docs/language/operators/cut.md) instead of `pick` (#2868) -* Add more background/context to Zed top-level language [README](docs/language/README.md) (#2866 #2878, #2901) -* Unify `from`, `split`, and `switch` syntax to the forms shown [here](docs/language/README.md) (#2871, #2896) +* Add more background/context to Zed top-level language [README](docs/language/_index.md) (#2866 #2878, #2901) +* Unify `from`, `split`, and `switch` syntax to the forms shown [here](docs/language/_index.md) (#2871, #2896) * Shapers can now cast values of the `null` type to any type (e.g., arrays or records) (#2882) * Fix an issue where [`join`](docs/language/operators/join.md) was failing to match on values of comparable types (e.g., `string` and `bstring`) (#2880, #2884) * Shapers can now cast a value to a `union` type (#2881) @@ -628,7 +628,7 @@ questions. ## v0.23.0 * zql: Add `week` as a unit for [time grouping with `every`](docs/language/functions/every.md) (#1374) -* zq: Fix an issue where a `null` value in a [JSON type definition](docs/integrations/zeek/README.md) caused a failure without an error message (#1377) +* zq: Fix an issue where a `null` value in a [JSON type definition](docs/integrations/zeek/_index.md) caused a failure without an error message (#1377) * zq: Add [`zst` format](docs/formats/csup.md) to `-i` and `-f` command-line help (#1384) * zq: ZNG spec and `zq` updates to introduce the beta ZNG storage format (#1375, #1415, #1394, #1457, #1512, #1523, #1529), also addressing the following: * New data type `bytes` for storing sequences of bytes encoded as base64 (#1315) @@ -644,11 +644,11 @@ questions. * zqd: Check and convert alpha ZNG filestores to beta ZNG (#1574, #1576) * zq: Fix an issue where spill-to-disk file names could collide (#1391) * zq: Allow the [`fuse` operator](docs/language/operators/fuse.md) to spill-to-disk to avoid memory limitations (#1355, #1402) -* zq: No longer require `_path` as a first column in a [JSON type definition](docs/integrations/zeek/README.md) (#1370) +* zq: No longer require `_path` as a first column in a [JSON type definition](docs/integrations/zeek/_index.md) (#1370) * zql: Improve ZQL docs for [aggregate functions](docs/language/operators/summarize.md) and grouping (#1385) * zql: Point links for developer docs at [pkg.go.dev](https://pkg.go.dev/) instead of [godoc.org](https://godoc.org/) (#1401) * zq: Add support for timestamps with signed timezone offsets (#1389) -* zq: Add a [JSON type definition](docs/integrations/zeek/README.md) for alert events in [Suricata EVE logs](https://suricata.readthedocs.io/en/suricata-5.0.2/output/eve/eve-json-output.html) (#1400) +* zq: Add a [JSON type definition](docs/integrations/zeek/_index.md) for alert events in [Suricata EVE logs](https://suricata.readthedocs.io/en/suricata-5.0.2/output/eve/eve-json-output.html) (#1400) * zq: Update the [ZNG over JSON (ZJSON)](docs/formats/zjson.md) spec and implementation (#1299) * zar: Use buffered streaming for archive import (#1397) * zq: Add an `ast` command that prints parsed ZQL as its underlying JSON object (#1416) @@ -695,7 +695,7 @@ questions. * zql: Fix broken links in documentation (#1321, #1339) * zst: Introduce the [ZST format](docs/formats/csup.md) for columnar data based on ZNG (#1268, #1338) * pcap: Fix an issue where certain pcapng files could fail import with a `bad option length` error (#1341) -* zql: [Document the `**` operator](docs/language/README.md#search-syntax) for type-specific searches that look within nested records (#1337) +* zql: [Document the `**` operator](docs/language/search-expressions.md) for type-specific searches that look within nested records (#1337) * zar: Change the archive data file layout to prepare for handing chunk files with overlapping ranges and improved S3 support (#1330) * zar: Support archive data files with overlapping time spans (#1348) * zqd: Add a page containing guidance for users that directly access the root `zqd` endpoint in a browser (#1350) @@ -789,7 +789,7 @@ questions. * zql: Add a new function `Time.trunc()` (#842) * zql: Support grouping by computed keys (#860) * zq: Change implementation of `every X` to use a computed groupby key (#893) -* zql: Clean up the [ZQL docs](docs/language/README.md) (#884) +* zql: Clean up the [ZQL docs](docs/language/_index.md) (#884) * zql: Change `cut` operator to emit any matching fields (#899) * zq: Allow output to an S3 bucket (#889) diff --git a/docs/_index.md b/docs/_index.md index ff45bd9c84..081d762cb9 100644 --- a/docs/_index.md +++ b/docs/_index.md @@ -5,7 +5,7 @@ heading: SuperDB --- SuperDB offers a new approach that makes it easier to manipulate and manage -your data. With its [super-structured data model](formats/README.md#2-a-super-structured-pattern), +your data. With its [super-structured data model](formats/_index.md#2-a-super-structured-pattern), messy JSON data can easily be given the fully-typed precision of relational tables without giving up JSON's uncanny ability to represent eclectic data. @@ -26,7 +26,7 @@ while for a technical user, SuperDB exposes its technical underpinnings in a gradual slope, providing as much detail as desired, packaged up in the easy-to-understand [Super JSON data format](formats/jsup.md) and -[SuperPipe language](language/README.md). +[SuperPipe language](language/_index.md). While `super` and its accompanying data formats are production quality, the project's [SuperDB data lake](commands/super-db.md) is a bit [earlier in development](commands/super-db.md#status). @@ -37,12 +37,12 @@ While `super` and its accompanying data formats are production quality, the proj a number of different elements of the system: * The [super data model](formats/zed.md) is the abstract definition of the data types and semantics that underlie the super-structured data formats. -* The [super data formats](formats/README.md) are a family of +* The [super data formats](formats/_index.md) are a family of [human-readable (Super JSON, JSUP)](formats/jsup.md), [sequential (Super Binary, BSUP)](formats/bsup.md), and [columnar (Super Columnar, CSUP)](formats/csup.md) formats that all adhere to the same abstract super data model. -* The [SuperPipe language](language/README.md) is the system's pipeline language for performing +* The [SuperPipe language](language/_index.md) is the system's pipeline language for performing queries, searches, analytics, transformations, or any of the above combined together. * A [SuperPipe query](language/overview.md) is a script that performs search and/or analytics. @@ -57,7 +57,7 @@ accessed via a [Git](https://git-scm.com/)-like API. ## Digging Deeper -The [SuperPipe language documentation](language/README.md) +The [SuperPipe language documentation](language/_index.md) is the best way to learn about `super` in depth. All of its examples use `super` commands run on the command line. Run `super -h` for a list of command options and online help. diff --git a/docs/commands/_index.md b/docs/commands/_index.md index 979c5678ca..f618316376 100644 --- a/docs/commands/_index.md +++ b/docs/commands/_index.md @@ -11,7 +11,7 @@ into, querying, and orchestrating SuperDB data lakes. These sub-commands are organized into further subcommands like the familiar command patterns of `docker` or `kubectl`. -All operations with these commands utilize the [super data model](../formats/README.md) -and provide querying via [SuperSQL](../language/README.md). +All operations with these commands utilize the [super data model](../formats/_index.md) +and provide querying via [SuperSQL](../language/_index.md). Built-in help for `super` and all sub-commands is always accessible with the `-h` flag. diff --git a/docs/commands/super-db.md b/docs/commands/super-db.md index a3bf860615..febae54da4 100644 --- a/docs/commands/super-db.md +++ b/docs/commands/super-db.md @@ -5,7 +5,7 @@ title: super db > **TL;DR** `super db` is a sub-command of `super` to manage and query SuperDB data lakes. > You can import data from a variety of formats and it will automatically -> be committed in [super-structured](../formats/README.md) +> be committed in [super-structured](../formats/_index.md) > format, providing full fidelity of the original format and the ability > to reconstruct the original data without loss of information. > @@ -16,7 +16,7 @@ title: super db

:::tip Status -While [`super`](super.md) and its accompanying [formats](../formats/README.md) +While [`super`](super.md) and its accompanying [formats](../formats/_index.md) are production quality, the SuperDB data lake is still fairly early in development and alpha quality. That said, SuperDB data lakes can be utilized quite effectively at small scale, @@ -657,7 +657,7 @@ according to configured policies and logic. ``` super db query [options] ``` -The `query` command runs a [SuperSQL](../language/README.md) query with data from a lake as input. +The `query` command runs a [SuperSQL](../language/_index.md) query with data from a lake as input. A query typically begins with a [`from` operator](../language/operators/from.md) indicating the pool and branch to use as input. diff --git a/docs/commands/super.md b/docs/commands/super.md index bf6790716a..ed3992ff4f 100644 --- a/docs/commands/super.md +++ b/docs/commands/super.md @@ -3,7 +3,7 @@ weight: 1 title: super --- -> **TL;DR** `super` is a command-line tool that uses [SuperSQL](../language/README.md) +> **TL;DR** `super` is a command-line tool that uses [SuperSQL](../language/_index.md) > to query a variety of data formats in files, over HTTP, or in [S3](../integrations/amazon-s3.md) > storage. Best performance is achieved when operating on data in binary formats such as > [Super Binary](../formats/bsup.md), [Super Columnar](../formats/csup.md), @@ -18,7 +18,7 @@ super [ options ] [ -c query ] input [ input ... ] `super` is a command-line tool for processing data in diverse input formats, providing data wrangling, search, analytics, and extensive transformations -using the [SuperSQL](../language/README.md) dialect of SQL. Any SQL query expression +using the [SuperSQL](../language/_index.md) dialect of SQL. Any SQL query expression may be extended with [pipe syntax](https://research.google/pubs/sql-has-problems-we-can-fix-them-pipe-syntax-in-sql/) to filter, transform, and/or analyze input data. Super's SQL pipes dialect is extensive, so much so that it can resemble @@ -28,10 +28,10 @@ The `super` command works with data from ephemeral sources like files and URLs. If you want to persist your data into a data lake for persistent storage, check out the [`super db`](super-db.md) set of commands. -By invoking the `-c` option, a query expressed in the [SuperSQL language](../language/README.md) +By invoking the `-c` option, a query expressed in the [SuperSQL language](../language/_index.md) may be specified and applied to the input stream. -The [super data model](../formats/zed.md) is based on [super-structured data](../formats/README.md#2-a-super-structured-pattern), meaning that all data +The [super data model](../formats/zed.md) is based on [super-structured data](../formats/_index.md#2-a-super-structured-pattern), meaning that all data is both strongly _and_ dynamically typed and need not conform to a homogeneous schema. The type structure is self-describing so it's easy to daisy-chain queries and inspect data at any point in a complex query or data pipeline. @@ -571,15 +571,15 @@ error("divide by zero") ## Examples -As you may have noticed, many examples of the [SuperSQL language](../language/README.md) +As you may have noticed, many examples of the [SuperSQL language](../language/_index.md) are illustrated using this pattern ``` echo | super -c - ``` -which is used throughout the [language documentation](../language/README.md) -and [operator reference](../language/operators/README.md). +which is used throughout the [language documentation](../language/_index.md) +and [operator reference](../language/operators/_index.md). -The language documentation and [tutorials directory](../tutorials/README.md) +The language documentation and [tutorials directory](../tutorials/_index.md) have many examples, but here are a few more simple `super` use cases. _Hello, world_ @@ -613,7 +613,7 @@ produces <[(int64,string)]> <|[string]|> ``` -_A simple [aggregation](../language/aggregates/README.md)_ +_A simple [aggregation](../language/aggregates/_index.md)_ ```mdtest-command echo '{key:"foo",val:1}{key:"bar",val:2}{key:"foo",val:3}' | super -z -c 'sum(val) by key | sort key' - diff --git a/docs/formats/csup.md b/docs/formats/csup.md index 2630b09a2c..133a6e7322 100644 --- a/docs/formats/csup.md +++ b/docs/formats/csup.md @@ -7,7 +7,7 @@ heading: Super Columnar Specification Super Columnar is a file format based on the [super data model](zed.md) where data is stacked to form columns. Its purpose is to provide for efficient analytics and search over -bounded-length sequences of [super-structured data](./README.md#2-a-super-structured-pattern) that is stored in columnar form. +bounded-length sequences of [super-structured data](./_index.md#2-a-super-structured-pattern) that is stored in columnar form. Like [Parquet](https://github.com/apache/parquet-format), Super Columnar provides an efficient representation for semi-structured data, diff --git a/docs/formats/zjson.md b/docs/formats/zjson.md index 4deb1df9ce..f4e8d506fa 100644 --- a/docs/formats/zjson.md +++ b/docs/formats/zjson.md @@ -60,7 +60,7 @@ Also, it is at the whim of a JSON implementation whether or not the order of object keys is preserved. While JSON is well suited for data exchange of generic information, it is not -sufficient for the [super-structured data model](./README.md#2-a-super-structured-pattern). +sufficient for the [super-structured data model](./_index.md#2-a-super-structured-pattern). That said, JSON can be used as an encoding format for super data with another layer of encoding on top of a JSON-based protocol. This allows clients like web apps or Electron apps to receive and understand Super JSON and, with the help of client diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index 492a4ee262..9688930bfe 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -12,7 +12,7 @@ record for archiving and analytics. This guide walks through two simple configurations of Fluentd with a Zed lake that can be used as reference for starting your own production configuration. As it's a data source important to many in the Zed community, log data from -[Zeek](./zeek/README.md) is used in this guide. The approach shown can be +[Zeek](./zeek/_index.md) is used in this guide. The approach shown can be easily adapted to any log data source. ## Software @@ -305,7 +305,7 @@ Example output: Notice quotes are no longer present around the values that contain IP addresses and times, since they are no longer stored as strings. With the data in this -shaped form, we could now invoke [Zed language](../language/README.md) +shaped form, we could now invoke [Zed language](../language/_index.md) functionality that leverages the richer data typing such as filtering `ip` values by CIDR block, e.g., diff --git a/docs/integrations/zeek/data-type-compatibility.md b/docs/integrations/zeek/data-type-compatibility.md index 6c050b1b50..7de64aac74 100644 --- a/docs/integrations/zeek/data-type-compatibility.md +++ b/docs/integrations/zeek/data-type-compatibility.md @@ -19,7 +19,7 @@ the types that may appear in Zeek logs. Zed tools maintain an internal Zed-typed representation of any Zeek data that is read or imported. Therefore, knowing the equivalent types will prove useful when performing operations in the -[Zed language](../../language/README.md) such as +[Zed language](../../language/_index.md) such as [type casting](../../language/shaping.md#cast) or looking at the data when output as Super JSON. diff --git a/docs/integrations/zeek/reading-zeek-log-formats.md b/docs/integrations/zeek/reading-zeek-log-formats.md index cdbac244cf..8d78a9304f 100644 --- a/docs/integrations/zeek/reading-zeek-log-formats.md +++ b/docs/integrations/zeek/reading-zeek-log-formats.md @@ -5,7 +5,7 @@ title: Reading Zeek Log Formats Zed is capable of reading both common Zeek log formats. This document provides guidance for what to expect when reading logs of these formats using -the Zed [command line tools](../../commands/README.md). +the Zed [command line tools](../../commands/_index.md). ## Zeek TSV diff --git a/docs/lake/format.md b/docs/lake/format.md index 3d30c67064..f223ac2700 100644 --- a/docs/lake/format.md +++ b/docs/lake/format.md @@ -31,7 +31,7 @@ to provide a universal data representation for all of these different approaches Also, while we are not currently focused on building a SQL engine for the Zed lake, it is most certainly possible to do so, as a Zed record type -[is analagous to](../formats/README.md#2-a-super-structured-pattern) +[is analagous to](../formats/_index.md#2-a-super-structured-pattern) a SQL table definition. SQL tables can essentially be dynamically projected via a table virtualization layer built on top of the Zed lake model. diff --git a/docs/language/_index.md b/docs/language/_index.md index cb7d545616..303b399ad1 100644 --- a/docs/language/_index.md +++ b/docs/language/_index.md @@ -12,5 +12,5 @@ The language documents: * describe the syntax of [expressions](expressions.md) and [search expressions](search-expressions.md), * explain [lateral subqueries](lateral-subqueries.md), * describe [shaping and type fusion](shaping.md), and -* enumerate the [operators](operators/README.md), [functions](functions/README.md), -and [aggregate functions](aggregates/README.md) in reference format. +* enumerate the [operators](operators/_index.md), [functions](functions/_index.md), +and [aggregate functions](aggregates/_index.md) in reference format. diff --git a/docs/language/conventions.md b/docs/language/conventions.md index 0359014178..5e9823a520 100644 --- a/docs/language/conventions.md +++ b/docs/language/conventions.md @@ -4,7 +4,7 @@ title: Conventions heading: Type Conventions --- -[Function](functions/README.md) arguments and [operator](operators/README.md) input values are all dynamically typed, +[Function](functions/_index.md) arguments and [operator](operators/_index.md) input values are all dynamically typed, yet certain functions expect certain specific [data types](data-types.md) or classes of data types. To this end, the function and operator prototypes in the Zed documentation include several type classes as follows: diff --git a/docs/language/data-types.md b/docs/language/data-types.md index 9283a74b47..ec789b127a 100644 --- a/docs/language/data-types.md +++ b/docs/language/data-types.md @@ -161,7 +161,7 @@ the scope of the Zed data model and language. That said, Zed provides flexible building blocks so systems can define their own schema versioning and schema management policies on top of these Zed primitives. -The [super-structured data model](../formats/README.md#2-a-super-structured-pattern) +The [super-structured data model](../formats/_index.md#2-a-super-structured-pattern) is a superset of relational tables and SuperPipe's type system can easily make this connection. As an example, consider this type definition for "employee": diff --git a/docs/language/expressions.md b/docs/language/expressions.md index c1059a44e7..47d423de70 100644 --- a/docs/language/expressions.md +++ b/docs/language/expressions.md @@ -240,14 +240,14 @@ produces ``` -Zed includes many [built-in functions](functions/README.md), some of which take +Zed includes many [built-in functions](functions/_index.md), some of which take a variable number of arguments. Zed also allows you to create [user-defined functions](statements.md#func-statements). ## Aggregate Function Calls -[Aggregate functions](aggregates/README.md) may be called within an expression. +[Aggregate functions](aggregates/_index.md) may be called within an expression. Unlike the aggregation context provided by a [summarizing group-by](operators/summarize.md), such calls in expression context yield an output value for each input value. diff --git a/docs/language/lateral-subqueries.md b/docs/language/lateral-subqueries.md index 1b7cdd2a7f..86294eafb9 100644 --- a/docs/language/lateral-subqueries.md +++ b/docs/language/lateral-subqueries.md @@ -187,5 +187,5 @@ produces ``` Similarly, a primitive value may be consistently produced by concluding the lateral scope with an operator such as [`head`](operators/head.md) or -[`tail`](operators/tail.md), or by applying certain [aggregate functions](aggregates/README.md) +[`tail`](operators/tail.md), or by applying certain [aggregate functions](aggregates/_index.md) such as done with [`sum`](aggregates/sum.md) above. diff --git a/docs/language/operators/summarize.md b/docs/language/operators/summarize.md index 6d7f4863fc..3edcc78b2a 100644 --- a/docs/language/operators/summarize.md +++ b/docs/language/operators/summarize.md @@ -14,7 +14,7 @@ ### Description In the first four forms, the `summarize` operator consumes all of its input, -applies an [aggregate function](../aggregates/README.md) to each input value +applies an [aggregate function](../aggregates/_index.md) to each input value optionally filtered by a `where` clause and/or organized with the group-by keys specified after the `by` keyword, and at the end of input produces one or more aggregations for each unique set of group-by key values. diff --git a/docs/language/overview.md b/docs/language/overview.md index 1a9daa3b1a..ff5e1d62ca 100644 --- a/docs/language/overview.md +++ b/docs/language/overview.md @@ -14,7 +14,7 @@ by a number of commands: command |> command | command | ... ``` However, in Zed, the entities that transform data are called -"[operators](operators/README.md)" instead of "commands" and unlike Unix pipelines, +"[operators](operators/_index.md)" instead of "commands" and unlike Unix pipelines, the streams of data in a Zed query are typed data sequences that adhere to the [Zed data model](../formats/zed.md). @@ -119,4 +119,4 @@ The following sections continue describing the Zed language. * [Lateral Subqueries](lateral-subqueries.md) * [Shaping and Type Fusion](shaping.md) -You may also be interested in the detailed reference materials on [operators](operators/README.md), [functions](functions/README.md), and [aggregate functions](aggregates/README.md), as well as the [conventions](conventions.md) for how they're described. +You may also be interested in the detailed reference materials on [operators](operators/_index.md), [functions](functions/_index.md), and [aggregate functions](aggregates/_index.md), as well as the [conventions](conventions.md) for how they're described. diff --git a/docs/language/pipeline-model.md b/docs/language/pipeline-model.md index 5bd3c25c9e..a8e1421558 100644 --- a/docs/language/pipeline-model.md +++ b/docs/language/pipeline-model.md @@ -6,7 +6,7 @@ title: Pipeline Model In SuperPipe, each operator takes its input from the output of its upstream operator beginning either with a data source or with an implied source. -All available operators are listed on the [reference page](operators/README.md). +All available operators are listed on the [reference page](operators/_index.md). ## Pipeline Sources @@ -104,7 +104,7 @@ in parallel on multiple threads). To establish a consistent sequence order, a [`merge` operator](operators/merge.md) may be applied at the output of the switch specifying a sort key upon which to order the upstream data. Often such order does not matter (e.g., when the output -of the switch hits an [aggregator](aggregates/README.md)), in which case it is typically more performant +of the switch hits an [aggregator](aggregates/_index.md)), in which case it is typically more performant to omit the merge (though the SuperDB runtime will often delete such unnecessary operations automatically as part optimizing queries when they are compiled). @@ -134,7 +134,7 @@ produces this case-sensitive output: "bar" "foo" ``` -But we can make the sort case-insensitive by applying a [function](functions/README.md) to the +But we can make the sort case-insensitive by applying a [function](functions/_index.md) to the input values with the expression `lower(this)`, which converts each value to lower-case for use in in the sort without actually modifying the input value, e.g., diff --git a/docs/language/search-expressions.md b/docs/language/search-expressions.md index 810f79feee..527d8f7e57 100644 --- a/docs/language/search-expressions.md +++ b/docs/language/search-expressions.md @@ -276,7 +276,7 @@ the "in" operator, e.g., #### Predicate Search Term -Any Boolean-valued [function](functions/README.md) like `is`, `has`, +Any Boolean-valued [function](functions/_index.md) like `is`, `has`, `grep`, etc. and any [comparison expression](expressions.md#comparisons) may be used as a search term and mixed into a search expression. diff --git a/docs/language/shaping.md b/docs/language/shaping.md index f29fb882a7..ed8a2fa033 100644 --- a/docs/language/shaping.md +++ b/docs/language/shaping.md @@ -12,7 +12,7 @@ a well-defined set of schemas, which combines the data into a unified store like a data warehouse. In Zed, this cleansing process is called "shaping" the data, and Zed leverages -its rich, [super-structured](../formats/README.md#2-a-super-structured-pattern) +its rich, [super-structured](../formats/_index.md#2-a-super-structured-pattern) type system to perform core aspects of data transformation. In a data model with nesting and multiple scalar types (such as Zed or JSON), shaping includes converting the type of leaf fields, adding or removing fields diff --git a/docs/tutorials/zed.md b/docs/tutorials/zed.md index bb6151f667..2fa2015b8b 100644 --- a/docs/tutorials/zed.md +++ b/docs/tutorials/zed.md @@ -318,7 +318,7 @@ the `zed` command. Some suggested next steps: 1. Dig deeper into SuperDB data lakes by having a look at the [`super db` command](../commands/super-db.md) documentation. 2. Get a better idea of ways you can query your data by looking at the -[Zed language documentation](../language/README.md). +[Zed language documentation](../language/_index.md). If you have any questions or run into any snags, join the friendly Zed community at the [Brim Data Slack workspace](https://www.brimdata.io/join-slack/). diff --git a/docs/tutorials/zq.md b/docs/tutorials/zq.md index f7b030491c..6561d09b77 100644 --- a/docs/tutorials/zq.md +++ b/docs/tutorials/zq.md @@ -5,10 +5,10 @@ heading: super Tutorial --- This tour provides new users of `super` an overview of the tool and -the [SuperPipe language](../language/README.md) +the [SuperPipe language](../language/_index.md) by walking through a number of examples on the command-line. This should get you started without having to read through all the gory details -of the [SuperPipe language](../language/README.md) or [`super` command-line usage](../commands/super.md). +of the [SuperPipe language](../language/_index.md) or [`super` command-line usage](../commands/super.md). We'll start with some simple one-liners on the command line where we feed some data to `super` with `echo` and specify `-` for `super` input to indicate @@ -410,7 +410,7 @@ preparation, union types are really quite powerful. They allow records with fields of different types or mixed-type arrays to be easily expressed while also having a very precise type definition. This is the essence of Zed's new -[super-structured data model](../formats/README.md#2-a-super-structured-pattern). +[super-structured data model](../formats/_index.md#2-a-super-structured-pattern). ## First-class Types @@ -1253,6 +1253,6 @@ of tricks to: clean data for analysis by `zq` or even export into other systems or for testing. If you'd like to learn more, feel free to read through the -[language docs](../language/README.md) in depth +[language docs](../language/_index.md) in depth or see how you can organize [data into a lake](../commands/super-db.md) using a git-like commit model. From 235dc368e7cdf5e1b03dd098dafc0337715f2e06 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Thu, 12 Dec 2024 16:02:55 -0800 Subject: [PATCH 05/18] Convert Zed lake auth doc to page bundle --- .../{zed-lake-auth.md => zed-lake-auth/_index.md} | 8 ++++---- .../{ => zed-lake-auth}/allowed-callback-urls.png | Bin .../api-allow-offline-access.png | Bin .../{ => zed-lake-auth}/api-name-identifier.png | Bin .../{ => zed-lake-auth}/application-name.png | Bin .../{ => zed-lake-auth}/click-api-settings.png | Bin .../click-application-settings.png | Bin .../{ => zed-lake-auth}/click-grant-types.png | Bin .../integrations/{ => zed-lake-auth}/create-api.png | Bin .../{ => zed-lake-auth}/create-application.png | Bin .../{ => zed-lake-auth}/device-code-grant-type.png | Bin .../expand-advanced-settings.png | Bin .../{ => zed-lake-auth}/login-flow-thumbnail.png | Bin .../save-application-settings.png | Bin 14 files changed, 4 insertions(+), 4 deletions(-) rename docs/integrations/{zed-lake-auth.md => zed-lake-auth/_index.md} (95%) rename docs/integrations/{ => zed-lake-auth}/allowed-callback-urls.png (100%) rename docs/integrations/{ => zed-lake-auth}/api-allow-offline-access.png (100%) rename docs/integrations/{ => zed-lake-auth}/api-name-identifier.png (100%) rename docs/integrations/{ => zed-lake-auth}/application-name.png (100%) rename docs/integrations/{ => zed-lake-auth}/click-api-settings.png (100%) rename docs/integrations/{ => zed-lake-auth}/click-application-settings.png (100%) rename docs/integrations/{ => zed-lake-auth}/click-grant-types.png (100%) rename docs/integrations/{ => zed-lake-auth}/create-api.png (100%) rename docs/integrations/{ => zed-lake-auth}/create-application.png (100%) rename docs/integrations/{ => zed-lake-auth}/device-code-grant-type.png (100%) rename docs/integrations/{ => zed-lake-auth}/expand-advanced-settings.png (100%) rename docs/integrations/{ => zed-lake-auth}/login-flow-thumbnail.png (100%) rename docs/integrations/{ => zed-lake-auth}/save-application-settings.png (100%) diff --git a/docs/integrations/zed-lake-auth.md b/docs/integrations/zed-lake-auth/_index.md similarity index 95% rename from docs/integrations/zed-lake-auth.md rename to docs/integrations/zed-lake-auth/_index.md index 79efc5f92b..8394d4135f 100644 --- a/docs/integrations/zed-lake-auth.md +++ b/docs/integrations/zed-lake-auth/_index.md @@ -4,11 +4,11 @@ title: Authentication Configuration heading: Configuring Authentication for a Zed Lake Service --- -A [SuperDB data lake service](../commands/super-db.md#serve) may be configured to require +A [SuperDB data lake service](../../commands/super-db.md#serve) may be configured to require user authentication to be accessed from clients such as the [Zui](https://zui.brimdata.io/) application, the -[`super db`](../commands/super.md) CLI commands, or the -[SuperDB Python client](../libraries/python.md). This document describes a simple +[`super db`](../../commands/super.md) CLI commands, or the +[SuperDB Python client](../../libraries/python.md). This document describes a simple [Auth0](https://auth0.com) configuration with accompanying `super db serve` flags that can be used as a starting point for creating similar configurations in your own environment. @@ -96,7 +96,7 @@ checkbox to enable the **Device Code** grant type. ## Zed Lake Service Configuration -1. Login to our Linux VM and [install](../install.md#building-from-source) +1. Login to our Linux VM and [install](../../install.md#building-from-source) the most recent Zed tools from source. ``` diff --git a/docs/integrations/allowed-callback-urls.png b/docs/integrations/zed-lake-auth/allowed-callback-urls.png similarity index 100% rename from docs/integrations/allowed-callback-urls.png rename to docs/integrations/zed-lake-auth/allowed-callback-urls.png diff --git a/docs/integrations/api-allow-offline-access.png b/docs/integrations/zed-lake-auth/api-allow-offline-access.png similarity index 100% rename from docs/integrations/api-allow-offline-access.png rename to docs/integrations/zed-lake-auth/api-allow-offline-access.png diff --git a/docs/integrations/api-name-identifier.png b/docs/integrations/zed-lake-auth/api-name-identifier.png similarity index 100% rename from docs/integrations/api-name-identifier.png rename to docs/integrations/zed-lake-auth/api-name-identifier.png diff --git a/docs/integrations/application-name.png b/docs/integrations/zed-lake-auth/application-name.png similarity index 100% rename from docs/integrations/application-name.png rename to docs/integrations/zed-lake-auth/application-name.png diff --git a/docs/integrations/click-api-settings.png b/docs/integrations/zed-lake-auth/click-api-settings.png similarity index 100% rename from docs/integrations/click-api-settings.png rename to docs/integrations/zed-lake-auth/click-api-settings.png diff --git a/docs/integrations/click-application-settings.png b/docs/integrations/zed-lake-auth/click-application-settings.png similarity index 100% rename from docs/integrations/click-application-settings.png rename to docs/integrations/zed-lake-auth/click-application-settings.png diff --git a/docs/integrations/click-grant-types.png b/docs/integrations/zed-lake-auth/click-grant-types.png similarity index 100% rename from docs/integrations/click-grant-types.png rename to docs/integrations/zed-lake-auth/click-grant-types.png diff --git a/docs/integrations/create-api.png b/docs/integrations/zed-lake-auth/create-api.png similarity index 100% rename from docs/integrations/create-api.png rename to docs/integrations/zed-lake-auth/create-api.png diff --git a/docs/integrations/create-application.png b/docs/integrations/zed-lake-auth/create-application.png similarity index 100% rename from docs/integrations/create-application.png rename to docs/integrations/zed-lake-auth/create-application.png diff --git a/docs/integrations/device-code-grant-type.png b/docs/integrations/zed-lake-auth/device-code-grant-type.png similarity index 100% rename from docs/integrations/device-code-grant-type.png rename to docs/integrations/zed-lake-auth/device-code-grant-type.png diff --git a/docs/integrations/expand-advanced-settings.png b/docs/integrations/zed-lake-auth/expand-advanced-settings.png similarity index 100% rename from docs/integrations/expand-advanced-settings.png rename to docs/integrations/zed-lake-auth/expand-advanced-settings.png diff --git a/docs/integrations/login-flow-thumbnail.png b/docs/integrations/zed-lake-auth/login-flow-thumbnail.png similarity index 100% rename from docs/integrations/login-flow-thumbnail.png rename to docs/integrations/zed-lake-auth/login-flow-thumbnail.png diff --git a/docs/integrations/save-application-settings.png b/docs/integrations/zed-lake-auth/save-application-settings.png similarity index 100% rename from docs/integrations/save-application-settings.png rename to docs/integrations/zed-lake-auth/save-application-settings.png From 9ff8037ccccb0122881c688750129b9fb6124c66 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Thu, 12 Dec 2024 16:17:14 -0800 Subject: [PATCH 06/18] Rename to index.md so Zed lake auth doc doesn't become a whole section --- docs/integrations/zed-lake-auth/{_index.md => index.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/integrations/zed-lake-auth/{_index.md => index.md} (100%) diff --git a/docs/integrations/zed-lake-auth/_index.md b/docs/integrations/zed-lake-auth/index.md similarity index 100% rename from docs/integrations/zed-lake-auth/_index.md rename to docs/integrations/zed-lake-auth/index.md From 627cf563c4beeb8441c8bad2a2184b21be87c6c3 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Thu, 12 Dec 2024 16:19:08 -0800 Subject: [PATCH 07/18] Convert Zed tutorial to page bundle --- docs/tutorials/{ => zed}/github1.bsup | Bin docs/tutorials/{ => zed}/github2.bsup | Bin docs/tutorials/{zed.md => zed/index.md} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename docs/tutorials/{ => zed}/github1.bsup (100%) rename docs/tutorials/{ => zed}/github2.bsup (100%) rename docs/tutorials/{zed.md => zed/index.md} (100%) diff --git a/docs/tutorials/github1.bsup b/docs/tutorials/zed/github1.bsup similarity index 100% rename from docs/tutorials/github1.bsup rename to docs/tutorials/zed/github1.bsup diff --git a/docs/tutorials/github2.bsup b/docs/tutorials/zed/github2.bsup similarity index 100% rename from docs/tutorials/github2.bsup rename to docs/tutorials/zed/github2.bsup diff --git a/docs/tutorials/zed.md b/docs/tutorials/zed/index.md similarity index 100% rename from docs/tutorials/zed.md rename to docs/tutorials/zed/index.md From 7ba51dc7ead4023dc24350338358665b9c94f709 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Thu, 12 Dec 2024 16:33:48 -0800 Subject: [PATCH 08/18] More link fixes --- .../{ => zed-lake-auth}/domain-client-id.png | Bin docs/tutorials/zed/index.md | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename docs/integrations/{ => zed-lake-auth}/domain-client-id.png (100%) diff --git a/docs/integrations/domain-client-id.png b/docs/integrations/zed-lake-auth/domain-client-id.png similarity index 100% rename from docs/integrations/domain-client-id.png rename to docs/integrations/zed-lake-auth/domain-client-id.png diff --git a/docs/tutorials/zed/index.md b/docs/tutorials/zed/index.md index a38ab961a2..0324f14702 100644 --- a/docs/tutorials/zed/index.md +++ b/docs/tutorials/zed/index.md @@ -9,7 +9,7 @@ analytics? This is where the `zed` command comes in. `zed` builds on the type system and language found in `zq` and adds a high performance data lake on top. > Note: `zed` is currently in alpha form. Check out its current status in the -> [`super db` command](../commands/super-db.md) documentation.. +> [`super db` command](../../commands/super-db.md) documentation.. ## Creating a Lake @@ -84,7 +84,7 @@ Our data has been committed. The `-use prs` argument in `zed load` tells With our data now loaded let's run a quick `count()` query to verify that we have the expected data. To do this we'll use the `zed query` command. To those -familiar with [`super`](../commands/super.md), `zed query` operates similarly except +familiar with [`super`](../../commands/super.md), `zed query` operates similarly except it doesn't accept file input arguments since it queries pools. ```bash @@ -316,9 +316,9 @@ $ zed query -Z 'min(created_at), max(created_at)' Obviously this is only the tip of the iceberg in terms of things that can be done with the `zed` command. Some suggested next steps: -1. Dig deeper into SuperDB data lakes by having a look at the [`super db` command](../commands/super-db.md) documentation. +1. Dig deeper into SuperDB data lakes by having a look at the [`super db` command](../../commands/super-db.md) documentation. 2. Get a better idea of ways you can query your data by looking at the -[Zed language documentation](../language/_index.md). +[Zed language documentation](../../language/_index.md). If you have any questions or run into any snags, join the friendly Zed community at the [Brim Data Slack workspace](https://www.brimdata.io/join-slack/). From 6539a3eb57e12acf661c4693ccbb696d01ccc7d2 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 08:18:44 -0800 Subject: [PATCH 09/18] Use new tip shortcode everywhere --- docs/commands/super-db.md | 28 +++++++-------- docs/commands/super.md | 4 +-- docs/formats/bsup.md | 8 ++--- docs/formats/csup.md | 36 +++++++++---------- docs/install.md | 4 +-- docs/integrations/amazon-s3.md | 4 +-- docs/integrations/fluentd.md | 9 ++--- docs/integrations/zed-lake-auth/index.md | 8 ++--- .../zeek/data-type-compatibility.md | 4 +-- docs/integrations/zeek/shaping-zeek-json.md | 4 +-- docs/language/functions/cast.md | 4 +-- docs/language/functions/fill.md | 4 +-- docs/language/functions/grok.md | 8 ++--- docs/language/functions/order.md | 8 ++--- docs/language/lateral-subqueries.md | 8 ++--- docs/language/operators/cut.md | 4 +-- docs/language/operators/file.md | 4 +-- docs/language/operators/from.md | 4 +-- docs/language/operators/join.md | 4 +-- docs/language/operators/load.md | 4 +-- docs/language/pipe-ambiguity.md | 6 ++-- docs/language/search-expressions.md | 12 +++---- docs/tutorials/join.md | 12 +++---- docs/tutorials/zq.md | 4 +-- 24 files changed, 97 insertions(+), 98 deletions(-) diff --git a/docs/commands/super-db.md b/docs/commands/super-db.md index febae54da4..b7bad70d03 100644 --- a/docs/commands/super-db.md +++ b/docs/commands/super-db.md @@ -15,7 +15,7 @@ title: super db

-:::tip Status +{{< tip "Status" >}} While [`super`](super.md) and its accompanying [formats](../formats/_index.md) are production quality, the SuperDB data lake is still fairly early in development and alpha quality. @@ -25,7 +25,7 @@ is deployed to manage the lake's data layout via the [lake API](../lake/api.md). Enhanced scalability with self-tuning configuration is under development. -::: +{{< /tip >}} ## The Lake Model @@ -153,7 +153,7 @@ running any `super db` lake command all pointing at the same storage endpoint and the lake's data footprint will always remain consistent as the endpoints all adhere to the consistency semantics of the lake. -:::tip caveat +{{< tip "Caveat" >}} Data consistency is not fully implemented yet for the S3 endpoint so only single-node access to S3 is available right now, though support for multi-node access is forthcoming. @@ -164,7 +164,7 @@ access to a local file system has been thoroughly tested and should be deemed reliable, i.e., you can run a direct-access instance of `super db` alongside a server instance of `super db` on the same file system and data consistency will be maintained. -::: +{{< /tip >}} ### Locating the Lake @@ -206,11 +206,11 @@ Each commit object is assigned a global ID. Similar to Git, commit objects are arranged into a tree and represent the entire commit history of the lake. -:::tip note +{{< tip "Note" >}} Technically speaking, Git can merge from multiple parents and thus Git commits form a directed acyclic graph instead of a tree; SuperDB does not currently support multiple parents in the commit object history. -::: +{{< /tip >}} A branch is simply a named pointer to a commit object in the lake and like a pool, a branch name can be any valid UTF-8 string. @@ -272,10 +272,10 @@ key. For example, on a pool with pool key `ts`, the query `ts == 100` will be optimized to scan only the data objects where the value `100` could be present. -:::tip note +{{< tip "Note" >}} The pool key will also serve as the primary key for the forthcoming CRUD semantics. -::: +{{< /tip >}} A pool also has a configured sort order, either ascending or descending and data is organized in the pool in accordance with this order. @@ -325,9 +325,9 @@ using that pool's "branches log" in a similar fashion, then its corresponding commit object can be used to construct the data of that branch at that past point in time. -:::tip note +{{< tip "Note" >}} Time travel using timestamps is a forthcoming feature. -::: +{{< /tip >}} ## `super db` Commands @@ -407,11 +407,11 @@ the [special value `this`](../language/pipeline-model.md#the-special-value-this) A newly created pool is initialized with a branch called `main`. -:::tip note +{{< tip "Note" >}} Lakes can be used without thinking about branches. When referencing a pool without a branch, the tooling presumes the "main" branch as the default, and everything can be done on main without having to think about branching. -::: +{{< /tip >}} ### Delete ``` @@ -582,9 +582,9 @@ that is stored in the commit journal for reference. These values may be specified as options to the [`load`](#load) command, and are also available in the [lake API](../lake/api.md) for automation. -:::tip note +{{< tip "Note" >}} The branchlog meta-query source is not yet implemented. -::: +{{< /tip >}} ### Ls ``` diff --git a/docs/commands/super.md b/docs/commands/super.md index 2feb835b8c..fadcd25e95 100644 --- a/docs/commands/super.md +++ b/docs/commands/super.md @@ -187,13 +187,13 @@ not desirable because (1) the Super JSON parser is not particularly performant a (2) all JSON numbers are floating point but the Super JSON parser will parse as JSON any number that appears without a decimal point as an integer type. -:::tip note +{{< tip "Note" >}} The reason `super` is not particularly performant for Super JSON is that the [Super Binary](../formats/bsup.md) or [Super Columnar](../formats/csup.md) formats are semantically equivalent to Super JSON but much more efficient and the design intent is that these efficient binary formats should be used in use cases where performance matters. Super JSON is typically used only when data needs to be human-readable in interactive settings or in automated tests. -::: +{{< /tip >}} To this end, `super` uses a heuristic to select between Super JSON and plain JSON when the `-i` option is not specified. Specifically, plain JSON is selected when the first values diff --git a/docs/formats/bsup.md b/docs/formats/bsup.md index 333f245d24..1bfacd5914 100644 --- a/docs/formats/bsup.md +++ b/docs/formats/bsup.md @@ -130,7 +130,7 @@ size decompression buffers in advance of decoding. Values for the `format` byte are defined in the [Super Binary compression format specification](./compression.md). -:::tip note +{{< tip "Note" >}} This arrangement of frames separating types and values allows for efficient scanning and parallelization. In general, values depend on type definitions but as long as all of the types are known when @@ -143,7 +143,7 @@ heuristics, e.g., knowing a filtering predicate can't be true based on a quick scan of the data perhaps using the Boyer-Moore algorithm to determine that a comparison with a string constant would not work for any value in the buffer. -::: +{{< /tip >}} Whether the payload was originally uncompressed or was decompressed, it is then interpreted according to the `T` bits of the frame code as a @@ -211,12 +211,12 @@ is further encoded as a "counted string", which is the `uvarint` encoding of the length of the string followed by that many bytes of UTF-8 encoded string data. -:::tip note +{{< tip "Note" >}} As defined by [Super JSON](jsup.md), a field name can be any valid UTF-8 string much like JSON objects can be indexed with arbitrary string keys (via index operator) even if the field names available to the dot operator are restricted by language syntax for identifiers. -::: +{{< /tip >}} The type ID follows the field name and is encoded as a `uvarint`. diff --git a/docs/formats/csup.md b/docs/formats/csup.md index 133a6e7322..de60c76920 100644 --- a/docs/formats/csup.md +++ b/docs/formats/csup.md @@ -64,12 +64,12 @@ then write the metadata into the reassembly section along with the trailer at the end. This allows a stream to be converted to a Super Columnar file in a single pass. -:::tip note +{{< tip "Note" >}} That said, the layout is flexible enough that an implementation may optimize the data layout with additional passes or by writing the output to multiple files then merging them together (or even leaving the Super Columnar entity as separate files). -::: +{{< /tip >}} ### The Data Section @@ -85,7 +85,7 @@ There is no information in the data section for how segments relate to one another or how they are reconstructed into columns. They are just blobs of Super Binary data. -:::tip note +{{< tip "Note" >}} Unlike Parquet, there is no explicit arrangement of the column chunks into row groups but rather they are allowed to grow at different rates so a high-volume column might be comprised of many segments while a low-volume @@ -93,9 +93,9 @@ column must just be one or several. This allows scans of low-volume record type (the "mice") to perform well amongst high-volume record types (the "elephants"), i.e., there are not a bunch of seeks with tiny reads of mice data interspersed throughout the elephants. -::: +{{< /tip >}} -:::tip TBD +{{< tip "TBD" >}} The mice/elephants model creates an interesting and challenging layout problem. If you let the row indexes get too far apart (call this "skew"), then you have to buffer very large amounts of data to keep the column data aligned. @@ -109,7 +109,7 @@ if you use lots of buffering on ingest, you can write the mice in front of the elephants so the read path requires less buffering to align columns. Or you can do two passes where you store segments in separate files then merge them at close according to an optimization plan. -::: +{{< /tip >}} ### The Reassembly Section @@ -117,7 +117,7 @@ The reassembly section provides the information needed to reconstruct column streams from segments, and in turn, to reconstruct the original values from column streams, i.e., to map columns back to composite values. -:::tip note +{{< tip "Note" >}} Of course, the reassembly section also provides the ability to extract just subsets of columns to be read and searched efficiently without ever needing to reconstruct the original rows. How well this performs is up to any particular @@ -127,7 +127,7 @@ Also, the reassembly section is in general vastly smaller than the data section so the goal here isn't to express information in cute and obscure compact forms but rather to represent data in an easy-to-digest, programmer-friendly form that leverages Super Binary. -::: +{{< /tip >}} The reassembly section is a Super Binary stream. Unlike Parquet, which uses an externally described schema @@ -147,9 +147,9 @@ A super type's integer position in this sequence defines its identifier encoded in the [super column](#the-super-column). This identifier is called the super ID. -:::tip note +{{< tip "Note" >}} Change the first N values to type values instead of nulls? -::: +{{< /tip >}} The next N+1 records contain reassembly information for each of the N super types where each record defines the column streams needed to reconstruct the original @@ -171,11 +171,11 @@ type signature: In the rest of this document, we will refer to this type as `` for shorthand and refer to the concept as a "segmap". -:::tip note +{{< tip "Note" >}} We use the type name "segmap" to emphasize that this information represents a set of byte ranges where data is stored and must be read from *rather than* the data itself. -::: +{{< /tip >}} #### The Super Column @@ -216,11 +216,11 @@ This simple top-down arrangement, along with the definition of the other column structures below, is all that is needed to reconstruct all of the original data. -:::tip note +{{< tip "Note" >}} Each row reassembly record has its own layout of columnar values and there is no attempt made to store like-typed columns from different schemas in the same physical column. -::: +{{< /tip >}} The notation `` refers to any instance of the five column types: * [``](#record-column), @@ -296,9 +296,9 @@ in the same column order implied by the union type, and * `tags` is a column of `int32` values where each subsequent value encodes the tag of the union type indicating which column the value falls within. -:::tip TBD +{{< tip "TBD" >}} Change code to conform to columns array instead of record{c0,c1,...} -::: +{{< /tip >}} The number of times each value of `tags` appears must equal the number of values in each respective column. @@ -350,14 +350,14 @@ data in the file, it will typically fit comfortably in memory and it can be very fast to scan the entire reassembly structure for any purpose. -:::tip Example +{{< tip "Example" >}} For a given query, a "scan planner" could traverse all the reassembly records to figure out which segments will be needed, then construct an intelligent plan for reading the needed segments and attempt to read them in mostly sequential order, which could serve as an optimizing intermediary between any underlying storage API and the Super Columnar decoding logic. -::: +{{< /tip >}} To decode the "next" row, its schema index is read from the root reassembly column stream. diff --git a/docs/install.md b/docs/install.md index fe1e8db596..075feadf89 100644 --- a/docs/install.md +++ b/docs/install.md @@ -40,11 +40,11 @@ This installs the `super` binary in your `$GOPATH/bin`. Once installed, run a [quick test](#quick-tests). -:::tip note +{{< tip "Note" >}} If you don't have Go installed, download and install it from the [Go install page](https://golang.org/doc/install). Go 1.23 or later is required. -::: +{{< /tip >}} ## Quick Tests diff --git a/docs/integrations/amazon-s3.md b/docs/integrations/amazon-s3.md index f377ce1453..a13caa192c 100644 --- a/docs/integrations/amazon-s3.md +++ b/docs/integrations/amazon-s3.md @@ -16,11 +16,11 @@ You must specify an AWS region via one of the following: You can create `~/.aws/config` by installing the [AWS CLI](https://aws.amazon.com/cli/) and running `aws configure`. -:::tip Note +{{< tip "Note" >}} If using S3-compatible storage that does not recognize the concept of regions, a region must still be specified, e.g., by providing a dummy value for `AWS_REGION`. -::: +{{< /tip >}} ## Credentials diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index 9688930bfe..97190d8350 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -81,13 +81,14 @@ The default settings when running `zed create` set the field and sort the stored data in descending order by that key. This configuration is ideal for Zeek log data. -:::tip Note +{{< tip "Note" >}} The [Zui](https://zui.brimdata.io/) desktop application automatically starts a Zed lake service when it launches. Therefore if you are using Zui you can skip the first set of commands shown above. The pool can be created from Zui by clicking **+**, selecting **New Pool**, then entering `ts` for the [pool key](../commands/super-db.md#pool-key). -::: +{{< /tip >}} + ### Fluentd @@ -366,7 +367,7 @@ leverage, you can reduce the lake's storage footprint by periodically running storage that contain the granular commits that have already been rolled into larger objects by compaction. -:::tip Note +{{< tip "Note" >}} As described in issue [super/4934](https://github.com/brimdata/super/issues/4934), even after running `zed vacuum`, some files related to commit history are currently still left behind below the lake storage path. The issue describes @@ -374,7 +375,7 @@ manual steps that can be taken to remove these files safely, if desired. However, if you find yourself needing to take these steps in your environment, please [contact us](#contact-us) as it will allow us to boost the priority of addressing the issue. -::: +{{< /tip >}} ## Ideas For Enhancement diff --git a/docs/integrations/zed-lake-auth/index.md b/docs/integrations/zed-lake-auth/index.md index 8394d4135f..0a6a4c2259 100644 --- a/docs/integrations/zed-lake-auth/index.md +++ b/docs/integrations/zed-lake-auth/index.md @@ -30,10 +30,10 @@ and then clicking the **Create API** button. 2. Enter any **Name** and URL **Identifier** for the API, then click the **Create** button. -:::tip +{{< tip "Tip" >}} Note the value you enter for the **Identifier** as you'll need it later for the Zed lake service configuration. -::: +{{< /tip >}} ![api-name-identifier](api-name-identifier.png) @@ -50,11 +50,11 @@ need it later for the Zed lake service configuration. 1. Begin creating a new application by clicking **Applications** in the left navigation menu and then clicking the **Create Application** button. -:::tip Note +{{< tip "Note" >}} Neither the "Zed lake (Test Application)" that was created for us automatically when we created our API nor the Default App that came with the trial are used in this configuration. -::: +{{< /tip >}} ![create-application](create-application.png) diff --git a/docs/integrations/zeek/data-type-compatibility.md b/docs/integrations/zeek/data-type-compatibility.md index 7de64aac74..b1fbb3379b 100644 --- a/docs/integrations/zeek/data-type-compatibility.md +++ b/docs/integrations/zeek/data-type-compatibility.md @@ -49,7 +49,7 @@ applicable to handling certain types. | [`vector`](https://docs.zeek.org/en/current/script-reference/types.html#type-vector) | [`array`](../../formats/zed.md#22-array | | | [`record`](https://docs.zeek.org/en/current/script-reference/types.html#type-record) | [`record`](../../formats/zed.md#21-record | See [`record` details](#record) | -:::tip Note +{{< tip "Note" >}} The [Zeek data types](https://docs.zeek.org/en/current/script-reference/types.html) page describes the types in the context of the [Zeek scripting language](https://docs.zeek.org/en/master/scripting/index.html). @@ -57,7 +57,7 @@ The Zeek types available in scripting are a superset of the data types that may appear in Zeek log files. The encodings of the types also differ in some ways between the two contexts. However, we link to this reference because there is no authoritative specification of the Zeek TSV log format. -::: +{{< /tip >}} ## Example diff --git a/docs/integrations/zeek/shaping-zeek-json.md b/docs/integrations/zeek/shaping-zeek-json.md index b574028ae7..73377ab5b5 100644 --- a/docs/integrations/zeek/shaping-zeek-json.md +++ b/docs/integrations/zeek/shaping-zeek-json.md @@ -193,11 +193,11 @@ specification. ... ``` -:::tip note +{{< tip "Note" >}} See [the role of `_path`](reading-zeek-log-formats.md#the-role-of-_path) for important details if you're using Zeek's built-in [ASCII logger](https://docs.zeek.org/en/current/scripts/base/frameworks/logging/writers/ascii.zeek.html) rather than the [JSON Streaming Logs](https://github.com/corelight/json-streaming-logs) package. -::: +{{< /tip >}} ### Zed Pipeline diff --git a/docs/language/functions/cast.md b/docs/language/functions/cast.md index 63819d1b66..3f6a4342b5 100644 --- a/docs/language/functions/cast.md +++ b/docs/language/functions/cast.md @@ -41,11 +41,11 @@ to match the output type's order but rather just modifies the leaf values. If a cast fails, an error is returned when casting to primitive types and the input value is returned when casting to complex types. -:::tip +{{< tip "Note" >}} Many users seeking to `cast` record values prefer to use the [`shape` function](./shape.md) which applies the `cast`, [`fill`](./fill.md), and [`order`](./order.md) functions simultaneously. -::: +{{< /tip >}} ### Examples diff --git a/docs/language/functions/fill.md b/docs/language/functions/fill.md index a5dd47897e..898c9b5d42 100644 --- a/docs/language/functions/fill.md +++ b/docs/language/functions/fill.md @@ -20,11 +20,11 @@ you want to be sure that all fields in a schema are present in a record. If `val` is not a record, it is returned unmodified. -:::tip +{{< tip "Tip" >}} Many users seeking the functionality of `fill` prefer to use the [`shape` function](./shape.md) which applies the `fill`, [`cast`](./cast.md), and [`order`](./order.md) functions simultaneously on a record. -::: +{{< /tip >}} ### Examples diff --git a/docs/language/functions/grok.md b/docs/language/functions/grok.md index 56cf02e474..c757014204 100644 --- a/docs/language/functions/grok.md +++ b/docs/language/functions/grok.md @@ -42,12 +42,12 @@ been published by Elastic and others that provide helpful guidance on becoming proficient in Grok. To help you adapt what you learn from these resources to the use of the `grok` function, review the tips below. -:::tip Note +{{< tip "Note" >}} As these represent areas of possible future SuperPipe enhancement, links to open issues are provided. If you find a functional gap significantly impacts your ability to use the `grok` function, please add a comment to the relevant issue describing your use case. -::: +{{< /tip >}} 1. Logstash's Grok offers an optional data type conversion syntax, e.g., @@ -111,7 +111,7 @@ issue describing your use case. avoid compatibility issues, we recommend building configurations starting from the RE2-based [included patterns](#included-patterns). -:::tip Note +{{< tip "Note" >}} If you absolutely require features of Logstash's Grok that are not currently present in SuperPipe, you can create a Logstash-based preprocessing pipeline that uses its @@ -121,7 +121,7 @@ and send its output as JSON to SuperPipe. Issue getting started. If you pursue this approach, please add a comment to the issue describing your use case or come talk to us on [community Slack](https://www.brimdata.io/join-slack/). -::: +{{< /tip >}} ### Debugging diff --git a/docs/language/functions/order.md b/docs/language/functions/order.md index 7d0e645c5f..94f0d7bac9 100644 --- a/docs/language/functions/order.md +++ b/docs/language/functions/order.md @@ -26,16 +26,16 @@ the empty record type, i.e., ``` order(val, <{}>) ``` -:::tip +{{< tip "Tip" >}} Many users seeking the functionality of `order` prefer to use the [`shape` function](./shape.md) which applies the `order`, [`cast`](./cast.md), and [`fill`](./fill.md) functions simultaneously on a record. -::: +{{< /tip >}} -:::tip Note +{{< tip "Note" >}} [Record expressions](../expressions.md#record-expressions) can also be used to reorder fields without specifying types ([example](../shaping.md#order)). -::: +{{< /tip >}} ### Examples diff --git a/docs/language/lateral-subqueries.md b/docs/language/lateral-subqueries.md index 86294eafb9..2510abd84a 100644 --- a/docs/language/lateral-subqueries.md +++ b/docs/language/lateral-subqueries.md @@ -9,10 +9,10 @@ The inner query may be _any_ pipeline operator sequence (excluding [`from` operators](operators/from.md)) and may refer to values from the outer sequence. -:::tip Note +{{< tip "Note" >}} This pattern rhymes with the SQL pattern of a "lateral join", which runs a subquery for each row of the outer query's results. -::: +{{< /tip >}} Lateral subqueries are created using the scoped form of the [`over` operator](operators/over.md). They may be nested to arbitrary depth @@ -125,9 +125,9 @@ parenthesized form: ( over [, ...] [with = [, ... [=]] |> ) ``` -:::tip +{{< tip "Note" >}} The parentheses disambiguate a lateral expression from a [lateral pipeline operator](operators/over.md). -::: +{{< /tip >}} This form must always include a [lateral scope](#lateral-scope) as indicated by ``. diff --git a/docs/language/operators/cut.md b/docs/language/operators/cut.md index 6c05d82b77..df28728cd7 100644 --- a/docs/language/operators/cut.md +++ b/docs/language/operators/cut.md @@ -76,11 +76,11 @@ echo '1 {a:1,b:2,c:3}' | super -z -c 'cut a,b' - ``` _Invoke a function while cutting to set a default value for a field_ -:::tip +{{< tip "Tip" >}} This can be helpful to transform data into a uniform record type, such as if the output will be exported in formats such as `csv` or `parquet` (see also: [`fuse`](fuse.md)). -::: +{{< /tip >}} ```mdtest-command echo '{a:1,b:null}{a:1,b:2}' | super -z -c 'cut a,b:=coalesce(b, 0)' - diff --git a/docs/language/operators/file.md b/docs/language/operators/file.md index b3fff30404..985c5721f5 100644 --- a/docs/language/operators/file.md +++ b/docs/language/operators/file.md @@ -6,7 +6,7 @@ `file` is a shorthand notation for `from`. See the [from operator](from.md) documentation for details. -:::tip Note +{{< tip "Note" >}} The `file` shorthand is exclusively for working with inputs to [`super`](../../commands/super.md) and is not available for use with [SuperDB data lakes](../../commands/super-db.md). -::: +{{< /tip >}} diff --git a/docs/language/operators/from.md b/docs/language/operators/from.md index 961d20e865..ae779c48c7 100644 --- a/docs/language/operators/from.md +++ b/docs/language/operators/from.md @@ -28,9 +28,9 @@ their data to its output. A data source can be * an HTTP, HTTPS, or S3 URI; or * the [`pass` operator](pass.md), to treat the upstream pipeline branch as a source. -:::tip Note +{{< tip "Note" >}} File paths and URIs may be followed by an optional [format](../../commands/super.md#input-formats) specifier. -::: +{{< /tip >}} Sourcing data from pools is only possible when querying a lake, such as via the [`super db` command](../../commands/super-db.md) or diff --git a/docs/language/operators/join.md b/docs/language/operators/join.md index 83c7bf6461..6b893919e1 100644 --- a/docs/language/operators/join.md +++ b/docs/language/operators/join.md @@ -14,13 +14,13 @@ | [anti|inner|left|right] join on = [[:=], ...] ``` -:::tip Note +{{< tip "Note" >}} The first `join` syntax shown above was more recently introduced and is in some ways similar to other languages such as SQL. The second was the original `join` syntax in SuperPipe. Most joins can be expressed using either syntax. See the [join tutorial](../../tutorials/join.md) for details. -::: +{{< /tip >}} ### Description diff --git a/docs/language/operators/load.md b/docs/language/operators/load.md index 4bf2a6c4d5..a78a84093a 100644 --- a/docs/language/operators/load.md +++ b/docs/language/operators/load.md @@ -8,11 +8,11 @@ load [@] [author ] [message ] [meta ] ``` -:::tip Note +{{< tip "Note" >}} The `load` operator is exclusively for working with pools in a [SuperDB data lake](../../commands/super-db.md) and is not available for use in [`super`](../../commands/super.md). -::: +{{< /tip >}} ### Description diff --git a/docs/language/pipe-ambiguity.md b/docs/language/pipe-ambiguity.md index adc6c17c1d..eeeb0da444 100644 --- a/docs/language/pipe-ambiguity.md +++ b/docs/language/pipe-ambiguity.md @@ -85,9 +85,7 @@ So when you want to run SuperSQL on old SQL queries that use top-level bitwise-OR expressions --- arguably a pretty obscure corner case --- just disable SuperSQL shortcuts and everything will work. -:::tip note - +{{< tip "Note" >}} Note that a config option to disable shortcuts is not yet implemented, but will be available in the future. - -::: +{{< /tip >}} diff --git a/docs/language/search-expressions.md b/docs/language/search-expressions.md index 527d8f7e57..6e964e2371 100644 --- a/docs/language/search-expressions.md +++ b/docs/language/search-expressions.md @@ -65,9 +65,9 @@ _ . : / % # @ ~ A glob must begin with one of these characters or `*` then may be followed by any of these characters, `*`, or digits `0` through `9`. -:::tip note +{{< tip "Note" >}} These rules do not allow for a leading digit. -::: +{{< /tip >}} For example, a prefix match is easily accomplished via `prefix*`, e.g., ```mdtest-command @@ -125,13 +125,13 @@ is a Boolean comparison between the product `a*b` and `c`. The search patterns described above can be combined with other "search terms" using Boolean logic to form search expressions. -:::tip note +{{< tip "Note" >}} When processing [Super Binary](../formats/bsup.md) data, the SuperDB runtime performs a multi-threaded Boyer-Moore scan over decompressed data buffers before parsing any data. This allows large buffers of data to be efficiently discarded and skipped when searching for rarely occurring values. For a [SuperDB data lake](../lake/format.md), a planned feature will use [Super Columnar](../formats/csup.md) files to further accelerate searches. -::: +{{< /tip >}} ### Search Terms @@ -228,12 +228,12 @@ is equivalent to where grep("foo", this) ``` -:::tip note +{{< tip "Note" >}} This equivalency between keyword search terms and grep semantics will change in the near future when we add support for full-text search. In this case, grep will still support substring match but keyword search will match segmented words from string fields. -::: +{{< /tip >}} #### Non-String Literal Search Term diff --git a/docs/tutorials/join.md b/docs/tutorials/join.md index 7badba8cad..081e645402 100644 --- a/docs/tutorials/join.md +++ b/docs/tutorials/join.md @@ -65,9 +65,9 @@ produces ## Left Join -:::tip note +{{< tip "Note" >}} In some databases a left join is called a _left outer join_. -::: +{{< /tip >}} By performing a left join that targets the same key fields, now all of our fruits will be shown in the results even if no one likes them (e.g., `avocado`). @@ -102,9 +102,9 @@ produces ## Right join -:::tip note +{{< tip "Note" >}} In SQL, a right join is called a _right outer join_. -::: +{{< /tip >}} Next we'll change the join type from `left` to `right`. Notice that this causes the `note` field from the right-hand input to appear in the joined results. @@ -132,9 +132,9 @@ produces ## Anti join -:::tip note +{{< tip "Note" >}} In some databases an anti join is called a _left anti join_. -::: +{{< /tip >}} The join type `anti` allows us to see which fruits are not liked by anyone. Note that with anti join only values from the left-hand input appear in the diff --git a/docs/tutorials/zq.md b/docs/tutorials/zq.md index 6561d09b77..2a24100ca8 100644 --- a/docs/tutorials/zq.md +++ b/docs/tutorials/zq.md @@ -43,12 +43,12 @@ To this end, if you want full JSON compatibility without having to delve into th details of Zed, just use the `-j` option with `zq` and this will tell `zq` to expect JSON values as input and produce JSON values as output, much like `jq`. -:::tip +{{< tip "Tip" >}} If your downstream JSON tooling expects only a single JSON value, we can use `-j` along with [`collect()`](../language/aggregates/collect.md) to aggregate multiple input values into an array. A `collect()` example is shown [later in this tutorial](#running-analytics). -::: +{{< /tip >}} ## `this` vs `.` From b5f93497a66099249ba91161ce77a9b7fcff00c2 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 08:59:24 -0800 Subject: [PATCH 10/18] Move Zeek integrations doc into Integrations section --- docs/integrations/zeek/{_index.md => index.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/integrations/zeek/{_index.md => index.md} (100%) diff --git a/docs/integrations/zeek/_index.md b/docs/integrations/zeek/index.md similarity index 100% rename from docs/integrations/zeek/_index.md rename to docs/integrations/zeek/index.md From e0234ad0e08564a7cc9748ba7b4748ddd07c7fc0 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 09:25:55 -0800 Subject: [PATCH 11/18] Remove consecutive blank lines --- docs/integrations/fluentd.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index 97190d8350..9b6c4a592e 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -89,7 +89,6 @@ by clicking **+**, selecting **New Pool**, then entering `ts` for the [pool key](../commands/super-db.md#pool-key). {{< /tip >}} - ### Fluentd Multiple approaches are available for From 84ed8e593186b7284e1f84a79906adfa64d62778 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 09:43:36 -0800 Subject: [PATCH 12/18] Fix broken links --- CHANGELOG.md | 12 ++++++------ docs/commands/super-db.md | 2 +- docs/integrations/fluentd.md | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 018e68b37c..f5a3c99801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ ## v1.16.0 * Improve ZNG scanning performance (#5101, #5103) * Improve the error message shown when `zq` is invoked with a single argument that's not a valid query and doesn't contain a source (#5119) -* Update [Zeek integration docs](docs/integrations/zeek/_index.md), including [reference shaper](docs/integrations/zeek/shaping-zeek-json.md) changes for [Zeek v6.2.0](https://github.com/zeek/zeek/releases/tag/v6.2.0) data (#5106) +* Update [Zeek integration docs](docs/integrations/zeek/index.md), including [reference shaper](docs/integrations/zeek/shaping-zeek-json.md) changes for [Zeek v6.2.0](https://github.com/zeek/zeek/releases/tag/v6.2.0) data (#5106) * [String literals](docs/language/expressions.md#formatted-string-literals) now use the "f-string" format `f"{ }"` instead of the previous `${ }` (#5123) * Prototype SQL support has been dropped from the Zed language (full SQL support is expected at a later date) (#5109) * Empty objects and arrays in JSON output are now consistently printed on a single line (#5127) @@ -150,7 +150,7 @@ ## v1.7.0 * Add [`regexp_replace()`](docs/language/functions/regexp_replace.md) function for replacing regular expression matches in a string (#4435, #4449) -* Add [documentation](docs/integrations/zed-lake-auth.md) showing how to configure Auth0 for authenticated access to a Zed lake service (#4439) +* Add [documentation](docs/integrations/zed-lake-auth/index.md) showing how to configure Auth0 for authenticated access to a Zed lake service (#4439) * Fix an issue where elements of map could not be accessed if the key was of a union type (#4447) * Allow [`head`](docs/language/operators/head.md) operator to accept an expression (#4451) * Allow [`tail`](docs/language/operators/tail.md) operator to accept an expression (#4464) @@ -454,7 +454,7 @@ questions. * Fix an issue where `len()` of a `null` array was evaluating to something greater than zero (#2761) * Fix an issue where `sort` with no fields was ignoring alias types and nested fields when picking a sort field (#2762) * Fix an issue where unexpected `cut: no record found` warnings were returned by `zed lake query` but not when the same data was queried via `zq` (#2764) -* Move and extend the [Zeek interoperability docs](docs/integrations/zeek/_index.md) (#2770, #2782, #2830) +* Move and extend the [Zeek interoperability docs](docs/integrations/zeek/index.md) (#2770, #2782, #2830) * Create endpoints in the Zed lake service API that correspond to underlying Zed lake operations, and expose them via `zapi` commands (#2741, #2774, #2786, #2775, #2794, #2795, #2796, #2920, #2925, #2928) * Fix an issue where `zq` would surface a syntax error when reading ZSON it had sent as output (#2792) * Add an `/events` endpoint to the API, which can be used by clients such as the Brim app to be notified of pool updates (#2791) @@ -628,7 +628,7 @@ questions. ## v0.23.0 * zql: Add `week` as a unit for [time grouping with `every`](docs/language/functions/every.md) (#1374) -* zq: Fix an issue where a `null` value in a [JSON type definition](docs/integrations/zeek/_index.md) caused a failure without an error message (#1377) +* zq: Fix an issue where a `null` value in a [JSON type definition](docs/integrations/zeek/index.md) caused a failure without an error message (#1377) * zq: Add [`zst` format](docs/formats/csup.md) to `-i` and `-f` command-line help (#1384) * zq: ZNG spec and `zq` updates to introduce the beta ZNG storage format (#1375, #1415, #1394, #1457, #1512, #1523, #1529), also addressing the following: * New data type `bytes` for storing sequences of bytes encoded as base64 (#1315) @@ -644,11 +644,11 @@ questions. * zqd: Check and convert alpha ZNG filestores to beta ZNG (#1574, #1576) * zq: Fix an issue where spill-to-disk file names could collide (#1391) * zq: Allow the [`fuse` operator](docs/language/operators/fuse.md) to spill-to-disk to avoid memory limitations (#1355, #1402) -* zq: No longer require `_path` as a first column in a [JSON type definition](docs/integrations/zeek/_index.md) (#1370) +* zq: No longer require `_path` as a first column in a [JSON type definition](docs/integrations/zeek/index.md) (#1370) * zql: Improve ZQL docs for [aggregate functions](docs/language/operators/summarize.md) and grouping (#1385) * zql: Point links for developer docs at [pkg.go.dev](https://pkg.go.dev/) instead of [godoc.org](https://godoc.org/) (#1401) * zq: Add support for timestamps with signed timezone offsets (#1389) -* zq: Add a [JSON type definition](docs/integrations/zeek/_index.md) for alert events in [Suricata EVE logs](https://suricata.readthedocs.io/en/suricata-5.0.2/output/eve/eve-json-output.html) (#1400) +* zq: Add a [JSON type definition](docs/integrations/zeek/index.md) for alert events in [Suricata EVE logs](https://suricata.readthedocs.io/en/suricata-5.0.2/output/eve/eve-json-output.html) (#1400) * zq: Update the [ZNG over JSON (ZJSON)](docs/formats/zjson.md) spec and implementation (#1299) * zar: Use buffered streaming for archive import (#1397) * zq: Add an `ast` command that prints parsed ZQL as its underlying JSON object (#1416) diff --git a/docs/commands/super-db.md b/docs/commands/super-db.md index b7bad70d03..1204522aa3 100644 --- a/docs/commands/super-db.md +++ b/docs/commands/super-db.md @@ -356,7 +356,7 @@ format. However, the `-f` option can be used to specify any supported super db auth login|logout|method|verify ``` Access to a lake can be secured with [Auth0 authentication](https://auth0.com/). -A [guide](../integrations/zed-lake-auth.md) is available with example configurations. +A [guide](../integrations/zed-lake-auth/index.md) is available with example configurations. Please reach out to us on our [community Slack](https://www.brimdata.io/join-slack/) if you have feedback on your experience or need additional help. diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index 9b6c4a592e..f8700871d2 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -12,7 +12,7 @@ record for archiving and analytics. This guide walks through two simple configurations of Fluentd with a Zed lake that can be used as reference for starting your own production configuration. As it's a data source important to many in the Zed community, log data from -[Zeek](./zeek/_index.md) is used in this guide. The approach shown can be +[Zeek](./zeek/index.md) is used in this guide. The approach shown can be easily adapted to any log data source. ## Software From affe7823bb58fbc062fd650949dcda361399e060 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 12:58:40 -0800 Subject: [PATCH 13/18] Make links work again inside tips --- docs/commands/super-db.md | 42 ++++++++++----- docs/commands/super.md | 6 ++- docs/formats/bsup.md | 12 +++-- docs/formats/csup.md | 54 ++++++++++++------- docs/install.md | 6 ++- docs/integrations/amazon-s3.md | 6 ++- docs/integrations/fluentd.md | 12 +++-- docs/integrations/zed-lake-auth/index.md | 12 +++-- .../zeek/data-type-compatibility.md | 6 ++- docs/integrations/zeek/shaping-zeek-json.md | 6 ++- docs/language/functions/cast.md | 6 ++- docs/language/functions/fill.md | 6 ++- docs/language/functions/grok.md | 12 +++-- docs/language/functions/order.md | 12 +++-- docs/language/lateral-subqueries.md | 12 +++-- docs/language/operators/cut.md | 6 ++- docs/language/operators/file.md | 6 ++- docs/language/operators/from.md | 6 ++- docs/language/operators/join.md | 6 ++- docs/language/operators/load.md | 6 ++- docs/language/pipe-ambiguity.md | 6 ++- docs/language/search-expressions.md | 18 ++++--- docs/tutorials/join.md | 18 ++++--- docs/tutorials/zq.md | 6 ++- 24 files changed, 192 insertions(+), 96 deletions(-) diff --git a/docs/commands/super-db.md b/docs/commands/super-db.md index 1204522aa3..a338a50389 100644 --- a/docs/commands/super-db.md +++ b/docs/commands/super-db.md @@ -15,7 +15,8 @@ title: super db

-{{< tip "Status" >}} +{{% tip "Status" %}} + While [`super`](super.md) and its accompanying [formats](../formats/_index.md) are production quality, the SuperDB data lake is still fairly early in development and alpha quality. @@ -25,7 +26,8 @@ is deployed to manage the lake's data layout via the [lake API](../lake/api.md). Enhanced scalability with self-tuning configuration is under development. -{{< /tip >}} + +{{% /tip %}} ## The Lake Model @@ -153,7 +155,8 @@ running any `super db` lake command all pointing at the same storage endpoint and the lake's data footprint will always remain consistent as the endpoints all adhere to the consistency semantics of the lake. -{{< tip "Caveat" >}} +{{% tip "Caveat" %}} + Data consistency is not fully implemented yet for the S3 endpoint so only single-node access to S3 is available right now, though support for multi-node access is forthcoming. @@ -164,7 +167,8 @@ access to a local file system has been thoroughly tested and should be deemed reliable, i.e., you can run a direct-access instance of `super db` alongside a server instance of `super db` on the same file system and data consistency will be maintained. -{{< /tip >}} + +{{% /tip %}} ### Locating the Lake @@ -206,11 +210,13 @@ Each commit object is assigned a global ID. Similar to Git, commit objects are arranged into a tree and represent the entire commit history of the lake. -{{< tip "Note" >}} +{{% tip "Note" %}} + Technically speaking, Git can merge from multiple parents and thus Git commits form a directed acyclic graph instead of a tree; SuperDB does not currently support multiple parents in the commit object history. -{{< /tip >}} + +{{% /tip %}} A branch is simply a named pointer to a commit object in the lake and like a pool, a branch name can be any valid UTF-8 string. @@ -272,10 +278,12 @@ key. For example, on a pool with pool key `ts`, the query `ts == 100` will be optimized to scan only the data objects where the value `100` could be present. -{{< tip "Note" >}} +{{% tip "Note" %}} + The pool key will also serve as the primary key for the forthcoming CRUD semantics. -{{< /tip >}} + +{{% /tip %}} A pool also has a configured sort order, either ascending or descending and data is organized in the pool in accordance with this order. @@ -325,9 +333,11 @@ using that pool's "branches log" in a similar fashion, then its corresponding commit object can be used to construct the data of that branch at that past point in time. -{{< tip "Note" >}} +{{% tip "Note" %}} + Time travel using timestamps is a forthcoming feature. -{{< /tip >}} + +{{% /tip %}} ## `super db` Commands @@ -407,11 +417,13 @@ the [special value `this`](../language/pipeline-model.md#the-special-value-this) A newly created pool is initialized with a branch called `main`. -{{< tip "Note" >}} +{{% tip "Note" %}} + Lakes can be used without thinking about branches. When referencing a pool without a branch, the tooling presumes the "main" branch as the default, and everything can be done on main without having to think about branching. -{{< /tip >}} + +{{% /tip %}} ### Delete ``` @@ -582,9 +594,11 @@ that is stored in the commit journal for reference. These values may be specified as options to the [`load`](#load) command, and are also available in the [lake API](../lake/api.md) for automation. -{{< tip "Note" >}} +{{% tip "Note" %}} + The branchlog meta-query source is not yet implemented. -{{< /tip >}} + +{{% /tip %}} ### Ls ``` diff --git a/docs/commands/super.md b/docs/commands/super.md index fadcd25e95..0c828bf15d 100644 --- a/docs/commands/super.md +++ b/docs/commands/super.md @@ -187,13 +187,15 @@ not desirable because (1) the Super JSON parser is not particularly performant a (2) all JSON numbers are floating point but the Super JSON parser will parse as JSON any number that appears without a decimal point as an integer type. -{{< tip "Note" >}} +{{% tip "Note" %}} + The reason `super` is not particularly performant for Super JSON is that the [Super Binary](../formats/bsup.md) or [Super Columnar](../formats/csup.md) formats are semantically equivalent to Super JSON but much more efficient and the design intent is that these efficient binary formats should be used in use cases where performance matters. Super JSON is typically used only when data needs to be human-readable in interactive settings or in automated tests. -{{< /tip >}} + +{{% /tip %}} To this end, `super` uses a heuristic to select between Super JSON and plain JSON when the `-i` option is not specified. Specifically, plain JSON is selected when the first values diff --git a/docs/formats/bsup.md b/docs/formats/bsup.md index 1bfacd5914..66dec48288 100644 --- a/docs/formats/bsup.md +++ b/docs/formats/bsup.md @@ -130,7 +130,8 @@ size decompression buffers in advance of decoding. Values for the `format` byte are defined in the [Super Binary compression format specification](./compression.md). -{{< tip "Note" >}} +{{% tip "Note" %}} + This arrangement of frames separating types and values allows for efficient scanning and parallelization. In general, values depend on type definitions but as long as all of the types are known when @@ -143,7 +144,8 @@ heuristics, e.g., knowing a filtering predicate can't be true based on a quick scan of the data perhaps using the Boyer-Moore algorithm to determine that a comparison with a string constant would not work for any value in the buffer. -{{< /tip >}} + +{{% /tip %}} Whether the payload was originally uncompressed or was decompressed, it is then interpreted according to the `T` bits of the frame code as a @@ -211,12 +213,14 @@ is further encoded as a "counted string", which is the `uvarint` encoding of the length of the string followed by that many bytes of UTF-8 encoded string data. -{{< tip "Note" >}} +{{% tip "Note" %}} + As defined by [Super JSON](jsup.md), a field name can be any valid UTF-8 string much like JSON objects can be indexed with arbitrary string keys (via index operator) even if the field names available to the dot operator are restricted by language syntax for identifiers. -{{< /tip >}} + +{{% /tip %}} The type ID follows the field name and is encoded as a `uvarint`. diff --git a/docs/formats/csup.md b/docs/formats/csup.md index de60c76920..a64b394511 100644 --- a/docs/formats/csup.md +++ b/docs/formats/csup.md @@ -64,12 +64,14 @@ then write the metadata into the reassembly section along with the trailer at the end. This allows a stream to be converted to a Super Columnar file in a single pass. -{{< tip "Note" >}} +{{% tip "Note" %}} + That said, the layout is flexible enough that an implementation may optimize the data layout with additional passes or by writing the output to multiple files then merging them together (or even leaving the Super Columnar entity as separate files). -{{< /tip >}} + +{{% /tip %}} ### The Data Section @@ -85,7 +87,8 @@ There is no information in the data section for how segments relate to one another or how they are reconstructed into columns. They are just blobs of Super Binary data. -{{< tip "Note" >}} +{{% tip "Note" %}} + Unlike Parquet, there is no explicit arrangement of the column chunks into row groups but rather they are allowed to grow at different rates so a high-volume column might be comprised of many segments while a low-volume @@ -93,9 +96,11 @@ column must just be one or several. This allows scans of low-volume record type (the "mice") to perform well amongst high-volume record types (the "elephants"), i.e., there are not a bunch of seeks with tiny reads of mice data interspersed throughout the elephants. -{{< /tip >}} -{{< tip "TBD" >}} +{{% /tip %}} + +{{% tip "TBD" %}} + The mice/elephants model creates an interesting and challenging layout problem. If you let the row indexes get too far apart (call this "skew"), then you have to buffer very large amounts of data to keep the column data aligned. @@ -109,7 +114,8 @@ if you use lots of buffering on ingest, you can write the mice in front of the elephants so the read path requires less buffering to align columns. Or you can do two passes where you store segments in separate files then merge them at close according to an optimization plan. -{{< /tip >}} + +{{% /tip %}} ### The Reassembly Section @@ -117,7 +123,8 @@ The reassembly section provides the information needed to reconstruct column streams from segments, and in turn, to reconstruct the original values from column streams, i.e., to map columns back to composite values. -{{< tip "Note" >}} +{{% tip "Note" %}} + Of course, the reassembly section also provides the ability to extract just subsets of columns to be read and searched efficiently without ever needing to reconstruct the original rows. How well this performs is up to any particular @@ -127,7 +134,8 @@ Also, the reassembly section is in general vastly smaller than the data section so the goal here isn't to express information in cute and obscure compact forms but rather to represent data in an easy-to-digest, programmer-friendly form that leverages Super Binary. -{{< /tip >}} + +{{% /tip %}} The reassembly section is a Super Binary stream. Unlike Parquet, which uses an externally described schema @@ -147,9 +155,11 @@ A super type's integer position in this sequence defines its identifier encoded in the [super column](#the-super-column). This identifier is called the super ID. -{{< tip "Note" >}} +{{% tip "Note" %}} + Change the first N values to type values instead of nulls? -{{< /tip >}} + +{{% /tip %}} The next N+1 records contain reassembly information for each of the N super types where each record defines the column streams needed to reconstruct the original @@ -171,11 +181,13 @@ type signature: In the rest of this document, we will refer to this type as `` for shorthand and refer to the concept as a "segmap". -{{< tip "Note" >}} +{{% tip "Note" %}} + We use the type name "segmap" to emphasize that this information represents a set of byte ranges where data is stored and must be read from *rather than* the data itself. -{{< /tip >}} + +{{% /tip %}} #### The Super Column @@ -216,11 +228,13 @@ This simple top-down arrangement, along with the definition of the other column structures below, is all that is needed to reconstruct all of the original data. -{{< tip "Note" >}} +{{% tip "Note" %}} + Each row reassembly record has its own layout of columnar values and there is no attempt made to store like-typed columns from different schemas in the same physical column. -{{< /tip >}} + +{{% /tip %}} The notation `` refers to any instance of the five column types: * [``](#record-column), @@ -296,9 +310,11 @@ in the same column order implied by the union type, and * `tags` is a column of `int32` values where each subsequent value encodes the tag of the union type indicating which column the value falls within. -{{< tip "TBD" >}} +{{% tip "TBD" %}} + Change code to conform to columns array instead of record{c0,c1,...} -{{< /tip >}} + +{{% /tip %}} The number of times each value of `tags` appears must equal the number of values in each respective column. @@ -350,14 +366,16 @@ data in the file, it will typically fit comfortably in memory and it can be very fast to scan the entire reassembly structure for any purpose. -{{< tip "Example" >}} +{{% tip "Example" %}} + For a given query, a "scan planner" could traverse all the reassembly records to figure out which segments will be needed, then construct an intelligent plan for reading the needed segments and attempt to read them in mostly sequential order, which could serve as an optimizing intermediary between any underlying storage API and the Super Columnar decoding logic. -{{< /tip >}} + +{{% /tip %}} To decode the "next" row, its schema index is read from the root reassembly column stream. diff --git a/docs/install.md b/docs/install.md index 075feadf89..0130874a6d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -40,11 +40,13 @@ This installs the `super` binary in your `$GOPATH/bin`. Once installed, run a [quick test](#quick-tests). -{{< tip "Note" >}} +{{% tip "Note" %}} + If you don't have Go installed, download and install it from the [Go install page](https://golang.org/doc/install). Go 1.23 or later is required. -{{< /tip >}} + +{{% /tip %}} ## Quick Tests diff --git a/docs/integrations/amazon-s3.md b/docs/integrations/amazon-s3.md index a13caa192c..671825a042 100644 --- a/docs/integrations/amazon-s3.md +++ b/docs/integrations/amazon-s3.md @@ -16,11 +16,13 @@ You must specify an AWS region via one of the following: You can create `~/.aws/config` by installing the [AWS CLI](https://aws.amazon.com/cli/) and running `aws configure`. -{{< tip "Note" >}} +{{% tip "Note" %}} + If using S3-compatible storage that does not recognize the concept of regions, a region must still be specified, e.g., by providing a dummy value for `AWS_REGION`. -{{< /tip >}} + +{{% /tip %}} ## Credentials diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index f8700871d2..cfe29c4e76 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -81,13 +81,15 @@ The default settings when running `zed create` set the field and sort the stored data in descending order by that key. This configuration is ideal for Zeek log data. -{{< tip "Note" >}} +{{% tip "Note" %}} + The [Zui](https://zui.brimdata.io/) desktop application automatically starts a Zed lake service when it launches. Therefore if you are using Zui you can skip the first set of commands shown above. The pool can be created from Zui by clicking **+**, selecting **New Pool**, then entering `ts` for the [pool key](../commands/super-db.md#pool-key). -{{< /tip >}} + +{{% /tip %}} ### Fluentd @@ -366,7 +368,8 @@ leverage, you can reduce the lake's storage footprint by periodically running storage that contain the granular commits that have already been rolled into larger objects by compaction. -{{< tip "Note" >}} +{{% tip "Note" %}} + As described in issue [super/4934](https://github.com/brimdata/super/issues/4934), even after running `zed vacuum`, some files related to commit history are currently still left behind below the lake storage path. The issue describes @@ -374,7 +377,8 @@ manual steps that can be taken to remove these files safely, if desired. However, if you find yourself needing to take these steps in your environment, please [contact us](#contact-us) as it will allow us to boost the priority of addressing the issue. -{{< /tip >}} + +{{% /tip %}} ## Ideas For Enhancement diff --git a/docs/integrations/zed-lake-auth/index.md b/docs/integrations/zed-lake-auth/index.md index 0a6a4c2259..6ddd770d71 100644 --- a/docs/integrations/zed-lake-auth/index.md +++ b/docs/integrations/zed-lake-auth/index.md @@ -30,10 +30,12 @@ and then clicking the **Create API** button. 2. Enter any **Name** and URL **Identifier** for the API, then click the **Create** button. -{{< tip "Tip" >}} +{{% tip "Tip" %}} + Note the value you enter for the **Identifier** as you'll need it later for the Zed lake service configuration. -{{< /tip >}} + +{{% /tip %}} ![api-name-identifier](api-name-identifier.png) @@ -50,11 +52,13 @@ need it later for the Zed lake service configuration. 1. Begin creating a new application by clicking **Applications** in the left navigation menu and then clicking the **Create Application** button. -{{< tip "Note" >}} +{{% tip "Note" %}} + Neither the "Zed lake (Test Application)" that was created for us automatically when we created our API nor the Default App that came with the trial are used in this configuration. -{{< /tip >}} + +{{% /tip %}} ![create-application](create-application.png) diff --git a/docs/integrations/zeek/data-type-compatibility.md b/docs/integrations/zeek/data-type-compatibility.md index b1fbb3379b..3f8f9a9619 100644 --- a/docs/integrations/zeek/data-type-compatibility.md +++ b/docs/integrations/zeek/data-type-compatibility.md @@ -49,7 +49,8 @@ applicable to handling certain types. | [`vector`](https://docs.zeek.org/en/current/script-reference/types.html#type-vector) | [`array`](../../formats/zed.md#22-array | | | [`record`](https://docs.zeek.org/en/current/script-reference/types.html#type-record) | [`record`](../../formats/zed.md#21-record | See [`record` details](#record) | -{{< tip "Note" >}} +{{% tip "Note" %}} + The [Zeek data types](https://docs.zeek.org/en/current/script-reference/types.html) page describes the types in the context of the [Zeek scripting language](https://docs.zeek.org/en/master/scripting/index.html). @@ -57,7 +58,8 @@ The Zeek types available in scripting are a superset of the data types that may appear in Zeek log files. The encodings of the types also differ in some ways between the two contexts. However, we link to this reference because there is no authoritative specification of the Zeek TSV log format. -{{< /tip >}} + +{{% /tip %}} ## Example diff --git a/docs/integrations/zeek/shaping-zeek-json.md b/docs/integrations/zeek/shaping-zeek-json.md index 73377ab5b5..6ae365d0ee 100644 --- a/docs/integrations/zeek/shaping-zeek-json.md +++ b/docs/integrations/zeek/shaping-zeek-json.md @@ -193,11 +193,13 @@ specification. ... ``` -{{< tip "Note" >}} +{{% tip "Note" %}} + See [the role of `_path`](reading-zeek-log-formats.md#the-role-of-_path) for important details if you're using Zeek's built-in [ASCII logger](https://docs.zeek.org/en/current/scripts/base/frameworks/logging/writers/ascii.zeek.html) rather than the [JSON Streaming Logs](https://github.com/corelight/json-streaming-logs) package. -{{< /tip >}} + +{{% /tip %}} ### Zed Pipeline diff --git a/docs/language/functions/cast.md b/docs/language/functions/cast.md index 3f6a4342b5..632a978b7b 100644 --- a/docs/language/functions/cast.md +++ b/docs/language/functions/cast.md @@ -41,11 +41,13 @@ to match the output type's order but rather just modifies the leaf values. If a cast fails, an error is returned when casting to primitive types and the input value is returned when casting to complex types. -{{< tip "Note" >}} +{{% tip "Note" %}} + Many users seeking to `cast` record values prefer to use the [`shape` function](./shape.md) which applies the `cast`, [`fill`](./fill.md), and [`order`](./order.md) functions simultaneously. -{{< /tip >}} + +{{% /tip %}} ### Examples diff --git a/docs/language/functions/fill.md b/docs/language/functions/fill.md index 898c9b5d42..3d125931bc 100644 --- a/docs/language/functions/fill.md +++ b/docs/language/functions/fill.md @@ -20,11 +20,13 @@ you want to be sure that all fields in a schema are present in a record. If `val` is not a record, it is returned unmodified. -{{< tip "Tip" >}} +{{% tip "Tip" %}} + Many users seeking the functionality of `fill` prefer to use the [`shape` function](./shape.md) which applies the `fill`, [`cast`](./cast.md), and [`order`](./order.md) functions simultaneously on a record. -{{< /tip >}} + +{{% /tip %}} ### Examples diff --git a/docs/language/functions/grok.md b/docs/language/functions/grok.md index c757014204..bd395832c0 100644 --- a/docs/language/functions/grok.md +++ b/docs/language/functions/grok.md @@ -42,12 +42,14 @@ been published by Elastic and others that provide helpful guidance on becoming proficient in Grok. To help you adapt what you learn from these resources to the use of the `grok` function, review the tips below. -{{< tip "Note" >}} +{{% tip "Note" %}} + As these represent areas of possible future SuperPipe enhancement, links to open issues are provided. If you find a functional gap significantly impacts your ability to use the `grok` function, please add a comment to the relevant issue describing your use case. -{{< /tip >}} + +{{% /tip %}} 1. Logstash's Grok offers an optional data type conversion syntax, e.g., @@ -111,7 +113,8 @@ issue describing your use case. avoid compatibility issues, we recommend building configurations starting from the RE2-based [included patterns](#included-patterns). -{{< tip "Note" >}} +{{% tip "Note" %}} + If you absolutely require features of Logstash's Grok that are not currently present in SuperPipe, you can create a Logstash-based preprocessing pipeline that uses its @@ -121,7 +124,8 @@ and send its output as JSON to SuperPipe. Issue getting started. If you pursue this approach, please add a comment to the issue describing your use case or come talk to us on [community Slack](https://www.brimdata.io/join-slack/). -{{< /tip >}} + +{{% /tip %}} ### Debugging diff --git a/docs/language/functions/order.md b/docs/language/functions/order.md index 94f0d7bac9..0dbf1de6dd 100644 --- a/docs/language/functions/order.md +++ b/docs/language/functions/order.md @@ -26,16 +26,20 @@ the empty record type, i.e., ``` order(val, <{}>) ``` -{{< tip "Tip" >}} +{{% tip "Tip" %}} + Many users seeking the functionality of `order` prefer to use the [`shape` function](./shape.md) which applies the `order`, [`cast`](./cast.md), and [`fill`](./fill.md) functions simultaneously on a record. -{{< /tip >}} -{{< tip "Note" >}} +{{% /tip %}} + +{{% tip "Note" %}} + [Record expressions](../expressions.md#record-expressions) can also be used to reorder fields without specifying types ([example](../shaping.md#order)). -{{< /tip >}} + +{{% /tip %}} ### Examples diff --git a/docs/language/lateral-subqueries.md b/docs/language/lateral-subqueries.md index 2510abd84a..1b51b50dd9 100644 --- a/docs/language/lateral-subqueries.md +++ b/docs/language/lateral-subqueries.md @@ -9,10 +9,12 @@ The inner query may be _any_ pipeline operator sequence (excluding [`from` operators](operators/from.md)) and may refer to values from the outer sequence. -{{< tip "Note" >}} +{{% tip "Note" %}} + This pattern rhymes with the SQL pattern of a "lateral join", which runs a subquery for each row of the outer query's results. -{{< /tip >}} + +{{% /tip %}} Lateral subqueries are created using the scoped form of the [`over` operator](operators/over.md). They may be nested to arbitrary depth @@ -125,9 +127,11 @@ parenthesized form: ( over [, ...] [with = [, ... [=]] |> ) ``` -{{< tip "Note" >}} +{{% tip "Note" %}} + The parentheses disambiguate a lateral expression from a [lateral pipeline operator](operators/over.md). -{{< /tip >}} + +{{% /tip %}} This form must always include a [lateral scope](#lateral-scope) as indicated by ``. diff --git a/docs/language/operators/cut.md b/docs/language/operators/cut.md index df28728cd7..1fd2c5a8c9 100644 --- a/docs/language/operators/cut.md +++ b/docs/language/operators/cut.md @@ -76,11 +76,13 @@ echo '1 {a:1,b:2,c:3}' | super -z -c 'cut a,b' - ``` _Invoke a function while cutting to set a default value for a field_ -{{< tip "Tip" >}} +{{% tip "Tip" %}} + This can be helpful to transform data into a uniform record type, such as if the output will be exported in formats such as `csv` or `parquet` (see also: [`fuse`](fuse.md)). -{{< /tip >}} + +{{% /tip %}} ```mdtest-command echo '{a:1,b:null}{a:1,b:2}' | super -z -c 'cut a,b:=coalesce(b, 0)' - diff --git a/docs/language/operators/file.md b/docs/language/operators/file.md index 985c5721f5..3e86ae1b0f 100644 --- a/docs/language/operators/file.md +++ b/docs/language/operators/file.md @@ -6,7 +6,9 @@ `file` is a shorthand notation for `from`. See the [from operator](from.md) documentation for details. -{{< tip "Note" >}} +{{% tip "Note" %}} + The `file` shorthand is exclusively for working with inputs to [`super`](../../commands/super.md) and is not available for use with [SuperDB data lakes](../../commands/super-db.md). -{{< /tip >}} + +{{% /tip %}} diff --git a/docs/language/operators/from.md b/docs/language/operators/from.md index ae779c48c7..f6b64719d1 100644 --- a/docs/language/operators/from.md +++ b/docs/language/operators/from.md @@ -28,9 +28,11 @@ their data to its output. A data source can be * an HTTP, HTTPS, or S3 URI; or * the [`pass` operator](pass.md), to treat the upstream pipeline branch as a source. -{{< tip "Note" >}} +{{% tip "Note" %}} + File paths and URIs may be followed by an optional [format](../../commands/super.md#input-formats) specifier. -{{< /tip >}} + +{{% /tip %}} Sourcing data from pools is only possible when querying a lake, such as via the [`super db` command](../../commands/super-db.md) or diff --git a/docs/language/operators/join.md b/docs/language/operators/join.md index 6b893919e1..e92e7ccc00 100644 --- a/docs/language/operators/join.md +++ b/docs/language/operators/join.md @@ -14,13 +14,15 @@ | [anti|inner|left|right] join on = [[:=], ...] ``` -{{< tip "Note" >}} +{{% tip "Note" %}} + The first `join` syntax shown above was more recently introduced and is in some ways similar to other languages such as SQL. The second was the original `join` syntax in SuperPipe. Most joins can be expressed using either syntax. See the [join tutorial](../../tutorials/join.md) for details. -{{< /tip >}} + +{{% /tip %}} ### Description diff --git a/docs/language/operators/load.md b/docs/language/operators/load.md index a78a84093a..e726962c39 100644 --- a/docs/language/operators/load.md +++ b/docs/language/operators/load.md @@ -8,11 +8,13 @@ load [@] [author ] [message ] [meta ] ``` -{{< tip "Note" >}} +{{% tip "Note" %}} + The `load` operator is exclusively for working with pools in a [SuperDB data lake](../../commands/super-db.md) and is not available for use in [`super`](../../commands/super.md). -{{< /tip >}} + +{{% /tip %}} ### Description diff --git a/docs/language/pipe-ambiguity.md b/docs/language/pipe-ambiguity.md index eeeb0da444..218ec117fa 100644 --- a/docs/language/pipe-ambiguity.md +++ b/docs/language/pipe-ambiguity.md @@ -85,7 +85,9 @@ So when you want to run SuperSQL on old SQL queries that use top-level bitwise-OR expressions --- arguably a pretty obscure corner case --- just disable SuperSQL shortcuts and everything will work. -{{< tip "Note" >}} +{{% tip "Note" %}} + Note that a config option to disable shortcuts is not yet implemented, but will be available in the future. -{{< /tip >}} + +{{% /tip %}} diff --git a/docs/language/search-expressions.md b/docs/language/search-expressions.md index 6e964e2371..399ff878d1 100644 --- a/docs/language/search-expressions.md +++ b/docs/language/search-expressions.md @@ -65,9 +65,11 @@ _ . : / % # @ ~ A glob must begin with one of these characters or `*` then may be followed by any of these characters, `*`, or digits `0` through `9`. -{{< tip "Note" >}} +{{% tip "Note" %}} + These rules do not allow for a leading digit. -{{< /tip >}} + +{{% /tip %}} For example, a prefix match is easily accomplished via `prefix*`, e.g., ```mdtest-command @@ -125,13 +127,15 @@ is a Boolean comparison between the product `a*b` and `c`. The search patterns described above can be combined with other "search terms" using Boolean logic to form search expressions. -{{< tip "Note" >}} +{{% tip "Note" %}} + When processing [Super Binary](../formats/bsup.md) data, the SuperDB runtime performs a multi-threaded Boyer-Moore scan over decompressed data buffers before parsing any data. This allows large buffers of data to be efficiently discarded and skipped when searching for rarely occurring values. For a [SuperDB data lake](../lake/format.md), a planned feature will use [Super Columnar](../formats/csup.md) files to further accelerate searches. -{{< /tip >}} + +{{% /tip %}} ### Search Terms @@ -228,12 +232,14 @@ is equivalent to where grep("foo", this) ``` -{{< tip "Note" >}} +{{% tip "Note" %}} + This equivalency between keyword search terms and grep semantics will change in the near future when we add support for full-text search. In this case, grep will still support substring match but keyword search will match segmented words from string fields. -{{< /tip >}} + +{{% /tip %}} #### Non-String Literal Search Term diff --git a/docs/tutorials/join.md b/docs/tutorials/join.md index 081e645402..7c186cd822 100644 --- a/docs/tutorials/join.md +++ b/docs/tutorials/join.md @@ -65,9 +65,11 @@ produces ## Left Join -{{< tip "Note" >}} +{{% tip "Note" %}} + In some databases a left join is called a _left outer join_. -{{< /tip >}} + +{{% /tip %}} By performing a left join that targets the same key fields, now all of our fruits will be shown in the results even if no one likes them (e.g., `avocado`). @@ -102,9 +104,11 @@ produces ## Right join -{{< tip "Note" >}} +{{% tip "Note" %}} + In SQL, a right join is called a _right outer join_. -{{< /tip >}} + +{{% /tip %}} Next we'll change the join type from `left` to `right`. Notice that this causes the `note` field from the right-hand input to appear in the joined results. @@ -132,9 +136,11 @@ produces ## Anti join -{{< tip "Note" >}} +{{% tip "Note" %}} + In some databases an anti join is called a _left anti join_. -{{< /tip >}} + +{{% /tip %}} The join type `anti` allows us to see which fruits are not liked by anyone. Note that with anti join only values from the left-hand input appear in the diff --git a/docs/tutorials/zq.md b/docs/tutorials/zq.md index 2a24100ca8..9237f4999e 100644 --- a/docs/tutorials/zq.md +++ b/docs/tutorials/zq.md @@ -43,12 +43,14 @@ To this end, if you want full JSON compatibility without having to delve into th details of Zed, just use the `-j` option with `zq` and this will tell `zq` to expect JSON values as input and produce JSON values as output, much like `jq`. -{{< tip "Tip" >}} +{{% tip "Tip" %}} + If your downstream JSON tooling expects only a single JSON value, we can use `-j` along with [`collect()`](../language/aggregates/collect.md) to aggregate multiple input values into an array. A `collect()` example is shown [later in this tutorial](#running-analytics). -{{< /tip >}} + +{{% /tip %}} ## `this` vs `.` From 8939f45da4875f545e3d12dfae10a673f47e24d9 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 13:36:08 -0800 Subject: [PATCH 14/18] Turn abs example back into an mdtest --- docs/language/functions/abs.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/language/functions/abs.md b/docs/language/functions/abs.md index c5d047e188..7d6f96d49a 100644 --- a/docs/language/functions/abs.md +++ b/docs/language/functions/abs.md @@ -15,12 +15,17 @@ must be a numeric type. ### Examples -{{< super-playground "yield abs(this)" >}} +Absolute value of a various numbers: +```mdtest-command +echo '1 -1 0 -1.0 -1(int8) 1(uint8) "foo"' | super -z -c 'yield abs(this)' - +``` + +```mdtest-output +1 1 --1 0 --1.0 --1(int8) +1. +1(int8) 1(uint8) -"foo" -{{}} +error({message:"abs: not a number",on:"foo"}) +``` From a58d801ddb75fc3501449fad2d206e8b751206bd Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 14:48:43 -0800 Subject: [PATCH 15/18] Move Zeek integrations docs to a top level section again for now --- docs/integrations/zeek/{index.md => _index.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/integrations/zeek/{index.md => _index.md} (100%) diff --git a/docs/integrations/zeek/index.md b/docs/integrations/zeek/_index.md similarity index 100% rename from docs/integrations/zeek/index.md rename to docs/integrations/zeek/_index.md From aa3a75b2acc93e21748e9932091c9c63ff196e16 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Fri, 13 Dec 2024 14:54:45 -0800 Subject: [PATCH 16/18] Fix links that broke from moving Zeek docs back to a top-level section --- CHANGELOG.md | 10 +++++----- docs/integrations/fluentd.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5a3c99801..30599d74db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ ## v1.16.0 * Improve ZNG scanning performance (#5101, #5103) * Improve the error message shown when `zq` is invoked with a single argument that's not a valid query and doesn't contain a source (#5119) -* Update [Zeek integration docs](docs/integrations/zeek/index.md), including [reference shaper](docs/integrations/zeek/shaping-zeek-json.md) changes for [Zeek v6.2.0](https://github.com/zeek/zeek/releases/tag/v6.2.0) data (#5106) +* Update [Zeek integration docs](docs/integrations/zeek/_index.md), including [reference shaper](docs/integrations/zeek/shaping-zeek-json.md) changes for [Zeek v6.2.0](https://github.com/zeek/zeek/releases/tag/v6.2.0) data (#5106) * [String literals](docs/language/expressions.md#formatted-string-literals) now use the "f-string" format `f"{ }"` instead of the previous `${ }` (#5123) * Prototype SQL support has been dropped from the Zed language (full SQL support is expected at a later date) (#5109) * Empty objects and arrays in JSON output are now consistently printed on a single line (#5127) @@ -454,7 +454,7 @@ questions. * Fix an issue where `len()` of a `null` array was evaluating to something greater than zero (#2761) * Fix an issue where `sort` with no fields was ignoring alias types and nested fields when picking a sort field (#2762) * Fix an issue where unexpected `cut: no record found` warnings were returned by `zed lake query` but not when the same data was queried via `zq` (#2764) -* Move and extend the [Zeek interoperability docs](docs/integrations/zeek/index.md) (#2770, #2782, #2830) +* Move and extend the [Zeek interoperability docs](docs/integrations/zeek/_index.md) (#2770, #2782, #2830) * Create endpoints in the Zed lake service API that correspond to underlying Zed lake operations, and expose them via `zapi` commands (#2741, #2774, #2786, #2775, #2794, #2795, #2796, #2920, #2925, #2928) * Fix an issue where `zq` would surface a syntax error when reading ZSON it had sent as output (#2792) * Add an `/events` endpoint to the API, which can be used by clients such as the Brim app to be notified of pool updates (#2791) @@ -628,7 +628,7 @@ questions. ## v0.23.0 * zql: Add `week` as a unit for [time grouping with `every`](docs/language/functions/every.md) (#1374) -* zq: Fix an issue where a `null` value in a [JSON type definition](docs/integrations/zeek/index.md) caused a failure without an error message (#1377) +* zq: Fix an issue where a `null` value in a [JSON type definition](docs/integrations/zeek/_index.md) caused a failure without an error message (#1377) * zq: Add [`zst` format](docs/formats/csup.md) to `-i` and `-f` command-line help (#1384) * zq: ZNG spec and `zq` updates to introduce the beta ZNG storage format (#1375, #1415, #1394, #1457, #1512, #1523, #1529), also addressing the following: * New data type `bytes` for storing sequences of bytes encoded as base64 (#1315) @@ -644,11 +644,11 @@ questions. * zqd: Check and convert alpha ZNG filestores to beta ZNG (#1574, #1576) * zq: Fix an issue where spill-to-disk file names could collide (#1391) * zq: Allow the [`fuse` operator](docs/language/operators/fuse.md) to spill-to-disk to avoid memory limitations (#1355, #1402) -* zq: No longer require `_path` as a first column in a [JSON type definition](docs/integrations/zeek/index.md) (#1370) +* zq: No longer require `_path` as a first column in a [JSON type definition](docs/integrations/zeek/_index.md) (#1370) * zql: Improve ZQL docs for [aggregate functions](docs/language/operators/summarize.md) and grouping (#1385) * zql: Point links for developer docs at [pkg.go.dev](https://pkg.go.dev/) instead of [godoc.org](https://godoc.org/) (#1401) * zq: Add support for timestamps with signed timezone offsets (#1389) -* zq: Add a [JSON type definition](docs/integrations/zeek/index.md) for alert events in [Suricata EVE logs](https://suricata.readthedocs.io/en/suricata-5.0.2/output/eve/eve-json-output.html) (#1400) +* zq: Add a [JSON type definition](docs/integrations/zeek/_index.md) for alert events in [Suricata EVE logs](https://suricata.readthedocs.io/en/suricata-5.0.2/output/eve/eve-json-output.html) (#1400) * zq: Update the [ZNG over JSON (ZJSON)](docs/formats/zjson.md) spec and implementation (#1299) * zar: Use buffered streaming for archive import (#1397) * zq: Add an `ast` command that prints parsed ZQL as its underlying JSON object (#1416) diff --git a/docs/integrations/fluentd.md b/docs/integrations/fluentd.md index cfe29c4e76..d98a0a8815 100644 --- a/docs/integrations/fluentd.md +++ b/docs/integrations/fluentd.md @@ -12,7 +12,7 @@ record for archiving and analytics. This guide walks through two simple configurations of Fluentd with a Zed lake that can be used as reference for starting your own production configuration. As it's a data source important to many in the Zed community, log data from -[Zeek](./zeek/index.md) is used in this guide. The approach shown can be +[Zeek](./zeek/_index.md) is used in this guide. The approach shown can be easily adapted to any log data source. ## Software From 38d12af7ad940a9cf530add2f9541f3740c2e7cc Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 16 Dec 2024 15:51:30 -0800 Subject: [PATCH 17/18] Add breadcrumb option --- docs/_index.md | 3 ++- docs/install.md | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_index.md b/docs/_index.md index 062ff7f927..ff82fadb17 100644 --- a/docs/_index.md +++ b/docs/_index.md @@ -1,7 +1,8 @@ --- weight: 1 -title: Docs +title: Introduction heading: SuperDB +breadcrumb: Docs --- SuperDB offers a new approach that makes it easier to manipulate and manage diff --git a/docs/install.md b/docs/install.md index 0130874a6d..1b91d023bd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,5 +1,4 @@ --- -weight: 2 title: Installation --- From b2c8c0f21d66677a781124f7b554fba585887794 Mon Sep 17 00:00:00 2001 From: Phil Rzewski Date: Wed, 18 Dec 2024 15:23:10 -0800 Subject: [PATCH 18/18] Adjust weights and titles --- docs/language/aggregates/_index.md | 4 ++-- docs/language/functions/_index.md | 1 + docs/language/operators/_index.md | 1 + docs/tutorials/join.md | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/language/aggregates/_index.md b/docs/language/aggregates/_index.md index 2f28163798..1513f23bad 100644 --- a/docs/language/aggregates/_index.md +++ b/docs/language/aggregates/_index.md @@ -1,6 +1,6 @@ --- -title: Aggregates -heading: Aggregate Functions +weight: 3 +title: Aggregate Functions --- Aggregate functions appear in either [summarization](../operators/summarize.md) diff --git a/docs/language/functions/_index.md b/docs/language/functions/_index.md index 27d2c54652..99ad7fd0c4 100644 --- a/docs/language/functions/_index.md +++ b/docs/language/functions/_index.md @@ -1,4 +1,5 @@ --- +weight: 2 title: Functions --- diff --git a/docs/language/operators/_index.md b/docs/language/operators/_index.md index e6192a512a..43036a1691 100644 --- a/docs/language/operators/_index.md +++ b/docs/language/operators/_index.md @@ -1,4 +1,5 @@ --- +weight: 1 title: Operators --- diff --git a/docs/tutorials/join.md b/docs/tutorials/join.md index 7c186cd822..419df63a34 100644 --- a/docs/tutorials/join.md +++ b/docs/tutorials/join.md @@ -1,6 +1,7 @@ --- weight: 3 -title: Join Overview +title: Join +heading: Join Overview --- This is a brief primer on the SuperPipe [`join` operator](../language/operators/join.md).