diff --git a/.github/workflows/fail-due-to-change-in-frozen-doc.yml b/.github/workflows/fail-due-to-change-in-frozen-doc.yml new file mode 100644 index 000000000..6c6ba3e16 --- /dev/null +++ b/.github/workflows/fail-due-to-change-in-frozen-doc.yml @@ -0,0 +1,26 @@ +name: fronzen-doc-check + +on: + pull_request: + push: + +jobs: + check-for-fronzen-doc-change: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.x + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Build Tools + run: | + set -e -x + ./site/hack/build.sh + - name: Check if frozen documentation was changed + run: | + set -e -x + ./site/hack/validate-doc-change.sh diff --git a/site/README.md b/site/README.md index 7f9a44a78..dd891b049 100644 --- a/site/README.md +++ b/site/README.md @@ -28,4 +28,16 @@ Serve site at [http://localhost:1313]() - `/content/` includes content for tool docs - `/data/` includes configuration for docs TOCs -More details: [Directory Structure Explained](https://gohugo.io/getting-started/directory-structure/) \ No newline at end of file +More details: [Directory Structure Explained](https://gohugo.io/getting-started/directory-structure/) + +### Generate documentation for new version +When creating a new version v0.1.0 of imgpkg +```bash +./hack/release-docs.sh imgpkg v0.1.0 +``` + +### Generate documentation for new patch +When creating a patch v0.1.1 for the version v0.1.0 of imgpkg +```bash +./hack/release-docs.sh imgpkg v0.1.1 v0.1.0 +``` diff --git a/site/config.yaml b/site/config.yaml index c84d391a7..94a2fe15f 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -1,97 +1,114 @@ ---- -baseURL: "https://carvel.dev" +baseURL: https://carvel.dev disablePathToLower: true languageCode: en-us -DefaultContentLanguage: "en" -title: "Carvel" -theme: "carvel" +DefaultContentLanguage: en +title: Carvel +theme: carvel frontmatter: - date: [":filename", ":default"] + date: + - :filename + - :default params: - products: [kapp-controller, ytt, kapp, kbld, imgpkg, vendir] + products: + - kapp-controller + - ytt + - kapp + - kbld + - imgpkg + - vendir + search: + app_id: BSVMS0V37R ytt: name: ytt root_link: /ytt/ - latest_docs_link: /ytt/docs/latest/ + latest_docs_link: /ytt/docs/v0.38.0/ github_url: https://github.com/vmware-tanzu/carvel-ytt search: true search_index_name: carvel - search_api_key: 582b1408d7a3fe145c0f84f48ca03755 + search_api_key: b20c4201c550557613a0c4771996b6c5 search_tag: ytt - versioning: false - version_latest: latest + versioning: true + version_latest: v0.38.0 versions: - - latest + - develop + - v0.38.0 kbld: name: kbld short_name: kbld root_link: /kbld/ - latest_docs_link: /kbld/docs/latest/ + latest_docs_link: /kbld/docs/v0.32.0/ github_url: https://github.com/vmware-tanzu/carvel-kbld search: true search_index_name: carvel - search_api_key: 582b1408d7a3fe145c0f84f48ca03755 + search_api_key: b20c4201c550557613a0c4771996b6c5 search_tag: kbld - versioning: false - version_latest: latest + versioning: true + version_latest: v0.32.0 versions: - - latest + - develop + - v0.32.0 kapp: name: kapp short_name: kapp root_link: /kapp/ - latest_docs_link: /kapp/docs/latest/ + latest_docs_link: /kapp/docs/v0.45.0/ github_url: https://github.com/vmware-tanzu/carvel-kapp search: true search_index_name: carvel - search_api_key: 582b1408d7a3fe145c0f84f48ca03755 + search_api_key: b20c4201c550557613a0c4771996b6c5 search_tag: kapp - versioning: false - version_latest: latest + versioning: true + version_latest: v0.45.0 versions: - - latest + - develop + - v0.45.0 + - v0.44.0 imgpkg: name: imgpkg short_name: imgpkg root_link: /imgpkg/ - latest_docs_link: /imgpkg/docs/latest/ + latest_docs_link: /imgpkg/docs/v0.24.0/ github_url: https://github.com/vmware-tanzu/carvel-imgpkg search: true search_index_name: carvel - search_api_key: 582b1408d7a3fe145c0f84f48ca03755 + search_api_key: b20c4201c550557613a0c4771996b6c5 search_tag: imgpkg - versioning: false - version_latest: latest + versioning: true + version_latest: v0.24.0 versions: - - latest + - develop + - v0.24.0 kapp-controller: name: kapp-controller short_name: kc root_link: /kapp-controller/ - latest_docs_link: /kapp-controller/docs/latest/ + latest_docs_link: /kapp-controller/docs/v0.32.0/ github_url: https://github.com/vmware-tanzu/carvel-kapp-controller search: true search_index_name: carvel - search_api_key: 582b1408d7a3fe145c0f84f48ca03755 + search_api_key: b20c4201c550557613a0c4771996b6c5 search_tag: kapp-controller - versioning: false - version_latest: latest + versioning: true + version_latest: v0.32.0 versions: - - latest + - develop + - v0.32.0 + - v0.31.0 vendir: name: vendir short_name: vendir root_link: /vendir/ - latest_docs_link: /vendir/docs/latest/ + latest_docs_link: /vendir/docs/v0.24.0/ github_url: https://github.com/vmware-tanzu/carvel-vendir search: true search_index_name: carvel - search_api_key: 582b1408d7a3fe145c0f84f48ca03755 + search_api_key: b20c4201c550557613a0c4771996b6c5 search_tag: vendir - versioning: false - version_latest: latest + versioning: true + version_latest: v0.24.0 versions: - - latest + - develop + - v0.24.0 links: twitter_url: https://twitter.com/carvel_dev github_url: https://github.com/vmware-tanzu/carvel @@ -110,4 +127,4 @@ markup: style: github tabWidth: 4 permalinks: - tags: "/blog/tags/:slug" + tags: /blog/tags/:slug diff --git a/site/content/imgpkg/docs/latest/_index.md b/site/content/imgpkg/docs/develop/_index.md similarity index 98% rename from site/content/imgpkg/docs/latest/_index.md rename to site/content/imgpkg/docs/develop/_index.md index c39a97e8d..392118164 100644 --- a/site/content/imgpkg/docs/latest/_index.md +++ b/site/content/imgpkg/docs/develop/_index.md @@ -2,7 +2,7 @@ title: "About imgpkg" toc: "false" cascade: - version: latest + version: develop toc: "true" type: docs layout: docs diff --git a/site/content/imgpkg/docs/latest/air-gapped-workflow.md b/site/content/imgpkg/docs/develop/air-gapped-workflow.md similarity index 100% rename from site/content/imgpkg/docs/latest/air-gapped-workflow.md rename to site/content/imgpkg/docs/develop/air-gapped-workflow.md diff --git a/site/content/imgpkg/docs/latest/auth.md b/site/content/imgpkg/docs/develop/auth.md similarity index 100% rename from site/content/imgpkg/docs/latest/auth.md rename to site/content/imgpkg/docs/develop/auth.md diff --git a/site/content/imgpkg/docs/latest/automation-workflow.md b/site/content/imgpkg/docs/develop/automation-workflow.md similarity index 100% rename from site/content/imgpkg/docs/latest/automation-workflow.md rename to site/content/imgpkg/docs/develop/automation-workflow.md diff --git a/site/content/imgpkg/docs/latest/basic-workflow.md b/site/content/imgpkg/docs/develop/basic-workflow.md similarity index 100% rename from site/content/imgpkg/docs/latest/basic-workflow.md rename to site/content/imgpkg/docs/develop/basic-workflow.md diff --git a/site/content/imgpkg/docs/latest/ca-certs-windows.md b/site/content/imgpkg/docs/develop/ca-certs-windows.md similarity index 100% rename from site/content/imgpkg/docs/latest/ca-certs-windows.md rename to site/content/imgpkg/docs/develop/ca-certs-windows.md diff --git a/site/content/imgpkg/docs/latest/commands.md b/site/content/imgpkg/docs/develop/commands.md similarity index 100% rename from site/content/imgpkg/docs/latest/commands.md rename to site/content/imgpkg/docs/develop/commands.md diff --git a/site/content/imgpkg/docs/latest/debugging.md b/site/content/imgpkg/docs/develop/debugging.md similarity index 100% rename from site/content/imgpkg/docs/latest/debugging.md rename to site/content/imgpkg/docs/develop/debugging.md diff --git a/site/content/imgpkg/docs/latest/install.md b/site/content/imgpkg/docs/develop/install.md similarity index 100% rename from site/content/imgpkg/docs/latest/install.md rename to site/content/imgpkg/docs/develop/install.md diff --git a/site/content/imgpkg/docs/latest/proxy.md b/site/content/imgpkg/docs/develop/proxy.md similarity index 100% rename from site/content/imgpkg/docs/latest/proxy.md rename to site/content/imgpkg/docs/develop/proxy.md diff --git a/site/content/imgpkg/docs/latest/resources.md b/site/content/imgpkg/docs/develop/resources.md similarity index 100% rename from site/content/imgpkg/docs/latest/resources.md rename to site/content/imgpkg/docs/develop/resources.md diff --git a/site/content/imgpkg/docs/latest/security.md b/site/content/imgpkg/docs/develop/security.md similarity index 100% rename from site/content/imgpkg/docs/latest/security.md rename to site/content/imgpkg/docs/develop/security.md diff --git a/site/content/imgpkg/docs/latest/working-directly-with-images.md b/site/content/imgpkg/docs/develop/working-directly-with-images.md similarity index 100% rename from site/content/imgpkg/docs/latest/working-directly-with-images.md rename to site/content/imgpkg/docs/develop/working-directly-with-images.md diff --git a/site/content/imgpkg/docs/v0.24.0/_index.md b/site/content/imgpkg/docs/v0.24.0/_index.md new file mode 100644 index 000000000..ffd9968b2 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/_index.md @@ -0,0 +1,21 @@ +--- +aliases: [/imgpkg/docs/latest/] +title: "About imgpkg" +toc: "false" +cascade: + version: v0.24.0 + toc: "true" + type: docs + layout: docs +--- + +`imgpkg` is a tool that allows users to store a set of arbitrary files as an OCI image. One of the driving use cases is to store Kubernetes configuration (plain YAML, ytt templates, Helm templates, etc.) in OCI registry as an image. + +`imgpkg`'s primary concept is a [bundle](resources.md#bundle), which is an OCI image that holds 0+ arbitrary files and 0+ references to dependent OCI images (which *may* also be [bundles](resources.md/#nested-bundle)). With this concept, `imgpkg` is able to copy bundles and their dependent images across registries (both online and offline). + +![Bundle diagram](/images/imgpkg/bundle-diagram.png) + +## Workflows + +- [Basic Workflow](basic-workflow.md) shows how to create, push, and pull bundles with a simple Kubernetes application +- [Air-gapped Workflow](air-gapped-workflow.md) shows how to copy bundles from one registry to another, to enable running Kubernetes applications without relying on external (public) registries diff --git a/site/content/imgpkg/docs/v0.24.0/air-gapped-workflow.md b/site/content/imgpkg/docs/v0.24.0/air-gapped-workflow.md new file mode 100644 index 000000000..a3d589126 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/air-gapped-workflow.md @@ -0,0 +1,154 @@ +--- +aliases: [/imgpkg/docs/latest/air-gapped-workflow] +title: Air-gapped Workflow +--- + +## Scenario + +You want to ensure Kubernetes application does not rely on images from external registries when deployed. + +This scenario _also_ applies when trying to ensure that all images are consolidated into a single registry, even if that registry is not air-gapped. + +## Prerequisites + +To complete this workflow you will need access to an OCI registry like Docker Hub, and optionally, +a Kubernetes cluster. (If you would like to use a local registry and Kubernetes cluster, try using [Kind](https://kind.sigs.k8s.io/docs/user/local-registry/)) + +If you would like to deploy the results of this scenario to your Kubernetes cluster, you will additionally need [`kbld`](/kbld) and kubectl. + +If any of your bundles contain [non-distributable layers](commands.md#non-distributable-or-foreign-layers) you will need to include +the `--include-non-distributable-layers` flag to each copy command in the examples provided. + +--- +## Step 1: Finding bundle in source registry + +If you have already pushed a bundle to the registry, continue to the next step. + +If you are trying to bundle your own or third-part software, you will need to create a bundle. Refer to basic workflow's ["Step 1: Creating the bundle" and "Step 2: Pushing the bundle to registry"](basic-workflow.md#step-1-creating-the-bundle). + +--- +## Step 2: Two methods of copying bundles + +You have two options how to transfer bundle from one registry to another: + +- Option 1: From a common location connected to both registries. This option is more efficient because only changed image layers will be transfered between registries. +- Option 2: With intermediate tarball. This option works best when registries have no common network access. + +### Option 1: From a location connected to both registries + +1. Get to a location that can access both registries + + This may be a server that has access to both internal and external networks. If there is no such location, you will have to use "Option 2" below. + +1. [Authenticate](auth.md) with both source, and destination registries + +1. Run following command to copy bundle from one registry to another: + + ```bash-plain + $ imgpkg copy -b index.docker.io/user1/simple-app-bundle:v1.0.0 --to-repo registry.corp.com/apps/simple-app-bundle + + copy | exporting 2 images... + copy | will export index.docker.io/user1/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + copy | will export index.docker.io/user1/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb + copy | exported 2 images + copy | importing 2 images... + copy | importing index.docker.io/user1/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb + -> registry.corp.com/apps/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb... + copy | importing index.docker.io/user1/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + -> registry.corp.com/apps/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0... + copy | imported 2 images + Succeeded + ``` + + The bundle, and all images referenced in the bundle, are copied to the destination registry. + + Flags used in the command: + * `-b` (`--bundle`) indicates the bundle location in the source registry + * `--to-repo` indicates the registry where the bundle and associated images should be copied to + +### Option 2: With intermediate tarball + +1. Get to a location that can access source registry + +1. [Authenticate with the source registry](auth.md) + +1. Save the bundle to a tarball + + ```bash-plain + $ imgpkg copy -b index.docker.io/user1/simple-app-bundle:v1.0.0 --to-tar /tmp/my-image.tar + + copy | exporting 2 images... + copy | will export index.docker.io/user1/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + copy | will export index.docker.io/user1/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb + copy | exported 2 images + copy | writing layers... + copy | done: file 'manifest.json' (13.71µs) + copy | done: file 'sha256-233f1d0dbdc8cf675af965df8639b0dfd4ef7542dfc9fcfd03bfc45c570b0e4d.tar.gz' (47.616µs) + copy | done: file 'sha256-8ece9ac45f2b7228b2ed95e9f407b4f0dc2ac74f93c62ff1156f24c53042ba54.tar.gz' (43.204905ms) + Succeeded + ``` + + Flags used in the command: + * `-b` (`--bundle`) indicates the bundle location in the source registry + * `--to-tar` indicates the local location to write a tar file containing the bundle assets + +1. Transfer the local tarball `/tmp/my-image.tar` to a location with access to the destination registry + +1. [Authenticate with the destination registry](auth.md) + +1. Import the bundle from your tarball to the destination registry: + + ```bash-plain + $ imgpkg copy --tar /tmp/my-image.tar --to-repo registry.corp.com/apps/simple-app-bundle + + copy | importing 2 images... + copy | importing index.docker.io/user1/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb -> registry.corp.com/apps/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb... + copy | importing index.docker.io/user1/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 -> registry.corp.com/apps/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0... + copy | imported 2 images + Succeeded + ``` + + The bundle, and all images referenced in the bundle, are copied to the destination registry. + + Flags used in the command: + * `--tar` indicates the path to a tar file containing the assets to be copied to a registry + * `--to-repo` indicates destination bundle location in the registry + +--- +## Step 3: Pulling bundle from destination registry + +1. [Authenticate with the destination registry](auth.md) + +1. Pull the bundle from the destination registry: + + ```bash-plain + $ imgpkg pull -b registry.corp.com/apps/simple-app-bundle:v1.0.0 -o /tmp/bundle + + Pulling image 'registry.corp.com/apps/simple-app-bundle@sha256:70225df0a05137ac385c95eb69f89ded3e7ef3a0c34db43d7274fd9eba3705bb' + Extracting layer 'sha256:233f1d0dbdc8cf675af965df8639b0dfd4ef7542dfc9fcfd03bfc45c570b0e4d' (1/1) + Locating image lock file images... + All images found in bundle repo; updating lock file: /tmp/bundle/.imgpkg/images.yml + + Succeeded + ``` + + Flags used in the command: + * `-b` (`--bundle`) indicates to pull a particular bundle from a registry + * `-o` (`--output`) indicates the local folder where the bundle will be unpacked + + Note that the `.imgpkg/images.yml` file was updated with the destination registry locations of the images. This happened because, in the prior step, the images referenced by the bundle were copied into the destination registry. + + ```bash-plain + $ cat /tmp/bundle/.imgpkg/images.yml + apiVersion: imgpkg.carvel.dev/v1alpha1 + kind: ImagesLock + images: + - image: registry.corp.com/apps/simple-app-bundle@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + annotations: + kbld.carvel.dev/id: docker.io/dkalinin/k8s-simple-app + ``` + +--- +## Step 4: Use pulled bundle contents + +Regardless which location the bundle is downloaded from, source registry or destination registry, use of the pulled bundle contents remains the same. Continue with ["Step 4: Use pulled bundle contents"](basic-workflow.md#step-4-use-pulled-bundle-contents) in the basic workflow. diff --git a/site/content/imgpkg/docs/v0.24.0/auth.md b/site/content/imgpkg/docs/v0.24.0/auth.md new file mode 100644 index 000000000..242466f96 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/auth.md @@ -0,0 +1,185 @@ +--- +aliases: [/imgpkg/docs/latest/auth] +title: Authentication +--- + +# Ordering + +imgpkg has multiple ways to provide authentication details to registries. + +The order at which imgpkg chooses which authentication details to use is the following: + +1. [Via Environment Variables](#via-environment-variables) +1. [Via IaaS](#via-iaas) +1. [Via Command Flags](#via-command-flags) +1. [Via Docker Config](#via-docker-config) + +## Via Environment Variables + +As of v0.7.0+, `imgpkg` can also use following environment variables: + +- `IMGPKG_REGISTRY_HOSTNAME` to specify registry hostname (e.g. gcr.io, docker.io, https://gcr.io, docker.io/v2/) + - As of v0.18.0+ `IMGPKG_REGISTRY_HOSTNAME` also supports providing glob wildcards. for e.g. `*.*.docker.io` will match `bar.foo.docker.io`. + - Note: if there is overlap between 2 HOSTNAMES, one using globbing and the other not, the HOSTNAME not using globbing will be applied. e.g. `IMGPKG_REGISTRY_HOSTNAME_0=*.docker.io` vs `IMGPKG_REGISTRY_HOSTNAME_1=foo.docker.io` for the image `foo.docker.io/image` will result in auth details from `IMGPKG_REGISTRY_HOSTNAME_1` being used. + - As of v0.18.0+ `IMGPKG_REGISTRY_HOSTNAME` also supports providing the fully qualified repository. for e.g. `gcr.io/repo/image`. +- `IMGPKG_REGISTRY_USERNAME` to specify registry username +- `IMGPKG_REGISTRY_PASSWORD` to specify registry password +- `IMGPKG_REGISTRY_IDENTITY_TOKEN` to authenticate the user and get an access token for the registry via an [oauth2 refresh token grant type](https://docs.docker.com/registry/spec/auth/oauth/). +- `IMGPKG_REGISTRY_REGISTRY_TOKEN` to specify the access token to be used in the Authorization Header as a [Bearer Token](https://docs.docker.com/registry/spec/auth/token/#using-the-bearer-token). + +Since you may need to provide multiple registry credentials, the environment variables above may be specified multiple times with a suffix of 1+ alphanumeric characters, + +e.g. If you had 2 registries you wish to provide authentication credentials for, you would require 2 sets of env variables. + +For Registry #1: + +``` +IMGPKG_REGISTRY_HOSTNAME_0=hostname.for.registry.1 +IMGPKG_REGISTRY_USERNAME_0=username +IMGPKG_REGISTRY_PASSWORD_0=password +``` + +For Registry #2: + +``` +IMGPKG_REGISTRY_HOSTNAME_1=hostname.for.registry.2 +IMGPKG_REGISTRY_IDENTITY_TOKEN_1=token +``` + +When imgpkg interacts with `hostname.for.registry.1`, it will use the env variables with the suffix `_0`. And when interacting with `hostname.for.registry.2`, it will use the env variables with the suffix `_1` + + +Note: Credentials provided via an env variable for a specific registry will take precedence over Command Flags. + +## Via IaaS + +As of v0.18.0+, `imgpkg` will attempt to authenticate itself via the underlying IaaS: + +This auth feature can be enabled/disabled via a feature flag (enabled by default): `IMGPKG_ENABLE_IAAS_AUTH=true|false` + +Below is a list of IaaS providers that imgpkg will authenticate with: + +- [GCP](https://cloud.google.com/compute/docs/metadata/overview) +- [AWS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) Note: If the AWS_SDK_LOAD_CONFIG environment variable is set to a truthy value the shared config file (~/.aws/config) will +also be loaded in addition to the shared credentials file (~/.aws/credentials). +- [Azure](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-managed-identities-work-vm) + +Note: When using Azure, a required configuration file needs to be provided via the flag `--registry-azure-cr-config ` or via the env variable `IMGPKG_REGISTRY_AZURE_CR_CONFIG=` + +See below the various configuration options allowed: +```yaml +{ + # The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13 + "cloud": "", + # The AAD Tenant ID for the Subscription that the cluster is deployed in + "tenantId": "TenantID", + # The ClientID for an AAD application with RBAC access to talk to Azure RM APIs + "aadClientId": "AADClientID", + # The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs + "aadClientSecret": "AADClientSecret", + # The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs + "aadClientCertPath": "AADClientCertPath", + # The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs + "aadClientCertPassword": "AADClientCertPassword", + # Use managed service identity for the virtual machine to access Azure ARM APIs + "useManagedIdentityExtension": false, + # UserAssignedIdentityID contains the Client ID of the user assigned MSI which is assigned to the underlying VMs. If empty the user assigned identity is not used. + # More details of the user assigned identity can be found at: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview + # For the user assigned identity specified here to be used, the UseManagedIdentityExtension has to be set to true. + "userAssignedIdentityID": "UserAssignedIdentityID", + # The ID of the Azure Subscription that the cluster is deployed in + "subscriptionId": "SubscriptionID", + # IdentitySystem indicates the identity provider. Relevant only to hybrid clouds (Azure Stack). + # Allowed values are 'azure_ad' (default), 'adfs'. + "identitySystem": "IdentitySystem", + # ResourceManagerEndpoint is the cloud's resource manager endpoint. If set, cloud provider queries this endpoint + # in order to generate an autorest.Environment instance instead of using one of the pre-defined Environments. + "resourceManagerEndpoint": "ResourceManagerEndpoint", + # The AAD Tenant ID for the Subscription that the network resources are deployed in + "networkResourceTenantID": "NetworkResourceTenantID", + # The ID of the Azure Subscription that the network resources are deployed in + "networkResourceSubscriptionID": "NetworkResourceSubscriptionID" +} +``` + + +## Via Command Flags + +You can explicitly specify credentials via command flags or associated environment variables. See `imgpkg push -h` for further details. + +- `--registry-username` (or `$IMGPKG_USERNAME`) +- `--registry-password` (or `$IMGPKG_PASSWORD`) +- `--registry-token` (or `$IMGPKG_TOKEN`): to specify the access token to be used in the Authorization Header as a [Bearer Token](https://docs.docker.com/registry/spec/auth/token/#using-the-bearer-token). +- `--registry-anon` (or `$IMGPKG_ANON=true`): used for anonymous access (commonly for pulling) + +## Via Docker config + +Even though `imgpkg` commands use registry APIs directly, by default it uses credentials stored in `~/.docker/config.json` which are typically generated via a `docker login` command. + +Example generated `~/.docker/config.json`: + +```json +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "dXNlcjpwYXNzd29yZA==" + }, + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.09.6 (darwin)" + } +} +``` + +where `dXNlcjpwYXNzd29yZA==` is `base64("username:password")`. + +## gcr.io + +- Create a service account with "Storage Admin" permissions for push access + - See [Permissions and Roles](https://cloud.google.com/container-registry/docs/access-control#permissions_and_roles) +- Download a JSON service account key and place it somewhere on filesystem (e.g. `/tmp/key`) + - See [Advanced authentication](https://cloud.google.com/container-registry/docs/advanced-authentication#json_key_file) +- Run `cat /tmp/key | docker login -u _json_key --password-stdin https://gcr.io` to authenticate + +## AWS ECR + +- Create an ECR repository +- Create an IAM user with an ECR policy that allows read/write + - See [Amazon ECR Policies](https://docs.aws.amazon.com/AmazonECR/latest/userguide/ecr_managed_policies.html) +- Run `aws configure` and specify access key ID, secret access key and region + - To install on Ubuntu, run `apt-get install pip3` and `pip3 install awscli` + - See [Installing the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) +- Run `eval $(aws ecr get-login --no-include-email)` to authenticate + - See [get-login command](https://docs.aws.amazon.com/cli/latest/reference/ecr/get-login.html) + +Example ECR policy from [Amazon ECR](https://docs.aws.amazon.com/AmazonECR/latest/userguide/ecr_managed_policies.html): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage" + ], + "Resource": "*" + } + ] +} +``` + +## Harbor + +You may have to provide `--registry-ca-cert-path` flag with a path to a CA certificate file for Harbor Registry API. diff --git a/site/content/imgpkg/docs/v0.24.0/automation-workflow.md b/site/content/imgpkg/docs/v0.24.0/automation-workflow.md new file mode 100644 index 000000000..0a6c87d5b --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/automation-workflow.md @@ -0,0 +1,141 @@ +--- +aliases: [/imgpkg/docs/latest/automation-workflow] +title: Automation Workflow +--- + +## Scenario + +When using an automated CI tool you might want to promote a given Bundle between steps of the pipeline + +## Prerequisites + +To complete this workflow you will need access to an OCI registry like Docker Hub. + +### Step 1: Creating the Bundle + +1. Prepare bundle contents + + The [examples/basic-step-1/](https://github.com/vmware-tanzu/carvel-imgpkg/tree/develop/examples/basic-step-1) + directory has a `config.yml` file, which contains a very simple Kubernetes application. Your application may have as + many configuration files as necessary in various formats such as plain YAML, ytt templates, Helm templates, etc. + + In our example `config.yml` includes an image reference to `docker.io/dkalinin/k8s-simple-app`. This reference does + not point to an exact image (via digest) meaning that it may change over time. To ensure we get precisely the bits we + expect, we will lock it down to an exact image next. + +1. Add `.imgpkg/` directory + + [examples/basic-step-2](https://github.com/vmware-tanzu/carvel-imgpkg/tree/develop/examples/basic-step-2) shows what + a `.imgpkg/` directory may look like. It contains: + + - **optional** [bundle.yml](resources.md#bundle-metadata): a file which records informational metadata + - **required** [images.yml](resources.md#imageslock): a file which records image references used by the + configuration + + ```bash-plain + examples/basic-step-2 + ├── .imgpkg + │   ├── bundle.yml + │   └── images.yml + └── config.yml + ``` + + Note that `.imgpkg/images.yml` contains a list of images, each with fully resolved digest reference ( + e.g `index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4...`) and a some additional metadata ( + e.g. `annotations` section). See [ImagesLock configuration](resources.md#imageslock-configuration) for details. + + ```yaml + apiVersion: imgpkg.carvel.dev/v1alpha1 + kind: ImagesLock + images: + - image: index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + annotations: + kbld.carvel.dev/id: docker.io/dkalinin/k8s-simple-app + ``` + +--- + +### Step 2: Creating the Bundle + +1. [Authenticate with a registry](auth.md) where we will push our bundle + +2. Push the bundle to the registry + + You can push the bundle with our specified contents to an OCI registry using the following command: + + ```bash-plain + $ imgpkg push -b index.docker.io/user1/simple-app-bundle:v1.0.0 -f examples/basic-step-2 --lock-output /tmp/bundle-lock.yml + + dir: . + dir: .imgpkg + file: .imgpkg/bundle.yml + file: .imgpkg/images.yml + file: config.yml + Pushed 'index.docker.io/user1/simple-app-bundle@sha256:5c2dafe3c70c13990190d643c91e9f67b8129b179257674888178868474f6511' + + Succeeded + ``` + + Flags used in the command: + - `-b` (`--bundle`) refers to a location for a bundle within an OCI registry + - `-f` (`--file`) indicates directory contents to include + - `--lock-output` indicates the destination of the [BundleLock](resources.md#bundlelock-configuration) file + +--- + +## Step 3: Promoting the BundleLock file + +Since in the previous step we generated a BundleLock we can promote this file and in the next steps of the pipeline we +can reference it. + +Examples of usage: + +1. Promote the Bundle to a different registry + + ```bash-plain + $ imgpkg copy --lock /tmp/bundle-lock.yml --to-repo production.registry.io/simple-app-bundle + copy | exporting 2 images... + copy | will export index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + copy | will export production.registry.io/simple-app-bundle@sha256:5c2dafe3c70c13990190d643c91e9f67b8129b179257674888178868474f6511 + copy | exported 2 images + copy | importing 2 images... + + 3.56 MiB / 3.57 MiB [========================================================================================================================================================================] 99.68% 8.80 MiB/s 0s + + copy | done uploading images + + Succeeded + ``` + + Flags used in the command: + - `--lock` refers to a location for a BundleLock file + - `--to-repo` indicates the destination Repository where the Bundle is copied to + +2. Download the Bundle contents to disk + + ```bash-plain + $ imgpkg pull --lock /tmp/bundle-lock.yml -o /tmp/simple-app-bundle + + Pulling image 'index.docker.io/user1/simple-app-bundle@sha256:ec3f870e958e404476b9ec67f28c598fa8f00f819b8ae05ee80d51bac9f35f5d' + Extracting layer 'sha256:7906b9650be657359ead106e354f2728e16c8f317e1d87f72b05b5c5ec3d89cc' (1/1) + + Locating image lock file images... + The bundle repo (index.docker.io/user1/simple-app-bundle@sha256:5c2dafe3c70c13990190d643c91e9f67b8129b179257674888178868474f6511) is hosting every image specified in the bundle's Images Lock file (.imgpkg/images.yml) + + Succeeded + ``` + + Flags used in the command: + - `--lock`e`) refers to a location for a BundleLock file + - `-o` (`--output`) indicates the destination directory on your local machine where the bundle contents will be + placed + + Bundle contents will be extracted into `/tmp/simple-app-bundle` directory: + + ```bash-plain + /tmp/simple-app-bundle + ├── .imgpkg + │   ├── bundle.yml + │   └── images.yml + └── config.yml + ``` diff --git a/site/content/imgpkg/docs/v0.24.0/basic-workflow.md b/site/content/imgpkg/docs/v0.24.0/basic-workflow.md new file mode 100644 index 000000000..f752ade58 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/basic-workflow.md @@ -0,0 +1,150 @@ +--- +aliases: [/imgpkg/docs/latest/basic-workflow] +title: Basic Workflow +--- + +## Scenario + +You want to create an immutable artifact containing Kubernetes configuration and images used in that configuration. Later, you want to grab that artifact and deploy it to Kubernetes. + +## Prerequisites + +To complete this workflow you will need access to an OCI registry like Docker Hub, and optionally, +a Kubernetes cluster. (If you would like to use a local registry and Kubernetes cluster, try using [Kind](https://kind.sigs.k8s.io/docs/user/local-registry/)) + +If you would like to deploy the results of this scenario to your Kubernetes cluster, you will additionally need [`kbld`](/kbld) and kubectl. + +## Step 1: Creating the bundle + +1. Prepare bundle contents + + The [examples/basic-step-1/](https://github.com/vmware-tanzu/carvel-imgpkg/tree/develop/examples/basic-step-1) directory has a `config.yml` file, which contains a very simple Kubernetes application. Your application may have as many configuration files as necessary in various formats such as plain YAML, ytt templates, Helm templates, etc. + + In our example `config.yml` includes an image reference to `docker.io/dkalinin/k8s-simple-app`. This reference does not point to an exact image (via digest) meaning that it may change over time. To ensure we get precisely the bits we expect, we will lock it down to an exact image next. + +1. Add `.imgpkg/` directory + + [examples/basic-step-2](https://github.com/vmware-tanzu/carvel-imgpkg/tree/develop/examples/basic-step-2) shows what a `.imgpkg/` directory may look like. It contains: + + - **optional** [bundle.yml](resources.md#bundle-metadata): a file which records informational metadata + - **required** [images.yml](resources.md#imageslock): a file which records image references used by the configuration + + ```bash-plain + examples/basic-step-2 + ├── .imgpkg + │   ├── bundle.yml + │   └── images.yml + └── config.yml + ``` + + Note that `.imgpkg/images.yml` contains a list of images, each with fully resolved digest reference (e.g `index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4...`) and a little bit of additional metadata (e.g. `annotations` section). See [ImagesLock configuration](resources.md#imageslock-configuration) for details. + + ```yaml + apiVersion: imgpkg.carvel.dev/v1alpha1 + kind: ImagesLock + images: + - image: index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + annotations: + kbld.carvel.dev/id: docker.io/dkalinin/k8s-simple-app + ``` + + This allows us to record the exact image that will be used by our Kubernetes configuration. We expect that `.imgpkg/images.yml` would be created either manually, or in an automated way. Our recommendation is to use [kbld](/kbld) to generate `.imgpkg/images.yml`: + + ```bash-plain + $ cd examples/basic-bundle/ + + $ kbld -f config.yml --imgpkg-lock-output .imgpkg/images.yml + ``` + +--- +## Step 2: Pushing the bundle to a registry + +1. [Authenticate with a registry](auth.md) where we will push our bundle + +1. Push the bundle to the registry + + You can push the bundle with our specified contents to an OCI registry using the following command: + + ```bash-plain + $ imgpkg push -b index.docker.io/user1/simple-app-bundle:v1.0.0 -f examples/basic-step-2 + + dir: . + dir: .imgpkg + file: .imgpkg/bundle.yml + file: .imgpkg/images.yml + file: config.yml + Pushed 'index.docker.io/user1/simple-app-bundle@sha256:ec3f870e958e404476b9ec67f28c598fa8f00f819b8ae05ee80d51bac9f35f5d' + + Succeeded + ``` + + Flags used in the command: + * `-b` (`--bundle`) refers to a location for a bundle within an OCI registry + * `-f` (`--file`) indicates directory contents to include + +1. The pushed bundle is now available at `index.docker.io/user1/simple-app-bundle:v1.0.0` + +--- +## Step 3: Pulling the bundle to registry + +Now that we have pushed a bundle to a registry, other users can pull it. + +1. [Authenticate with the registry](auth.md) from which we'll pull our bundle + +1. Download the bundle by running the following command: + + ```bash-plain + $ imgpkg pull -b index.docker.io/user1/simple-app-bundle:v1.0.0 -o /tmp/simple-app-bundle + + Pulling image 'index.docker.io/user1/simple-app-bundle@sha256:ec3f870e958e404476b9ec67f28c598fa8f00f819b8ae05ee80d51bac9f35f5d' + Extracting layer 'sha256:7906b9650be657359ead106e354f2728e16c8f317e1d87f72b05b5c5ec3d89cc' (1/1) + Locating image lock file images... + One or more images not found in bundle repo; skipping lock file update + + Succeeded + ``` + + Flags used in the command: + * `-b` (`--bundle`) refers to a location for a bundle within an OCI registry + * `-o` (`--output`) indicates the destination directory on your local machine where the bundle contents will be placed + + Bundle contents will be extracted into `/tmp/simple-app-bundle` directory: + + ```bash-plain + /tmp/simple-app-bundle + ├── .imgpkg + │   ├── bundle.yml + │   └── images.yml + └── config.yml + ``` + + __Note:__ The message `One or more images not found in bundle repo; skipping lock file update` is expected, and indicates that `/tmp/simple-app-bundle/.imgpkg/images.yml` (ImagesLock configuration) was not modified. + + If imgpkg had been able to find all images that were referenced in the [ImagesLock configuration](resources.md#imageslock-configuration) in the registry where bundle is located, then it would update `.imgpkg/images.yml` file to point to the registry-local locations. + + See what happens to the lock file if you run the same pull command after [copying](air-gapped-workflow.md#option-1-from-a-location-connected-to-both-registries) the bundle to another registry! + +--- +## Step 4: Use pulled bundle contents + +1. Now that we have have pulled bundle contents to a local directory, we can deploy Kubernetes configuration: + + Before we apply Kubernetes configuration, let's use [kbld](/kbld) to ensure that Kubernetes configuration uses exact image reference from `.imgpkg/images.yml`. (You can of course use other tools to take advantage of data stored in `.imgpkg/images.yml`). + + ```bash-plain + $ cd /tmp/simple-app-bundle/ + + $ kbld -f ./config.yml -f .imgpkg/images.yml | kubectl apply -f- + + resolve | final: docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 -> index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + resolve | final: index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 -> index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + + service/simple-app configured + deployment/simple-app configured + ``` + + kbld found `docker.io/dkalinin/k8s-simple-app` in Kubernetes configuration and replaced it with `index.docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0` before forwarding configuration to kubectl. + +## Next steps + +In this workflow we saw how to publish and download a bundle to distribute a Kubernetes application. Next, follow the [Air-gapped workflow](air-gapped-workflow.md) to see how we can use the `imgpkg copy` command to copy a bundle between registries. diff --git a/site/content/imgpkg/docs/v0.24.0/ca-certs-windows.md b/site/content/imgpkg/docs/v0.24.0/ca-certs-windows.md new file mode 100644 index 000000000..2f0404248 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/ca-certs-windows.md @@ -0,0 +1,28 @@ +--- +aliases: [/imgpkg/docs/latest/ca-certs-windows] +title: CA Certs on Windows +--- + +## Known issue verifying certificates on Windows + +If you are using imgpkg v0.19.0 or earlier, and use imgpkg with a registry over https, you will likely encounter the following error: +``` +imgpkg: Error: Fetching image: + Get "https://some.registry/v2/": x509: certificate signed by unknown authority +``` + +imgpkg v0.20.0+ supports loading Windows root ca certs. Meaning, that imgpkg is able to verify registry certificates signed by a trusted certificate authority! + +## Known issue providing custom ca certificates on Windows + +imgpkg allows specifying the `--registry-ca-cert-path` flag as a way to add custom ca certificates to use when verifying a registry server certificate. + +However, on Windows, the entire set of ca certificates to use during verify is loaded from the flag (Windows root ca store is skipped in this case). +Meaning that if you are targeting multiple registries, and some are signed with a trusted certificate authority and others signed with a custom ca certificate, +both ca certificates will need to be provided. (via the `--registry-ca-cert-path` flag) + +An example workflow: +1. Build a single ca certificate file (containing multiple ca certificates) from a trusted source. e.g. [extract ca certs provided by Mozilla](https://github.com/curl/curl/blob/4d2f8006777d6354d9b62eae38ebd0a0256d0f94/lib/firefox-db2pem.sh) +1. Provide that single ca certificate file to imgpkg. `--registry-ca-cert-path ./mozilla-ca-certs.pem` +1. Provide any additional custom ca certificates to imgpkg. `--registry-ca-cert-path ./dev-registry.pem` + diff --git a/site/content/imgpkg/docs/v0.24.0/commands.md b/site/content/imgpkg/docs/v0.24.0/commands.md new file mode 100644 index 000000000..58a6140b7 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/commands.md @@ -0,0 +1,140 @@ +--- +aliases: [/imgpkg/docs/latest/commands] +title: Commands +--- + +## Push + +### Overview + +`push` command allows users to create a bundle in the registry from files and/or directories on their local file systems. For example, + +```bash-plain +$ imgpkg push -b index.docker.io/k8slt/sample-bundle -f my-bundle/ +``` + +will push a bundle contents containing in the `my-bundle/` directory to `index.docker.io/k8slt/sample-bundle`. + +Use the `-b`/`--bundle` flag to specify the destination of the push. If the specified destination does not include a tag, the artifact will be pushed with the default tag `:latest`. + +The `-f` flag can be used multiple times to add different files or directories to the bundle. + +### Generating a BundleLock + +`push` command can output a [`BundleLock` configuration](resources.md#bundlelock-configuration) for users that would like a deterministic reference to a pushed bundle. For example, running: + +```bash-plain +$ impgpkg push -b index.docker.io/k8slt/sample-bundle:v0.1.0 -f my-bundle/ --lock-output +/tmp/bundle.lock.yml +``` + +will create `/tmp/bundle.lock.yml` with BundleLock configuration. If another bundle image in the repository is later given the same tag (`v0.1.0`), the BundleLock configuration will continue to provide immutable reference (via digest) to the original pushed bundle. + +--- +## Pull + +### Overview + +After pushing bundles to a registry, users can retrieve them with `imgpkg pull`. For example, + +```bash-plain +$ imgpkg pull -b index.docker.io/k8slt/sample-bundle -o my-bundle/ +``` + +will pull a bundle from `index.docker.io/k8slt/sample-bundle` and extract its contents into the `my-bundle/` directory, which gets created if it does not already exist. + +When pulling a bundle, imgpkg ensures that the referenced images are updated to account for any relocations. It will search for each referenced image by digest in the same repository as the bundle. If all referenced digests are found, imgpkg will update image references in the bundle's [`.imgpkg/images.yml` file](resources.md#imgpkg-directory). If any of the digests are not found in the repository, imgpkg will not update any references. + +### Pulling via lock file + +[BundleLock configuration](resources.md#bundlelock-configuration) can be used as input to the pull command via the `--lock` flag. + +```bash-plain +$ imgpkg pull --lock bundle.lock.yml -o my-bundle/ +``` + +### Pulling nested bundles + +If pulling a bundle that references another bundle (via it's ImagesLock file), in order to *also* pull down the contents of every nested bundle, use the `--recursive` flag. + +```bash-plain +$ imgpkg pull --recursive -b bundle-with-nested-bundles +``` + +Contents of *every* nested bundle are written to the 'parent' bundle's `.imgpkg/bundles` directory, namespaced by the bundle's sha256. + +For e.g. pulling a bundle with a nested bundle having sha of `123` would result in: +``` +parent-bundle-path/.imgpkg/bundles/sha256-123/ +``` + +--- +## Copy + +### Overview + +The `copy` command copies a bundle from a registry to another registry (as long as both registries are accessible from where the command is running): + +```bash-plain +$ imgpkg copy -b index.docker.io/k8slt/sample-bundle --to-repo registry.corp.com/user2/sample-bundle-name +``` + +Alternatively `copy` can copy a bundle between registries which are not both accessible from a single location, by creating an intermediate tarball: + +```bash-plain +$ imgpkg copy -b index.docker.io/k8slt/sample-bundle --to-tar=/Volumes/secure-thumb/bundle.tar +# ... take thumb driver to a different location ... +$ imgpkg copy --tar=/Volumes/secure-thumb/bundle.tar --to-repo registry.corp.com/user2/sample-bundle-name +``` + +In either case, the bundle image and all dependent images are copied to the destination location `registry.corp.com/user2/sample-bundle-name`. + +### Copying via lock file + +[BundleLock configuration](resources.md#bundlelock-configuration) can be used as input to the copy command via the `--lock` flag. + +```bash-plain +$ imgpkg copy --lock bundle.lock.yml --to-repo registry.corp.com/user2/sample-bundle-name --lock-output /tmp/new-bundle.lock.yml +``` + +### Non-Distributable or Foreign Layers + +Some images contain layers which should not be uploaded when copying, such as a proprietary base image. +Instead, to comply with license requirements, it is expected to get them directly from the source registry. +These layers are interchangeably known as +[Non-Distributable](https://github.com/opencontainers/image-spec/blob/79b036d80240ae530a8de15e1d21c7ab9292c693/layer.md#non-distributable-layers) +(by the OCI) or +[Foreign](https://docs.docker.com/registry/spec/manifest-v2-2/) (by Docker) and denoted in the layer's MediaType. + +By default, imgpkg will not relocate any layers marked as non-distributable. + +This can cause issues when dealing with [air-gapped environments](air-gapped-workflow.md) as they may be unable to reach the external registries. +To allow this use case, imgpkg supports the `--include-non-distributable-layers` flag to copy all layers, even those marked as non-distributable. + +Note that usage of this flag shall not preclude your obligation to comply with the terms of the image license(s). + +### Image Signatures + +Starting on version v0.9.0 `imgpkg` can copy Signature created by [cosign](https://github.com/sigstore/cosign). By +default `imgpkg` will not search for Signatures for Images. To enable the search and copy of the signatures the +flag `--cosign-signatures` needs to be provided to copy command + +```bash-plain +$ imgpkg copy -b index.docker.io/k8slt/sample-bundle --to-repo some.repo.io/some-bundle --cosign-signatures +``` + +This feature will work while copying to a different repository as well as copying to a tarball. + +--- + +## Tag + +`imgpkg tag` supports a `list` subcommand that allows users to list the image tags pushed to registries. The command features an `--image`/`-i` option that allows a user to specify an image name. + +An example of this is shown below: + +```bash-plain +$ imgpkg tag list -i index.docker.io/k8slt/image +``` + +The output shows the names of all tags associated with the image, along with its digest. diff --git a/site/content/imgpkg/docs/v0.24.0/debugging.md b/site/content/imgpkg/docs/v0.24.0/debugging.md new file mode 100644 index 000000000..9000ae487 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/debugging.md @@ -0,0 +1,46 @@ +--- +aliases: [/imgpkg/docs/latest/debugging] +title: Debugging +--- + +## Debugging + +In the process of communicating with remote OCI registries, it is possible that an error will occur. In order to help debug an error situation, use the `--debug` command line argument. Specifying this argument will output detailed logs of all communications between `imgpkg` and the OCI registries. + +> This feature is available in v0.20.0 and later + +As an example, consider this pull command along with the additional information logged. The record of all HTTP communication will be displayed to assist in resolving a problem or error condition. + +```bash-plain +imgpkg pull -b registry.example.com/foo/bar:latest -o temp --debug + +2021/09/21 20:50:12 --> GET https://registry.example.com/v2/ +2021/09/21 20:50:12 GET /v2/ HTTP/1.1 +Host: registry.example.com +User-Agent: Go-http-client/1.1 +Accept-Encoding: gzip + +2021/09/21 20:50:12 <-- 401 https://registry.example.com/v2/ (207.596107ms) +2021/09/21 20:50:12 HTTP/1.1 401 Unauthorized +Content-Length: 76 +Connection: keep-alive +Content-Type: application/json; charset=utf-8 +Date: Wed, 22 Sep 2021 02:50:12 GMT +Docker-Distribution-Api-Version: registry/2.0 +Set-Cookie: sid=f9752c01ce47ab50791d4a845a78d996; Path=/; HttpOnly; Secure +Strict-Transport-Security: max-age=31536000; includeSubDomains +Www-Authenticate: Bearer realm="https://registry.example.com/service/token",service="harbor-registry" +X-Request-Id: 2fe97b25-ca40-4012-9105-bbf8284995b6 + +{"errors":[{"code":"UNAUTHORIZED","message":"unauthorized: unauthorized"}]} + +2021/09/21 20:50:12 --> GET https://registry.example.com/service/token?scope=repository%3Afoo%2Fbar%3Apull&service=harbor-registry [body redacted: basic token response contains credentials] +2021/09/21 20:50:12 GET /service/token?scope=repository%3Afoo%2Fbar%3Apull&service=harbor-registry HTTP/1.1 +Host: registry.example.com +User-Agent: go-containerregistry/v0.6.0 +Authorization: +Accept-Encoding: gzip +... +``` + +> Note that sensitive information, such as basic authentication parameters and Authorization strings, are not displayed. diff --git a/site/content/imgpkg/docs/v0.24.0/install.md b/site/content/imgpkg/docs/v0.24.0/install.md new file mode 100644 index 000000000..058244d52 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/install.md @@ -0,0 +1,58 @@ +--- +aliases: [/imgpkg/docs/latest/install] +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ imgpkg version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ imgpkg version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/vmware-tanzu/homebrew-carvel](https://github.com/vmware-tanzu/homebrew-carvel). + +```bash +$ brew tap vmware-tanzu/carvel +$ brew install imgpkg +$ imgpkg version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/vmware-tanzu/carvel-imgpkg/releases), for example for 'imgpkg-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/imgpkg-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/imgpkg-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/imgpkg-darwin-amd64 /usr/local/bin/imgpkg + +# Make binary executable +$ chmod +x /usr/local/bin/imgpkg + +# Check its version +$ imgpkg version +``` diff --git a/site/content/imgpkg/docs/v0.24.0/proxy.md b/site/content/imgpkg/docs/v0.24.0/proxy.md new file mode 100644 index 000000000..0998e7dc2 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/proxy.md @@ -0,0 +1,33 @@ +--- +aliases: [/imgpkg/docs/latest/proxy] +title: Proxy +--- + +## Using Proxy + +When using `imgpkg` to connect with a registry via a proxy you will need to provide one of following environment variables + +- `HTTP_PROXY` or `http_proxy` when using the flag `--registry-insecure` +- `HTTPS_PROXY` or `https_proxy` when the communication with the registry need to be using TLS + +### No TLS example + +Assuming the proxy to access the registry is located in `http://proxy.company.com` + +When executing `imgpkg` do the following: +```bash +export http_proxy=http://proxy.company.com + +imgpkg pull -b registry.company.com/my-image@sha256:265d4a5ed8bf0df27d1107edb00b70e658ee9aa5acb3f37336c5a17db634481e -o folder --registry-insecure +``` + +### TLS example + +Assuming the proxy to access the registry is located in `https://proxy.company.com` + +When executing `imgpkg` do the following: +```bash +export https_proxy=https://proxy.company.com + +imgpkg pull -b registry.company.com/my-image@sha256:265d4a5ed8bf0df27d1107edb00b70e658ee9aa5acb3f37336c5a17db634481e -o folder +``` diff --git a/site/content/imgpkg/docs/v0.24.0/resources.md b/site/content/imgpkg/docs/v0.24.0/resources.md new file mode 100644 index 000000000..a1ac2b5eb --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/resources.md @@ -0,0 +1,140 @@ +--- +aliases: [/imgpkg/docs/latest/resources] +title: Resources +--- + +## Image + +An OCI image is an artifact that lives within an OCI registry (such as DockerHub). It can contain an arbitrary number of files. + +--- +## Bundle + +A bundle is an OCI image that holds 0+ arbitrary files _and_ 0+ references to dependent OCI images (which *may* also be [bundles](#nested-bundle)). By tracking dependent images, imgpkg can copy bundles across registries. + +Referenced images are stored within the [`.imgpkg` directory](#imgpkg-directory) at the root level of the bundle image. + +![Bundle diagram](/images/imgpkg/bundle-diagram.png) + +Implementation note: A bundle OCI image has the `dev.carvel.imgpkg.bundle` [label](https://docs.docker.com/config/labels-custom-metadata/) set. + +--- +## `.imgpkg` directory + +`.imgpkg` directory contains metadata files describing bundle: + +- `images.yml` (required) contains [ImagesLock configuration](#imageslock-configuration) that describes 0+ dependent OCI images. Consumers of bundles can rely on this file being always present. + +- `bundle.yml` (optional) file contains [Bundle configuration](#bundle-configuration) that contains details about bundle authors, associated websites, etc. + +Restrictions for location of `.imgpkg` directory: + +- Only one `.imgpkg` directory is allowed across all directories provided via `-f` to the `push` command. This restriction ensures there is a single source of bundle metadata and referenced images. + +- The `.imgpkg` directory must be a direct child of one of the input directories. This prevents any confusion around the scope of the `.imgpkg` metadata. + +--- +## Bundle configuration + +Used by bundle creators to store general information about the bundle. Stored in `.imgpkg/bundle.yml`. + +Example: + +```yaml +apiVersion: imgpkg.carvel.dev/v1alpha1 +kind: Bundle +metadata: + name: my-bundle +authors: +- name: Full Name + email: name@example.com +websites: +- url: example.com +``` + +- `authors` (array of structs; optional) + - `name` (string) Author name + - `email` (string) Author email +- `websites` (array of structs; optional) + - `url` (string) Related website URL + +--- +## ImagesLock configuration + +An ImagesLock configuration is used to track a collection of image references. + +Bundle's `.imgpkg/images.yml` contains ImagesLock configuration. That's how bundle knows which OCI images it references. When copying a bundle `imgpkg` uses this configuration to know which images to copy. + +It can be conveniently generated with [kbld](/kbld): + +```bash-plain +$ kbld -f config.yml --imgpkg-lock-output .imgpkg/images.yml +``` + +Example: + +```yaml +apiVersion: imgpkg.carvel.dev/v1alpha1 +kind: ImagesLock +images: +- image: docker.io/user1/my-app@sha256:42462d0cb227497976754bb67348bdd7471c7bd159819d6bd63fdf479eb7eb19 + annotations: + kbld.carvel.dev/id: "my-app:v1" +- image: gcr.io/projectX/controller@sha256:6ecba6f14373a449f8d54fa4286f57fb8ef37c4ffa637969551f2fda52672206 +``` + +- `images` (array of images): 0+ images + - `image` (string; required) digest reference to OCI image (tag references are not allowed) + - `annotations` (map[string]string; optional) arbitrary additional data about image reference. Expected to be used by tools that create or read ImagesLock configuration. Example: [kbld](/kbld) uses annotations to store an identifier that can later tell it which location(s) within a Kubernetes configuration to update with the digest reference. + +Advanced non-bundle use: See [copying via lock files](commands.md#copying-via-lock-file). + +--- +## BundleLock configuration + +Stores a digest reference to a bundle (as well as the tag it was pushed with). + +This configuration is generated by the `--lock-output` flag during a `push` command. + +```yaml +$ imgpkg push -b ... --lock-output /tmp/lock.yml + +$ cat /tmp/lock.yml + +apiVersion: imgpkg.carvel.dev/v1alpha1 +kind: BundleLock +bundle: + image: docker.io/my-app@sha256:b12026c7a0a6a1756a82a2a74ac759e9a7036523faca0e33dbddebc214e097df + tag: v1.0 +``` + +--- +## Nested Bundle + +A nested bundle is a bundle referenced from a 'parent' bundle in its `ImagesLock` configuration. + +Having a bundle 'reference' another bundle is no different from referencing any other OCI image. The copy and pull commands work the same as dealing with any OCI image. + +![Nested Bundle diagram](/images/imgpkg/nested-bundle-diagram.png) + +One key difference between nested bundles and other OCI images, is the directory structure when `imgpkg pull` writes the nested bundle's content to disk. + +For further details refer to [pulling a nested bundle.](commands.md#pulling-nested-bundles) + +--- +## Locations OCI Image + +`imgpkg` when copying Bundles and Images now creates a new OCI Images that will act as a Cache that contain information +about the Images that were copied and if these Images are a Bundle or not. This OCI Image will contain a single layer +with a single file `image-locations.yml` at the root of the image. This is the file structure + +```yaml +apiVersion: imgpkg.carvel.dev/v1alpha1 +kind: ImageLocations +images: +- image: some.image.io/test@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + isBundle: true +``` + +The OCI Image will be pushed into the same repository as the Bundle and will have the tag +`sha256-{Bundle SHA}.image-locations.imgpkg` diff --git a/site/content/imgpkg/docs/v0.24.0/security.md b/site/content/imgpkg/docs/v0.24.0/security.md new file mode 100644 index 000000000..ca69db28d --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/security.md @@ -0,0 +1,8 @@ +--- +aliases: [/imgpkg/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `imgpkg`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/imgpkg/docs/v0.24.0/working-directly-with-images.md b/site/content/imgpkg/docs/v0.24.0/working-directly-with-images.md new file mode 100644 index 000000000..cdac04918 --- /dev/null +++ b/site/content/imgpkg/docs/v0.24.0/working-directly-with-images.md @@ -0,0 +1,8 @@ +--- +aliases: [/imgpkg/docs/latest/working-directly-with-images] +title: Working directly with images +--- + +In rare cases imgpkg's [bundle](resources.md#bundle) concept is not wanted (or necessary). imgpkg provides a `--image` flag for push, pull and copy commands. When the `--image` flag is used, there is no need for a `.imgpkg` directory to store metadata. + +For most use cases, we suggest using the bundle concept and `--bundle` flag. diff --git a/site/content/kapp-controller/docs/latest/_index.md b/site/content/kapp-controller/docs/develop/_index.md similarity index 97% rename from site/content/kapp-controller/docs/latest/_index.md rename to site/content/kapp-controller/docs/develop/_index.md index d7df95cb9..6a852213c 100644 --- a/site/content/kapp-controller/docs/latest/_index.md +++ b/site/content/kapp-controller/docs/develop/_index.md @@ -2,7 +2,7 @@ title: "About kapp-controller" toc: "false" cascade: - version: latest + version: develop toc: "true" type: docs layout: docs @@ -32,4 +32,4 @@ With its layered approach, kapp-controller can be used as: Use kapp-controller's Package Management features along with Carvel's imgpkg bundles to distribute Package Repositories that can be added to cluster to provide a catalog of software for users to install. Package Repositries can be automatically updated ensuring users always have access to latest versions of software. Package Repositories and Packages can also be relocated and run in air-gapped environments. #### Reliable and ready for production! -kapp-controller has been hardened and is in use on production Kubernetes clusters. Learn more through [case studies](/blog/casestudy-modernizing-the-us-army) on our blog. \ No newline at end of file +kapp-controller has been hardened and is in use on production Kubernetes clusters. Learn more through [case studies](/blog/casestudy-modernizing-the-us-army) on our blog. diff --git a/site/content/kapp-controller/docs/latest/air-gapped-workflow.md b/site/content/kapp-controller/docs/develop/air-gapped-workflow.md similarity index 100% rename from site/content/kapp-controller/docs/latest/air-gapped-workflow.md rename to site/content/kapp-controller/docs/develop/air-gapped-workflow.md diff --git a/site/content/kapp-controller/docs/latest/app-examples.md b/site/content/kapp-controller/docs/develop/app-examples.md similarity index 100% rename from site/content/kapp-controller/docs/latest/app-examples.md rename to site/content/kapp-controller/docs/develop/app-examples.md diff --git a/site/content/kapp-controller/docs/latest/app-overview.md b/site/content/kapp-controller/docs/develop/app-overview.md similarity index 93% rename from site/content/kapp-controller/docs/latest/app-overview.md rename to site/content/kapp-controller/docs/develop/app-overview.md index f7a41860e..e634f6599 100644 --- a/site/content/kapp-controller/docs/latest/app-overview.md +++ b/site/content/kapp-controller/docs/develop/app-overview.md @@ -20,7 +20,7 @@ Full App CR spec can be found [here](app-spec.md). App CR supports multiple source for fetching configuration and OCI images to give developers flexibility. - `inline`: specify one or more files within resource -- `imgpkgBundle`: download [imgpkg bundle](https://carvel.dev/imgpkg/docs/latest/resources/#bundle) from registry (available in v0.17.0+) +- `imgpkgBundle`: download [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) from registry (available in v0.17.0+) - `image`: download Docker image from registry - `http`: download file at URL - `git`: clone Git repository @@ -69,4 +69,4 @@ App CR supports multiple templating, overlaying, and data transformation tools t App CR uses Carvel's `kapp` CLI to deploy. -- `kapp`: uses [kapp](/kapp) to deploy resources \ No newline at end of file +- `kapp`: uses [kapp](/kapp) to deploy resources diff --git a/site/content/kapp-controller/docs/develop/app-spec.md b/site/content/kapp-controller/docs/develop/app-spec.md new file mode 100644 index 000000000..252c64943 --- /dev/null +++ b/site/content/kapp-controller/docs/develop/app-spec.md @@ -0,0 +1,332 @@ +--- +title: App CR spec +--- + +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App + +metadata: + name: simple-app + # namespace is going to be used as a default namespace during kapp deploy + namespace: ns + +spec: + # pauses _future_ reconcilation; does _not_ affect + # currently running reconciliation (optional; default=false) + paused: true + + # cancels current and future reconciliations (optional; default=false) + canceled: true + + # Deletion requests for the App will result in the App CR being + # deleted, but its associated resources will not be deleted + # (optional; default=false; v0.18.0+) + noopDelete: true + + # specifies that app should be deployed authenticated via + # given service account, found in this namespace (optional; v0.6.0+) + serviceAccountName: sa-name + + # specifies the length of time to wait, in time + unit + # format, before reconciling. Always >= 30s. If value below + # 30s is specified, 30s will be used. (optional; v0.9.0+; default=30s) + syncPeriod: 1m + + # specifies that app should be deployed to destination cluster; + # by default, cluster is same as where this resource resides (optional; v0.5.0+) + cluster: + # specifies namespace in destination cluster (optional) + namespace: ns2 + # specifies secret containing kubeconfig (required) + kubeconfigSecretRef: + # specifies secret name within app's namespace (required) + name: cluster1 + # specifies key that contains kubeconfig (optional) + key: value + + # Fetch must have one or more directives + fetch: + # pull content from within this resource; or other resources in the cluster + - inline: + # specifies mapping of paths to their content; + # not recommended for sensitive values as CR is not encrypted (optional) + paths: + dir/file.ext: file-content + # specifies content via secrets and config maps; + # data values are recommended to be placed in secrets (optional) + pathsFrom: + - secretRef: + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + + # pulls content from Docker/OCI registry + - image: + # Docker image url; unqualified, tagged, or + # digest references supported (required) + url: host.com/username/image:v0.1.0 + # secret with auth details (optional) + secretRef: + name: secret-name + # grab only portion of image (optional) + subPath: inside-dir/dir2 + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + + # pulls imgpkg bundle from Docker/OCI registry (v0.17.0+) + - imgpkgBundle: + # Docker image url; unqualified, tagged, or + # digest references supported (required) + image: host.com/username/image:v0.1.0 + # secret with auth details (optional) + secretRef: + name: secret-name + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + + # uses http library to fetch file + - http: + # http and https url are supported; + # plain file, tgz and tar types are supported (required) + url: https://host.com/archive.tgz + # checksum to verify after download (optional) + sha256: 0a12cdef83... + # secret to provide auth details (optional) + secretRef: + name: secret-name + # grab only portion of download (optional) + subPath: inside-dir/dir2 + + # uses git to clone repository + - git: + # http or ssh urls are supported (required) + url: https://github.com/k14s/k8s-simple-app-example + # branch, tag, commit; origin is the name of the remote (required) + ref: origin/develop + # secret with auth details. allowed keys: ssh-privatekey, ssh-knownhosts, username, password (optional) + # (if ssh-knownhosts is not specified, git will not perform strict host checking) + secretRef: + name: secret-name + # grab only portion of repository (optional) + subPath: config-step-2-template + # skip lfs download (optional) + lfsSkipSmudge: true + # specifies a strategy to resolve to an explicit ref (optional; v0.24.0+) + refSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + + # uses helm fetch to fetch specified chart + - helmChart: + name: stable/nginx + # (optional) + version: "0.1.0" + # (optional) + repository: + # repository url; + # scheme of oci:// will fetch experimental helm oci chart (v0.19.0+) + # (required) + url: https://... + # (optional) + secretRef: + name: secret-name + + # Template must have one or more directives + template: + # use ytt to template configuration + - ytt: + # ignores comments that ytt doesn't recognize + # (optional; default=false) + ignoreUnknownComments: true + # forces strict mode https://github.com/k14s/ytt/blob/develop/docs/strict.md + # (optional; default=false) + strict: true + # specify additional files, including data values (optional) + inline: + # specifies content inline within resource; + # not recommended for sensitive values as CR is not encrypted (optional) + paths: + # mapping of paths to their content + dir/file.ext: | + file-content + file-content + # specified content via secrets and config maps; + # data values are recommended to be placed in secrets (optional) + pathsFrom: + - secretRef: + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + # lists paths to provide to ytt explicitly (optional) + paths: + # - must be quoted when included with paths + - "-" + - dir/common + - dir/nested/app + # control metadata about input files passed to ytt (optional; v0.18.0+) + # see https://carvel.dev/ytt/docs/latest/file-marks/ for more details + fileMarks: + - file-content:type=yaml-plain + - dir/common/bom**/*:type=text-plain + - dir/nested/app/file.txt:exclude=true + - dir/common/generated.go.txt:path=gen.go.txt + # provide values via ytt's --data-values-file (optional; v0.19.0-alpha.9) + valuesFrom: + - secretRef: + name: secret-name + - configMapRef: + name: cfgmap-name + - path: values/shared.yml + + # use kbld to resolve image references to use digests + - kbld: + # lists paths to use explicitly (optional; v0.13.0+) + # - must be quoted when included with paths + paths: + - "-" + - .imgpkg/images.yml + + # use helm template command to render helm chart + - helmTemplate: + # path to chart (optional; v0.13.0+) + path: some-chart/ + # set name explicitly, default is App CR's name (optional; v0.13.0+) + name: custom-name + # set namespace explicitly, default is App CR's namespace (optional; v0.13.0+) + namespace: custom-ns + # one or more secrets, config maps, paths that provide values (optional) + valuesFrom: + - secretRef: + name: secret-name + - configMapRef: + name: cfgmap-name + - path: values/shared.yml + + # use sops to decrypt *.sops.yml files (optional; v0.11.0+) + - sops: + # use PGP to decrypt files (required) + pgp: + # secret with private armored PGP private keys (required) + privateKeysSecretRef: + # (required) + name: pgp-secrets + # lists paths to decrypt explicitly (optional; v0.13.0+) + paths: + - all-secrets/ + - prod-secrets/prod.sops.yml + + # Deploy must have one directive + deploy: + # use kapp to deploy resources + - kapp: + # override namespace for all resources (optional) + intoNs: another-ns1 + # provide custom namespace override mapping (optional) + mapNs: ["ns1=another-ns1"] + # pass through options to kapp deploy (optional) + rawOptions: ["--apply-concurrency=10"] + # configuration for inspect command (optional) + # as of kapp-controller v0.31.0, inspect is disabled by default + # add rawOptions or use an empty inspect config like `inspect: {}` to enable it + inspect: + # pass through options to kapp inspect (optional) + rawOptions: ["--json=true"] + # configuration for delete command (optional) + delete: + # pass through options to kapp delete (optional) + rawOptions: ["--apply-ignored=true"] + +# status is popuated by the controller +status: + # populated based on metadata.generation when controller + # observes a change to the resource; if this value is + # out of data, other status fields do not reflect latest state + observedGeneration: 1 + + conditions: + # "Reconciling" indicates that fetch/template/deploy is happening; + # it does not mean that any resource has changed + - type: Reconciling + status: "True" + # "ReconcileFailed" indicates that one of the stages failed + - type: ReconcileFailed + status: "True" + # "ReconcileSucceeded" indicates that all stages succeeded + - type: ReconcileSucceeded + status: "True" + + fetch: + exitCode: 0 + error: "..." + stderr: "..." + startedAt: "2019-11-07T16:37:23Z" + updatedAt: "2019-11-07T16:37:23Z" + + template: + exitCode: 0 + error: "..." + stderr: "..." + updatedAt: "2019-11-07T16:37:23Z" + + deploy: + exitCode: 0 + error: "..." + stderr: "..." + finished: true + startedAt: "2019-11-07T16:37:23Z" + stdout: |- + Changes + Namespace Name Kind Conds. Age Op Wait to Rs Ri + Op: 0 create, 0 delete, 0 update, 0 noop + Wait to: 0 reconcile, 0 delete, 0 noop + Succeeded + updatedAt: "2019-11-07T16:37:23Z" + + inspect: + exitCode: 0 + error: "..." + stderr: "..." + stdout: |- + Resources in app 'simple-app-ctrl' + Namespace Name Kind Owner Conds. Rs Ri Age + default simple-app Deployment kapp 2/2 t ok - 7d + default L simple-app-6b6b4fcd97 ReplicaSet cluster - ok - 7d + default L.. simple-app-6b6b4fcd97-kwclv Pod cluster 4/4 t ok - 7d + default simple-app Service kapp - ok - 7d + default L simple-app Endpoints cluster - ok - 7d + Rs: Reconcile state + Ri: Reconcile information + 5 resources + Succeeded + updatedAt: "2019-11-07T16:37:23Z" +``` diff --git a/site/content/kapp-controller/docs/latest/controller-config.md b/site/content/kapp-controller/docs/develop/controller-config.md similarity index 100% rename from site/content/kapp-controller/docs/latest/controller-config.md rename to site/content/kapp-controller/docs/develop/controller-config.md diff --git a/site/content/kapp-controller/docs/latest/debugging-crs.md b/site/content/kapp-controller/docs/develop/debugging-crs.md similarity index 100% rename from site/content/kapp-controller/docs/latest/debugging-crs.md rename to site/content/kapp-controller/docs/develop/debugging-crs.md diff --git a/site/content/kapp-controller/docs/latest/debugging-kc.md b/site/content/kapp-controller/docs/develop/debugging-kc.md similarity index 100% rename from site/content/kapp-controller/docs/latest/debugging-kc.md rename to site/content/kapp-controller/docs/develop/debugging-kc.md diff --git a/site/content/kapp-controller/docs/latest/dev.md b/site/content/kapp-controller/docs/develop/dev.md similarity index 100% rename from site/content/kapp-controller/docs/latest/dev.md rename to site/content/kapp-controller/docs/develop/dev.md diff --git a/site/content/kapp-controller/docs/latest/faq.md b/site/content/kapp-controller/docs/develop/faq.md similarity index 99% rename from site/content/kapp-controller/docs/latest/faq.md rename to site/content/kapp-controller/docs/develop/faq.md index 416abce26..de7a52d26 100644 --- a/site/content/kapp-controller/docs/latest/faq.md +++ b/site/content/kapp-controller/docs/develop/faq.md @@ -93,4 +93,4 @@ This is the recommended workflow: For more details, see: - [ytt: Export Schema in OpenAPI Format](../../../ytt/docs/latest/how-to-export-schema.md). - [ytt: Configuring Data Values via command line flags](../../../ytt/docs/latest/ytt-data-values.md#configuring-data-values-via-command-line-flags) -- [@ytt:yaml module](../../../ytt/docs/latest/lang-ref-ytt.md#yaml) \ No newline at end of file +- [@ytt:yaml module](../../../ytt/docs/latest/lang-ref-ytt.md#yaml) diff --git a/site/content/kapp-controller/docs/latest/install.md b/site/content/kapp-controller/docs/develop/install.md similarity index 97% rename from site/content/kapp-controller/docs/latest/install.md rename to site/content/kapp-controller/docs/develop/install.md index 887fe2ae0..81c96b225 100644 --- a/site/content/kapp-controller/docs/latest/install.md +++ b/site/content/kapp-controller/docs/develop/install.md @@ -62,7 +62,7 @@ version of kapp-controller). ``` 3. Set the `IMGPKG_ENABLE_IAAS_AUTH` [environment - variable](https://carvel.dev/imgpkg/docs/latest/auth/#via-iaas) to false. + variable](/imgpkg/docs/latest/auth/#via-iaas) to false. ### Kubernetes versions < 1.20 diff --git a/site/content/kapp-controller/docs/latest/oss-packages.md b/site/content/kapp-controller/docs/develop/oss-packages.md similarity index 100% rename from site/content/kapp-controller/docs/latest/oss-packages.md rename to site/content/kapp-controller/docs/develop/oss-packages.md diff --git a/site/content/kapp-controller/docs/latest/package-consumer-concepts.md b/site/content/kapp-controller/docs/develop/package-consumer-concepts.md similarity index 100% rename from site/content/kapp-controller/docs/latest/package-consumer-concepts.md rename to site/content/kapp-controller/docs/develop/package-consumer-concepts.md diff --git a/site/content/kapp-controller/docs/latest/package-install-extensions.md b/site/content/kapp-controller/docs/develop/package-install-extensions.md similarity index 100% rename from site/content/kapp-controller/docs/latest/package-install-extensions.md rename to site/content/kapp-controller/docs/develop/package-install-extensions.md diff --git a/site/content/kapp-controller/docs/latest/packaging-artifact-formats.md b/site/content/kapp-controller/docs/develop/packaging-artifact-formats.md similarity index 100% rename from site/content/kapp-controller/docs/latest/packaging-artifact-formats.md rename to site/content/kapp-controller/docs/develop/packaging-artifact-formats.md diff --git a/site/content/kapp-controller/docs/latest/packaging-gitops.md b/site/content/kapp-controller/docs/develop/packaging-gitops.md similarity index 100% rename from site/content/kapp-controller/docs/latest/packaging-gitops.md rename to site/content/kapp-controller/docs/develop/packaging-gitops.md diff --git a/site/content/kapp-controller/docs/latest/packaging-tutorial.md b/site/content/kapp-controller/docs/develop/packaging-tutorial.md similarity index 91% rename from site/content/kapp-controller/docs/latest/packaging-tutorial.md rename to site/content/kapp-controller/docs/develop/packaging-tutorial.md index 6a9eb848e..03f640246 100644 --- a/site/content/kapp-controller/docs/latest/packaging-tutorial.md +++ b/site/content/kapp-controller/docs/develop/packaging-tutorial.md @@ -171,10 +171,10 @@ EOF ``` ## Creating a Package: Structuring our contents -We'll create an [imgpkg bundle](https://carvel.dev/imgpkg/docs/latest/resources/#bundle) +We'll create an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) that contains the package contents: the configuration (config.yml and values.yml from the previous step) and a reference to the greeter app image (docker.io/dkalinin/k8s-simple-app@sha256:...). -The [package bundle format](https://carvel.dev/kapp-controller/docs/latest/packaging-artifact-formats/#package-contents-bundle) describes the purpose of each directory +The [package bundle format](packaging-artifact-formats.md#package-contents-bundle) describes the purpose of each directory used in this section of the tutorial as well as general recommendations. Let's create a directory with our configuration files: @@ -190,7 +190,7 @@ mkdir -p package-contents/.imgpkg kbld -f package-contents/config/ --imgpkg-lock-output package-contents/.imgpkg/images.yml ``` -For more on using kbld to populate the .imgpkg directory with an ImagesLock, and why it is useful, see the [imgpkg docs on the subject](https://carvel.dev/imgpkg/docs/latest/resources/#imageslock-configuration). +For more on using kbld to populate the .imgpkg directory with an ImagesLock, and why it is useful, see the [imgpkg docs on the subject](/imgpkg/docs/latest/resources/#imageslock-configuration). Once these files have been added, our package contents bundle is ready to be pushed! @@ -299,17 +299,17 @@ EOF This Package contains some metadata fields specific to the version, such as releaseNotes and a valuesSchema. The valuesSchema shows what configurable properties exist for the version. This will help when users want to install this package and want to know what can be configured. The other main component of this CR is the template section. -This section informs kapp-controller of the actions required to install the packaged software, so take a look at the [app-spec](https://carvel.dev/kapp-controller/docs/latest/app-spec/) section to learn more about each of the template sections. For this example, we have chosen a basic setup that will fetch the imgpkg bundle we created in the previous section, run the templates stored inside through ytt, apply kbld transformations, and then deploy the resulting manifests with kapp. +This section informs kapp-controller of the actions required to install the packaged software, so take a look at the [app-spec](app-spec.md) section to learn more about each of the template sections. For this example, we have chosen a basic setup that will fetch the imgpkg bundle we created in the previous section, run the templates stored inside through ytt, apply kbld transformations, and then deploy the resulting manifests with kapp. There will also be validations run on the Package CR, so ensure that spec.refName and spec.version are not empty and that metadata.name is `.`. These validations are done to encourage a naming scheme that keeps package version names unique. ## Creating a Package Repository -A [package repository](https://carvel.dev/kapp-controller/docs/latest/packaging/#package-repository) +A [package repository](packaging.md#package-repository) is a collection of packages (more specifically a collection of Package and PackageMetadata CRs). -Our recommended way to make a package repository is via an [imgpkg bundle](https://carvel.dev/imgpkg/docs/latest/resources/#bundle). +Our recommended way to make a package repository is via an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle). -The [PackageRepository bundle format](https://carvel.dev/kapp-controller/docs/latest/packaging-artifact-formats/#package-repository-bundle) describes purpose of each directory and general recommendations. +The [PackageRepository bundle format](packaging-artifact-formats.md#package-repository-bundle) describes purpose of each directory and general recommendations. Lets start by creating the needed directories: @@ -371,7 +371,7 @@ EOF ``` (See our -[demo video](https://www.youtube.com/watch?v=PmwkicgEKQE) and [website](https://carvel.dev/kapp-controller/docs/latest/private-registry-auth) for more typical usage with an external repository.) +[demo video](https://www.youtube.com/watch?v=PmwkicgEKQE) and [website](private-registry-auth.md) for more typical usage with an external repository.) This PackageRepository CR will allow kapp-controller to install any of the packages found within the `${REPO_HOST}/packages/my-pkg-repo:1.0.0` imgpkg bundle, which we @@ -446,11 +446,11 @@ EOF This CR references the Package we created in the previous sections using the package’s `refName` and `version` fields (see yaml from step 7). Do note, the `versionSelection` property has a constraints subproperty to give more control over which versions are chosen for installation. -More information on PackageInstall versioning can be found [here](https://carvel.dev/kapp-controller/docs/latest/packaging/#versioning-packageinstalls). +More information on PackageInstall versioning can be found [here](packaging.md#versioning-packageinstalls). This yaml snippet also contains a Kubernetes secret, which is referenced by the PackageInstall. This secret is used to provide customized values to the package installation’s templating steps. Consumers can discover more details on the configurable properties of a package by inspecting the Package CR’s valuesSchema. -Finally, to install the above package, we will also need to create `default-ns-sa` service account (refer to [Security model](https://carvel.dev/kapp-controller/docs/latest/security-model/) +Finally, to install the above package, we will also need to create `default-ns-sa` service account (refer to [Security model](security-model.md) for explanation of how service accounts are used) that give kapp-controller privileges to create resources in the default namespace: ```bash kapp deploy -a default-ns-rbac -f https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/develop/examples/rbac/default-ns.yml -y @@ -480,4 +480,4 @@ curl localhost:3000 Visit [carvel.dev](https://carvel.dev/) to learn more about Carvel tools. -See the full docs for [Package Management with kapp-controller](https://carvel.dev/kapp-controller/docs/latest/packaging/) +See the full docs for [Package Management with kapp-controller](packaging.md) diff --git a/site/content/kapp-controller/docs/latest/packaging.md b/site/content/kapp-controller/docs/develop/packaging.md similarity index 100% rename from site/content/kapp-controller/docs/latest/packaging.md rename to site/content/kapp-controller/docs/develop/packaging.md diff --git a/site/content/kapp-controller/docs/latest/private-registry-auth.md b/site/content/kapp-controller/docs/develop/private-registry-auth.md similarity index 100% rename from site/content/kapp-controller/docs/latest/private-registry-auth.md rename to site/content/kapp-controller/docs/develop/private-registry-auth.md diff --git a/site/content/kapp-controller/docs/latest/security-model.md b/site/content/kapp-controller/docs/develop/security-model.md similarity index 100% rename from site/content/kapp-controller/docs/latest/security-model.md rename to site/content/kapp-controller/docs/develop/security-model.md diff --git a/site/content/kapp-controller/docs/latest/security.md b/site/content/kapp-controller/docs/develop/security.md similarity index 100% rename from site/content/kapp-controller/docs/latest/security.md rename to site/content/kapp-controller/docs/develop/security.md diff --git a/site/content/kapp-controller/docs/latest/sops.md b/site/content/kapp-controller/docs/develop/sops.md similarity index 100% rename from site/content/kapp-controller/docs/latest/sops.md rename to site/content/kapp-controller/docs/develop/sops.md diff --git a/site/content/kapp-controller/docs/latest/startup.md b/site/content/kapp-controller/docs/develop/startup.md similarity index 100% rename from site/content/kapp-controller/docs/latest/startup.md rename to site/content/kapp-controller/docs/develop/startup.md diff --git a/site/content/kapp-controller/docs/latest/walkthrough.md b/site/content/kapp-controller/docs/develop/walkthrough.md similarity index 100% rename from site/content/kapp-controller/docs/latest/walkthrough.md rename to site/content/kapp-controller/docs/develop/walkthrough.md diff --git a/site/content/kapp-controller/docs/v0.31.0/_index.md b/site/content/kapp-controller/docs/v0.31.0/_index.md new file mode 100644 index 000000000..159de1ee0 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/_index.md @@ -0,0 +1,36 @@ +--- + +title: "About kapp-controller" +toc: "false" +cascade: + version: v0.31.0 + toc: "true" + type: docs + layout: docs +--- + +kapp-controller provides declarative APIs to customize, install, and update your Kubernetes applications and packages. It is a part of the Carvel toolkit and follows core Carvel design principles. Get started with the [tutorial](packaging-tutorial.md)! + +#### Choice for authors; consistency for consumers +Kubernetes configuration takes many forms -- plain YAML configurations, Helm charts, ytt templates, jsonnet templates, etc. +Software running on Kubernetes lives in many different places: a Git repository, an archive over HTTP, a Helm repository, etc. + +kapp-controller provides software authors flexibility to choose their own configuration tools, while providing software consumers with consistent declarative APIs to customize, install, and update software on Kubernetes from various sources. + +#### Lightweight and composable +kapp-controller breaks down the installation of applications and packages into three easy to understand steps: +- Fetch: get configuration and OCI images from various sources including a Git repository, a local ConfigMap, a Helm chart, an OCI registry, etc. +- Template: take user provided values to customize software using ytt templates, helm templates, and more +- Deploy: create/update resources on the cluster + +#### GitOps and Continuous Delivery +With its layered approach, kapp-controller can be used as: +- Continuous delivery for Kubernetes applications using [App CR](app-spec.md) +- Kubernetes Package Management using [Package CR and supplementary CRs](packaging.md) +- Managing applications and packages using GitOps + +#### Share software and build distributions +Use kapp-controller's Package Management features along with Carvel's imgpkg bundles to distribute Package Repositories that can be added to cluster to provide a catalog of software for users to install. Package Repositries can be automatically updated ensuring users always have access to latest versions of software. Package Repositories and Packages can also be relocated and run in air-gapped environments. + +#### Reliable and ready for production! +kapp-controller has been hardened and is in use on production Kubernetes clusters. Learn more through [case studies](/blog/casestudy-modernizing-the-us-army) on our blog. diff --git a/site/content/kapp-controller/docs/v0.31.0/air-gapped-workflow.md b/site/content/kapp-controller/docs/v0.31.0/air-gapped-workflow.md new file mode 100644 index 000000000..819f333eb --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/air-gapped-workflow.md @@ -0,0 +1,75 @@ +--- + +title: Install Packages in an air-gapped (offline) environment +--- + +The documentation below covers topics from the [imgpkg air-gapped workflow docs](/imgpkg/docs/latest/air-gapped-workflow) +more concisely in order to focus on applying these workflows to kapp-controller package repositories. + +## Scenario + +You have a [PackageRepository](packaging#packagerepository-cr) in an [imgpkg bundle format](/imgpkg/docs/latest/resources/#bundle) +in an external OCI registry that you would like to move into an OCI registry in an air-gapped environment. Once relocated, you would +like to deploy the bundle as part of a PackageRepository to a Kubernetes cluster. + +## Prerequisites + +In order to go through this process of moving an imgpkg bundle to an air-gapped environment, you will need to have [imgpkg](/imgpkg) +installed. More information on installing Carvel tools, including `imgpkg`, can be found [here](/#whole-suite). + +## Copy PackageRepository bundle to new location + +Most of the steps documented for the [imgpkg air-gapped workflow docs](/imgpkg/docs/latest/air-gapped-workflow#step-1-finding-bundle-in-source-registry) +still apply in the case of working with kapp-controller package repositories. A summary of these docs is that you will need to copy your package repository +bundle with `imgpkg` via one of the following options: + +- Option 1: From a common location connected to both registries. This option is more efficient because only changed image layers will be transfered between registries. +- Option 2: With intermediate tarball. This option works best when registries have no common network access. + +More detailed documents for [`Option 1`](/imgpkg/docs/latest/air-gapped-workflow/#option-1-from-a-location-connected-to-both-registries) and +[`Option 2`](/imgpkg/docs/latest/air-gapped-workflow/#option-2-with-intermediate-tarball) can be found at the attached links. + +A summary of steps for relocating a package repository bundle to an air-gapped environment are documented for both options below: + +For `Option 1`: +* Get to a location that can access both registries. If there is no such location, you will have to use `Option 2` steps. +* [Authenticate](/imgpkg/docs/latest/auth.md) with both source and destination registries +* Run `imgpkg copy -b index.docker.io/user1/simple-app-bundle:v1.0.0 --to-repo final-registry.corp.com/apps/simple-app-bundle` + +For `Option 2`: +* Get to a location that can access the source registry +* [Authenticate](/imgpkg/docs/latest/auth.md) with the source registry +* Run `imgpkg copy -b index.docker.io/user1/simple-app-bundle:v1.0.0 --to-tar /tmp/my-image.tar` +* Make sure the tar file is in a location that has access to the destination registry +* Authenticate with the destination registry +* Run `imgpkg copy --tar /tmp/my-image.tar --to-repo final-registry.corp.com/apps/simple-app-bundle` + +## Use Relocated Bundle or Image with PackageRepository + +Once you have relocated the package repository bundle into the destination OCI registry in your air-gapped environment, you can +now reference the relocated bundle in a PackageRepository definition: + +```yaml +--- +apiVersion: install.package.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: simple-package-repository +spec: + fetch: + imgpkgBundle: + image: final-registry.corp.com/apps/simple-app-bundle +``` + +In the event your PackageRepository needs authentication to pull the bundle, you can read more about kapp-controller's +[private authentication workflows using secretgen-controller](private-registry-auth.md) or [without secretgen-controller](private-registry-auth.md#packagerepository-authentication-without-secretgen-controller). + +After applying the PackageRepository definition above to your Kubernetes cluster, you will be able to check that the PackageRepository and +its associated Packages were successfully deployed by checking the PackageRepository status: + +```bash +$ kubectl get packagerepository/simple-package-repository +``` + +You will see a message of `Reconcile Succeeded` in the `DESCRIPTION` column of the output from `kubectl` if the PackageRepository was deloyed +successfully. You can also run `kubectl get packages` to see that all Packages were introduced successfully. diff --git a/site/content/kapp-controller/docs/v0.31.0/app-examples.md b/site/content/kapp-controller/docs/v0.31.0/app-examples.md new file mode 100644 index 000000000..41ec811e7 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/app-examples.md @@ -0,0 +1,108 @@ +--- + +title: Example Usage +--- + +Below are some example App CRs showing common ways our users have used App CRs. Full App CR spec can be found [here](app-spec.md). + +## Gitops with an app +In this example a user wants to keep their app up to date with changes to the source Git repo + +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: simple-app +spec: + serviceAccountName: default + fetch: + - git: + url: https://github.com/k14s/k8s-simple-app-example + ref: origin/develop + subPath: config-step-2-template + + template: + - ytt: {} + + deploy: + - kapp: {} +``` + +## Gitops with a Helm chart +In this example a user wants to keep their cluster up to date with the latest version of a Helm chart fetched from a Git repo +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: nginx-helm +pec: + fetch: + - git: + url: https://github.com/bitnami/charts + ref: origin/master + subPath: bitnami/nginx + + template: + - helmTemplate: + valuesFrom: + - secretRef: + name: nginx-values + + deploy: + - kapp: {} +``` + +## Install a Helm chart +In this example a user wants to keep their cluster up to date with the latest version of a Helm chart +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: concourse-helm +spec: + fetch: + - helmChart: + name: stable/concourse + + template: + - helmTemplate: + valuesFrom: + - secretRef: + name: concourse-values + + deploy: + - kapp: {} +``` + +## Customize a Helm chart by adding an overlay +In this example a user wants to use `helm template`, but then modify the resulting YAML by adding their own add their own `ytt overlay` +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: concourse-helm +spec: + fetch: + - git: + url: https://github.com/bitnami/charts + ref: origin/master + subPath: bitnami/nginx + + template: + - helmTemplate: {} + - ytt: + ignoreUnknownComments: true + inline: + paths: + remove-lb.yml: | + #@ load("@ytt:overlay", "overlay") + #@overlay/match by=overlay.subset({"kind":"Service","metadata":{"name":"nginx"}}) + --- + spec: + type: ClusterIP + #@overlay/remove + externalTrafficPolicy: + + deploy: + - kapp: {} +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/app-overview.md b/site/content/kapp-controller/docs/v0.31.0/app-overview.md new file mode 100644 index 000000000..aaee01a3c --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/app-overview.md @@ -0,0 +1,73 @@ +--- + +title: App CR High Level Overview +--- + +## Overview +kapp-controller provides a declarative way to install, manage, and upgrade applications on a Kubernetes cluster using the App CRD. Get started by installing the [latest release of kapp-controller](install.md). + +## App +An App is a set of Kubernetes resources. These resources could span any number of namespaces or could be cluster-wide (e.g. CRDs). An App is represented in kapp-controller using a App CR. + +The App CR comprises of three main sections: +- spec.fetch -- declare source for fetching configuration and OCI images +- spec.template -- declare templating tool and values +- spec.deploy -- declare deployment tool and any deploy specific configuration + +Full App CR spec can be found [here](app-spec.md). + +### spec.fetch + +App CR supports multiple source for fetching configuration and OCI images to give developers flexibility. + +- `inline`: specify one or more files within resource +- `imgpkgBundle`: download [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) from registry (available in v0.17.0+) +- `image`: download Docker image from registry +- `http`: download file at URL +- `git`: clone Git repository +- `helmChart`: fetch Helm chart from Helm repository + +For each fetch source, App CR supports specifying Secret resources that will be used for authenticating with the source. kapp-controller does not check for `type` value of Secret resource. + +#### `image` and `imgpkgBundle` authentication + +Allowed secret keys: + +- `username` and `password` +- `token`: Alternative to username/password authentication + +Also supports [dockerconfigjson secret type](https://kubernetes.io/docs/concepts/configuration/secret/#docker-config-secrets) (v0.19.0+) + + +#### `git` authentication + +Allowed secret keys: + +- `ssh-privatekey`: PEM-encoded key that will be provided to SSH +- `ssh-knownhosts`: Optional, set of known hosts allowed to connect (if not specified, all hosts are allowed) +- `username` and `password`: Alternative to private key authentication + +#### `http` and `helmChart` authentication + +Allowed secret keys: + +- `username` and `password` + + +### spec.template + +App CR supports multiple templating, overlaying, and data transformation tools to give developers flexibility. + +- `helmTemplate`: uses `helm template` command to render chart +- `ytt`: uses [ytt](/ytt) to rended templates +- `kbld`: uses [kbld](/kbld) to resolve image URLs to include digests +- `kustomize`: (not implemented yet) uses kustomize to render configuration +- `jsonnnet`: (not implemented yet) renders jsonnet files +- `sops`: uses [sops](https://github.com/mozilla/sops) to decrypt secrets. [More details](sops.md). Available in v0.11.0+. + +--- +### spec.deploy + +App CR uses Carvel's `kapp` CLI to deploy. + +- `kapp`: uses [kapp](/kapp) to deploy resources diff --git a/site/content/kapp-controller/docs/latest/app-spec.md b/site/content/kapp-controller/docs/v0.31.0/app-spec.md similarity index 99% rename from site/content/kapp-controller/docs/latest/app-spec.md rename to site/content/kapp-controller/docs/v0.31.0/app-spec.md index 96e46426d..6d94de1a3 100644 --- a/site/content/kapp-controller/docs/latest/app-spec.md +++ b/site/content/kapp-controller/docs/v0.31.0/app-spec.md @@ -1,4 +1,5 @@ --- + title: App CR spec --- @@ -37,7 +38,7 @@ spec: # by default, cluster is same as where this resource resides (optional; v0.5.0+) cluster: # specifies kapp namespace in destination cluster (optional) - # see {{< ref "/kapp/docs/latest/state-namespace" >}} + # see {{< ref "/kapp/docs/v0.44.0/state-namespace" >}} namespace: ns2 # specifies secret containing kubeconfig (required) kubeconfigSecretRef: diff --git a/site/content/kapp-controller/docs/v0.31.0/controller-config.md b/site/content/kapp-controller/docs/v0.31.0/controller-config.md new file mode 100644 index 000000000..323a43a1e --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/controller-config.md @@ -0,0 +1,88 @@ +--- + +title: Configuring the Controller +--- + +kapp-controller exposes the ability to configure the controller via a +Secret (available in v0.22.0+) or ConfigMap (available in v0.14.0+), +which kapp controller will look for and apply as part of its [startup processes](startup.md). + +The controller configuration was originally only available in a ConfigMap +format, but as of v0.22.0 it is recommended to use a Secret since there +may be sensitive information stored in the config (e.g. proxy information including passwords). + +## Controller Configuration Spec + +```yaml +apiVersion: v1 +kind: Secret +metadata: + # Name must be `kapp-controller-config` for kapp controller to pick it up + name: kapp-controller-config + + # Namespace must match the namespace kapp-controller is deployed to + namespace: kapp-controller + +stringData: + # A cert chain of trusted ca certs. These will be added to the system-wide + # cert pool of trusted ca's (optional) + caCerts: | + -----BEGIN CERTIFICATE----- + MIIEXTCCAsWgAwIBAgIQDqAvoGhrmyB/EvhjT/efWzANBgkqhkiG9w0BAQsFADA4 + MQwwCgYDVQQGEwNVU0ExFjAUBgNVBAoTDUNsb3VkIEZvdW5kcnkxEDAOBgNVBAMT + B2Jvc2gtY2EwHhcNMjAxMjIzMTY1OTAxWhcNMjExMjIzMTY1OTAxWjA4MQwwCgYD + VQQGEwNVU0ExFjAUBgNVBAoTDUNsb3VkIEZvdW5kcnkxEDAOBgNVBAMTB2Jvc2gt + Y2EwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCsMTj5yHLez8jzONu1 + tv+u0dqzt8UdWCtUtHCDkIiNJIcB3PkGG7x/LvZ0bMydWeFcBq0g15tfG6N6vHnF + 4p2E9nSe0XjEEnxEkmtdpoFVPZdHTBgc6H5LOMshPH1ARWpuvBnDb87oVinIZBaf + 7BjhUQcRoGtsomk/R9Ke9FB4rMZUfuY/7CC8lDyP5Y02VeTAUimK6/WfDh3VPB3e + vQfXKJY0Ba5s43fIdudV+fcuKDut01oKmiBL6IHLRSrZKta5mg4fgimst6nJ4xvU + SWqYWS4yMxf6pOrTHPjbKUqXqbK4Reh+oQoE12WJZ3NvXr1GoDzt1xzTNzUpUVws + nQm5Fo9H07mkjKeu8gOrOBQ2FqaK+eZ5FFNV7kToVQj2KVTEbLLcTrF454jhsoSd + EOlqVUjtfxGz0dGEuy+IgMvSSjtky7eI08jdBWMiOThQvR3n0Q6TXF/wBwCEfgDa + 4eVeziaUGPXUsefR2+2ZCQ6Z31SmtUGECciCKmKtZTekKCUCAwEAAaNjMGEwDgYD + VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFDwRpmIKYZvr + lKqROus2Ae6gSKkDMB8GA1UdIwQYMBaAFDwRpmIKYZvrlKqROus2Ae6gSKkDMA0G + CSqGSIb3DQEBCwUAA4IBgQA/LX15Qb7v/og06XB27TPl9StGBiewrb0WdHEz9H16 + eN926TwxWKUr6QcbGg6UbNfLUfMC3VicCDMTQCSNhBTUXm+4pKcJsTyM9/Sk/e4U + 5+l3FTgxXs+3mEoYJy16QlkU1XDr1q6Myo9Kc38d1yUW9OPxBV4Ur3+12uk5ElSC + jZu7l+ox2FLds1TmYBhRR/2Jdbm5aoamh4FVpkmDgGedjERREymvnOIMkhWyUfWE + L8Sxa2d8427cBieiEP4foLgjWKr2+diCDrBymU/pz/ZMRRpvUc2uFV005/vmDedK + xQACQ8ZWBYWzNCV4C0Y5AS1PETxbocZ09Yw6K1XyVveEp8aQ/ROMkAUOObhMD45W + GZNwewGU/V7kclDgMwq6R1VXr5R7NtK9V96vi6ZaujoJKvF1PFpZ/IHWcfFkpVoy + Fu8L5PIkg4weBW+87kp+CCseEXPUplpqQCAnmVJdvilK6vgKc7T+vzbET8LNw7NX + mHOVA3CR2w+yUhN4uiCI1aY= + -----END CERTIFICATE----- + + # The url/ip of a proxy for kapp controller to use when making network + # requests (optional) + httpProxy: proxy-svc.proxy-server.svc.cluster.local:80 + + + # The url/ip of a tls capable proxy for kapp controller to use when + # making network requests (optional) + httpsProxy: "" + + # A comma delimited list of domain names which kapp controller should + # bypass the proxy for when making requests (optional) + noProxy: "github.com,docker.io" + + # A comma delimited list of domain names for which kapp controller, when + # fetching images or imgpkgBundles, will skip TLS verification. (optional) + dangerousSkipTLSVerify: "private-registry.com,insecure-registry.com" +``` + +## Config Shorthands + +kapp-controller v0.30.0+ supports a shorthand for easily adding the `KUBERNETES_SERVICE_HOST` +environment variable to kapp-controller's `noProxy` controller config property. This can help +when a Kubernetes cluster is configured with a proxy and the kapp-controller-config is created +with the http and https proxy URL. In this case, kapp-controller fails to communicate with the +Kubernetes API server. + +To make this configuration simpler, the `noProxy` property will interpret `KAPPCTRL_KUBERNETES_SERVICE_HOST` +as the value of `KUBERNETES_SERVICE_HOST` (typically 10.96.9.1) environment variable in the kapp-controller pod. + +```yaml +noProxy: "github.com,docker.io,KAPPCTRL_KUBERNETES_SERVICE_HOST" +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/debugging-crs.md b/site/content/kapp-controller/docs/v0.31.0/debugging-crs.md new file mode 100644 index 000000000..0f8736601 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/debugging-crs.md @@ -0,0 +1,126 @@ +--- + +title: Debugging CRs +--- + +Running into issues deploying any of the kapp-controller CRs? This page will help with commonly encountered issues. + +If you can't find what you are looking for here, please reach out to us on #carvel. We love hearing from users and are keen to help you resolve any issues! + +## Debugging kapp-controller CRs + +### Reconcile failed +Your first alert to a failure +will come from the tool(s) (e.g. kapp or kubectl) you are using to deploy +kapp-controller CRs. `kapp` will more immediately tell you if a resource you are +creating or updating fails, but you will need to verify with a `kubectl get` if +using `kubectl` to create/update. + +You can verify a failure occurred by running a `kubectl get` for the resource +you encountered the failure with. You can then see in the `DESCRIPTION` column +of the output of `kubectl get` if the reconciliation process for the resource +failed. An example of this is below: + +``` +NAMESPACE NAME PACKAGE NAME PACKAGE VERSION DESCRIPTION AGE +foo instl-pkg-test-fail pkg.fail.carvel.dev 1.0.0 Reconcile failed: Error (see .status.usefulErrorMessage for details) 12s +``` + +### status.usefulErrorMessage +Once you have confirmed an error occurred, you can review the status of the CR +for more information. + +Apps, PackageInstalls, and PackageRepsitories all feature a status property +named `usefulErrorMessage`. `usefulErrorMessage` which contains an error message +from kapp-controller or the stderr from the underlying tool used by +kapp-controller (i.e. vendir, imgpkg, kbld, ytt, kapp, or helm). + +`usefulErrorMessage` will be located at the bottom of the statuses if running a +`kubectl get` or `kubectl describe` to view more information about a failure. + +`usefulErrorMessage` can also be accessed more directly through `kubectl` like +in the following examples: + +``` +# App errors +$ kubectl get apps/simple-app -o=jsonpath={.status.usefulErrorMessage} -n namespace + +# PackageInstall errors +$ kubectl get packageinstall simple-app -o=jsonpath={.status.usefulErrorMessage} -n namespace + +# PackageRepository errors (cluster scoped so no namespace) +$ kubectl get packagerepository repo -o=jsonpath={.status.usefulErrorMessage} +``` + +### Errors from underlying tools (App CR and PackageInstall CR) + +Failures can arise from fetch, template, deploy, or delete steps for an App CR. +These failures correspond to issues with runtime information declared in the App +CR's spec. kapp-controller creates an App CR for every PackageInstall + +Errors are reported as stderr from associated tools used in kapp-controller +(i.e. vendir, imgpkg, kbld, ytt, kapp, and helm) or as direct messages from +kapp-controller (e.g. when an App uses a ServiceAccount that doesn't exist). + +When a failure occurs with an App CR, you can find further details in the App +CR's `DESCRIPTION` column by running `kubectl get apps/simple-app -n namespace`: + +``` +NAME DESCRIPTION SINCE-DEPLOY AGE +simple-app Delete failed: Preparing kapp: Getting service account: serviceaccounts "default-ns-sa" not found 3s 56m +``` + +In the case above, the error message shown is coming directly from +kapp-controller, so all the information for the failure should be presented in +the description column. This commonly occurs when references used by +kapp-controller (e.g. secrets, configmaps, serviceaccounts) are not found by +kapp-controller. + +In cases where the error message does not originate from kapp-controller (e.g. a +failed fetch event for a git repository), the stderr from the underlying tool +(i.e. vendir, imgpkg, kbld, ytt, kapp, and helm) is shown in the App's status. + +In the App status, there is a field called `usefulErrorMessage` that displays +the stderr for a failure during App reconciliation. + +This `usefulErrorMessage` field can be found by running `kubectl get +apps/simple-app -o=jsonpath={.status.usefulErrorMessage} -n namespace`. The +kubectl command will return the stderr output from the App status to help you +further understand the reason for the App failure. + +The `usefulErrorMessage` can be helpful in pointing out where errors occurred +from inputs in the App spec and also pinpoint the resource that caused a +deployment failure. However, Apps will not surface errors of resources they are +deploying to Kubernetes and further debugging of resources deployed by an App +may be needed. + +### Debugging PackageInstall CRs + +Failures for PackageInstalls can be viewed directly via the `usefulErrorMessage` +property of the PackageInstall's status. This `usefulErrorMessage` property +comes from an App CR that is created as a result of creating a PackageInstall. +More information on interpreting the error message from `usefulErrorMessage` can +be found under the [Errors from underlying tools](#Errors from underlying tools (App CR and PackageInstall CR)). The underlying App +CR will have the same name as the PackageInstall that you create. + +You can also inpect the Package CR referenced by the PackageInstall CR for issues. You can view the Package details by running the following command: + +``` +$ kubectl describe package/ +``` + +You can then view the `.template.spec` of the Package to see if there are any +issues with the inputs of the Package. These inputs are eventually used to +create the App for the PackageInstall and can lead to failures. + +### Debugging PackageRepository CRs + +Failures for PackageRepositories can be viewed directly via the +`usefulErrorMessage` property of the PackageRepository's status. More information [here](status.usefulErrorMessage) + +A common source of errors is being unable to fetch the PackageRepository +contents. Please check the `.spec.fetch` portion of the PackageRepository spec for issues related to this. + +Is the registry you are fetching from require authentication? If so, check out [authenticating to private registries](private-registry-auth.md) + +You can also fetch the PackageRepository `imgpkg` bundle or image separately and inspect format of Package resources. diff --git a/site/content/kapp-controller/docs/v0.31.0/debugging-kc.md b/site/content/kapp-controller/docs/v0.31.0/debugging-kc.md new file mode 100644 index 000000000..60e8751e6 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/debugging-kc.md @@ -0,0 +1,13 @@ +--- + +title: Debugging kapp-controller +--- + +The following flags can be used to debug the kapp-controller deployment. Use of these flags are **strongly discouraged in a production setting**. + +## `--dangerous-enable-pprof=true` + +This flag enables [Go's pprof server](https://golang.org/pkg/net/http/pprof/) within kapp-controller process. It runs on `0.0.0.0:6060`. It allows to inspect running Go process in various ways, for example: + +- list goroutines: `http://x.x.x.x/debug/pprof/goroutine?debug=2` +- collect CPU samples: `go tool pprof x.x.x.x/debug/pprof/profile?seconds=60` (useful commands: top10, tree) diff --git a/site/content/kapp-controller/docs/v0.31.0/dev.md b/site/content/kapp-controller/docs/v0.31.0/dev.md new file mode 100644 index 000000000..90623a017 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/dev.md @@ -0,0 +1,24 @@ +--- + +title: Development & Deploy +--- + +Install ytt, kbld, kapp beforehand (https://carvel.dev). + +``` +./hack/build.sh # to build locally + +# add `-v image_repo=docker.io/username/kapp-controller` with your registry to ytt invocation inside +./hack/deploy.sh # to deploy + +export KAPPCTRL_E2E_NAMESPACE=kappctrl-test +./hack/test-all.sh +``` + +## Release + +``` +# Bump version in cmd/controller/main.go +# Commit +./hack/build-release.sh +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/faq.md b/site/content/kapp-controller/docs/v0.31.0/faq.md new file mode 100644 index 000000000..c6ee48e5e --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/faq.md @@ -0,0 +1,97 @@ +--- + +title: FAQ +--- + +## App CR + +This section covers questions for users directly using the [App](app-spec.md) +custom resource. + +### How can I control App CR reconciliation (pause, force, adjust frequency...)? + +You can set and unset spec.paused +([example](https://github.com/vmware-tanzu/carvel-kapp-controller/blob/d94984a77fa907ac5ecc681e9a842b9877766a6b/test/e2e/pause_test.go#L91)) +or fiddle with spec.syncPeriod ([example]( +https://github.com/vmware-tanzu/carvel-kapp-controller/blob/d94984a77fa907ac5ecc681e9a842b9877766a6b/test/e2e/app_secret_configmap_reconcile_test.go#L133)), which +defaults to 30 seconds. + +### How can I tell which version of kapp-controller is installed? + +kapp-controller sets the annotation `kapp-controller.carvel.dev/version` on the deployment to the version deployed, +so e.g. `kubectl describe deployment kapp-controller -n kapp-controller | grep version` will show the installed version. + +## Package Management CRs + +This section covers questions for users directly using the [Package Management CRs](packaging.md) +custom resource. + +### How does kapp-controller handle PackageInstall when a PackageRepository is removed from the cluster? + +If a PackageInstall has been installed successfully from a Package that is part +of a PackageRepository, and if that PacakgeRepository is ever deleted after the +successful install, the PackageInstall will eventually report the following +error: `Reconcile failed: Expected to find at least one version, but did not`. +This error occurs due to the regular syncing of a PackageInstall with its +Package. + +Even though the error above is reported, the Package will still be installed and +should work as expected. It can also still be uninstalled by deleting the +PackageInstall. The PackageRepository can be recreated and the PackageInstall +will sync and reconcile without any updates needed to resolve the error. + +### How can I generate the valuesSchema from my ytt schema? + +If you are using `ytt` as your Package's templating option and have [defined a schema](../../../ytt/docs/latest/how-to-write-schema), you can use `ytt` to generate your `valuesSchema` (which is in OpenAPI v3 format) for you. + +This is the recommended workflow: + +1. Create an OpenAPI Document from a Data Values Schema file: + + ```bash + $ ytt -f schema.yml --data-values-schema-inspect -o openapi-v3 >schema-openapi.yml + ``` + + which will produce... + + ```yaml + #! schema-openapi.yml + openapi: 3.0.0 + info: + version: 1.0.0 + title: Openapi schema generated from ytt schema + paths: {} + components: + schemas: + dataValues: + type: object + properties: + namespace: + type: string + default: fluent-bit + ``` + +2. Turn your Package CR into a `ytt` template, so that you can insert the schema definition in the right spot, automatically: + + `package-template.yml` + ```yaml + #@ load("@ytt:data", "data") + #@ load("@ytt:yaml", "yaml") + ... + kind: Package + spec: + valuesSchema: + openAPIv3: #@ yaml.decode(data.values.openapi)["components"]["schemas"]["dataValues"] + ... + ``` + + and render with the output from the ytt schema inspect: + + ```bash + $ ytt -f package-template.yml --data-value-file openapi=schema-openapi.yml > package.yml + ``` + +For more details, see: +- [ytt: Export Schema in OpenAPI Format](../../../ytt/docs/latest/how-to-export-schema.md). +- [ytt: Configuring Data Values via command line flags](../../../ytt/docs/latest/ytt-data-values.md#configuring-data-values-via-command-line-flags) +- [@ytt:yaml module](../../../ytt/docs/latest/lang-ref-ytt.md#yaml) diff --git a/site/content/kapp-controller/docs/v0.31.0/install.md b/site/content/kapp-controller/docs/v0.31.0/install.md new file mode 100644 index 000000000..e582a4bf3 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/install.md @@ -0,0 +1,76 @@ +--- + +title: Install +--- + +Grab the latest copy of YAML from the [Releases page](https://github.com/vmware-tanzu/carvel-kapp-controller/releases) and use your favorite deployment tool (such as [kapp](/kapp) or kubectl) to install it. + +Example: + +```bash +$ kapp deploy -a kc -f https://github.com/vmware-tanzu/carvel-kapp-controller/releases/latest/download/release.yml +``` + +or + +```bash +$ kubectl apply -f https://github.com/vmware-tanzu/carvel-kapp-controller/releases/latest/download/release.yml +``` + +## Specific Environments and Distributions +Some kubernetes distributions require specific setup. +Notes below capture the wisdom of our collective community - we +appreciate your corrections and contributions to help everyone install +kapp-controller everywhere. + +### Openshift +1. Explicitly set resource packageinstalls/finalizers for kapp controller cluster role to access (else the kapp controller fails to create packageinstalls). +``` +kind: ClusterRole +metadata: + name: kapp-controller-cluster-role +rules: +- apiGroups: + - packaging.carvel.dev + resources: + ... + - packageinstalls/finalizers +``` +2. Bind the kapp-controller cluster role to a security context constraint that allows uids/gids that kapp deployment uses +(currently uid 1000; value given for `runAsUser` in the release.yaml for your +version of kapp-controller). +The security context constraint you provide should allow kapp-controller's uid +to run and should not have root privileges. +``` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kapp-controller-cluster-role +rules: +- apiGroups: + - security.openshift.io + resourceNames: + - my-nonroot-security-context-contstraint + resources: + - securitycontextconstraints + verbs: + - use +``` +3. Set the `IMGPKG_ENABLE_IAAS_AUTH` [environment + variable](/imgpkg/docs/latest/auth/#via-iaas) to false. + + +### Kubernetes versions < 1.20 +Starting in kapp-controller 0.31.0 we have upgraded our underlying kubernetes +libraries which will try to use APIs that don't exist on clusters v1.19 and +earlier. + +Those using k8s v1.19 and earlier will see a repeating error message such as the one below, because +our libraries are hardcoded to watch `v1beta1.PriorityLevelConfiguration` and that won't exist on your cluster. +``` +k8s.io/client-go@v0.22.4/tools/cache/reflector.go:167: Failed to watch *v1beta1.PriorityLevelConfiguration: failed to list *v1beta1.PriorityLevelConfiguration: the server could not find the requested resource (get prioritylevelconfigurations.flowcontrol.apiserver.k8s.io) +``` +While kapp-controller will still work, your logs may fill at a remarkable pace. + +To disable these APIs, set the deployment config variable +`enable_api_priority_and_fairness` to false. diff --git a/site/content/kapp-controller/docs/v0.31.0/oss-packages.md b/site/content/kapp-controller/docs/v0.31.0/oss-packages.md new file mode 100644 index 000000000..4012235a5 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/oss-packages.md @@ -0,0 +1,24 @@ +--- + +title: OSS Carvel Packages +--- + +This page provides a list of Carvel Packages and Package Repositories that are available to open source users. + +Do you have a Package or Package Repository you'd like to add to this list? Please make a PR with details to our [docs](https://github.com/vmware-tanzu/carvel/main/site/content/kapp-controller/docs/latest/oss-packages.md). + +## Tanzu Community Edition +Tanzu Community Edition provides several open source [Carvel Packages](https://tanzucommunityedition.io/packages/). These are actively contributed to and maintained by contributors to Tanzu Community Edition. A list of the Package CRs can be found [here](https://github.com/vmware-tanzu/community-edition/tree/main/addons/packages). You can add the Package Repository to your cluster by creating a PackageRepository CR. + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: tce-repo +spec: + fetch: + imgpkgBundle: + # Check out the latest version from Tanzu Community Edition docs + image: projects.registry.vmware.com/tce/main:0.9.1 +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/package-consumer-concepts.md b/site/content/kapp-controller/docs/v0.31.0/package-consumer-concepts.md new file mode 100644 index 000000000..4bbe1c1c3 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/package-consumer-concepts.md @@ -0,0 +1,166 @@ +--- + +title: Concepts for Package Consumers +--- + +## Namespacing + +### Overview + +In the packaging APIs, all the CRs are namespaced, which can create a lot of +duplication when wanting to share packages across the cluster. To account for +this, kapp-controller accepts a flag `-packaging-global-namespace`, which +configures kapp-controller to treat the provided namespace as a global namespace +for packaging resources. This means any Package and PackageMetadata CRs within +that namespace will be included in all other namespaces on the cluster, without +duplicating them. This does not apply to PackageRepositories or PackageInstalls. + +### Collisions + +When there is a conflict, the locally namespaced resources will take precedence +over the global ones. A conflict for Packages is defined as having the same +`spec.refName` and `spec.version`, while for PackageMetadatas it is defined as +having the same `metadata.name`. For example, if there is a globally available +PackageMetadata created from the following YAML: + +```yaml +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + name: simple-app.corp.com + namespace: +spec: + categories: + - demo + displayName: Simple App + longDescription: Simple app consisting of a k8s deployment and service + shortDescription: Simple app for demoing +``` + +and then a new locally available PackageMetadata is created from this YAML, + +```yaml +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + name: simple-app.corp.com + namespace: +spec: + categories: + - demo + displayName: Simple App Override + longDescription: My locally available version of the Simple App package + shortDescription: Simple App with some edits +``` + +a user listing the PackageMetadatas will see the second CR, and not the first. + +### Annotations + +For client discoverability, the namespace should also be present as an +annotation on the PackageRepository CRD under the +`packaging.carvel.dev/global-namespace`. Kapp controller's release +YAML comes preconfigured with this annotation. + +(upcoming) If users would like to exclude the global packages from their namespace, the +annotation `packaging.carvel.dev/exclude-global-packages` can be added to +the namespace. + +## Using PackageInstall's Version Selection + +The following sections cover aspects of how to approach version selection when using PackageInstalls. + +### Constraints + +PackageInstalls offer a property called `constraints` under +`.spec.packageRef.versionSelection`. This `constraints` property can be +used to select a specific version of a Package CR to install or include a set of +conditions to pick a version. This `constraints` property is based on semver +ranges and more details on conditions that can be included with `constraints` +can be found [here](https://github.com/blang/semver#ranges). + +To select a specific version of a Package CR to use with a PackageInstall, the +full version (i.e. `.spec.version` from a Package CR) can be included in the +`constraints` property, such as what is shown below: + +```yaml +packageRef: + refName: fluent-bit.vmware.com + versionSelection: + constraints: "1.5.3" +``` + +The example above will result in version 1.5.3 of the Package being installed. + +An example of using a condition to select a Package CR with `constraints` is shown below: + +```yaml +packageRef: + refName: fluent-bit.vmware.com + versionSelection: + constraints: ">1.5.3" +``` + +The above constraint will result in any version greater than `1.5.3` of the +Package being installed. It will also automatically update to the latest +versions of the Package as they become available on the cluster. + +### Prereleases + +When creating a PackageInstall, by default prereleases are not included by +kapp-controller when considering which versions of a Package CR to install. To +include prereleases when creating a PackageInstall, the following can be +added to the spec: + +```yaml +versionSelection: + constraints: "3.0.0-rc.1" + prereleases: {} +``` + +Specifying `prereleases: {}` will make kapp-controller consider all available +prereleases when seeing if a Package CR is available to be installed. + +To filter by releases containing certain substrings, there is an `identifiers` +property under `prereleases` that can be used to only include certain +prereleases that contain the identifier, such as what is shown below: + +```yaml +versionSelection: + constraints: "3.0.0" + prereleases: + identifiers: [rc] +``` + +Multiple identifiers can be specified to include multiple types of pre-releases +(e.g. `identifiers: [rc, beta]`). + +### Downgrading + +In v0.25.0+ of kapp-controller, PackageInstalls feature an annotation to allow +PackageInstalls to be downgraded to previous versions of a Package. By default, +kapp-controller does not allow downgrading to a previous version of a Package to +protect against certain scenarios (e.g. the latest version of a Package being removed +resulting in a unintended reconciliation where the PackageInstall picks up a lower +Package version that is now the latest version). + +If downgrading to a previous version is desired, adding the annotation +`packaging.carvel.dev/downgradable: ""` to a PackageInstall will allow for +explicit or automated ways of downgrading the PackageInstall to a lower version. + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: pkg-demo + annotations: + packaging.carvel.dev/downgradable: "" +spec: + packageRef: + refName: simple-app.corp.com + versionSelection: + constraints: >=1.0.0 +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/package-install-extensions.md b/site/content/kapp-controller/docs/v0.31.0/package-install-extensions.md new file mode 100644 index 000000000..48c2ff050 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/package-install-extensions.md @@ -0,0 +1,65 @@ +--- + +title: Overlays with PackageInstall +--- + +PackageInstalls expose the ability to customize package installation using +annotations recognized by kapp-controller. + +## Adding Paths to YTT (overlays) + +Since it is impossible for package configuration and exposed data values to meet +every consumer's use case, we have added an annotation which enables +consumers to extend the package configuration with custom ytt paths. The most +likely use case for this is providing overlays to tweak configuration that is +not exposed via data values, but it can be used to provide any kind of ytt file. + +The extension annotation is called `ext.packaging.carvel.dev/ytt-paths-from-secret-name` +and can be suffixed with a `.X`, where X is some number, to allow for specifying +it multiple times. For example, + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: fluent-bit + namespace: my-ns + annotations: + ext.packaging.carvel.dev/ytt-paths-from-secret-name.0: my-overlay-secret +spec: + serviceAccountName: fluent-bit-sa + packageRef: + refName: fluent-bit.vmware.com + versionSelection: + constraints: ">v1.5.3" + prereleases: {} + values: + - secretRef: + name: fluent-bit-values + +``` + +will include the overlay stored in the secret `my-overlay-secret` during the +templating steps of the package. This will allow users to further customize a +package installation in advanced cases. + +Example secret resource with a ytt overlay that adds a label to all Namespaces added by this package: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-overlay-secret + namespace: my-ns +stringData: + add-ns-label.yml: | + #@ load("@ytt:overlay", "overlay") + #@overlay/match by=overlay.subset({"kind":"Namespace"}),expects="1+" + --- + metadata: + #@overlay/match missing_ok=True + labels: + #@overlay/match missing_ok=True + custom-lbl: custom-lbl-value +``` + diff --git a/site/content/kapp-controller/docs/v0.31.0/packaging-artifact-formats.md b/site/content/kapp-controller/docs/v0.31.0/packaging-artifact-formats.md new file mode 100644 index 000000000..21e03299b --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/packaging-artifact-formats.md @@ -0,0 +1,66 @@ +--- + +title: Artifact formats +--- + +## Package Contents Bundle + +A package bundle is an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) that +holds package contents such as Kubernetes YAML configuration, ytt templates, +Helm templates, etc. + +Filesystem structure used for package bundle creation: + +```bash +my-pkg/ +└── .imgpkg/ + └── images.yml +└── config/ + └── deployment.yml + └── service.yml + └── ingress.yml +``` + +- `.imgpkg/` directory (required) is a standard directory for any imgpkg bundle + - `images.yml` file (required) contains container image refs used by configuration (typically generated with `kbld`) +- `config/` directory (optional) should contain arbitrary package contents such as Kubernetes YAML configuration, ytt templates, Helm templates, etc. + - Recommendations: + - Group Kubernetes configuration into a single directory (`config/` is our + recommendation for the name) so that it could be easily referenced in the + Package CR (e.g. using `ytt` template step against single directory) + +See [Creating a package](packaging-tutorial.md#creating-a-package) for example creation steps. + +## Package Repository Bundle + +A package repository bundle is an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) that holds PackageMetadata and Package CRs. + +Filesystem structure used for package repository bundle creation: + +```bash +my-pkg-repo/ +└── .imgpkg/ + └── images.yml +└── packages/ + └── simple-app.corp.com + └── metadata.yml + └── 1.0.0.yml + └── 1.2.0.yml +``` + +- `.imgpkg/` directory (required) is a standard directory for any imgpkg bundle + - `images.yml` file (required) contains package bundle refs used by Package CRs (typically generated with `kbld`) +- `packages/` directory (required) should contain zero or more YAML files describing available packages + - Each file may contain one or more PackageMetadata or Package CRs (using standard YAML document separator) + - Files may be grouped in directories or kept flat + - File names do not have any special meaning + - Recommendations: + - Separate packages in to directories with the package name + - Keep each PackageMetadata CR in a `metadata.yml` file in the package's + directory. + - Keep each versioned package in a file with the version name inside the package's + directory + - Always have a PackageMetadata CR if you have Package CRs + +See [Creating a Package Repository](packaging-tutorial.md#creating-a-package-repository) for example creation steps. + diff --git a/site/content/kapp-controller/docs/v0.31.0/packaging-gitops.md b/site/content/kapp-controller/docs/v0.31.0/packaging-gitops.md new file mode 100644 index 000000000..2ebd2a281 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/packaging-gitops.md @@ -0,0 +1,250 @@ +--- + +title: Package Management with GitOps +--- + +As you begin working with the [package management APIs](packaging.md) for kapp-controller, you may +be wondering how to use kapp-controller's gitops features to manage kapp-controller packages. This +section will cover an example gitops workflow using kapp-controller's package management resources. + +### GitOps Scenario + +An example gitops scenario with kapp-controller could be that a user wants to install a subset of +[Packages](packaging.md#package) from a [PackageRepository](packaging.md#package-repository). The +user wants to define which PackageRepository and Packages to install by defining these resources in +a git repository. With this approach, a user can manage resources in a declarative fashion and track +the history of changes to a Kubernetes cluster. + +After making changes to the main branch of the git repository, the user expects that kapp-controller +will sync these resources to the Kubernetes cluster where kapp-controller is installed. The user also +expects kapp-controller to sync these resources based on updates (e.g. PackageRepository or Package version upgrades) +to the main branch of the git repository. + +### Package Management GitOps Example + +To start, a user should already have kapp-controller installed on a Kubernetes cluster and have +a git repository available. + +First, a user can start by defining an [App custom resource](app-overview.md) like below. **NOTE:** +A user will also need to create a serviceaccount with associated RBAC permissions for the App to use. + +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: pkg-gitops-example + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-rule.create-order: "upsert after upserting rbac" + kapp.k14s.io/change-rule.delete-order: "delete before deleting rbac" +spec: + serviceAccountName: pkg-gitops-app-sa + fetch: + - git: + url: https://github.com/user/my-pkg-gitops-repo + ref: origin/main + subPath: packaging + + template: + - ytt: {} + + deploy: + - kapp: {} +``` + +The App will be pointed at the git repository branch where kapp-controller resources +(e.g. PackageRepository and Packages) will be defined. Read more on setting the App +up with a private git repository [here](app-overview.md#git-authentication). + +By default, an App custom resource will sync the cluster with its fetch source every +30 seconds to prevent the cluster state from drifting from its source of truth, which +is a git repository in this case. + +**NOTE:** The App should be managed separately from any additional kapp-controller resources +stored in a git repository for a gitops workflow. One potential example could be storing the +App definition and associated RBAC in the same git repository it is fetching from and have a +CI/CD process redeploy only the App when a change is made to the App itself versus other resourcees +in the repository. For simplicity in the example above, the user is deploying the App with +`kubectl` or `kapp` manually. + +After creating the App, a user can define a PackageRepository like below: + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: tce-repository + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-group: "tce-repo" +spec: + fetch: + imgpkgBundle: + image: projects.registry.vmware.com/tce/main:0.10.0 +``` + +This PackageRepository will install the [Tanzu Community Edition](https://tanzucommunityedition.io/) +packages on the cluster where kapp-controller is installed. + +Next, a user can pick which Packages to install on the cluster by defining [PackageInstall](packaging.md#packageinstall) +resources. The Tanzu Community Edition repository offers several Packages that are documented +[here](https://tanzucommunityedition.io/docs/latest/package-management/). + +For our gitops example, let's say the user is installing [cert-manager](https://cert-manager.io/docs/) +and [contour](https://projectcontour.io/) on the cluster. To do this, a user could define the following +PackageInstall along with associated RBAC. **NOTE:** The example below gives cluster admin permissions +to the serviceaccount. Make sure to assess appropriate RBAC needed for your specific PackageInstalls. + +RBAC: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pkg-gitops-pkgi-sa + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-group: "packageinstall-setup" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-admin-cluster-role + annotations: + kapp.k14s.io/change-group: "packageinstall-setup" +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-admin-cluster-role-binding + annotations: + kapp.k14s.io/change-group: "packageinstall-setup" +subjects: +- kind: ServiceAccount + name: pkg-gitops-pkgi-sa + namespace: pkg-gitops +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin-cluster-role +``` + +PackageInstalls: + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: cert-manager + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-group: "cert-manager" + kapp.k14s.io/change-rule.create-order: "upsert after upserting packageinstall-setup" + kapp.k14s.io/change-rule.delete-order: "delete before deleting packageinstall-setup" +spec: + serviceAccountName: pkg-gitops-pkgi-sa + packageRef: + refName: cert-manager.community.tanzu.vmware.com + versionSelection: + constraints: 1.5.4 +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: contour + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-rule.create-order: "upsert after upserting cert-manager" + kapp.k14s.io/change-rule.delete-order: "delete before deleting packageinstall-setup" +spec: + serviceAccountName: pkg-gitops-pkgi-sa + packageRef: + refName: contour.community.tanzu.vmware.com + versionSelection: + constraints: 1.17.1 +``` + +The structure of the git repository might look like the example below. In this structure, +the App definition is stored under a folder called app. The App definition above uses a +property called `subPath` to tell kapp-controller to only fetch and sync resources found +under the packaging folder of this git repository. The packaging folder will contain the +PackageRepository, PackageInstalls, and associated RBAC. + +``` +. +├── app +│ ├── app.yml +│ ├── rbac.yml +└── packaging + ├── packageinstalls.yml + ├── rbac.yml + └── repo.yml +``` + +The user can then check in and commit the App, PackageRepository, RBAC, and PackageInstalls to +the main branch of the git repository and push up the resources. + +To deploy the PackageRepository, RBAC, and PackageInstalls, create the App by running the following +commands: + +```shell +# Use kubectl +kubectl apply -f app/ +# Use kapp +kapp deploy -a pkg-gitops -f app/ +``` + +Once committed, the App custom resource created will create the PackageRepository, RBAC, and PackageInstalls +on the cluster. + +The user can view the status of the deployment through the App as well by running the following: + +```shell +kubectl get apps/pkg-gitops-example -n pkg-gitops +``` + +When the App's status is `Reconcile succeeded`, cert-manager and contour should be installed on the +cluster. This can be verified by running the following command: + +```shell +kubectl get pkgi -n pkg-gitops +``` + +### Making an Update + +When it's time to make an update to Packages installed on your cluster, a user can simply +open a pull request to the main branch of the git repositoy, make necessary changes in the +pull request review, and then merge when ready to introduce the change to the cluster. + +To expand on the example above, a user may want to upgrade contour to a later version (e.g. 1.17.2). +To do this, check out the git repository, edit the version used for the PackageInstall like below, +and then commit the change to the main branch. + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: contour + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-rule.create-order: "upsert after upserting cert-manager" + kapp.k14s.io/change-rule.delete-order: "delete before deleting packageinstall-setup" +spec: + serviceAccountName: pkg-gitops-pkgi-sa + packageRef: + refName: contour.community.tanzu.vmware.com + versionSelection: + constraints: 1.17.2 +``` + +Once committed, the App custom resource will eventually sync in the changes. The change can be +verified by running the following command and checking in the kubectl output that the contour +PackageInstall is now using version 1.17.2: + +```shell +kubectl get pkgi/contour -n pkg-gitops +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/packaging-tutorial.md b/site/content/kapp-controller/docs/v0.31.0/packaging-tutorial.md new file mode 100644 index 000000000..40ed4ffc5 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/packaging-tutorial.md @@ -0,0 +1,484 @@ +--- + +title: "Tutorial: Create and Install a Package" +--- + +[//]: # (Generated from katacoda content using 'carvel/tutorials/copy-katacoda-to-static.sh') + +## Get Started With Katacoda +Make a katacoda account and take our interactive tutorial [here](https://katacoda.com/carvel/scenarios/kapp-controller-package-management) +## Or Follow Our Tutorial Below +You can spin up your favorite [playground](https://www.katacoda.com/courses/kubernetes/playground) and follow the steps below. +Note the below steps are from the linked katacoda tutorial so your environment may differ slightly. + +## Installing kapp-controller dependencies + +We'll be using [Carvel](https://carvel.dev/) tools throughout this tutorial, so first we'll install +[ytt](https://carvel.dev/ytt/), [kbld](https://carvel.dev/kbld/), +[kapp](https://carvel.dev/kapp/), [imgpkg](https://carvel.dev/imgpkg/), and [vendir](https://carvel.dev/vendir/). + +Install the whole tool suite with the script below: + +(Note: we are temporarily overriding kapp-controller's version to jump to ytt +0.38.0, in order to include the recent OpenAPI Schema feature in this tutorial) +```bash +wget -O- https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/fc5458fe2102d67e85116c26534a35e265b28125/hack/install-deps.sh | \ +sed 's/ytt_version=v0.35.1/ytt_version=v0.38.0/' | \ +sed 's/0aa78f7b5f5a0a4c39bddfed915172880344270809c26b9844e9d0cbf6437030/2ca800c561464e0b252e5ee5cacff6aa53831e65e2fb9a09cf388d764013c40d/' | \ +bash +``` + + +## Optional: explore kapp + +Before we install kapp-controller with [kapp](https://carvel.dev/kapp/), you may be interested in seeing +a different example of how kapp works. + +You can skip this step if you want to get straight to kapp-controller. + +### Using kapp to install a cronjob + +First clone the GitHub repository for examples: + +```bash +git clone https://github.com/vmware-tanzu/carvel-kapp +``` + +Then deploy a CronJob to the Kubernetes cluster in this environment: + +```bash +kapp deploy -a hellocron -f carvel-kapp/examples/jobs/cron-job.yml -y +``` + +Now take a look at the Kubernetes resources being managed by kapp: + +```bash +kapp ls +``` + +```bash +kapp inspect -a hellocron --tree +``` + +We scheduled our CronJob to output a hello message every minute, so if you're +patient you'll see new messages appended to the logs: + +```bash +kapp logs -f -a hellocron +``` + +When you're done watching the logs you can use control-c (`^C`) to quit. + +Because this was an optional interlude, we can use kapp to uninstall the CronJob before proceeding: +```bash +kapp delete -a hellocron -y +``` +## I believe I was promised kapp-controller? + +Use kapp to install kapp-controller (reconciliation may take a moment, which you +could use to read about [kubernetes controller reconciliation loops](https://kubernetes.io/docs/concepts/architecture/controller/)): + +```bash +kapp deploy -a kc -f https://github.com/vmware-tanzu/carvel-kapp-controller/releases/download/v0.21.0/release.yml -y +``` + +Gaze upon the splendor! + +```bash +kubectl get all -n kapp-controller +``` + +The kapp deployment is managing a replicaset which owns a service and a pod. The +pod is running kapp-controller, which is a kubernetes controller +running its own reconciliation loop. + +kapp-controller introduces new Custom Resource (CR) types we'll use throughout this +tutorial, including PackageRepositories and PackageInstalls. + +```bash +kubectl api-resources --api-group packaging.carvel.dev +``` + +You can see other kapp-controller CRs in other groups: + +```bash +kubectl api-resources --api-group data.packaging.carvel.dev +``` + +```bash +kubectl api-resources --api-group kappctrl.k14s.io +``` +## Creating a Package: Templating our config + +We will be using [ytt](https://carvel.dev/ytt/) templates that describe a simple Kubernetes Deployment and Service. +These templates will install a simple greeter app with a templated hello message. + +Create a config.yml: + +```bash +cat > config.yml << EOF +#@ load("@ytt:data", "data") + +#@ def labels(): +simple-app: "" +#@ end + +--- +apiVersion: v1 +kind: Service +metadata: + namespace: default + name: simple-app +spec: + ports: + - port: #@ data.values.svc_port + targetPort: #@ data.values.app_port + selector: #@ labels() +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: default + name: simple-app +spec: + selector: + matchLabels: #@ labels() + template: + metadata: + labels: #@ labels() + spec: + containers: + - name: simple-app + image: docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + env: + - name: HELLO_MSG + value: #@ data.values.hello_msg +EOF +``` + +and put our schema into values.yml: + +```bash +cat > values.yml <<- EOF +#@data/values-schema +--- +#@schema/desc "Port number for the service." +svc_port: 80 +#@schema/desc "Target port for the application." +app_port: 80 +#@schema/desc "Name used in hello message from app when app is pinged." +hello_msg: stranger +EOF +``` + +## Creating a Package: Structuring our contents +We'll create an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) +that contains the package contents: the configuration (config.yml and values.yml from the previous step) and a reference to the greeter app image (docker.io/dkalinin/k8s-simple-app@sha256:...). + +The [package bundle format](packaging-artifact-formats.md#package-contents-bundle) describes the purpose of each directory +used in this section of the tutorial as well as general recommendations. + +Let's create a directory with our configuration files: +```bash +mkdir -p package-contents/config/ +cp config.yml package-contents/config/config.yml +cp values.yml package-contents/config/values.yml +``` + +Once we have the configuration figured out, let’s use kbld to record which container images are used: +```bash +mkdir -p package-contents/.imgpkg +kbld -f package-contents/config/ --imgpkg-lock-output package-contents/.imgpkg/images.yml +``` + +For more on using kbld to populate the .imgpkg directory with an ImagesLock, and why it is useful, see the [imgpkg docs on the subject](/imgpkg/docs/latest/resources/#imageslock-configuration). + +Once these files have been added, our package contents bundle is ready to be pushed! + +For the purpose of this tutorial, we will run an unsecured local docker +registry. In the real world please be safe and use appropriate security +measures. + +```bash +docker run -d -p 5000:5000 --restart=always --name registry registry:2 +``` + +From the terminal we can access this registry as `localhost:5000` but within the +cluster we'll need the IP Address. To emphasize that you would +normally use a repo host such as dockerhub or harbor we will store the IP +address in a variable: + +```bash +export REPO_HOST="`ifconfig | grep -A1 docker | grep inet | cut -f10 -d' '`:5000" +``` + +Now we can publish our bundle to our registry: + +```bash +imgpkg push -b ${REPO_HOST}/packages/simple-app:1.0.0 -f package-contents/ +``` + + +You can verify that we pushed something called `packages/simple-app` by checking the Docker registry catalog: + +```bash +curl ${REPO_HOST}/v2/_catalog +``` + +## Creating the Custom Resources + +To finish creating a package, we need to create two CRs. The first CR is the PackageMetadata CR, which will contain high level information and descriptions about our package. + +When creating this CR, the api will validate that the PackageMetadata’s name is a fully qualified name: It must have at least three segments separated by `.` and cannot have a trailing `.`. + +We'll make a conformant `metadata.yml` file: + +```bash +cat > metadata.yml << EOF +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + # This will be the name of our package + name: simple-app.corp.com +spec: + displayName: "Simple App" + longDescription: "Simple app consisting of a k8s deployment and service" + shortDescription: "Simple app for demoing" + categories: + - demo +EOF +``` + +Now we need to create a Package CR. +This CR contains versioned instructions and metadata used to install packaged software that fits the description provided in the PackageMetadata CR we just saved in `metadata.yml`. + +In order to create the Package CR with our OpenAPI Schema, we will export from +our ytt schema: + +```bash +ytt -f package-contents/config/values.yml --data-values-schema-inspect -o openapi-v3 > schema-openapi.yml +``` + +That command creates an OpenAPI document, from which we really only need the +`components.schema` section for our Package CR. + + +```bash +cat > package-template.yml << EOF +#@ load("@ytt:data", "data") # for reading data values (generated via ytt's data-values-schema-inspect mode). +#@ load("@ytt:yaml", "yaml") # for dynamically decoding the output of ytt's data-values-schema-inspect +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: #@ "simple-app.corp.com." + data.values.version +spec: + refName: simple-app.corp.com + version: #@ data.values.version + releaseNotes: | + Initial release of the simple app package + valuesSchema: + openAPIv3: #@ yaml.decode(data.values.openapi)["components"]["schemas"]["dataValues"] + template: + spec: + fetch: + - imgpkgBundle: + image: #@ "${REPO_HOST}/packages/simple-app:" + data.values.version + template: + - ytt: + paths: + - "config/" + - kbld: + paths: + - "-" + - ".imgpkg/images.yml" + deploy: + - kapp: {} +EOF +``` + +This Package contains some metadata fields specific to the version, such as releaseNotes and a valuesSchema. The valuesSchema shows what configurable properties exist for the version. This will help when users want to install this package and want to know what can be configured. + +The other main component of this CR is the template section. +This section informs kapp-controller of the actions required to install the packaged software, so take a look at the [app-spec](app-spec.md) section to learn more about each of the template sections. For this example, we have chosen a basic setup that will fetch the imgpkg bundle we created in the previous section, run the templates stored inside through ytt, apply kbld transformations, and then deploy the resulting manifests with kapp. + +There will also be validations run on the Package CR, so ensure that spec.refName and spec.version are not empty and that metadata.name is `.`. +These validations are done to encourage a naming scheme that keeps package version names unique. +## Creating a Package Repository + +A [package repository](packaging.md#package-repository) +is a collection of packages (more specifically a collection of Package and PackageMetadata CRs). +Our recommended way to make a package repository is via an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle). + +The [PackageRepository bundle format](packaging-artifact-formats.md#package-repository-bundle) describes purpose of each directory and general recommendations. + +Lets start by creating the needed directories: + +```bash +mkdir -p my-pkg-repo/.imgpkg my-pkg-repo/packages/simple-app.corp.com +``` + +we can copy our CR YAMLs from the previous step in to the proper packages +subdirectory. Note that we are declaring the version and the openAPI schema file +to ytt. + +```bash +ytt -f package-template.yml --data-value-file openapi=schema-openapi.yml -v version="1.0.0" > my-pkg-repo/packages/simple-app.corp.com/1.0.0.yml +cp metadata.yml my-pkg-repo/packages/simple-app.corp.com +``` + +Next, let’s use kbld to record which package bundles are used: + +```bash +kbld -f my-pkg-repo/packages/ --imgpkg-lock-output my-pkg-repo/.imgpkg/images.yml +``` + +With the bundle metadata files present, we can push our bundle to whatever OCI +registry we plan to distribute it from, which for this tutorial will just be our +same REPO_HOST. + +```bash +imgpkg push -b ${REPO_HOST}/packages/my-pkg-repo:1.0.0 -f my-pkg-repo +``` + +The package repository is pushed! + +You can verify by checking the Docker registry catalog: + +```bash +curl ${REPO_HOST}/v2/_catalog +``` + +In the next steps we'll act as the package consumer, showing an example of adding and using a PackageRepository with kapp-controller. + +## Adding a PackageRepository + +kapp-controller needs to know which packages are available to install. +One way to let it know about available packages is by creating a package repository. +To do this, we need a PackageRepository CR: + +```bash +cat > repo.yml << EOF +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: simple-package-repository +spec: + fetch: + imgpkgBundle: + image: ${REPO_HOST}/packages/my-pkg-repo:1.0.0 +EOF +``` + +(See our +[demo video](https://www.youtube.com/watch?v=PmwkicgEKQE) and [website](private-registry-auth.md) for more typical usage with an external repository.) + +This PackageRepository CR will allow kapp-controller to install any of the +packages found within the `${REPO_HOST}/packages/my-pkg-repo:1.0.0` imgpkg bundle, which we +stored in our docker OCI registry previously. + +We can use kapp to apply it to the cluster: +```bash +kapp deploy -a repo -f repo.yml -y +``` + +Check for the success of reconciliation to see the repository become available: +```bash +watch kubectl get packagerepository +``` + +Once the simple-package-repository has a "**Reconcile succeeded**" description, +we're ready to continue! You can exit the watch by hitting control-c or +clicking: `^C` + +Once the deploy has finished, we are able to list the package metadatas to see, at a high level, which packages are now available within our namespace: +```bash +kubectl get packagemetadatas +``` + +If there are numerous available packages, each with many versions, this list can become a bit unwieldy, so we can also list the packages with a particular name using the --field-selector option on kubectl get. +```bash +kubectl get packages --field-selector spec.refName=simple-app.corp.com +``` + +From here, if we are interested, we can further inspect each version to discover +information such as release notes, installation steps, licenses, etc. For +example: +```bash +kubectl get package simple-app.corp.com.1.0.0 -o yaml +``` + + +## Installing a Package + +Once we have the packages available for installation (as seen via `kubectl get packages`), +we need to let kapp-controller know which package we want to install. +To do this, we will need to create a PackageInstall CR (and a secret to hold the values used by our package): + +```bash +cat > pkginstall.yml << EOF +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: pkg-demo +spec: + serviceAccountName: default-ns-sa + packageRef: + refName: simple-app.corp.com + versionSelection: + constraints: 1.0.0 + values: + - secretRef: + name: pkg-demo-values +--- +apiVersion: v1 +kind: Secret +metadata: + name: pkg-demo-values +stringData: + values.yml: | + --- + hello_msg: "to all my katacoda friends" +EOF +``` + + +This CR references the Package we created in the previous sections using the package’s `refName` and `version` fields (see yaml from step 7). +Do note, the `versionSelection` property has a constraints subproperty to give more control over which versions are chosen for installation. +More information on PackageInstall versioning can be found [here](packaging.md#versioning-packageinstalls). + +This yaml snippet also contains a Kubernetes secret, which is referenced by the PackageInstall. This secret is used to provide customized values to the package installation’s templating steps. Consumers can discover more details on the configurable properties of a package by inspecting the Package CR’s valuesSchema. + +Finally, to install the above package, we will also need to create `default-ns-sa` service account (refer to [Security model](security-model.md) +for explanation of how service accounts are used) that give kapp-controller privileges to create resources in the default namespace: +```bash +kapp deploy -a default-ns-rbac -f https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/develop/examples/rbac/default-ns.yml -y +``` + +Apply the PackageInstall using kapp: +```bash +kapp deploy -a pkg-demo -f pkginstall.yml -y +``` + +After the deploy has finished, kapp-controller will have installed the package in the cluster. We can verify this by checking the pods to see that we have a workload pod running. The output should show a single running pod which is part of simple-app: +```bash +kubectl get pods +``` + +Once the pod is ready, you can use kubectl’s port forwarding to verify the customized hello message has been used in the workload: +```bash +kubectl port-forward service/simple-app 3000:80 & +``` + +Now if we make a request against our service, we can see that our `hello_msg` +values is being used: +```bash +curl localhost:3000 +``` +## Congratulations! + +Visit [carvel.dev](https://carvel.dev/) to learn more about Carvel tools. + +See the full docs for [Package Management with kapp-controller](packaging.md) diff --git a/site/content/kapp-controller/docs/v0.31.0/packaging.md b/site/content/kapp-controller/docs/v0.31.0/packaging.md new file mode 100644 index 000000000..77b0ae1d2 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/packaging.md @@ -0,0 +1,296 @@ +--- + +title: Package Management +--- + + +## Overview + +kapp-controller provides a declarative way to install, manage, and upgrade packages on a Kubernetes cluster. It leverages the PackageRepository, PackageMetadata, Package, and PackageInstall CRDs to do so. Get started by installing the [latest release of kapp-controller](install.md). + +## Concepts & CustomResourceDefinitions + +### Package + +A package is a combination of configuration metadata and OCI images that informs the package manager what software it holds and how to install itself onto a Kubernetes cluster. For example, an nginx-ingress package would instruct the package manager where to download the nginx container image, how to configure the associated Deployment, and install it into a cluster. + +A Package is represented in kapp-controller using a Package CR. The Package CR is created for every new version of a package and it carries information about how to fetch, template, and deploy the package. A Package CR is a namespaced resource by default. [Learn more](package-consumer-concepts.md#namespacing) about how to share a Package CR across all namespaces within a cluster. + +```yaml +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + # Must be of the form '.' (Note the period) + name: fluent-bit.carvel.dev.1.5.3 + # The namespace this package is available in + namespace: my-ns +spec: + # The name of the PackageMetadata associated with this version + # Must be a valid PackageMetadata name (see PackageMetadata CR for details) + # Cannot be empty + refName: fluent-bit.carvel.dev + # Package version; Referenced by PackageInstall; + # Must be valid semver (required) + # Cannot be empty + version: 1.5.3 + # Version release notes (optional; string) + releaseNotes: "Fixed some bugs" + # System requirements needed to install the package. + # Note: these requirements will not be verified by kapp-controller on + # installation. (optional; string) + capacityRequirementsDescription: "RAM: 10GB" + # Description of the licenses that apply to the package software + # (optional; Array of strings) + licenses: + - "Apache 2.0" + - "MIT" + # Timestamp of release (iso8601 formatted string; optional) + releasedAt: 2021-05-05T18:57:06Z + # valuesSchema can be used to show template values that + # can be configured by users when a Package is installed. + # These values should be specified in an OpenAPI schema format. (optional) + valuesSchema: + # openAPIv3 key can be used to declare template values in OpenAPIv3 + # format. Read more on using ytt to generate this schema: + # https://carvel.dev/kapp-controller/docs/latest/packaging-tutorial/#creating-the-custom-resources + openAPIv3: + title: fluent-bit.carvel.dev.1.5.3 values schema + examples: + - namespace: fluent-bit + properties: + namespace: + type: string + description: Namespace where fluent-bit will be installed. + default: fluent-bit + examples: + - fluent-bit + # App template used to create the underlying App CR. + # See 'App CR Spec' docs for more info + template: + spec: + fetch: + - imgpkgBundle: + image: registry.tkg.vmware.run/tkg-fluent-bit@sha256:... + template: + - ytt: + paths: + - config/ + - kbld: + paths: + # - must be quoted when included with paths + - "-" + - .imgpkg/images.yml + deploy: + - kapp: {} +``` + +### Package Metadata + +Package Metadata are attributes of a single package that do not change frequently and that are shared across multiple versions of a single package. It contains information similar to a project's README.md. + +It is represented in kapp-controller by a PackageMetadata CR. A PackageMetadata CR is a namespaced resource by default. [Learn more](package-consumer-concepts.md#namespacing) about how to share a PackageMetadata CR across all namespaces within a cluster. + +```yaml +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + # Must consist of at least three segments separated by a '.' + # Cannot have a trailing '.' + name: fluent-bit.vmware.com + # The namespace this package metadata is available in + namespace: my-ns +spec: + # Human friendly name of the package (optional; string) + displayName: "Fluent Bit" + # Long description of the package (optional; string) + longDescription: "Fluent bit is an open source..." + # Short desription of the package (optional; string) + shortDescription: "Log processing and forwarding" + # Base64 encoded icon (optional; string) + iconSVGBase64: YXNmZGdlcmdlcg== + # Name of the entity distributing the package (optional; string) + providerName: VMware + # List of maintainer info for the package. + # Currently only supports the name key. (optional; array of maintner info) + maintainers: + - name: "Person 1" + - name: "Person 2" + # Classifiers of the package (optional; Array of strings) + categories: + - "logging" + - "daemon-set" + # Description of the support available for the package (optional; string) + supportDescription: "..." +``` + +### Package Repository + +A package repository is a collection of packages and their metadata. Similar to a maven repository or a rpm repository, adding a package repository to a cluster gives users of that cluster the ability to install any of the packages from that repository. + +It is represented in kapp-controller by a PackageRepository CR. A PackageRepository CR is a namespaced resource by default. [Learn more](package-consumer-concepts.md#namespacing) about how to share a PackageRepository CR across all namespaces within a cluster. + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + # Any user-chosen name that describes package repository + name: basic.carvel.dev + # The namespace to make packages available to + namespace: my-ns +spec: + # pauses _future_ reconcilation; does _not_ affect + # currently running reconciliation (optional; default=false) + paused: true + # specifies the length of time to wait, in time + unit + # format, before reconciling.(optional; default=10m) + syncPeriod: 1m + # Must have only one directive. + fetch: + # pull content from within this resource; or other resources in the cluster + inline: # NOTE: inline fetch available since v 0.31.0 + # specifies mapping of paths to their content; + # not recommended for sensitive values as CR is not encrypted (optional) + paths: + dir/file.ext: file-content + # specifies content via secrets and config maps; + # data values are recommended to be placed in secrets (optional) + pathsFrom: + - secretRef: + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + # pulls imgpkg bundle from Docker/OCI registry + imgpkgBundle: + # Docker image url; unqualified, tagged, or + # digest references supported (required) + image: host.com/username/image:v0.1.0 + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + # pulls image containing packages from Docker/OCI registry + image: + # Image url; unqualified, tagged, or + # digest references supported (required) + url: host.com/username/image:v0.1.0 + # grab only portion of image (optional) + subPath: inside-dir/dir2 + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + # uses http library to fetch file containing packages + http: + # http and https url are supported; + # plain file, tgz and tar types are supported (required) + url: https://host.com/archive.tgz + # checksum to verify after download (optional) + sha256: 0a12cdef83... + # grab only portion of download (optional) + subPath: inside-dir/dir2 + # uses git to clone repository containing package list + git: + # http or ssh urls are supported (required) + url: https://github.com/k14s/k8s-simple-app-example + # branch, tag, commit; origin is the name of the remote (required) + ref: origin/develop + # grab only portion of repository (optional) + subPath: config-step-2-template + # skip lfs download (optional) + lfsSkipSmudge: true + # specifies a strategy to resolve to an explicit ref (optional; v0.24.0+) + refSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] +``` + +### Package Install + +A Package Install is an actual installation of a package and its underlying resources on a Kubernetes cluster. It is represented in kapp-controller by a PackageInstall CR. A PackageInstall CR must reference a Package CR. + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: fluent-bit + # The namespace to install the package in to + namespace: my-ns +spec: + # pauses _future_ reconcilation; does _not_ affect + # currently running reconciliation (optional; default=false) + paused: true + # cancels current and future reconciliations (optional; default=false) + canceled: true + # Deletion requests for the PackageInstall/App will result in + # the PackageInstall/App CR being deleted, but its associated + # resources will not be deleted (optional; default=false) + noopDelete: true + # specifies the length of time to wait, in time + unit + # format, before reconciling.(optional; default=10m) + syncPeriod: 1m + # specifies that Package should be deployed to destination cluster; + # by default, cluster is same as where this resource resides (optional) + cluster: + # specifies namespace in destination cluster (optional) + namespace: ns2 + # specifies secret containing kubeconfig (required) + kubeconfigSecretRef: + # specifies secret name within app's namespace (required) + name: cluster1 + # specifies key that contains kubeconfig (optional) + key: value + # specifies service account that will be used to install underlying package contents + serviceAccountName: fluent-bit-sa + packageRef: + # Specifies the name of the package to install (required) + refName: fluent-bit.vmware.com + # Selects version of a package based on constraints provided (optional) + # Either version or versionSelection is required. + versionSelection: + # Constraint to limit acceptable versions of a package; + # Latest version satisfying the constraint is chosen; + # Newly available, acceptable later versions are picked up and installed automatically. (optional) + constraints: ">v1.5.3" + # Include prereleases when selecting version. (optional) + prereleases: {} + # Values to be included in package's templating step + # (currently only included in the first templating step) (optional) + values: + - secretRef: + name: fluent-bit-values +# Populated by the controller +status: + packageRef: + # Kubernetes resource name of the package chosen against the constraints + name: fluent-bit.tkg.vmware.com.v1.5.3 + # Derived from the underlying App's Status + conditions: + - type: ValuesSchemaCheckFailed + - type: ReconcileSucceeded + - type: ReconcileFailed + - type: Reconciling +``` + +**Note:** Values will only be included in the first templating step of the package, +though we intend to improve this experience in later releases. diff --git a/site/content/kapp-controller/docs/v0.31.0/private-registry-auth.md b/site/content/kapp-controller/docs/v0.31.0/private-registry-auth.md new file mode 100644 index 000000000..42edf9543 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/private-registry-auth.md @@ -0,0 +1,217 @@ +--- + +title: Authenticating to Private Registries +--- + +## Scenario + +As a package consumer you may need to provide registry credentials if you are consuming package repository (and/or packages) from a registry that requires authenticated access. That may involve providing registry credentials to multiple parts of the system: + +1. credentials for pulling package repository bundle (via PackageRepository CR) + - consumed by imgpkg running inside kapp-controller Pod +1. credentials for pulling package contents bundle (via PackageInstall CR) + - consumed by imgpkg running inside kapp-controller Pod +1. credentials for pulling container images used by the package + - credentials consumed by Kubelets + - e.g. needed by cert-manager controller Pod +1. credentials for pulling container images used by packages operator + - credentials consumed by Kubelets + - e.g. needed by Kafka cluster Pods created for KafkaInstance CR + +Providing credentials manually to each one of these parts of the system can become a hassle. kapp-controller v0.24.0+ when installed together with [secretgen-controller](https://github.com/vmware-tanzu/carvel-secretgen-controller) v0.5.0+ allow package consumers and package authors to simplify such configuration. + +Note that if you are using an IaaS provided Kubernetes cluster already preauthenticated with an IaaS provided registry, then there is no need to provide credentials manually in the cluster. kapp-controller v0.25.0+ is able to automatically pick up provided credentials to satisfy first two bullet points above. Last two bullet points are already satisfied by the Kubernetes kubelet. + +## secretgen-controller's placeholder secrets and SecretExport CR + +For this specific use case, secretgen-controller allows package consumer to specify registry credentials in one namespace and allows to export that secret to the entire cluster (or subset of namespaces) via [SecretExport CR](https://github.com/vmware-tanzu/carvel-secretgen-controller/blob/develop/docs/secret-export.md#secretexport-and-secretrequest). Registry credentials could be consumed in different namespaces via "placeholder secrets". + +A placeholder secret is: +- plain Kubernetes Secret +- with `kubernetes.io/dockerconfigjson` type (more about this secret type [here](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials)) +- has `secretgen.carvel.dev/image-pull-secret` annotation + +secretgen-controller will populate placeholder Secrets with a combined registry credentials. For example: + +- within `reg-creds` Namespace + - Secret `dockerhub-reg` includes DockerHub credentials for `index.docker.io` domain + - SecretExport CR `dockerhub-reg` specifies that same-named secret will be exported to all namespaces + - Secret `corp-reg` includes registry credentials for `registry.corp.com` domain + - SecretExport CR `corp-reg` specifies that same-named secret will be exported to all namespaces +- within `cert-manager-install` Namespace + - Secret `reg-creds` has `secretgen.carvel.dev/image-pull-secret` annotation indicating to secretgen to continuously ensure that this secret is filled with combination of registry credentials that allow export to this namespace (in this case both `dockerhub-reg` and `corp-reg`) + +Known limitation: Currently Secrets with type `kubernetes.io/dockerconfigjson` do not allow specifying multiple credentials for the same domain, hence you cannot provide multiple credentials for the same registry. + +**Warning** Since SecretExport CR allows you to export registry credentials to other namespaces, they will become visible to users of such namespaces. We **strongly recommend** to ensure that registry credentials you are exporting only allow read-only access to the registry and are minimally scoped within the registry. + +## kapp-controller CRs and placeholder secrets + +As of kapp-controller v0.24.0+, PackageRepository and PackageInstall CRs automatically create placeholder secrets for `image` and `imgpkgBunle` fetch types, if no explicit `secretRef.name` is provided. (These placeholder secrets are named as `-fetch-`.) If secretgen-controller is present on the cluster, these secrets will be populated with combined registry credentials; otherwise, they will remain empty. + +## Package authoring and placeholder secrets + +We encourage all package authors to include placeholder secrets within your package configuration already preconfigured to be used by your Deployments, StatefulSets, DaemonSets, Pods, etc (and any other resources that consume image pull secrets). This removes a need for package consumers to worry about configuring packages in any special way if it's being consumed from a registry that requires authentication. Note that even if you are distributing package repository from a registry that support anonymous access, package consumers may still copy it (via imgpkg copy) into a private registry that does require authentication. + +Note: In future we could provide a feature to automatically inject placeholder secrets as part of package installation (e.g. via Pod webhook); however, that is a bit more intrusive, hence we are recommending explicit usage of placeholder secrets for now. + +Example of a placeholder secret package authors should add next other resources: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: reg-creds + annotations: + secretgen.carvel.dev/image-pull-secret: "" +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: e30K +``` + +Note: `e30K` is base64 encoded `{}`. Valid `.dockerconfigjson` value is required when creating a Secret. + +## Operator writing and placeholder secrets + +If you are an owner of an operator, similar to the above section, we encourage you to create a placeholder secret for Pods (or other resources that consume image pull secrets) that may be created by your operator in other namespaces. More general operator packaging docs will come soon. + +--- +## Bringing it all together + +- Ensure kapp-controller v0.24.0+ is installed + +- Install secretgen-controller v0.5.0+ + + ```bash + kapp deploy -a sg -f https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/download/v0.5.0/release.yml + ``` + +- Create registry credential Secret and use SecretExport CR to make it available for all namespaces (Note: if you use `kubectl create secret docker-registry` and you want to auth with DockerHub, please specify `--docker-server=index.docker.io` explicitly instead of relying on default server value.) + + ```yaml + --- + apiVersion: v1 + kind: Secret + metadata: + name: reg-creds # could be any name + namespace: secrets-ns # could be any namespace + type: kubernetes.io/dockerconfigjson # needs to be this type + stringData: + .dockerconfigjson: | + { + "auths": { + "index.docker.io": { + "username": "user...", + "password": "password...", + "auth": "" + } + } + } + + --- + apiVersion: secretgen.carvel.dev/v1alpha1 + kind: SecretExport + metadata: + name: reg-creds # must match source secret name + namespace: secrets-ns # must match source secret namespace + spec: + toNamespaces: + - "*" # star means export is available for all namespaces + ``` + +- Use PackageRepository and PackageInstall CRs without specifying secrets explicitly + + ```yaml + --- + apiVersion: packaging.carvel.dev/v1alpha1 + kind: PackageRepository + metadata: + name: e2e-repo.test.carvel.dev + namespace: kapp-controller-packaging-global + spec: + fetch: + imgpkgBundle: + image: k14stest/private-repo@sha256:ddd93b... + --- + apiVersion: packaging.carvel.dev/v1alpha1 + kind: PackageInstall + metadata: + name: pkg-demo + spec: + serviceAccountName: default-ns-sa + packageRef: + refName: pkg.test.carvel.dev + versionSelection: + constraints: 1.0.0 + ``` + +Assuming registry credentials specified are correct and both package repository bundle and package contents bundle use the same registry + +--- +## Manual configuration (without secretgen-controller) + +### PackageRepository + +If the registry containing the PackageRepository imgpkg bundle or image is private and secretgen-controller is not installed on your cluster, a secretRef can be added to the fetch stage for PackageRepository CR. For example: + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: simple-package-repository +spec: + fetch: + imgpkgBundle: + image: k8slt/corp-com-pkg-repo:1.0.0 + secretRef: + name: my-registry-creds +``` + +This secret will need to be located in the namespace where the PackageRepository +is created and be in the format described in the [fetch docs](config.md#image-authentication). + +### PackageInstall + +As of kapp-controller v0.23.0, support for adding an annotation on the PackageInstall was added to allow users to set a secret on the PackageInstall's underlying App custom resource. Before creating a PackageInstall, users can look at the Package definition that they want to install and see what fetch stages a Package has defined like below: + +```yaml +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: simple-app.corp.com.1.0.0 +spec: + refName: simple-app.corp.com + version: 1.0.0 + template: + spec: + fetch: + - imgpkgBundle: + image: registry.corp.com/packages/simple-app:1.0.0 + # ... +``` + +In the example above, the Package has a single fetch stage to retrieve an imgpkg bundle. To use a PackageInstall +to specify what secret to use for this fetch stage, an annotation is added to the PackageInstall as shown below: + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: simple-app-with-secret + annotations: + ext.packaging.carvel.dev/fetch-0-secret-name: simple-app-secret +spec: + serviceAccountName: default-ns-sa + packageRef: + refName: simple-app.corp.com + versionSelection: + constraints: 1.0.0 +``` + +The annotation shown above `ext.packaging.carvel.dev/fetch-0-secret-name: simple-app-secret` has a format that allows users to specify the specific fetch stage by how it is defined in the Package definition. In this case, the PackageInstall being created will add a secretRef to the App's first fetch stage (i.e. `fetch-0-secret-name`) for the imgpkg bundle. If the Package definition had an additional fetch stage, the secret annotation could be added in the following format: `ext.packaging.carvel.dev/fetch-1-secret-name: simple-app-additional-secret`. + +To use this annotation with a PackageInstall, associated secrets will need to be located in the namespace where the PackageInstall is created and be in the format described in the [fetch docs](config.md#image-authentication). diff --git a/site/content/kapp-controller/docs/v0.31.0/security-model.md b/site/content/kapp-controller/docs/v0.31.0/security-model.md new file mode 100644 index 000000000..5655a710a --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/security-model.md @@ -0,0 +1,67 @@ +--- + +title: Security Model +--- + +## App CR privileges + +kapp-controller container runs with a service account (named +`kapp-controller-sa` inside `kapp-controller` namespace) that has access to all +service accounts and secrets in the cluster. This service account *is not* used +for deployment of app resources. + +Each App CR *must* specify either a + +- service account (via `spec.serviceAccountName`) +- or, Secret with kubeconfig contents for some cluster (via `spec.cluster.kubeconfigSecretRef.name`) + +forcing App CR owner to explicitly provide needed privileges for management of +app resources. This avoids a problem of privilege escalation commonly found in +other general resource controllers which rely on a shared service account (often +requiring cluster admin privileges) to deploy resources. + +Since App CR only allows to reference service account or kubeconfig Secret +within the same namespace where App CR is located, kapp-controller is well +suited for multi-tenant use where different users of App CRD have varied level +of access (e.g. some may have cluster level privileges, and other may only have +access to one or more namespace). + +Example: + +- User A has been granted access to namespace `a` (and no other namespace or + cluster level access). User A can create an App CR with a service account + located in namespace `a` to deploy resources into namespace `a`. It _is not_ + possible for user A to create an App CR that would install cluster-wide + resources or place resources into another namespace. (e.g. a user that just + deploys web application to their namespace) + +- User B has been granted access to namespace `b` and ability to manage + specifically named CRD (single scoped cluster-wide privilege). User B can + create an App CR with a service account located in namespace `b` that installs + app into namespace `b` and also manages single CRD lifecycle. (e.g. a user + that manages another controller for other users) + +## Minimum ServiceAccount Permissions + +For users managing App and PackageInstall CR privileges via a service account, +the verbs in the role below are needed for working with ConfigMaps. + +```yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: app-ip-cr-role +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "update", "delete"] +``` + +These permissions are needed because of how `kapp` tracks information about apps +it manages, which is via storing information in a ConfigMap. So even if your App +or PackageInstall CR does not create ConfigMaps, the service account will +still need permissions for working with ConfigMaps. + +The ConfigMap permissions above are needed in addition to any other +resource/verb combinations needed to deploy all resources created by the App and +PackageInstall CRs. diff --git a/site/content/kapp-controller/docs/v0.31.0/security.md b/site/content/kapp-controller/docs/v0.31.0/security.md new file mode 100644 index 000000000..8a06a0f80 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/security.md @@ -0,0 +1,8 @@ +--- + +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `kapp-controller`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/kapp-controller/docs/v0.31.0/sops.md b/site/content/kapp-controller/docs/v0.31.0/sops.md new file mode 100644 index 000000000..1a58819cf --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/sops.md @@ -0,0 +1,146 @@ +--- + +title: Sops +--- + +Available in v0.11.0+. + +Storing _encrypted_ secrets next to your configuration (within a Git repo or other artifacts) is one way to manage secret lifecycle. kapp-controller integrates with [Mozilla's SOPS](https://github.com/mozilla/sops) to decrypt secret material in fetched configuration. + +## Prepare your keys +Sops shipped with kapp-controller includes support for encryption via both [GPG](https://gnupg.org/) and [age](https://github.com/FiloSottile/age). +Note that the Sops project recommends Age. + +### using GPG + +```bash +$ gpg --gen-key +... + +$ gpg --list-secret-keys --keyid-format LONG +/root/.gnupg/secring.gpg +------------------------ +sec 4096R/B464DFD255C6B9F8 2020-10-03 +uid test test (test) +ssb 4096R/FEE37B8E2098EDFC 2020-10-03 +``` + +(Note: `B464DFD255C6B9F8` is the ID to be used later) + +### using Age +You may find [this screencast](https://asciinema.org/a/431605) helpful. +```bash +$ age-keygen -o key.txt +Public key: age12345... + +$ chmod 600 key.txt +``` +(Note: the public key `age12345...` will be used later) + +## Encrypt contents + +kapp-controller expects that encrypted files have `.sops.yml` extension (or `.sops.yml`). + +You can start by creating an unencrypted yaml: +```bash +# Unencrypted file +$ cat secret.yml +apiVersion: v1 +kind: Secret +metadata: + name: my-sec +data: + password: my-password +``` + +Then encrypt file with your public key from the previous step, to be later decrypted by kapp-controller. + +### using GPG +```bash +$ sops --encrypt --pgp B464DFD255C6B9F8 secret.yml > secret.sops.yml + +# Delete unencrypted file +$ rm secret.yml +``` + +### using Age +```bash +$ SOPS_AGE_KEY_FILE=./key.txt sops --encrypt --age age12345... secret.yml > secret.sops.yml +``` + +## Import private key into Kubernetes +Make a secret that includes the private key and import it into your +cluster. It will be referenced by App CR. + +### using GPG +Extract PGP private key from your GPG installation: + +```bash +$ gpg --armor --export-secret-keys B464DFD255C6B9F8 > my.pk +``` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: pgp-key + namespace: default +stringData: + # value of this is contents of my.pk + my.pk: | + -----BEGIN PGP PRIVATE KEY BLOCK----- + Version: GnuPG v1 + ... +``` + +### using Age +Cat the contents of your local key.txt to the body of a stringData block also named +key.txt: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: age-key +stringData: + key.txt: | + # created: + # public key: age12345... + AGE-SECRET-KEY-HUNTER2ORSOMETHINGWHENEVERYOUTYPEYOURPASSWORDIJUSTSEEHUNTER2 +``` + +## Decrypt in App CR + +Configure App CR to decrypt contents. Assuming, in this example, your git repo contains `secret.sops.yml`, it would be decrypted into `secret.yml` file. + +### using GPG +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: config-with-sops + namespace: default +spec: + serviceAccountName: default-ns + fetch: + - git: + ... + template: + - sops: + pgp: + privateKeysSecretRef: + name: pgp-key + - ytt: {} + deploy: + - kapp: {} +``` + +### using Age +Nearly identical to the example above, but with a sops.age key: +```yaml +spec: + template: + - sops: + age: + privateKeysSecretRef: + name: age-key +``` diff --git a/site/content/kapp-controller/docs/v0.31.0/startup.md b/site/content/kapp-controller/docs/v0.31.0/startup.md new file mode 100644 index 000000000..08d73b54c --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/startup.md @@ -0,0 +1,31 @@ +--- + +Title: Kapp Controller Startup +--- + +(v0.14.0+) + +The startup of kapp controller consists of two processes: +controllerinit and controller. + +## The controllerinit Process + +This is the main process for the kapp controller binary. Since the binary is +the entrypoint for the docker image, kapp controller will be PID 1 +and is therefore responsible for reaping any zombie processes, so the process +begins by starting a thread to reap any zombies that appear. More on PID 1 and +zombie processes can be found [here][1]. + +Next, the process will look for the [controller Secret or ConfigMap][2] and apply any system level +configuration specified within. + +Finally, the process will fork to the same binary with a new flag, `--internal-controller`, +starting a second process, which runs the actual kubernetes controller. + +## The controller Process + +This process is responsible for running the app reconciler, which handles the fetch, +template, and deploy aspects of kapp controller. + +[1]: https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/ +[2]: controller-config.md diff --git a/site/content/kapp-controller/docs/v0.31.0/walkthrough.md b/site/content/kapp-controller/docs/v0.31.0/walkthrough.md new file mode 100644 index 000000000..9ba9764a4 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.31.0/walkthrough.md @@ -0,0 +1,205 @@ +--- + +title: Install an Application +--- + +This walkthrough demonstrates how to install a simple example application, an HTTP server, on Kubernetes with kapp-controller. We will use `examples/simple-app-git` directory as our YAML configuration. + +You can use `kubectl` (or another tool) to deploy the YAML examples below. We've chosen [kapp](/kapp). + +- Start by [installing](install.md) kapp-controller onto your cluster + +- Install [examples/default-ns-rbac.yml](https://github.com/vmware-tanzu/carvel-kapp-controller/blob/develop/examples/rbac/default-ns.yml). It creates `default-ns-sa` service account that allows to change any + resource within the `default` namespace. This will be used by App CR below. + + ```bash-plain + $ kapp deploy -a default-ns-rbac -f https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/develop/examples/rbac/default-ns.yml + ``` + +- Install [examples/simple-app-git/1.yml](https://github.com/vmware-tanzu/carvel-kapp-controller/blob/develop/examples/simple-app-git/1.yml) App CR. It specifies how to fetch, template, and deploy our example application. + + ```bash-plain + $ kapp deploy -a simple-app -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/1.yml + # or... kubectl apply -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/1.yml + + Changes + + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app App - - create reconcile - - + + Op: 1 create, 0 delete, 0 update, 0 noop + Wait to: 1 reconcile, 0 delete, 0 noop + + Continue? [yN]: y + + 5:20:27PM: ---- applying 1 changes [0/1 done] ---- + 5:20:27PM: create app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:20:27PM: ---- waiting on 1 changes [0/1 done] ---- + 5:20:27PM: ongoing: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:20:33PM: ok: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:20:33PM: ---- applying complete [1/1 done] ---- + 5:20:33PM: ---- waiting complete [1/1 done] ---- + + Succeeded + ``` + +- Check out `kubectl get app` output to see that app is deployed. + +- Additionally, let's check status of our App CR. It shows the overall status of the application, including the latest deploy output (`status.deploy.stdout`) and latest inspect output (`status.inspect.stdout`). Based on the inspect output we can see that our app included a `Deployment` and a `Service`. (Note: As of kapp-controller v0.31.0, inspect is disabled by default. See [App CR spec](app-spec.md) for more details.) + + ```bash-plain + $ kapp inspect -a simple-app --status + # or... kubectl get app simple-app -oyaml + + Resources in app 'simple-app' + + Namespace default + Name simple-app + Kind App + Status conditions: + - status: "True" + type: ReconcileSucceeded + deploy: + exitCode: 0 + finished: true + startedAt: "2019-12-02T22:20:28Z" + stdout: |- + Changes + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app Deployment - - create reconcile - - + ^ simple-app Service - - create reconcile - - + Op: 2 create, 0 delete, 0 update, 0 noop + Wait to: 2 reconcile, 0 delete, 0 noop + 10:20:28PM: ---- applying 2 changes [0/2 done] ---- + 10:20:28PM: create service/simple-app (v1) namespace: default + 10:20:28PM: create deployment/simple-app (apps/v1) namespace: default + 10:20:29PM: ---- waiting on 2 changes [0/2 done] ---- + 10:20:29PM: ok: reconcile service/simple-app (v1) namespace: default + 10:20:29PM: ongoing: reconcile deployment/simple-app (apps/v1) namespace: default + 10:20:29PM: ^ Waiting for 1 unavailable replicas + 10:20:29PM: L ok: waiting on replicaset/simple-app-6fb57f844b (apps/v1) namespace: default + 10:20:29PM: L ongoing: waiting on pod/simple-app-6fb57f844b-jk7d8 (v1) namespace: default + 10:20:29PM: ^ Pending: ContainerCreating + 10:20:29PM: ---- waiting on 1 changes [1/2 done] ---- + 10:20:31PM: ok: reconcile deployment/simple-app (apps/v1) namespace: default + 10:20:31PM: ---- applying complete [2/2 done] ---- + 10:20:31PM: ---- waiting complete [2/2 done] ---- + Succeeded + updatedAt: "2019-12-02T22:20:31Z" + fetch: + exitCode: 0 + startedAt: "2019-12-02T22:20:27Z" + updatedAt: "2019-12-02T22:20:27Z" + inspect: + exitCode: 0 + stdout: |- + Resources in app 'simple-app-ctrl' + Namespace Name Kind Owner Conds. Rs Ri Age + default simple-app Deployment kapp 2/2 t ok - 4s + default L simple-app-6fb57f844b ReplicaSet cluster - ok - 4s + default L.. simple-app-6fb57f844b-jk7d8 Pod cluster 4/4 t ok - 4s + default simple-app Service kapp - ok - 4s + default L simple-app Endpoints cluster - ok - 4s + Rs: Reconcile state + Ri: Reconcile information + 5 resources + Succeeded + updatedAt: "2019-12-02T22:20:32Z" + observedGeneration: 2 + template: + exitCode: 0 + updatedAt: "2019-12-02T22:20:28Z" + + 1 resources + + Succeeded + ``` + +- Update simple-app App CR to reconfigure simple-app. In this example we are changing data values for ytt templates. + + ```bash-plain + $ kapp deploy -a simple-app -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/2.yml -c + # or... kubectl apply -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/2.yml + + --- update app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + ... + 23, 23 template: + 24 - - ytt: {} + 24 + - ytt: + 25 + inline: + 26 + pathsFrom: + 27 + - secretRef: + 28 + name: simple-app-values + 25, 29 status: + 26, 30 conditions: + --- create secret/simple-app-values (v1) namespace: default + 0 + apiVersion: v1 + 1 + kind: Secret + 2 + metadata: + 3 + labels: + 4 + kapp.k14s.io/app: "1575325198404867000" + 5 + kapp.k14s.io/association: v1.7a671029ad7db07aa797301eac59e9ad + 6 + name: simple-app-values + 7 + namespace: default + 8 + stringData: + 9 + values2.yml: | + 10 + #@data/values + 11 + --- + 12 + hello_msg: updated + 13 + + + Changes + + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app App 1/1 t 2m update reconcile ok - + ^ simple-app-values Secret - - create reconcile - - + + Op: 1 create, 0 delete, 1 update, 0 noop + Wait to: 2 reconcile, 0 delete, 0 noop + + Continue? [yN]: y + + 5:23:13PM: ---- applying 2 changes [0/2 done] ---- + 5:23:13PM: update app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:23:13PM: create secret/simple-app-values (v1) namespace: default + 5:23:14PM: ---- waiting on 2 changes [0/2 done] ---- + 5:23:14PM: ongoing: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:23:14PM: ok: reconcile secret/simple-app-values (v1) namespace: default + 5:23:14PM: ---- waiting on 1 changes [1/2 done] ---- + 5:23:17PM: ok: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:23:17PM: ---- applying complete [2/2 done] ---- + 5:23:17PM: ---- waiting complete [2/2 done] ---- + + Succeeded + ``` + +- Delete simple-app App CR + + ```bash-plain + $ kapp delete -a simple-app + # or... kubectl delete -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/2.yml + + Changes + + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app App 1/1 t 6m delete delete ok - + ^ simple-app-values Secret - 3m delete delete ok - + + Op: 0 create, 2 delete, 0 update, 0 noop + Wait to: 0 reconcile, 2 delete, 0 noop + + Continue? [yN]: y + + 5:26:25PM: ---- applying 2 changes [0/2 done] ---- + 5:26:25PM: delete secret/simple-app-values (v1) namespace: default + 5:26:25PM: delete app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:26:26PM: ---- waiting on 2 changes [0/2 done] ---- + 5:26:26PM: ok: delete secret/simple-app-values (v1) namespace: default + 5:26:26PM: ongoing: delete app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:26:26PM: ---- waiting on 1 changes [1/2 done] ---- + 5:26:30PM: ok: delete app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:26:30PM: ---- applying complete [2/2 done] ---- + 5:26:30PM: ---- waiting complete [2/2 done] ---- + + Succeeded + ``` diff --git a/site/content/kapp-controller/docs/v0.32.0/_index.md b/site/content/kapp-controller/docs/v0.32.0/_index.md new file mode 100644 index 000000000..de2460fc0 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/_index.md @@ -0,0 +1,36 @@ +--- +aliases: [/kapp-controller/docs/latest/] +title: "About kapp-controller" +toc: "false" +cascade: + version: v0.32.0 + toc: "true" + type: docs + layout: docs +--- + +kapp-controller provides declarative APIs to customize, install, and update your Kubernetes applications and packages. It is a part of the Carvel toolkit and follows core Carvel design principles. Get started with the [tutorial](packaging-tutorial.md)! + +#### Choice for authors; consistency for consumers +Kubernetes configuration takes many forms -- plain YAML configurations, Helm charts, ytt templates, jsonnet templates, etc. +Software running on Kubernetes lives in many different places: a Git repository, an archive over HTTP, a Helm repository, etc. + +kapp-controller provides software authors flexibility to choose their own configuration tools, while providing software consumers with consistent declarative APIs to customize, install, and update software on Kubernetes from various sources. + +#### Lightweight and composable +kapp-controller breaks down the installation of applications and packages into three easy to understand steps: +- Fetch: get configuration and OCI images from various sources including a Git repository, a local ConfigMap, a Helm chart, an OCI registry, etc. +- Template: take user provided values to customize software using ytt templates, helm templates, and more +- Deploy: create/update resources on the cluster + +#### GitOps and Continuous Delivery +With its layered approach, kapp-controller can be used as: +- Continuous delivery for Kubernetes applications using [App CR](app-spec.md) +- Kubernetes Package Management using [Package CR and supplementary CRs](packaging.md) +- Managing applications and packages using GitOps + +#### Share software and build distributions +Use kapp-controller's Package Management features along with Carvel's imgpkg bundles to distribute Package Repositories that can be added to cluster to provide a catalog of software for users to install. Package Repositries can be automatically updated ensuring users always have access to latest versions of software. Package Repositories and Packages can also be relocated and run in air-gapped environments. + +#### Reliable and ready for production! +kapp-controller has been hardened and is in use on production Kubernetes clusters. Learn more through [case studies](/blog/casestudy-modernizing-the-us-army) on our blog. diff --git a/site/content/kapp-controller/docs/v0.32.0/air-gapped-workflow.md b/site/content/kapp-controller/docs/v0.32.0/air-gapped-workflow.md new file mode 100644 index 000000000..098144811 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/air-gapped-workflow.md @@ -0,0 +1,75 @@ +--- +aliases: [/kapp-controller/docs/latest/air-gapped-workflow] +title: Install Packages in an air-gapped (offline) environment +--- + +The documentation below covers topics from the [imgpkg air-gapped workflow docs](/imgpkg/docs/latest/air-gapped-workflow) +more concisely in order to focus on applying these workflows to kapp-controller package repositories. + +## Scenario + +You have a [PackageRepository](packaging#packagerepository-cr) in an [imgpkg bundle format](/imgpkg/docs/latest/resources/#bundle) +in an external OCI registry that you would like to move into an OCI registry in an air-gapped environment. Once relocated, you would +like to deploy the bundle as part of a PackageRepository to a Kubernetes cluster. + +## Prerequisites + +In order to go through this process of moving an imgpkg bundle to an air-gapped environment, you will need to have [imgpkg](/imgpkg) +installed. More information on installing Carvel tools, including `imgpkg`, can be found [here](/#whole-suite). + +## Copy PackageRepository bundle to new location + +Most of the steps documented for the [imgpkg air-gapped workflow docs](/imgpkg/docs/latest/air-gapped-workflow#step-1-finding-bundle-in-source-registry) +still apply in the case of working with kapp-controller package repositories. A summary of these docs is that you will need to copy your package repository +bundle with `imgpkg` via one of the following options: + +- Option 1: From a common location connected to both registries. This option is more efficient because only changed image layers will be transfered between registries. +- Option 2: With intermediate tarball. This option works best when registries have no common network access. + +More detailed documents for [`Option 1`](/imgpkg/docs/latest/air-gapped-workflow/#option-1-from-a-location-connected-to-both-registries) and +[`Option 2`](/imgpkg/docs/latest/air-gapped-workflow/#option-2-with-intermediate-tarball) can be found at the attached links. + +A summary of steps for relocating a package repository bundle to an air-gapped environment are documented for both options below: + +For `Option 1`: +* Get to a location that can access both registries. If there is no such location, you will have to use `Option 2` steps. +* [Authenticate](/imgpkg/docs/latest/auth.md) with both source and destination registries +* Run `imgpkg copy -b index.docker.io/user1/simple-app-bundle:v1.0.0 --to-repo final-registry.corp.com/apps/simple-app-bundle` + +For `Option 2`: +* Get to a location that can access the source registry +* [Authenticate](/imgpkg/docs/latest/auth.md) with the source registry +* Run `imgpkg copy -b index.docker.io/user1/simple-app-bundle:v1.0.0 --to-tar /tmp/my-image.tar` +* Make sure the tar file is in a location that has access to the destination registry +* Authenticate with the destination registry +* Run `imgpkg copy --tar /tmp/my-image.tar --to-repo final-registry.corp.com/apps/simple-app-bundle` + +## Use Relocated Bundle or Image with PackageRepository + +Once you have relocated the package repository bundle into the destination OCI registry in your air-gapped environment, you can +now reference the relocated bundle in a PackageRepository definition: + +```yaml +--- +apiVersion: install.package.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: simple-package-repository +spec: + fetch: + imgpkgBundle: + image: final-registry.corp.com/apps/simple-app-bundle +``` + +In the event your PackageRepository needs authentication to pull the bundle, you can read more about kapp-controller's +[private authentication workflows using secretgen-controller](private-registry-auth.md) or [without secretgen-controller](private-registry-auth.md#packagerepository-authentication-without-secretgen-controller). + +After applying the PackageRepository definition above to your Kubernetes cluster, you will be able to check that the PackageRepository and +its associated Packages were successfully deployed by checking the PackageRepository status: + +```bash +$ kubectl get packagerepository/simple-package-repository +``` + +You will see a message of `Reconcile Succeeded` in the `DESCRIPTION` column of the output from `kubectl` if the PackageRepository was deloyed +successfully. You can also run `kubectl get packages` to see that all Packages were introduced successfully. diff --git a/site/content/kapp-controller/docs/v0.32.0/app-examples.md b/site/content/kapp-controller/docs/v0.32.0/app-examples.md new file mode 100644 index 000000000..1477b0557 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/app-examples.md @@ -0,0 +1,108 @@ +--- +aliases: [/kapp-controller/docs/latest/app-examples] +title: Example Usage +--- + +Below are some example App CRs showing common ways our users have used App CRs. Full App CR spec can be found [here](app-spec.md). + +## Gitops with an app +In this example a user wants to keep their app up to date with changes to the source Git repo + +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: simple-app +spec: + serviceAccountName: default + fetch: + - git: + url: https://github.com/k14s/k8s-simple-app-example + ref: origin/develop + subPath: config-step-2-template + + template: + - ytt: {} + + deploy: + - kapp: {} +``` + +## Gitops with a Helm chart +In this example a user wants to keep their cluster up to date with the latest version of a Helm chart fetched from a Git repo +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: nginx-helm +pec: + fetch: + - git: + url: https://github.com/bitnami/charts + ref: origin/master + subPath: bitnami/nginx + + template: + - helmTemplate: + valuesFrom: + - secretRef: + name: nginx-values + + deploy: + - kapp: {} +``` + +## Install a Helm chart +In this example a user wants to keep their cluster up to date with the latest version of a Helm chart +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: concourse-helm +spec: + fetch: + - helmChart: + name: stable/concourse + + template: + - helmTemplate: + valuesFrom: + - secretRef: + name: concourse-values + + deploy: + - kapp: {} +``` + +## Customize a Helm chart by adding an overlay +In this example a user wants to use `helm template`, but then modify the resulting YAML by adding their own add their own `ytt overlay` +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: concourse-helm +spec: + fetch: + - git: + url: https://github.com/bitnami/charts + ref: origin/master + subPath: bitnami/nginx + + template: + - helmTemplate: {} + - ytt: + ignoreUnknownComments: true + inline: + paths: + remove-lb.yml: | + #@ load("@ytt:overlay", "overlay") + #@overlay/match by=overlay.subset({"kind":"Service","metadata":{"name":"nginx"}}) + --- + spec: + type: ClusterIP + #@overlay/remove + externalTrafficPolicy: + + deploy: + - kapp: {} +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/app-overview.md b/site/content/kapp-controller/docs/v0.32.0/app-overview.md new file mode 100644 index 000000000..f38290451 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/app-overview.md @@ -0,0 +1,73 @@ +--- +aliases: [/kapp-controller/docs/latest/app-overview] +title: App CR High Level Overview +--- + +## Overview +kapp-controller provides a declarative way to install, manage, and upgrade applications on a Kubernetes cluster using the App CRD. Get started by installing the [latest release of kapp-controller](install.md). + +## App +An App is a set of Kubernetes resources. These resources could span any number of namespaces or could be cluster-wide (e.g. CRDs). An App is represented in kapp-controller using a App CR. + +The App CR comprises of three main sections: +- spec.fetch -- declare source for fetching configuration and OCI images +- spec.template -- declare templating tool and values +- spec.deploy -- declare deployment tool and any deploy specific configuration + +Full App CR spec can be found [here](app-spec.md). + +### spec.fetch + +App CR supports multiple source for fetching configuration and OCI images to give developers flexibility. + +- `inline`: specify one or more files within resource +- `imgpkgBundle`: download [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) from registry (available in v0.17.0+) +- `image`: download Docker image from registry +- `http`: download file at URL +- `git`: clone Git repository +- `helmChart`: fetch Helm chart from Helm repository + +For each fetch source, App CR supports specifying Secret resources that will be used for authenticating with the source. kapp-controller does not check for `type` value of Secret resource. + +#### `image` and `imgpkgBundle` authentication + +Allowed secret keys: + +- `username` and `password` +- `token`: Alternative to username/password authentication + +Also supports [dockerconfigjson secret type](https://kubernetes.io/docs/concepts/configuration/secret/#docker-config-secrets) (v0.19.0+) + + +#### `git` authentication + +Allowed secret keys: + +- `ssh-privatekey`: PEM-encoded key that will be provided to SSH +- `ssh-knownhosts`: Optional, set of known hosts allowed to connect (if not specified, all hosts are allowed) +- `username` and `password`: Alternative to private key authentication + +#### `http` and `helmChart` authentication + +Allowed secret keys: + +- `username` and `password` + + +### spec.template + +App CR supports multiple templating, overlaying, and data transformation tools to give developers flexibility. + +- `helmTemplate`: uses `helm template` command to render chart +- `ytt`: uses [ytt](/ytt) to rended templates +- `kbld`: uses [kbld](/kbld) to resolve image URLs to include digests +- `kustomize`: (not implemented yet) uses kustomize to render configuration +- `jsonnnet`: (not implemented yet) renders jsonnet files +- `sops`: uses [sops](https://github.com/mozilla/sops) to decrypt secrets. [More details](sops.md). Available in v0.11.0+. + +--- +### spec.deploy + +App CR uses Carvel's `kapp` CLI to deploy. + +- `kapp`: uses [kapp](/kapp) to deploy resources diff --git a/site/content/kapp-controller/docs/v0.32.0/app-spec.md b/site/content/kapp-controller/docs/v0.32.0/app-spec.md new file mode 100644 index 000000000..c91034fc6 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/app-spec.md @@ -0,0 +1,333 @@ +--- +aliases: [/kapp-controller/docs/latest/app-spec] +title: App CR spec +--- + +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App + +metadata: + name: simple-app + # namespace is going to be used as a default namespace during kapp deploy + namespace: ns + +spec: + # pauses _future_ reconcilation; does _not_ affect + # currently running reconciliation (optional; default=false) + paused: true + + # cancels current and future reconciliations (optional; default=false) + canceled: true + + # Deletion requests for the App will result in the App CR being + # deleted, but its associated resources will not be deleted + # (optional; default=false; v0.18.0+) + noopDelete: true + + # specifies that app should be deployed authenticated via + # given service account, found in this namespace (optional; v0.6.0+) + serviceAccountName: sa-name + + # specifies the length of time to wait, in time + unit + # format, before reconciling. Always >= 30s. If value below + # 30s is specified, 30s will be used. (optional; v0.9.0+; default=30s) + syncPeriod: 1m + + # specifies that app should be deployed to destination cluster; + # by default, cluster is same as where this resource resides (optional; v0.5.0+) + cluster: + # specifies namespace in destination cluster (optional) + namespace: ns2 + # specifies secret containing kubeconfig (required) + kubeconfigSecretRef: + # specifies secret name within app's namespace (required) + name: cluster1 + # specifies key that contains kubeconfig (optional) + key: value + + # Fetch must have one or more directives + fetch: + # pull content from within this resource; or other resources in the cluster + - inline: + # specifies mapping of paths to their content; + # not recommended for sensitive values as CR is not encrypted (optional) + paths: + dir/file.ext: file-content + # specifies content via secrets and config maps; + # data values are recommended to be placed in secrets (optional) + pathsFrom: + - secretRef: + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + + # pulls content from Docker/OCI registry + - image: + # Docker image url; unqualified, tagged, or + # digest references supported (required) + url: host.com/username/image:v0.1.0 + # secret with auth details (optional) + secretRef: + name: secret-name + # grab only portion of image (optional) + subPath: inside-dir/dir2 + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + + # pulls imgpkg bundle from Docker/OCI registry (v0.17.0+) + - imgpkgBundle: + # Docker image url; unqualified, tagged, or + # digest references supported (required) + image: host.com/username/image:v0.1.0 + # secret with auth details (optional) + secretRef: + name: secret-name + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + + # uses http library to fetch file + - http: + # http and https url are supported; + # plain file, tgz and tar types are supported (required) + url: https://host.com/archive.tgz + # checksum to verify after download (optional) + sha256: 0a12cdef83... + # secret to provide auth details (optional) + secretRef: + name: secret-name + # grab only portion of download (optional) + subPath: inside-dir/dir2 + + # uses git to clone repository + - git: + # http or ssh urls are supported (required) + url: https://github.com/k14s/k8s-simple-app-example + # branch, tag, commit; origin is the name of the remote (required) + ref: origin/develop + # secret with auth details. allowed keys: ssh-privatekey, ssh-knownhosts, username, password (optional) + # (if ssh-knownhosts is not specified, git will not perform strict host checking) + secretRef: + name: secret-name + # grab only portion of repository (optional) + subPath: config-step-2-template + # skip lfs download (optional) + lfsSkipSmudge: true + # specifies a strategy to resolve to an explicit ref (optional; v0.24.0+) + refSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + + # uses helm fetch to fetch specified chart + - helmChart: + name: stable/nginx + # (optional) + version: "0.1.0" + # (optional) + repository: + # repository url; + # scheme of oci:// will fetch experimental helm oci chart (v0.19.0+) + # (required) + url: https://... + # (optional) + secretRef: + name: secret-name + + # Template must have one or more directives + template: + # use ytt to template configuration + - ytt: + # ignores comments that ytt doesn't recognize + # (optional; default=false) + ignoreUnknownComments: true + # forces strict mode https://github.com/k14s/ytt/blob/develop/docs/strict.md + # (optional; default=false) + strict: true + # specify additional files, including data values (optional) + inline: + # specifies content inline within resource; + # not recommended for sensitive values as CR is not encrypted (optional) + paths: + # mapping of paths to their content + dir/file.ext: | + file-content + file-content + # specified content via secrets and config maps; + # data values are recommended to be placed in secrets (optional) + pathsFrom: + - secretRef: + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + # lists paths to provide to ytt explicitly (optional) + paths: + # - must be quoted when included with paths + - "-" + - dir/common + - dir/nested/app + # control metadata about input files passed to ytt (optional; v0.18.0+) + # see https://carvel.dev/ytt/docs/latest/file-marks/ for more details + fileMarks: + - file-content:type=yaml-plain + - dir/common/bom**/*:type=text-plain + - dir/nested/app/file.txt:exclude=true + - dir/common/generated.go.txt:path=gen.go.txt + # provide values via ytt's --data-values-file (optional; v0.19.0-alpha.9) + valuesFrom: + - secretRef: + name: secret-name + - configMapRef: + name: cfgmap-name + - path: values/shared.yml + + # use kbld to resolve image references to use digests + - kbld: + # lists paths to use explicitly (optional; v0.13.0+) + # - must be quoted when included with paths + paths: + - "-" + - .imgpkg/images.yml + + # use helm template command to render helm chart + - helmTemplate: + # path to chart (optional; v0.13.0+) + path: some-chart/ + # set name explicitly, default is App CR's name (optional; v0.13.0+) + name: custom-name + # set namespace explicitly, default is App CR's namespace (optional; v0.13.0+) + namespace: custom-ns + # one or more secrets, config maps, paths that provide values (optional) + valuesFrom: + - secretRef: + name: secret-name + - configMapRef: + name: cfgmap-name + - path: values/shared.yml + + # use sops to decrypt *.sops.yml files (optional; v0.11.0+) + - sops: + # use PGP to decrypt files (required) + pgp: + # secret with private armored PGP private keys (required) + privateKeysSecretRef: + # (required) + name: pgp-secrets + # lists paths to decrypt explicitly (optional; v0.13.0+) + paths: + - all-secrets/ + - prod-secrets/prod.sops.yml + + # Deploy must have one directive + deploy: + # use kapp to deploy resources + - kapp: + # override namespace for all resources (optional) + intoNs: another-ns1 + # provide custom namespace override mapping (optional) + mapNs: ["ns1=another-ns1"] + # pass through options to kapp deploy (optional) + rawOptions: ["--apply-concurrency=10"] + # configuration for inspect command (optional) + # as of kapp-controller v0.31.0, inspect is disabled by default + # add rawOptions or use an empty inspect config like `inspect: {}` to enable it + inspect: + # pass through options to kapp inspect (optional) + rawOptions: ["--json=true"] + # configuration for delete command (optional) + delete: + # pass through options to kapp delete (optional) + rawOptions: ["--apply-ignored=true"] + +# status is popuated by the controller +status: + # populated based on metadata.generation when controller + # observes a change to the resource; if this value is + # out of data, other status fields do not reflect latest state + observedGeneration: 1 + + conditions: + # "Reconciling" indicates that fetch/template/deploy is happening; + # it does not mean that any resource has changed + - type: Reconciling + status: "True" + # "ReconcileFailed" indicates that one of the stages failed + - type: ReconcileFailed + status: "True" + # "ReconcileSucceeded" indicates that all stages succeeded + - type: ReconcileSucceeded + status: "True" + + fetch: + exitCode: 0 + error: "..." + stderr: "..." + startedAt: "2019-11-07T16:37:23Z" + updatedAt: "2019-11-07T16:37:23Z" + + template: + exitCode: 0 + error: "..." + stderr: "..." + updatedAt: "2019-11-07T16:37:23Z" + + deploy: + exitCode: 0 + error: "..." + stderr: "..." + finished: true + startedAt: "2019-11-07T16:37:23Z" + stdout: |- + Changes + Namespace Name Kind Conds. Age Op Wait to Rs Ri + Op: 0 create, 0 delete, 0 update, 0 noop + Wait to: 0 reconcile, 0 delete, 0 noop + Succeeded + updatedAt: "2019-11-07T16:37:23Z" + + inspect: + exitCode: 0 + error: "..." + stderr: "..." + stdout: |- + Resources in app 'simple-app-ctrl' + Namespace Name Kind Owner Conds. Rs Ri Age + default simple-app Deployment kapp 2/2 t ok - 7d + default L simple-app-6b6b4fcd97 ReplicaSet cluster - ok - 7d + default L.. simple-app-6b6b4fcd97-kwclv Pod cluster 4/4 t ok - 7d + default simple-app Service kapp - ok - 7d + default L simple-app Endpoints cluster - ok - 7d + Rs: Reconcile state + Ri: Reconcile information + 5 resources + Succeeded + updatedAt: "2019-11-07T16:37:23Z" +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/controller-config.md b/site/content/kapp-controller/docs/v0.32.0/controller-config.md new file mode 100644 index 000000000..076dc4d79 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/controller-config.md @@ -0,0 +1,88 @@ +--- +aliases: [/kapp-controller/docs/latest/controller-config] +title: Configuring the Controller +--- + +kapp-controller exposes the ability to configure the controller via a +Secret (available in v0.22.0+) or ConfigMap (available in v0.14.0+), +which kapp controller will look for and apply as part of its [startup processes](startup.md). + +The controller configuration was originally only available in a ConfigMap +format, but as of v0.22.0 it is recommended to use a Secret since there +may be sensitive information stored in the config (e.g. proxy information including passwords). + +## Controller Configuration Spec + +```yaml +apiVersion: v1 +kind: Secret +metadata: + # Name must be `kapp-controller-config` for kapp controller to pick it up + name: kapp-controller-config + + # Namespace must match the namespace kapp-controller is deployed to + namespace: kapp-controller + +stringData: + # A cert chain of trusted ca certs. These will be added to the system-wide + # cert pool of trusted ca's (optional) + caCerts: | + -----BEGIN CERTIFICATE----- + MIIEXTCCAsWgAwIBAgIQDqAvoGhrmyB/EvhjT/efWzANBgkqhkiG9w0BAQsFADA4 + MQwwCgYDVQQGEwNVU0ExFjAUBgNVBAoTDUNsb3VkIEZvdW5kcnkxEDAOBgNVBAMT + B2Jvc2gtY2EwHhcNMjAxMjIzMTY1OTAxWhcNMjExMjIzMTY1OTAxWjA4MQwwCgYD + VQQGEwNVU0ExFjAUBgNVBAoTDUNsb3VkIEZvdW5kcnkxEDAOBgNVBAMTB2Jvc2gt + Y2EwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCsMTj5yHLez8jzONu1 + tv+u0dqzt8UdWCtUtHCDkIiNJIcB3PkGG7x/LvZ0bMydWeFcBq0g15tfG6N6vHnF + 4p2E9nSe0XjEEnxEkmtdpoFVPZdHTBgc6H5LOMshPH1ARWpuvBnDb87oVinIZBaf + 7BjhUQcRoGtsomk/R9Ke9FB4rMZUfuY/7CC8lDyP5Y02VeTAUimK6/WfDh3VPB3e + vQfXKJY0Ba5s43fIdudV+fcuKDut01oKmiBL6IHLRSrZKta5mg4fgimst6nJ4xvU + SWqYWS4yMxf6pOrTHPjbKUqXqbK4Reh+oQoE12WJZ3NvXr1GoDzt1xzTNzUpUVws + nQm5Fo9H07mkjKeu8gOrOBQ2FqaK+eZ5FFNV7kToVQj2KVTEbLLcTrF454jhsoSd + EOlqVUjtfxGz0dGEuy+IgMvSSjtky7eI08jdBWMiOThQvR3n0Q6TXF/wBwCEfgDa + 4eVeziaUGPXUsefR2+2ZCQ6Z31SmtUGECciCKmKtZTekKCUCAwEAAaNjMGEwDgYD + VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFDwRpmIKYZvr + lKqROus2Ae6gSKkDMB8GA1UdIwQYMBaAFDwRpmIKYZvrlKqROus2Ae6gSKkDMA0G + CSqGSIb3DQEBCwUAA4IBgQA/LX15Qb7v/og06XB27TPl9StGBiewrb0WdHEz9H16 + eN926TwxWKUr6QcbGg6UbNfLUfMC3VicCDMTQCSNhBTUXm+4pKcJsTyM9/Sk/e4U + 5+l3FTgxXs+3mEoYJy16QlkU1XDr1q6Myo9Kc38d1yUW9OPxBV4Ur3+12uk5ElSC + jZu7l+ox2FLds1TmYBhRR/2Jdbm5aoamh4FVpkmDgGedjERREymvnOIMkhWyUfWE + L8Sxa2d8427cBieiEP4foLgjWKr2+diCDrBymU/pz/ZMRRpvUc2uFV005/vmDedK + xQACQ8ZWBYWzNCV4C0Y5AS1PETxbocZ09Yw6K1XyVveEp8aQ/ROMkAUOObhMD45W + GZNwewGU/V7kclDgMwq6R1VXr5R7NtK9V96vi6ZaujoJKvF1PFpZ/IHWcfFkpVoy + Fu8L5PIkg4weBW+87kp+CCseEXPUplpqQCAnmVJdvilK6vgKc7T+vzbET8LNw7NX + mHOVA3CR2w+yUhN4uiCI1aY= + -----END CERTIFICATE----- + + # The url/ip of a proxy for kapp controller to use when making network + # requests (optional) + httpProxy: proxy-svc.proxy-server.svc.cluster.local:80 + + + # The url/ip of a tls capable proxy for kapp controller to use when + # making network requests (optional) + httpsProxy: "" + + # A comma delimited list of domain names which kapp controller should + # bypass the proxy for when making requests (optional) + noProxy: "github.com,docker.io" + + # A comma delimited list of domain names for which kapp controller, when + # fetching images or imgpkgBundles, will skip TLS verification. (optional) + dangerousSkipTLSVerify: "private-registry.com,insecure-registry.com" +``` + +## Config Shorthands + +kapp-controller v0.30.0+ supports a shorthand for easily adding the `KUBERNETES_SERVICE_HOST` +environment variable to kapp-controller's `noProxy` controller config property. This can help +when a Kubernetes cluster is configured with a proxy and the kapp-controller-config is created +with the http and https proxy URL. In this case, kapp-controller fails to communicate with the +Kubernetes API server. + +To make this configuration simpler, the `noProxy` property will interpret `KAPPCTRL_KUBERNETES_SERVICE_HOST` +as the value of `KUBERNETES_SERVICE_HOST` (typically 10.96.9.1) environment variable in the kapp-controller pod. + +```yaml +noProxy: "github.com,docker.io,KAPPCTRL_KUBERNETES_SERVICE_HOST" +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/debugging-crs.md b/site/content/kapp-controller/docs/v0.32.0/debugging-crs.md new file mode 100644 index 000000000..9823d7444 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/debugging-crs.md @@ -0,0 +1,126 @@ +--- +aliases: [/kapp-controller/docs/latest/debugging-crs] +title: Debugging CRs +--- + +Running into issues deploying any of the kapp-controller CRs? This page will help with commonly encountered issues. + +If you can't find what you are looking for here, please reach out to us on #carvel. We love hearing from users and are keen to help you resolve any issues! + +## Debugging kapp-controller CRs + +### Reconcile failed +Your first alert to a failure +will come from the tool(s) (e.g. kapp or kubectl) you are using to deploy +kapp-controller CRs. `kapp` will more immediately tell you if a resource you are +creating or updating fails, but you will need to verify with a `kubectl get` if +using `kubectl` to create/update. + +You can verify a failure occurred by running a `kubectl get` for the resource +you encountered the failure with. You can then see in the `DESCRIPTION` column +of the output of `kubectl get` if the reconciliation process for the resource +failed. An example of this is below: + +``` +NAMESPACE NAME PACKAGE NAME PACKAGE VERSION DESCRIPTION AGE +foo instl-pkg-test-fail pkg.fail.carvel.dev 1.0.0 Reconcile failed: Error (see .status.usefulErrorMessage for details) 12s +``` + +### status.usefulErrorMessage +Once you have confirmed an error occurred, you can review the status of the CR +for more information. + +Apps, PackageInstalls, and PackageRepsitories all feature a status property +named `usefulErrorMessage`. `usefulErrorMessage` which contains an error message +from kapp-controller or the stderr from the underlying tool used by +kapp-controller (i.e. vendir, imgpkg, kbld, ytt, kapp, or helm). + +`usefulErrorMessage` will be located at the bottom of the statuses if running a +`kubectl get` or `kubectl describe` to view more information about a failure. + +`usefulErrorMessage` can also be accessed more directly through `kubectl` like +in the following examples: + +``` +# App errors +$ kubectl get apps/simple-app -o=jsonpath={.status.usefulErrorMessage} -n namespace + +# PackageInstall errors +$ kubectl get packageinstall simple-app -o=jsonpath={.status.usefulErrorMessage} -n namespace + +# PackageRepository errors (cluster scoped so no namespace) +$ kubectl get packagerepository repo -o=jsonpath={.status.usefulErrorMessage} +``` + +### Errors from underlying tools (App CR and PackageInstall CR) + +Failures can arise from fetch, template, deploy, or delete steps for an App CR. +These failures correspond to issues with runtime information declared in the App +CR's spec. kapp-controller creates an App CR for every PackageInstall + +Errors are reported as stderr from associated tools used in kapp-controller +(i.e. vendir, imgpkg, kbld, ytt, kapp, and helm) or as direct messages from +kapp-controller (e.g. when an App uses a ServiceAccount that doesn't exist). + +When a failure occurs with an App CR, you can find further details in the App +CR's `DESCRIPTION` column by running `kubectl get apps/simple-app -n namespace`: + +``` +NAME DESCRIPTION SINCE-DEPLOY AGE +simple-app Delete failed: Preparing kapp: Getting service account: serviceaccounts "default-ns-sa" not found 3s 56m +``` + +In the case above, the error message shown is coming directly from +kapp-controller, so all the information for the failure should be presented in +the description column. This commonly occurs when references used by +kapp-controller (e.g. secrets, configmaps, serviceaccounts) are not found by +kapp-controller. + +In cases where the error message does not originate from kapp-controller (e.g. a +failed fetch event for a git repository), the stderr from the underlying tool +(i.e. vendir, imgpkg, kbld, ytt, kapp, and helm) is shown in the App's status. + +In the App status, there is a field called `usefulErrorMessage` that displays +the stderr for a failure during App reconciliation. + +This `usefulErrorMessage` field can be found by running `kubectl get +apps/simple-app -o=jsonpath={.status.usefulErrorMessage} -n namespace`. The +kubectl command will return the stderr output from the App status to help you +further understand the reason for the App failure. + +The `usefulErrorMessage` can be helpful in pointing out where errors occurred +from inputs in the App spec and also pinpoint the resource that caused a +deployment failure. However, Apps will not surface errors of resources they are +deploying to Kubernetes and further debugging of resources deployed by an App +may be needed. + +### Debugging PackageInstall CRs + +Failures for PackageInstalls can be viewed directly via the `usefulErrorMessage` +property of the PackageInstall's status. This `usefulErrorMessage` property +comes from an App CR that is created as a result of creating a PackageInstall. +More information on interpreting the error message from `usefulErrorMessage` can +be found under the [Errors from underlying tools](#Errors from underlying tools (App CR and PackageInstall CR)). The underlying App +CR will have the same name as the PackageInstall that you create. + +You can also inpect the Package CR referenced by the PackageInstall CR for issues. You can view the Package details by running the following command: + +``` +$ kubectl describe package/ +``` + +You can then view the `.template.spec` of the Package to see if there are any +issues with the inputs of the Package. These inputs are eventually used to +create the App for the PackageInstall and can lead to failures. + +### Debugging PackageRepository CRs + +Failures for PackageRepositories can be viewed directly via the +`usefulErrorMessage` property of the PackageRepository's status. More information [here](status.usefulErrorMessage) + +A common source of errors is being unable to fetch the PackageRepository +contents. Please check the `.spec.fetch` portion of the PackageRepository spec for issues related to this. + +Is the registry you are fetching from require authentication? If so, check out [authenticating to private registries](private-registry-auth.md) + +You can also fetch the PackageRepository `imgpkg` bundle or image separately and inspect format of Package resources. diff --git a/site/content/kapp-controller/docs/v0.32.0/debugging-kc.md b/site/content/kapp-controller/docs/v0.32.0/debugging-kc.md new file mode 100644 index 000000000..e6d3a4cf6 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/debugging-kc.md @@ -0,0 +1,13 @@ +--- +aliases: [/kapp-controller/docs/latest/debugging-kc] +title: Debugging kapp-controller +--- + +The following flags can be used to debug the kapp-controller deployment. Use of these flags are **strongly discouraged in a production setting**. + +## `--dangerous-enable-pprof=true` + +This flag enables [Go's pprof server](https://golang.org/pkg/net/http/pprof/) within kapp-controller process. It runs on `0.0.0.0:6060`. It allows to inspect running Go process in various ways, for example: + +- list goroutines: `http://x.x.x.x/debug/pprof/goroutine?debug=2` +- collect CPU samples: `go tool pprof x.x.x.x/debug/pprof/profile?seconds=60` (useful commands: top10, tree) diff --git a/site/content/kapp-controller/docs/v0.32.0/dev.md b/site/content/kapp-controller/docs/v0.32.0/dev.md new file mode 100644 index 000000000..7eed6c3cf --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/dev.md @@ -0,0 +1,24 @@ +--- +aliases: [/kapp-controller/docs/latest/dev] +title: Development & Deploy +--- + +Install ytt, kbld, kapp beforehand (https://carvel.dev). + +``` +./hack/build.sh # to build locally + +# add `-v image_repo=docker.io/username/kapp-controller` with your registry to ytt invocation inside +./hack/deploy.sh # to deploy + +export KAPPCTRL_E2E_NAMESPACE=kappctrl-test +./hack/test-all.sh +``` + +## Release + +``` +# Bump version in cmd/controller/main.go +# Commit +./hack/build-release.sh +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/faq.md b/site/content/kapp-controller/docs/v0.32.0/faq.md new file mode 100644 index 000000000..593d69d1c --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/faq.md @@ -0,0 +1,97 @@ +--- +aliases: [/kapp-controller/docs/latest/faq] +title: FAQ +--- + +## App CR + +This section covers questions for users directly using the [App](app-spec.md) +custom resource. + +### How can I control App CR reconciliation (pause, force, adjust frequency...)? + +You can set and unset spec.paused +([example](https://github.com/vmware-tanzu/carvel-kapp-controller/blob/d94984a77fa907ac5ecc681e9a842b9877766a6b/test/e2e/pause_test.go#L91)) +or fiddle with spec.syncPeriod ([example]( +https://github.com/vmware-tanzu/carvel-kapp-controller/blob/d94984a77fa907ac5ecc681e9a842b9877766a6b/test/e2e/app_secret_configmap_reconcile_test.go#L133)), which +defaults to 30 seconds. + +### How can I tell which version of kapp-controller is installed? + +kapp-controller sets the annotation `kapp-controller.carvel.dev/version` on the deployment to the version deployed, +so e.g. `kubectl describe deployment kapp-controller -n kapp-controller | grep version` will show the installed version. + +## Package Management CRs + +This section covers questions for users directly using the [Package Management CRs](packaging.md) +custom resource. + +### How does kapp-controller handle PackageInstall when a PackageRepository is removed from the cluster? + +If a PackageInstall has been installed successfully from a Package that is part +of a PackageRepository, and if that PacakgeRepository is ever deleted after the +successful install, the PackageInstall will eventually report the following +error: `Reconcile failed: Expected to find at least one version, but did not`. +This error occurs due to the regular syncing of a PackageInstall with its +Package. + +Even though the error above is reported, the Package will still be installed and +should work as expected. It can also still be uninstalled by deleting the +PackageInstall. The PackageRepository can be recreated and the PackageInstall +will sync and reconcile without any updates needed to resolve the error. + +### How can I generate the valuesSchema from my ytt schema? + +If you are using `ytt` as your Package's templating option and have [defined a schema](../../../ytt/docs/latest/how-to-write-schema), you can use `ytt` to generate your `valuesSchema` (which is in OpenAPI v3 format) for you. + +This is the recommended workflow: + +1. Create an OpenAPI Document from a Data Values Schema file: + + ```bash + $ ytt -f schema.yml --data-values-schema-inspect -o openapi-v3 >schema-openapi.yml + ``` + + which will produce... + + ```yaml + #! schema-openapi.yml + openapi: 3.0.0 + info: + version: 1.0.0 + title: Openapi schema generated from ytt schema + paths: {} + components: + schemas: + dataValues: + type: object + properties: + namespace: + type: string + default: fluent-bit + ``` + +2. Turn your Package CR into a `ytt` template, so that you can insert the schema definition in the right spot, automatically: + + `package-template.yml` + ```yaml + #@ load("@ytt:data", "data") + #@ load("@ytt:yaml", "yaml") + ... + kind: Package + spec: + valuesSchema: + openAPIv3: #@ yaml.decode(data.values.openapi)["components"]["schemas"]["dataValues"] + ... + ``` + + and render with the output from the ytt schema inspect: + + ```bash + $ ytt -f package-template.yml --data-value-file openapi=schema-openapi.yml > package.yml + ``` + +For more details, see: +- [ytt: Export Schema in OpenAPI Format](../../../ytt/docs/latest/how-to-export-schema.md). +- [ytt: Configuring Data Values via command line flags](../../../ytt/docs/latest/ytt-data-values.md#configuring-data-values-via-command-line-flags) +- [@ytt:yaml module](../../../ytt/docs/latest/lang-ref-ytt.md#yaml) diff --git a/site/content/kapp-controller/docs/v0.32.0/install.md b/site/content/kapp-controller/docs/v0.32.0/install.md new file mode 100644 index 000000000..7266945e5 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/install.md @@ -0,0 +1,81 @@ +--- +aliases: [/kapp-controller/docs/latest/install] +title: Install +--- + +Grab the latest copy of YAML from the [Releases page](https://github.com/vmware-tanzu/carvel-kapp-controller/releases) and use your favorite deployment tool (such as [kapp](/kapp) or kubectl) to install it. + +Example: + +```bash +$ kapp deploy -a kc -f https://github.com/vmware-tanzu/carvel-kapp-controller/releases/latest/download/release.yml +``` + +or + +```bash +$ kubectl apply -f https://github.com/vmware-tanzu/carvel-kapp-controller/releases/latest/download/release.yml +``` + +## Specific Environments and Distributions +Some kubernetes distributions require specific setup. +Notes below capture the wisdom of our collective community - we +appreciate your corrections and contributions to help everyone install +kapp-controller everywhere. + +### Openshift +1. Explicitly set resource packageinstalls/finalizers for kapp controller cluster role to access (else the kapp controller fails to create packageinstalls). + + ``` + kind: ClusterRole + metadata: + name: kapp-controller-cluster-role + rules: + - apiGroups: + - packaging.carvel.dev + resources: + ... + - packageinstalls/finalizers + ``` + +2. Bind the kapp-controller cluster role to a security context constraint allowing uids/gids that kapp deployment uses +(currently uid 1000; value given for `runAsUser` in the release.yaml for your +version of kapp-controller). + + **Note:** The security context constraint you provide should allow kapp-controller's uid + to run and should not have root privileges. + + ``` + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: kapp-controller-cluster-role + rules: + - apiGroups: + - security.openshift.io + resourceNames: + - my-nonroot-security-context-contstraint + resources: + - securitycontextconstraints + verbs: + - use + ``` + +3. Set the `IMGPKG_ENABLE_IAAS_AUTH` [environment + variable](/imgpkg/docs/latest/auth/#via-iaas) to false. + + +### Kubernetes versions < 1.20 +Starting in kapp-controller 0.31.0 we have upgraded our underlying kubernetes +libraries which will try to use APIs that don't exist on clusters v1.19 and +earlier. + +Those using k8s v1.19 and earlier will see a repeating error message such as the one below, because +our libraries are hardcoded to watch `v1beta1.PriorityLevelConfiguration` and that won't exist on your cluster. +``` +k8s.io/client-go@v0.22.4/tools/cache/reflector.go:167: Failed to watch *v1beta1.PriorityLevelConfiguration: failed to list *v1beta1.PriorityLevelConfiguration: the server could not find the requested resource (get prioritylevelconfigurations.flowcontrol.apiserver.k8s.io) +``` +While kapp-controller will still work, your logs may fill at a remarkable pace. + +To disable these APIs, set the deployment config variable +`enable_api_priority_and_fairness` to false. diff --git a/site/content/kapp-controller/docs/v0.32.0/oss-packages.md b/site/content/kapp-controller/docs/v0.32.0/oss-packages.md new file mode 100644 index 000000000..9cee0da02 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/oss-packages.md @@ -0,0 +1,24 @@ +--- +aliases: [/kapp-controller/docs/latest/oss-packages] +title: OSS Carvel Packages +--- + +This page provides a list of Carvel Packages and Package Repositories that are available to open source users. + +Do you have a Package or Package Repository you'd like to add to this list? Please make a PR with details to our [docs](https://github.com/vmware-tanzu/carvel/main/site/content/kapp-controller/docs/latest/oss-packages.md). + +## Tanzu Community Edition +Tanzu Community Edition provides several open source [Carvel Packages](https://tanzucommunityedition.io/packages/). These are actively contributed to and maintained by contributors to Tanzu Community Edition. A list of the Package CRs can be found [here](https://github.com/vmware-tanzu/community-edition/tree/main/addons/packages). You can add the Package Repository to your cluster by creating a PackageRepository CR. + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: tce-repo +spec: + fetch: + imgpkgBundle: + # Check out the latest version from Tanzu Community Edition docs + image: projects.registry.vmware.com/tce/main:0.9.1 +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/package-consumer-concepts.md b/site/content/kapp-controller/docs/v0.32.0/package-consumer-concepts.md new file mode 100644 index 000000000..0c60a2095 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/package-consumer-concepts.md @@ -0,0 +1,166 @@ +--- +aliases: [/kapp-controller/docs/latest/package-consumer-concepts] +title: Concepts for Package Consumers +--- + +## Namespacing + +### Overview + +In the packaging APIs, all the CRs are namespaced, which can create a lot of +duplication when wanting to share packages across the cluster. To account for +this, kapp-controller accepts a flag `-packaging-global-namespace`, which +configures kapp-controller to treat the provided namespace as a global namespace +for packaging resources. This means any Package and PackageMetadata CRs within +that namespace will be included in all other namespaces on the cluster, without +duplicating them. This does not apply to PackageRepositories or PackageInstalls. + +### Collisions + +When there is a conflict, the locally namespaced resources will take precedence +over the global ones. A conflict for Packages is defined as having the same +`spec.refName` and `spec.version`, while for PackageMetadatas it is defined as +having the same `metadata.name`. For example, if there is a globally available +PackageMetadata created from the following YAML: + +```yaml +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + name: simple-app.corp.com + namespace: +spec: + categories: + - demo + displayName: Simple App + longDescription: Simple app consisting of a k8s deployment and service + shortDescription: Simple app for demoing +``` + +and then a new locally available PackageMetadata is created from this YAML, + +```yaml +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + name: simple-app.corp.com + namespace: +spec: + categories: + - demo + displayName: Simple App Override + longDescription: My locally available version of the Simple App package + shortDescription: Simple App with some edits +``` + +a user listing the PackageMetadatas will see the second CR, and not the first. + +### Annotations + +For client discoverability, the namespace should also be present as an +annotation on the PackageRepository CRD under the +`packaging.carvel.dev/global-namespace`. Kapp controller's release +YAML comes preconfigured with this annotation. + +(upcoming) If users would like to exclude the global packages from their namespace, the +annotation `packaging.carvel.dev/exclude-global-packages` can be added to +the namespace. + +## Using PackageInstall's Version Selection + +The following sections cover aspects of how to approach version selection when using PackageInstalls. + +### Constraints + +PackageInstalls offer a property called `constraints` under +`.spec.packageRef.versionSelection`. This `constraints` property can be +used to select a specific version of a Package CR to install or include a set of +conditions to pick a version. This `constraints` property is based on semver +ranges and more details on conditions that can be included with `constraints` +can be found [here](https://github.com/blang/semver#ranges). + +To select a specific version of a Package CR to use with a PackageInstall, the +full version (i.e. `.spec.version` from a Package CR) can be included in the +`constraints` property, such as what is shown below: + +```yaml +packageRef: + refName: fluent-bit.vmware.com + versionSelection: + constraints: "1.5.3" +``` + +The example above will result in version 1.5.3 of the Package being installed. + +An example of using a condition to select a Package CR with `constraints` is shown below: + +```yaml +packageRef: + refName: fluent-bit.vmware.com + versionSelection: + constraints: ">1.5.3" +``` + +The above constraint will result in any version greater than `1.5.3` of the +Package being installed. It will also automatically update to the latest +versions of the Package as they become available on the cluster. + +### Prereleases + +When creating a PackageInstall, by default prereleases are not included by +kapp-controller when considering which versions of a Package CR to install. To +include prereleases when creating a PackageInstall, the following can be +added to the spec: + +```yaml +versionSelection: + constraints: "3.0.0-rc.1" + prereleases: {} +``` + +Specifying `prereleases: {}` will make kapp-controller consider all available +prereleases when seeing if a Package CR is available to be installed. + +To filter by releases containing certain substrings, there is an `identifiers` +property under `prereleases` that can be used to only include certain +prereleases that contain the identifier, such as what is shown below: + +```yaml +versionSelection: + constraints: "3.0.0" + prereleases: + identifiers: [rc] +``` + +Multiple identifiers can be specified to include multiple types of pre-releases +(e.g. `identifiers: [rc, beta]`). + +### Downgrading + +In v0.25.0+ of kapp-controller, PackageInstalls feature an annotation to allow +PackageInstalls to be downgraded to previous versions of a Package. By default, +kapp-controller does not allow downgrading to a previous version of a Package to +protect against certain scenarios (e.g. the latest version of a Package being removed +resulting in a unintended reconciliation where the PackageInstall picks up a lower +Package version that is now the latest version). + +If downgrading to a previous version is desired, adding the annotation +`packaging.carvel.dev/downgradable: ""` to a PackageInstall will allow for +explicit or automated ways of downgrading the PackageInstall to a lower version. + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: pkg-demo + annotations: + packaging.carvel.dev/downgradable: "" +spec: + packageRef: + refName: simple-app.corp.com + versionSelection: + constraints: >=1.0.0 +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/package-install-extensions.md b/site/content/kapp-controller/docs/v0.32.0/package-install-extensions.md new file mode 100644 index 000000000..56d7cf833 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/package-install-extensions.md @@ -0,0 +1,65 @@ +--- +aliases: [/kapp-controller/docs/latest/package-install-extensions] +title: Overlays with PackageInstall +--- + +PackageInstalls expose the ability to customize package installation using +annotations recognized by kapp-controller. + +## Adding Paths to YTT (overlays) + +Since it is impossible for package configuration and exposed data values to meet +every consumer's use case, we have added an annotation which enables +consumers to extend the package configuration with custom ytt paths. The most +likely use case for this is providing overlays to tweak configuration that is +not exposed via data values, but it can be used to provide any kind of ytt file. + +The extension annotation is called `ext.packaging.carvel.dev/ytt-paths-from-secret-name` +and can be suffixed with a `.X`, where X is some number, to allow for specifying +it multiple times. For example, + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: fluent-bit + namespace: my-ns + annotations: + ext.packaging.carvel.dev/ytt-paths-from-secret-name.0: my-overlay-secret +spec: + serviceAccountName: fluent-bit-sa + packageRef: + refName: fluent-bit.vmware.com + versionSelection: + constraints: ">v1.5.3" + prereleases: {} + values: + - secretRef: + name: fluent-bit-values + +``` + +will include the overlay stored in the secret `my-overlay-secret` during the +templating steps of the package. This will allow users to further customize a +package installation in advanced cases. + +Example secret resource with a ytt overlay that adds a label to all Namespaces added by this package: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-overlay-secret + namespace: my-ns +stringData: + add-ns-label.yml: | + #@ load("@ytt:overlay", "overlay") + #@overlay/match by=overlay.subset({"kind":"Namespace"}),expects="1+" + --- + metadata: + #@overlay/match missing_ok=True + labels: + #@overlay/match missing_ok=True + custom-lbl: custom-lbl-value +``` + diff --git a/site/content/kapp-controller/docs/v0.32.0/packaging-artifact-formats.md b/site/content/kapp-controller/docs/v0.32.0/packaging-artifact-formats.md new file mode 100644 index 000000000..f7bc5003b --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/packaging-artifact-formats.md @@ -0,0 +1,66 @@ +--- +aliases: [/kapp-controller/docs/latest/packaging-artifact-formats] +title: Artifact formats +--- + +## Package Contents Bundle + +A package bundle is an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) that +holds package contents such as Kubernetes YAML configuration, ytt templates, +Helm templates, etc. + +Filesystem structure used for package bundle creation: + +```bash +my-pkg/ +└── .imgpkg/ + └── images.yml +└── config/ + └── deployment.yml + └── service.yml + └── ingress.yml +``` + +- `.imgpkg/` directory (required) is a standard directory for any imgpkg bundle + - `images.yml` file (required) contains container image refs used by configuration (typically generated with `kbld`) +- `config/` directory (optional) should contain arbitrary package contents such as Kubernetes YAML configuration, ytt templates, Helm templates, etc. + - Recommendations: + - Group Kubernetes configuration into a single directory (`config/` is our + recommendation for the name) so that it could be easily referenced in the + Package CR (e.g. using `ytt` template step against single directory) + +See [Creating a package](packaging-tutorial.md#creating-a-package) for example creation steps. + +## Package Repository Bundle + +A package repository bundle is an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) that holds PackageMetadata and Package CRs. + +Filesystem structure used for package repository bundle creation: + +```bash +my-pkg-repo/ +└── .imgpkg/ + └── images.yml +└── packages/ + └── simple-app.corp.com + └── metadata.yml + └── 1.0.0.yml + └── 1.2.0.yml +``` + +- `.imgpkg/` directory (required) is a standard directory for any imgpkg bundle + - `images.yml` file (required) contains package bundle refs used by Package CRs (typically generated with `kbld`) +- `packages/` directory (required) should contain zero or more YAML files describing available packages + - Each file may contain one or more PackageMetadata or Package CRs (using standard YAML document separator) + - Files may be grouped in directories or kept flat + - File names do not have any special meaning + - Recommendations: + - Separate packages in to directories with the package name + - Keep each PackageMetadata CR in a `metadata.yml` file in the package's + directory. + - Keep each versioned package in a file with the version name inside the package's + directory + - Always have a PackageMetadata CR if you have Package CRs + +See [Creating a Package Repository](packaging-tutorial.md#creating-a-package-repository) for example creation steps. + diff --git a/site/content/kapp-controller/docs/v0.32.0/packaging-gitops.md b/site/content/kapp-controller/docs/v0.32.0/packaging-gitops.md new file mode 100644 index 000000000..538c3f97c --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/packaging-gitops.md @@ -0,0 +1,250 @@ +--- +aliases: [/kapp-controller/docs/latest/packaging-gitops] +title: Package Management with GitOps +--- + +As you begin working with the [package management APIs](packaging.md) for kapp-controller, you may +be wondering how to use kapp-controller's gitops features to manage kapp-controller packages. This +section will cover an example gitops workflow using kapp-controller's package management resources. + +### GitOps Scenario + +An example gitops scenario with kapp-controller could be that a user wants to install a subset of +[Packages](packaging.md#package) from a [PackageRepository](packaging.md#package-repository). The +user wants to define which PackageRepository and Packages to install by defining these resources in +a git repository. With this approach, a user can manage resources in a declarative fashion and track +the history of changes to a Kubernetes cluster. + +After making changes to the main branch of the git repository, the user expects that kapp-controller +will sync these resources to the Kubernetes cluster where kapp-controller is installed. The user also +expects kapp-controller to sync these resources based on updates (e.g. PackageRepository or Package version upgrades) +to the main branch of the git repository. + +### Package Management GitOps Example + +To start, a user should already have kapp-controller installed on a Kubernetes cluster and have +a git repository available. + +First, a user can start by defining an [App custom resource](app-overview.md) like below. **NOTE:** +A user will also need to create a serviceaccount with associated RBAC permissions for the App to use. + +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: pkg-gitops-example + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-rule.create-order: "upsert after upserting rbac" + kapp.k14s.io/change-rule.delete-order: "delete before deleting rbac" +spec: + serviceAccountName: pkg-gitops-app-sa + fetch: + - git: + url: https://github.com/user/my-pkg-gitops-repo + ref: origin/main + subPath: packaging + + template: + - ytt: {} + + deploy: + - kapp: {} +``` + +The App will be pointed at the git repository branch where kapp-controller resources +(e.g. PackageRepository and Packages) will be defined. Read more on setting the App +up with a private git repository [here](app-overview.md#git-authentication). + +By default, an App custom resource will sync the cluster with its fetch source every +30 seconds to prevent the cluster state from drifting from its source of truth, which +is a git repository in this case. + +**NOTE:** The App should be managed separately from any additional kapp-controller resources +stored in a git repository for a gitops workflow. One potential example could be storing the +App definition and associated RBAC in the same git repository it is fetching from and have a +CI/CD process redeploy only the App when a change is made to the App itself versus other resourcees +in the repository. For simplicity in the example above, the user is deploying the App with +`kubectl` or `kapp` manually. + +After creating the App, a user can define a PackageRepository like below: + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: tce-repository + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-group: "tce-repo" +spec: + fetch: + imgpkgBundle: + image: projects.registry.vmware.com/tce/main:0.10.0 +``` + +This PackageRepository will install the [Tanzu Community Edition](https://tanzucommunityedition.io/) +packages on the cluster where kapp-controller is installed. + +Next, a user can pick which Packages to install on the cluster by defining [PackageInstall](packaging.md#packageinstall) +resources. The Tanzu Community Edition repository offers several Packages that are documented +[here](https://tanzucommunityedition.io/docs/latest/package-management/). + +For our gitops example, let's say the user is installing [cert-manager](https://cert-manager.io/docs/) +and [contour](https://projectcontour.io/) on the cluster. To do this, a user could define the following +PackageInstall along with associated RBAC. **NOTE:** The example below gives cluster admin permissions +to the serviceaccount. Make sure to assess appropriate RBAC needed for your specific PackageInstalls. + +RBAC: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pkg-gitops-pkgi-sa + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-group: "packageinstall-setup" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-admin-cluster-role + annotations: + kapp.k14s.io/change-group: "packageinstall-setup" +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-admin-cluster-role-binding + annotations: + kapp.k14s.io/change-group: "packageinstall-setup" +subjects: +- kind: ServiceAccount + name: pkg-gitops-pkgi-sa + namespace: pkg-gitops +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin-cluster-role +``` + +PackageInstalls: + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: cert-manager + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-group: "cert-manager" + kapp.k14s.io/change-rule.create-order: "upsert after upserting packageinstall-setup" + kapp.k14s.io/change-rule.delete-order: "delete before deleting packageinstall-setup" +spec: + serviceAccountName: pkg-gitops-pkgi-sa + packageRef: + refName: cert-manager.community.tanzu.vmware.com + versionSelection: + constraints: 1.5.4 +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: contour + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-rule.create-order: "upsert after upserting cert-manager" + kapp.k14s.io/change-rule.delete-order: "delete before deleting packageinstall-setup" +spec: + serviceAccountName: pkg-gitops-pkgi-sa + packageRef: + refName: contour.community.tanzu.vmware.com + versionSelection: + constraints: 1.17.1 +``` + +The structure of the git repository might look like the example below. In this structure, +the App definition is stored under a folder called app. The App definition above uses a +property called `subPath` to tell kapp-controller to only fetch and sync resources found +under the packaging folder of this git repository. The packaging folder will contain the +PackageRepository, PackageInstalls, and associated RBAC. + +``` +. +├── app +│ ├── app.yml +│ ├── rbac.yml +└── packaging + ├── packageinstalls.yml + ├── rbac.yml + └── repo.yml +``` + +The user can then check in and commit the App, PackageRepository, RBAC, and PackageInstalls to +the main branch of the git repository and push up the resources. + +To deploy the PackageRepository, RBAC, and PackageInstalls, create the App by running the following +commands: + +```shell +# Use kubectl +kubectl apply -f app/ +# Use kapp +kapp deploy -a pkg-gitops -f app/ +``` + +Once committed, the App custom resource created will create the PackageRepository, RBAC, and PackageInstalls +on the cluster. + +The user can view the status of the deployment through the App as well by running the following: + +```shell +kubectl get apps/pkg-gitops-example -n pkg-gitops +``` + +When the App's status is `Reconcile succeeded`, cert-manager and contour should be installed on the +cluster. This can be verified by running the following command: + +```shell +kubectl get pkgi -n pkg-gitops +``` + +### Making an Update + +When it's time to make an update to Packages installed on your cluster, a user can simply +open a pull request to the main branch of the git repositoy, make necessary changes in the +pull request review, and then merge when ready to introduce the change to the cluster. + +To expand on the example above, a user may want to upgrade contour to a later version (e.g. 1.17.2). +To do this, check out the git repository, edit the version used for the PackageInstall like below, +and then commit the change to the main branch. + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: contour + namespace: pkg-gitops + annotations: + kapp.k14s.io/change-rule.create-order: "upsert after upserting cert-manager" + kapp.k14s.io/change-rule.delete-order: "delete before deleting packageinstall-setup" +spec: + serviceAccountName: pkg-gitops-pkgi-sa + packageRef: + refName: contour.community.tanzu.vmware.com + versionSelection: + constraints: 1.17.2 +``` + +Once committed, the App custom resource will eventually sync in the changes. The change can be +verified by running the following command and checking in the kubectl output that the contour +PackageInstall is now using version 1.17.2: + +```shell +kubectl get pkgi/contour -n pkg-gitops +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/packaging-tutorial.md b/site/content/kapp-controller/docs/v0.32.0/packaging-tutorial.md new file mode 100644 index 000000000..88bdda7d4 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/packaging-tutorial.md @@ -0,0 +1,484 @@ +--- +aliases: [/kapp-controller/docs/latest/packaging-tutorial] +title: "Tutorial: Create and Install a Package" +--- + +[//]: # (Generated from katacoda content using 'carvel/tutorials/copy-katacoda-to-static.sh') + +## Get Started With Katacoda +Make a katacoda account and take our interactive tutorial [here](https://katacoda.com/carvel/scenarios/kapp-controller-package-management) +## Or Follow Our Tutorial Below +You can spin up your favorite [playground](https://www.katacoda.com/courses/kubernetes/playground) and follow the steps below. +Note the below steps are from the linked katacoda tutorial so your environment may differ slightly. + +## Installing kapp-controller dependencies + +We'll be using [Carvel](https://carvel.dev/) tools throughout this tutorial, so first we'll install +[ytt](https://carvel.dev/ytt/), [kbld](https://carvel.dev/kbld/), +[kapp](https://carvel.dev/kapp/), [imgpkg](https://carvel.dev/imgpkg/), and [vendir](https://carvel.dev/vendir/). + +Install the whole tool suite with the script below: + +(Note: we are temporarily overriding kapp-controller's version to jump to ytt +0.38.0, in order to include the recent OpenAPI Schema feature in this tutorial) +```bash +wget -O- https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/fc5458fe2102d67e85116c26534a35e265b28125/hack/install-deps.sh | \ +sed 's/ytt_version=v0.35.1/ytt_version=v0.38.0/' | \ +sed 's/0aa78f7b5f5a0a4c39bddfed915172880344270809c26b9844e9d0cbf6437030/2ca800c561464e0b252e5ee5cacff6aa53831e65e2fb9a09cf388d764013c40d/' | \ +bash +``` + + +## Optional: explore kapp + +Before we install kapp-controller with [kapp](https://carvel.dev/kapp/), you may be interested in seeing +a different example of how kapp works. + +You can skip this step if you want to get straight to kapp-controller. + +### Using kapp to install a cronjob + +First clone the GitHub repository for examples: + +```bash +git clone https://github.com/vmware-tanzu/carvel-kapp +``` + +Then deploy a CronJob to the Kubernetes cluster in this environment: + +```bash +kapp deploy -a hellocron -f carvel-kapp/examples/jobs/cron-job.yml -y +``` + +Now take a look at the Kubernetes resources being managed by kapp: + +```bash +kapp ls +``` + +```bash +kapp inspect -a hellocron --tree +``` + +We scheduled our CronJob to output a hello message every minute, so if you're +patient you'll see new messages appended to the logs: + +```bash +kapp logs -f -a hellocron +``` + +When you're done watching the logs you can use control-c (`^C`) to quit. + +Because this was an optional interlude, we can use kapp to uninstall the CronJob before proceeding: +```bash +kapp delete -a hellocron -y +``` +## I believe I was promised kapp-controller? + +Use kapp to install kapp-controller (reconciliation may take a moment, which you +could use to read about [kubernetes controller reconciliation loops](https://kubernetes.io/docs/concepts/architecture/controller/)): + +```bash +kapp deploy -a kc -f https://github.com/vmware-tanzu/carvel-kapp-controller/releases/download/v0.21.0/release.yml -y +``` + +Gaze upon the splendor! + +```bash +kubectl get all -n kapp-controller +``` + +The kapp deployment is managing a replicaset which owns a service and a pod. The +pod is running kapp-controller, which is a kubernetes controller +running its own reconciliation loop. + +kapp-controller introduces new Custom Resource (CR) types we'll use throughout this +tutorial, including PackageRepositories and PackageInstalls. + +```bash +kubectl api-resources --api-group packaging.carvel.dev +``` + +You can see other kapp-controller CRs in other groups: + +```bash +kubectl api-resources --api-group data.packaging.carvel.dev +``` + +```bash +kubectl api-resources --api-group kappctrl.k14s.io +``` +## Creating a Package: Templating our config + +We will be using [ytt](https://carvel.dev/ytt/) templates that describe a simple Kubernetes Deployment and Service. +These templates will install a simple greeter app with a templated hello message. + +Create a config.yml: + +```bash +cat > config.yml << EOF +#@ load("@ytt:data", "data") + +#@ def labels(): +simple-app: "" +#@ end + +--- +apiVersion: v1 +kind: Service +metadata: + namespace: default + name: simple-app +spec: + ports: + - port: #@ data.values.svc_port + targetPort: #@ data.values.app_port + selector: #@ labels() +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: default + name: simple-app +spec: + selector: + matchLabels: #@ labels() + template: + metadata: + labels: #@ labels() + spec: + containers: + - name: simple-app + image: docker.io/dkalinin/k8s-simple-app@sha256:4c8b96d4fffdfae29258d94a22ae4ad1fe36139d47288b8960d9958d1e63a9d0 + env: + - name: HELLO_MSG + value: #@ data.values.hello_msg +EOF +``` + +and put our schema into values.yml: + +```bash +cat > values.yml <<- EOF +#@data/values-schema +--- +#@schema/desc "Port number for the service." +svc_port: 80 +#@schema/desc "Target port for the application." +app_port: 80 +#@schema/desc "Name used in hello message from app when app is pinged." +hello_msg: stranger +EOF +``` + +## Creating a Package: Structuring our contents +We'll create an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle) +that contains the package contents: the configuration (config.yml and values.yml from the previous step) and a reference to the greeter app image (docker.io/dkalinin/k8s-simple-app@sha256:...). + +The [package bundle format](packaging-artifact-formats.md#package-contents-bundle) describes the purpose of each directory +used in this section of the tutorial as well as general recommendations. + +Let's create a directory with our configuration files: +```bash +mkdir -p package-contents/config/ +cp config.yml package-contents/config/config.yml +cp values.yml package-contents/config/values.yml +``` + +Once we have the configuration figured out, let’s use kbld to record which container images are used: +```bash +mkdir -p package-contents/.imgpkg +kbld -f package-contents/config/ --imgpkg-lock-output package-contents/.imgpkg/images.yml +``` + +For more on using kbld to populate the .imgpkg directory with an ImagesLock, and why it is useful, see the [imgpkg docs on the subject](/imgpkg/docs/latest/resources/#imageslock-configuration). + +Once these files have been added, our package contents bundle is ready to be pushed! + +For the purpose of this tutorial, we will run an unsecured local docker +registry. In the real world please be safe and use appropriate security +measures. + +```bash +docker run -d -p 5000:5000 --restart=always --name registry registry:2 +``` + +From the terminal we can access this registry as `localhost:5000` but within the +cluster we'll need the IP Address. To emphasize that you would +normally use a repo host such as dockerhub or harbor we will store the IP +address in a variable: + +```bash +export REPO_HOST="`ifconfig | grep -A1 docker | grep inet | cut -f10 -d' '`:5000" +``` + +Now we can publish our bundle to our registry: + +```bash +imgpkg push -b ${REPO_HOST}/packages/simple-app:1.0.0 -f package-contents/ +``` + + +You can verify that we pushed something called `packages/simple-app` by checking the Docker registry catalog: + +```bash +curl ${REPO_HOST}/v2/_catalog +``` + +## Creating the Custom Resources + +To finish creating a package, we need to create two CRs. The first CR is the PackageMetadata CR, which will contain high level information and descriptions about our package. + +When creating this CR, the api will validate that the PackageMetadata’s name is a fully qualified name: It must have at least three segments separated by `.` and cannot have a trailing `.`. + +We'll make a conformant `metadata.yml` file: + +```bash +cat > metadata.yml << EOF +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + # This will be the name of our package + name: simple-app.corp.com +spec: + displayName: "Simple App" + longDescription: "Simple app consisting of a k8s deployment and service" + shortDescription: "Simple app for demoing" + categories: + - demo +EOF +``` + +Now we need to create a Package CR. +This CR contains versioned instructions and metadata used to install packaged software that fits the description provided in the PackageMetadata CR we just saved in `metadata.yml`. + +In order to create the Package CR with our OpenAPI Schema, we will export from +our ytt schema: + +```bash +ytt -f package-contents/config/values.yml --data-values-schema-inspect -o openapi-v3 > schema-openapi.yml +``` + +That command creates an OpenAPI document, from which we really only need the +`components.schema` section for our Package CR. + + +```bash +cat > package-template.yml << EOF +#@ load("@ytt:data", "data") # for reading data values (generated via ytt's data-values-schema-inspect mode). +#@ load("@ytt:yaml", "yaml") # for dynamically decoding the output of ytt's data-values-schema-inspect +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: #@ "simple-app.corp.com." + data.values.version +spec: + refName: simple-app.corp.com + version: #@ data.values.version + releaseNotes: | + Initial release of the simple app package + valuesSchema: + openAPIv3: #@ yaml.decode(data.values.openapi)["components"]["schemas"]["dataValues"] + template: + spec: + fetch: + - imgpkgBundle: + image: #@ "${REPO_HOST}/packages/simple-app:" + data.values.version + template: + - ytt: + paths: + - "config/" + - kbld: + paths: + - "-" + - ".imgpkg/images.yml" + deploy: + - kapp: {} +EOF +``` + +This Package contains some metadata fields specific to the version, such as releaseNotes and a valuesSchema. The valuesSchema shows what configurable properties exist for the version. This will help when users want to install this package and want to know what can be configured. + +The other main component of this CR is the template section. +This section informs kapp-controller of the actions required to install the packaged software, so take a look at the [app-spec](app-spec.md) section to learn more about each of the template sections. For this example, we have chosen a basic setup that will fetch the imgpkg bundle we created in the previous section, run the templates stored inside through ytt, apply kbld transformations, and then deploy the resulting manifests with kapp. + +There will also be validations run on the Package CR, so ensure that spec.refName and spec.version are not empty and that metadata.name is `.`. +These validations are done to encourage a naming scheme that keeps package version names unique. +## Creating a Package Repository + +A [package repository](packaging.md#package-repository) +is a collection of packages (more specifically a collection of Package and PackageMetadata CRs). +Our recommended way to make a package repository is via an [imgpkg bundle](/imgpkg/docs/latest/resources/#bundle). + +The [PackageRepository bundle format](packaging-artifact-formats.md#package-repository-bundle) describes purpose of each directory and general recommendations. + +Lets start by creating the needed directories: + +```bash +mkdir -p my-pkg-repo/.imgpkg my-pkg-repo/packages/simple-app.corp.com +``` + +we can copy our CR YAMLs from the previous step in to the proper packages +subdirectory. Note that we are declaring the version and the openAPI schema file +to ytt. + +```bash +ytt -f package-template.yml --data-value-file openapi=schema-openapi.yml -v version="1.0.0" > my-pkg-repo/packages/simple-app.corp.com/1.0.0.yml +cp metadata.yml my-pkg-repo/packages/simple-app.corp.com +``` + +Next, let’s use kbld to record which package bundles are used: + +```bash +kbld -f my-pkg-repo/packages/ --imgpkg-lock-output my-pkg-repo/.imgpkg/images.yml +``` + +With the bundle metadata files present, we can push our bundle to whatever OCI +registry we plan to distribute it from, which for this tutorial will just be our +same REPO_HOST. + +```bash +imgpkg push -b ${REPO_HOST}/packages/my-pkg-repo:1.0.0 -f my-pkg-repo +``` + +The package repository is pushed! + +You can verify by checking the Docker registry catalog: + +```bash +curl ${REPO_HOST}/v2/_catalog +``` + +In the next steps we'll act as the package consumer, showing an example of adding and using a PackageRepository with kapp-controller. + +## Adding a PackageRepository + +kapp-controller needs to know which packages are available to install. +One way to let it know about available packages is by creating a package repository. +To do this, we need a PackageRepository CR: + +```bash +cat > repo.yml << EOF +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: simple-package-repository +spec: + fetch: + imgpkgBundle: + image: ${REPO_HOST}/packages/my-pkg-repo:1.0.0 +EOF +``` + +(See our +[demo video](https://www.youtube.com/watch?v=PmwkicgEKQE) and [website](private-registry-auth.md) for more typical usage with an external repository.) + +This PackageRepository CR will allow kapp-controller to install any of the +packages found within the `${REPO_HOST}/packages/my-pkg-repo:1.0.0` imgpkg bundle, which we +stored in our docker OCI registry previously. + +We can use kapp to apply it to the cluster: +```bash +kapp deploy -a repo -f repo.yml -y +``` + +Check for the success of reconciliation to see the repository become available: +```bash +watch kubectl get packagerepository +``` + +Once the simple-package-repository has a "**Reconcile succeeded**" description, +we're ready to continue! You can exit the watch by hitting control-c or +clicking: `^C` + +Once the deploy has finished, we are able to list the package metadatas to see, at a high level, which packages are now available within our namespace: +```bash +kubectl get packagemetadatas +``` + +If there are numerous available packages, each with many versions, this list can become a bit unwieldy, so we can also list the packages with a particular name using the --field-selector option on kubectl get. +```bash +kubectl get packages --field-selector spec.refName=simple-app.corp.com +``` + +From here, if we are interested, we can further inspect each version to discover +information such as release notes, installation steps, licenses, etc. For +example: +```bash +kubectl get package simple-app.corp.com.1.0.0 -o yaml +``` + + +## Installing a Package + +Once we have the packages available for installation (as seen via `kubectl get packages`), +we need to let kapp-controller know which package we want to install. +To do this, we will need to create a PackageInstall CR (and a secret to hold the values used by our package): + +```bash +cat > pkginstall.yml << EOF +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: pkg-demo +spec: + serviceAccountName: default-ns-sa + packageRef: + refName: simple-app.corp.com + versionSelection: + constraints: 1.0.0 + values: + - secretRef: + name: pkg-demo-values +--- +apiVersion: v1 +kind: Secret +metadata: + name: pkg-demo-values +stringData: + values.yml: | + --- + hello_msg: "to all my katacoda friends" +EOF +``` + + +This CR references the Package we created in the previous sections using the package’s `refName` and `version` fields (see yaml from step 7). +Do note, the `versionSelection` property has a constraints subproperty to give more control over which versions are chosen for installation. +More information on PackageInstall versioning can be found [here](packaging.md#versioning-packageinstalls). + +This yaml snippet also contains a Kubernetes secret, which is referenced by the PackageInstall. This secret is used to provide customized values to the package installation’s templating steps. Consumers can discover more details on the configurable properties of a package by inspecting the Package CR’s valuesSchema. + +Finally, to install the above package, we will also need to create `default-ns-sa` service account (refer to [Security model](security-model.md) +for explanation of how service accounts are used) that give kapp-controller privileges to create resources in the default namespace: +```bash +kapp deploy -a default-ns-rbac -f https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/develop/examples/rbac/default-ns.yml -y +``` + +Apply the PackageInstall using kapp: +```bash +kapp deploy -a pkg-demo -f pkginstall.yml -y +``` + +After the deploy has finished, kapp-controller will have installed the package in the cluster. We can verify this by checking the pods to see that we have a workload pod running. The output should show a single running pod which is part of simple-app: +```bash +kubectl get pods +``` + +Once the pod is ready, you can use kubectl’s port forwarding to verify the customized hello message has been used in the workload: +```bash +kubectl port-forward service/simple-app 3000:80 & +``` + +Now if we make a request against our service, we can see that our `hello_msg` +values is being used: +```bash +curl localhost:3000 +``` +## Congratulations! + +Visit [carvel.dev](https://carvel.dev/) to learn more about Carvel tools. + +See the full docs for [Package Management with kapp-controller](packaging.md) diff --git a/site/content/kapp-controller/docs/v0.32.0/packaging.md b/site/content/kapp-controller/docs/v0.32.0/packaging.md new file mode 100644 index 000000000..ff741d8b9 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/packaging.md @@ -0,0 +1,296 @@ +--- +aliases: [/kapp-controller/docs/latest/packaging] +title: Package Management +--- + + +## Overview + +kapp-controller provides a declarative way to install, manage, and upgrade packages on a Kubernetes cluster. It leverages the PackageRepository, PackageMetadata, Package, and PackageInstall CRDs to do so. Get started by installing the [latest release of kapp-controller](install.md). + +## Concepts & CustomResourceDefinitions + +### Package + +A package is a combination of configuration metadata and OCI images that informs the package manager what software it holds and how to install itself onto a Kubernetes cluster. For example, an nginx-ingress package would instruct the package manager where to download the nginx container image, how to configure the associated Deployment, and install it into a cluster. + +A Package is represented in kapp-controller using a Package CR. The Package CR is created for every new version of a package and it carries information about how to fetch, template, and deploy the package. A Package CR is a namespaced resource by default. [Learn more](package-consumer-concepts.md#namespacing) about how to share a Package CR across all namespaces within a cluster. + +```yaml +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + # Must be of the form '.' (Note the period) + name: fluent-bit.carvel.dev.1.5.3 + # The namespace this package is available in + namespace: my-ns +spec: + # The name of the PackageMetadata associated with this version + # Must be a valid PackageMetadata name (see PackageMetadata CR for details) + # Cannot be empty + refName: fluent-bit.carvel.dev + # Package version; Referenced by PackageInstall; + # Must be valid semver (required) + # Cannot be empty + version: 1.5.3 + # Version release notes (optional; string) + releaseNotes: "Fixed some bugs" + # System requirements needed to install the package. + # Note: these requirements will not be verified by kapp-controller on + # installation. (optional; string) + capacityRequirementsDescription: "RAM: 10GB" + # Description of the licenses that apply to the package software + # (optional; Array of strings) + licenses: + - "Apache 2.0" + - "MIT" + # Timestamp of release (iso8601 formatted string; optional) + releasedAt: 2021-05-05T18:57:06Z + # valuesSchema can be used to show template values that + # can be configured by users when a Package is installed. + # These values should be specified in an OpenAPI schema format. (optional) + valuesSchema: + # openAPIv3 key can be used to declare template values in OpenAPIv3 + # format. Read more on using ytt to generate this schema: + # https://carvel.dev/kapp-controller/docs/latest/packaging-tutorial/#creating-the-custom-resources + openAPIv3: + title: fluent-bit.carvel.dev.1.5.3 values schema + examples: + - namespace: fluent-bit + properties: + namespace: + type: string + description: Namespace where fluent-bit will be installed. + default: fluent-bit + examples: + - fluent-bit + # App template used to create the underlying App CR. + # See 'App CR Spec' docs for more info + template: + spec: + fetch: + - imgpkgBundle: + image: registry.tkg.vmware.run/tkg-fluent-bit@sha256:... + template: + - ytt: + paths: + - config/ + - kbld: + paths: + # - must be quoted when included with paths + - "-" + - .imgpkg/images.yml + deploy: + - kapp: {} +``` + +### Package Metadata + +Package Metadata are attributes of a single package that do not change frequently and that are shared across multiple versions of a single package. It contains information similar to a project's README.md. + +It is represented in kapp-controller by a PackageMetadata CR. A PackageMetadata CR is a namespaced resource by default. [Learn more](package-consumer-concepts.md#namespacing) about how to share a PackageMetadata CR across all namespaces within a cluster. + +```yaml +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + # Must consist of at least three segments separated by a '.' + # Cannot have a trailing '.' + name: fluent-bit.vmware.com + # The namespace this package metadata is available in + namespace: my-ns +spec: + # Human friendly name of the package (optional; string) + displayName: "Fluent Bit" + # Long description of the package (optional; string) + longDescription: "Fluent bit is an open source..." + # Short desription of the package (optional; string) + shortDescription: "Log processing and forwarding" + # Base64 encoded icon (optional; string) + iconSVGBase64: YXNmZGdlcmdlcg== + # Name of the entity distributing the package (optional; string) + providerName: VMware + # List of maintainer info for the package. + # Currently only supports the name key. (optional; array of maintner info) + maintainers: + - name: "Person 1" + - name: "Person 2" + # Classifiers of the package (optional; Array of strings) + categories: + - "logging" + - "daemon-set" + # Description of the support available for the package (optional; string) + supportDescription: "..." +``` + +### Package Repository + +A package repository is a collection of packages and their metadata. Similar to a maven repository or a rpm repository, adding a package repository to a cluster gives users of that cluster the ability to install any of the packages from that repository. + +It is represented in kapp-controller by a PackageRepository CR. A PackageRepository CR is a namespaced resource by default. [Learn more](package-consumer-concepts.md#namespacing) about how to share a PackageRepository CR across all namespaces within a cluster. + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + # Any user-chosen name that describes package repository + name: basic.carvel.dev + # The namespace to make packages available to + namespace: my-ns +spec: + # pauses _future_ reconcilation; does _not_ affect + # currently running reconciliation (optional; default=false) + paused: true + # specifies the length of time to wait, in time + unit + # format, before reconciling.(optional; default=10m) + syncPeriod: 1m + # Must have only one directive. + fetch: + # pull content from within this resource; or other resources in the cluster + inline: # NOTE: inline fetch available since v 0.31.0 + # specifies mapping of paths to their content; + # not recommended for sensitive values as CR is not encrypted (optional) + paths: + dir/file.ext: file-content + # specifies content via secrets and config maps; + # data values are recommended to be placed in secrets (optional) + pathsFrom: + - secretRef: + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + # pulls imgpkg bundle from Docker/OCI registry + imgpkgBundle: + # Docker image url; unqualified, tagged, or + # digest references supported (required) + image: host.com/username/image:v0.1.0 + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + # pulls image containing packages from Docker/OCI registry + image: + # Image url; unqualified, tagged, or + # digest references supported (required) + url: host.com/username/image:v0.1.0 + # grab only portion of image (optional) + subPath: inside-dir/dir2 + # specifies a strategy to choose a tag (optional; v0.24.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">1.0.0 <3.0.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] + # uses http library to fetch file containing packages + http: + # http and https url are supported; + # plain file, tgz and tar types are supported (required) + url: https://host.com/archive.tgz + # checksum to verify after download (optional) + sha256: 0a12cdef83... + # grab only portion of download (optional) + subPath: inside-dir/dir2 + # uses git to clone repository containing package list + git: + # http or ssh urls are supported (required) + url: https://github.com/k14s/k8s-simple-app-example + # branch, tag, commit; origin is the name of the remote (required) + ref: origin/develop + # grab only portion of repository (optional) + subPath: config-step-2-template + # skip lfs download (optional) + lfsSkipSmudge: true + # specifies a strategy to resolve to an explicit ref (optional; v0.24.0+) + refSelection: + semver: + # list of semver constraints (see https://carvel.dev/vendir/docs/latest/versions/ for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.24.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.24.0+) + identifiers: [beta, rc] +``` + +### Package Install + +A Package Install is an actual installation of a package and its underlying resources on a Kubernetes cluster. It is represented in kapp-controller by a PackageInstall CR. A PackageInstall CR must reference a Package CR. + +```yaml +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: fluent-bit + # The namespace to install the package in to + namespace: my-ns +spec: + # pauses _future_ reconcilation; does _not_ affect + # currently running reconciliation (optional; default=false) + paused: true + # cancels current and future reconciliations (optional; default=false) + canceled: true + # Deletion requests for the PackageInstall/App will result in + # the PackageInstall/App CR being deleted, but its associated + # resources will not be deleted (optional; default=false) + noopDelete: true + # specifies the length of time to wait, in time + unit + # format, before reconciling.(optional; default=10m) + syncPeriod: 1m + # specifies that Package should be deployed to destination cluster; + # by default, cluster is same as where this resource resides (optional) + cluster: + # specifies namespace in destination cluster (optional) + namespace: ns2 + # specifies secret containing kubeconfig (required) + kubeconfigSecretRef: + # specifies secret name within app's namespace (required) + name: cluster1 + # specifies key that contains kubeconfig (optional) + key: value + # specifies service account that will be used to install underlying package contents + serviceAccountName: fluent-bit-sa + packageRef: + # Specifies the name of the package to install (required) + refName: fluent-bit.vmware.com + # Selects version of a package based on constraints provided (optional) + # Either version or versionSelection is required. + versionSelection: + # Constraint to limit acceptable versions of a package; + # Latest version satisfying the constraint is chosen; + # Newly available, acceptable later versions are picked up and installed automatically. (optional) + constraints: ">v1.5.3" + # Include prereleases when selecting version. (optional) + prereleases: {} + # Values to be included in package's templating step + # (currently only included in the first templating step) (optional) + values: + - secretRef: + name: fluent-bit-values +# Populated by the controller +status: + packageRef: + # Kubernetes resource name of the package chosen against the constraints + name: fluent-bit.tkg.vmware.com.v1.5.3 + # Derived from the underlying App's Status + conditions: + - type: ValuesSchemaCheckFailed + - type: ReconcileSucceeded + - type: ReconcileFailed + - type: Reconciling +``` + +**Note:** Values will only be included in the first templating step of the package, +though we intend to improve this experience in later releases. diff --git a/site/content/kapp-controller/docs/v0.32.0/private-registry-auth.md b/site/content/kapp-controller/docs/v0.32.0/private-registry-auth.md new file mode 100644 index 000000000..008a87105 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/private-registry-auth.md @@ -0,0 +1,217 @@ +--- +aliases: [/kapp-controller/docs/latest/private-registry-auth] +title: Authenticating to Private Registries +--- + +## Scenario + +As a package consumer you may need to provide registry credentials if you are consuming package repository (and/or packages) from a registry that requires authenticated access. That may involve providing registry credentials to multiple parts of the system: + +- credentials for pulling package repository bundle (via PackageRepository CR) + - consumed by imgpkg running inside kapp-controller Pod +- credentials for pulling package contents bundle (via PackageInstall CR) + - consumed by imgpkg running inside kapp-controller Pod +- credentials for pulling container images used by the package + - credentials consumed by Kubelets + - e.g. needed by cert-manager controller Pod +- credentials for pulling container images used by packages operator + - credentials consumed by Kubelets + - e.g. needed by Kafka cluster Pods created for KafkaInstance CR + +Providing credentials manually to each one of these parts of the system can become a hassle. kapp-controller v0.24.0+ when installed together with [secretgen-controller](https://github.com/vmware-tanzu/carvel-secretgen-controller) v0.5.0+ allow package consumers and package authors to simplify such configuration. + +Note that if you are using an IaaS provided Kubernetes cluster already preauthenticated with an IaaS provided registry, then there is no need to provide credentials manually in the cluster. kapp-controller v0.25.0+ is able to automatically pick up provided credentials to satisfy first two bullet points above. Last two bullet points are already satisfied by the Kubernetes kubelet. + +## secretgen-controller's placeholder secrets and SecretExport CR + +For this specific use case, secretgen-controller allows package consumer to specify registry credentials in one namespace and allows to export that secret to the entire cluster (or subset of namespaces) via [SecretExport CR](https://github.com/vmware-tanzu/carvel-secretgen-controller/blob/develop/docs/secret-export.md#secretexport-and-secretrequest). Registry credentials could be consumed in different namespaces via "placeholder secrets". + +A placeholder secret is: +- plain Kubernetes Secret +- with `kubernetes.io/dockerconfigjson` type (more about this secret type [here](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials)) +- has `secretgen.carvel.dev/image-pull-secret` annotation + +secretgen-controller will populate placeholder Secrets with a combined registry credentials. For example: + +- within `reg-creds` Namespace + - Secret `dockerhub-reg` includes DockerHub credentials for `index.docker.io` domain + - SecretExport CR `dockerhub-reg` specifies that same-named secret will be exported to all namespaces + - Secret `corp-reg` includes registry credentials for `registry.corp.com` domain + - SecretExport CR `corp-reg` specifies that same-named secret will be exported to all namespaces +- within `cert-manager-install` Namespace + - Secret `reg-creds` has `secretgen.carvel.dev/image-pull-secret` annotation indicating to secretgen to continuously ensure that this secret is filled with combination of registry credentials that allow export to this namespace (in this case both `dockerhub-reg` and `corp-reg`) + +Known limitation: Currently Secrets with type `kubernetes.io/dockerconfigjson` do not allow specifying multiple credentials for the same domain, hence you cannot provide multiple credentials for the same registry. + +**Warning** Since SecretExport CR allows you to export registry credentials to other namespaces, they will become visible to users of such namespaces. We **strongly recommend** to ensure that registry credentials you are exporting only allow read-only access to the registry and are minimally scoped within the registry. + +## kapp-controller CRs and placeholder secrets + +As of kapp-controller v0.24.0+, PackageRepository and PackageInstall CRs automatically create placeholder secrets for `image` and `imgpkgBunle` fetch types, if no explicit `secretRef.name` is provided. (These placeholder secrets are named as `-fetch-`.) If secretgen-controller is present on the cluster, these secrets will be populated with combined registry credentials; otherwise, they will remain empty. + +## Package authoring and placeholder secrets + +We encourage all package authors to include placeholder secrets within your package configuration already preconfigured to be used by your Deployments, StatefulSets, DaemonSets, Pods, etc (and any other resources that consume image pull secrets). This removes a need for package consumers to worry about configuring packages in any special way if it's being consumed from a registry that requires authentication. Note that even if you are distributing package repository from a registry that support anonymous access, package consumers may still copy it (via imgpkg copy) into a private registry that does require authentication. + +Note: In future we could provide a feature to automatically inject placeholder secrets as part of package installation (e.g. via Pod webhook); however, that is a bit more intrusive, hence we are recommending explicit usage of placeholder secrets for now. + +Example of a placeholder secret package authors should add next other resources: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: reg-creds + annotations: + secretgen.carvel.dev/image-pull-secret: "" +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: e30K +``` + +Note: `e30K` is base64 encoded `{}`. Valid `.dockerconfigjson` value is required when creating a Secret. + +## Operator writing and placeholder secrets + +If you are an owner of an operator, similar to the above section, we encourage you to create a placeholder secret for Pods (or other resources that consume image pull secrets) that may be created by your operator in other namespaces. More general operator packaging docs will come soon. + +--- +## Bringing it all together + +- Ensure kapp-controller v0.24.0+ is installed + +- Install secretgen-controller v0.5.0+ + + ```bash + kapp deploy -a sg -f https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/download/v0.5.0/release.yml + ``` + +- Create registry credential Secret and use SecretExport CR to make it available for all namespaces (Note: if you use `kubectl create secret docker-registry` and you want to auth with DockerHub, please specify `--docker-server=index.docker.io` explicitly instead of relying on default server value.) + + ```yaml + --- + apiVersion: v1 + kind: Secret + metadata: + name: reg-creds # could be any name + namespace: secrets-ns # could be any namespace + type: kubernetes.io/dockerconfigjson # needs to be this type + stringData: + .dockerconfigjson: | + { + "auths": { + "index.docker.io": { + "username": "user...", + "password": "password...", + "auth": "" + } + } + } + + --- + apiVersion: secretgen.carvel.dev/v1alpha1 + kind: SecretExport + metadata: + name: reg-creds # must match source secret name + namespace: secrets-ns # must match source secret namespace + spec: + toNamespaces: + - "*" # star means export is available for all namespaces + ``` + +- Use PackageRepository and PackageInstall CRs without specifying secrets explicitly + + ```yaml + --- + apiVersion: packaging.carvel.dev/v1alpha1 + kind: PackageRepository + metadata: + name: e2e-repo.test.carvel.dev + namespace: kapp-controller-packaging-global + spec: + fetch: + imgpkgBundle: + image: k14stest/private-repo@sha256:ddd93b... + --- + apiVersion: packaging.carvel.dev/v1alpha1 + kind: PackageInstall + metadata: + name: pkg-demo + spec: + serviceAccountName: default-ns-sa + packageRef: + refName: pkg.test.carvel.dev + versionSelection: + constraints: 1.0.0 + ``` + +Assuming registry credentials specified are correct and both package repository bundle and package contents bundle use the same registry + +--- +## Manual configuration (without secretgen-controller) + +### PackageRepository + +If the registry containing the PackageRepository imgpkg bundle or image is private and secretgen-controller is not installed on your cluster, a secretRef can be added to the fetch stage for PackageRepository CR. For example: + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageRepository +metadata: + name: simple-package-repository +spec: + fetch: + imgpkgBundle: + image: k8slt/corp-com-pkg-repo:1.0.0 + secretRef: + name: my-registry-creds +``` + +This secret will need to be located in the namespace where the PackageRepository +is created and be in the format described in the [fetch docs](config.md#image-authentication). + +### PackageInstall + +As of kapp-controller v0.23.0, support for adding an annotation on the PackageInstall was added to allow users to set a secret on the PackageInstall's underlying App custom resource. Before creating a PackageInstall, users can look at the Package definition that they want to install and see what fetch stages a Package has defined like below: + +```yaml +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package +metadata: + name: simple-app.corp.com.1.0.0 +spec: + refName: simple-app.corp.com + version: 1.0.0 + template: + spec: + fetch: + - imgpkgBundle: + image: registry.corp.com/packages/simple-app:1.0.0 + # ... +``` + +In the example above, the Package has a single fetch stage to retrieve an imgpkg bundle. To use a PackageInstall +to specify what secret to use for this fetch stage, an annotation is added to the PackageInstall as shown below: + +```yaml +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: simple-app-with-secret + annotations: + ext.packaging.carvel.dev/fetch-0-secret-name: simple-app-secret +spec: + serviceAccountName: default-ns-sa + packageRef: + refName: simple-app.corp.com + versionSelection: + constraints: 1.0.0 +``` + +The annotation shown above `ext.packaging.carvel.dev/fetch-0-secret-name: simple-app-secret` has a format that allows users to specify the specific fetch stage by how it is defined in the Package definition. In this case, the PackageInstall being created will add a secretRef to the App's first fetch stage (i.e. `fetch-0-secret-name`) for the imgpkg bundle. If the Package definition had an additional fetch stage, the secret annotation could be added in the following format: `ext.packaging.carvel.dev/fetch-1-secret-name: simple-app-additional-secret`. + +To use this annotation with a PackageInstall, associated secrets will need to be located in the namespace where the PackageInstall is created and be in the format described in the [fetch docs](config.md#image-authentication). diff --git a/site/content/kapp-controller/docs/v0.32.0/security-model.md b/site/content/kapp-controller/docs/v0.32.0/security-model.md new file mode 100644 index 000000000..0c168c4d7 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/security-model.md @@ -0,0 +1,67 @@ +--- +aliases: [/kapp-controller/docs/latest/security-model] +title: Security Model +--- + +## App CR privileges + +kapp-controller container runs with a service account (named +`kapp-controller-sa` inside `kapp-controller` namespace) that has access to all +service accounts and secrets in the cluster. This service account *is not* used +for deployment of app resources. + +Each App CR *must* specify either a + +- service account (via `spec.serviceAccountName`) +- or, Secret with kubeconfig contents for some cluster (via `spec.cluster.kubeconfigSecretRef.name`) + +forcing App CR owner to explicitly provide needed privileges for management of +app resources. This avoids a problem of privilege escalation commonly found in +other general resource controllers which rely on a shared service account (often +requiring cluster admin privileges) to deploy resources. + +Since App CR only allows to reference service account or kubeconfig Secret +within the same namespace where App CR is located, kapp-controller is well +suited for multi-tenant use where different users of App CRD have varied level +of access (e.g. some may have cluster level privileges, and other may only have +access to one or more namespace). + +Example: + +- User A has been granted access to namespace `a` (and no other namespace or + cluster level access). User A can create an App CR with a service account + located in namespace `a` to deploy resources into namespace `a`. It _is not_ + possible for user A to create an App CR that would install cluster-wide + resources or place resources into another namespace. (e.g. a user that just + deploys web application to their namespace) + +- User B has been granted access to namespace `b` and ability to manage + specifically named CRD (single scoped cluster-wide privilege). User B can + create an App CR with a service account located in namespace `b` that installs + app into namespace `b` and also manages single CRD lifecycle. (e.g. a user + that manages another controller for other users) + +## Minimum ServiceAccount Permissions + +For users managing App and PackageInstall CR privileges via a service account, +the verbs in the role below are needed for working with ConfigMaps. + +```yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: app-ip-cr-role +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "update", "delete"] +``` + +These permissions are needed because of how `kapp` tracks information about apps +it manages, which is via storing information in a ConfigMap. So even if your App +or PackageInstall CR does not create ConfigMaps, the service account will +still need permissions for working with ConfigMaps. + +The ConfigMap permissions above are needed in addition to any other +resource/verb combinations needed to deploy all resources created by the App and +PackageInstall CRs. diff --git a/site/content/kapp-controller/docs/v0.32.0/security.md b/site/content/kapp-controller/docs/v0.32.0/security.md new file mode 100644 index 000000000..4a7b0542c --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/security.md @@ -0,0 +1,8 @@ +--- +aliases: [/kapp-controller/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `kapp-controller`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/kapp-controller/docs/v0.32.0/sops.md b/site/content/kapp-controller/docs/v0.32.0/sops.md new file mode 100644 index 000000000..8279a345d --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/sops.md @@ -0,0 +1,146 @@ +--- +aliases: [/kapp-controller/docs/latest/sops] +title: Sops +--- + +Available in v0.11.0+. + +Storing _encrypted_ secrets next to your configuration (within a Git repo or other artifacts) is one way to manage secret lifecycle. kapp-controller integrates with [Mozilla's SOPS](https://github.com/mozilla/sops) to decrypt secret material in fetched configuration. + +## Prepare your keys +Sops shipped with kapp-controller includes support for encryption via both [GPG](https://gnupg.org/) and [age](https://github.com/FiloSottile/age). +Note that the Sops project recommends Age. + +### using GPG + +```bash +$ gpg --gen-key +... + +$ gpg --list-secret-keys --keyid-format LONG +/root/.gnupg/secring.gpg +------------------------ +sec 4096R/B464DFD255C6B9F8 2020-10-03 +uid test test (test) +ssb 4096R/FEE37B8E2098EDFC 2020-10-03 +``` + +(Note: `B464DFD255C6B9F8` is the ID to be used later) + +### using Age +You may find [this screencast](https://asciinema.org/a/431605) helpful. +```bash +$ age-keygen -o key.txt +Public key: age12345... + +$ chmod 600 key.txt +``` +(Note: the public key `age12345...` will be used later) + +## Encrypt contents + +kapp-controller expects that encrypted files have `.sops.yml` extension (or `.sops.yml`). + +You can start by creating an unencrypted yaml: +```bash +# Unencrypted file +$ cat secret.yml +apiVersion: v1 +kind: Secret +metadata: + name: my-sec +data: + password: my-password +``` + +Then encrypt file with your public key from the previous step, to be later decrypted by kapp-controller. + +### using GPG +```bash +$ sops --encrypt --pgp B464DFD255C6B9F8 secret.yml > secret.sops.yml + +# Delete unencrypted file +$ rm secret.yml +``` + +### using Age +```bash +$ SOPS_AGE_KEY_FILE=./key.txt sops --encrypt --age age12345... secret.yml > secret.sops.yml +``` + +## Import private key into Kubernetes +Make a secret that includes the private key and import it into your +cluster. It will be referenced by App CR. + +### using GPG +Extract PGP private key from your GPG installation: + +```bash +$ gpg --armor --export-secret-keys B464DFD255C6B9F8 > my.pk +``` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: pgp-key + namespace: default +stringData: + # value of this is contents of my.pk + my.pk: | + -----BEGIN PGP PRIVATE KEY BLOCK----- + Version: GnuPG v1 + ... +``` + +### using Age +Cat the contents of your local key.txt to the body of a stringData block also named +key.txt: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: age-key +stringData: + key.txt: | + # created: + # public key: age12345... + AGE-SECRET-KEY-HUNTER2ORSOMETHINGWHENEVERYOUTYPEYOURPASSWORDIJUSTSEEHUNTER2 +``` + +## Decrypt in App CR + +Configure App CR to decrypt contents. Assuming, in this example, your git repo contains `secret.sops.yml`, it would be decrypted into `secret.yml` file. + +### using GPG +```yaml +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: config-with-sops + namespace: default +spec: + serviceAccountName: default-ns + fetch: + - git: + ... + template: + - sops: + pgp: + privateKeysSecretRef: + name: pgp-key + - ytt: {} + deploy: + - kapp: {} +``` + +### using Age +Nearly identical to the example above, but with a sops.age key: +```yaml +spec: + template: + - sops: + age: + privateKeysSecretRef: + name: age-key +``` diff --git a/site/content/kapp-controller/docs/v0.32.0/startup.md b/site/content/kapp-controller/docs/v0.32.0/startup.md new file mode 100644 index 000000000..a6e0492e9 --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/startup.md @@ -0,0 +1,31 @@ +--- +aliases: [/kapp-controller/docs/latest/startup] +Title: Kapp Controller Startup +--- + +(v0.14.0+) + +The startup of kapp controller consists of two processes: +controllerinit and controller. + +## The controllerinit Process + +This is the main process for the kapp controller binary. Since the binary is +the entrypoint for the docker image, kapp controller will be PID 1 +and is therefore responsible for reaping any zombie processes, so the process +begins by starting a thread to reap any zombies that appear. More on PID 1 and +zombie processes can be found [here][1]. + +Next, the process will look for the [controller Secret or ConfigMap][2] and apply any system level +configuration specified within. + +Finally, the process will fork to the same binary with a new flag, `--internal-controller`, +starting a second process, which runs the actual kubernetes controller. + +## The controller Process + +This process is responsible for running the app reconciler, which handles the fetch, +template, and deploy aspects of kapp controller. + +[1]: https://blog.phusion.nl/2015/01/20/docker-and-the-pid-1-zombie-reaping-problem/ +[2]: controller-config.md diff --git a/site/content/kapp-controller/docs/v0.32.0/walkthrough.md b/site/content/kapp-controller/docs/v0.32.0/walkthrough.md new file mode 100644 index 000000000..7e6f3e74f --- /dev/null +++ b/site/content/kapp-controller/docs/v0.32.0/walkthrough.md @@ -0,0 +1,205 @@ +--- +aliases: [/kapp-controller/docs/latest/walkthrough] +title: Install an Application +--- + +This walkthrough demonstrates how to install a simple example application, an HTTP server, on Kubernetes with kapp-controller. We will use `examples/simple-app-git` directory as our YAML configuration. + +You can use `kubectl` (or another tool) to deploy the YAML examples below. We've chosen [kapp](/kapp). + +- Start by [installing](install.md) kapp-controller onto your cluster + +- Install [examples/default-ns-rbac.yml](https://github.com/vmware-tanzu/carvel-kapp-controller/blob/develop/examples/rbac/default-ns.yml). It creates `default-ns-sa` service account that allows to change any + resource within the `default` namespace. This will be used by App CR below. + + ```bash-plain + $ kapp deploy -a default-ns-rbac -f https://raw.githubusercontent.com/vmware-tanzu/carvel-kapp-controller/develop/examples/rbac/default-ns.yml + ``` + +- Install [examples/simple-app-git/1.yml](https://github.com/vmware-tanzu/carvel-kapp-controller/blob/develop/examples/simple-app-git/1.yml) App CR. It specifies how to fetch, template, and deploy our example application. + + ```bash-plain + $ kapp deploy -a simple-app -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/1.yml + # or... kubectl apply -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/1.yml + + Changes + + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app App - - create reconcile - - + + Op: 1 create, 0 delete, 0 update, 0 noop + Wait to: 1 reconcile, 0 delete, 0 noop + + Continue? [yN]: y + + 5:20:27PM: ---- applying 1 changes [0/1 done] ---- + 5:20:27PM: create app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:20:27PM: ---- waiting on 1 changes [0/1 done] ---- + 5:20:27PM: ongoing: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:20:33PM: ok: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:20:33PM: ---- applying complete [1/1 done] ---- + 5:20:33PM: ---- waiting complete [1/1 done] ---- + + Succeeded + ``` + +- Check out `kubectl get app` output to see that app is deployed. + +- Additionally, let's check status of our App CR. It shows the overall status of the application, including the latest deploy output (`status.deploy.stdout`) and latest inspect output (`status.inspect.stdout`). Based on the inspect output we can see that our app included a `Deployment` and a `Service`. (Note: As of kapp-controller v0.31.0, inspect is disabled by default. See [App CR spec](app-spec.md) for more details.) + + ```bash-plain + $ kapp inspect -a simple-app --status + # or... kubectl get app simple-app -oyaml + + Resources in app 'simple-app' + + Namespace default + Name simple-app + Kind App + Status conditions: + - status: "True" + type: ReconcileSucceeded + deploy: + exitCode: 0 + finished: true + startedAt: "2019-12-02T22:20:28Z" + stdout: |- + Changes + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app Deployment - - create reconcile - - + ^ simple-app Service - - create reconcile - - + Op: 2 create, 0 delete, 0 update, 0 noop + Wait to: 2 reconcile, 0 delete, 0 noop + 10:20:28PM: ---- applying 2 changes [0/2 done] ---- + 10:20:28PM: create service/simple-app (v1) namespace: default + 10:20:28PM: create deployment/simple-app (apps/v1) namespace: default + 10:20:29PM: ---- waiting on 2 changes [0/2 done] ---- + 10:20:29PM: ok: reconcile service/simple-app (v1) namespace: default + 10:20:29PM: ongoing: reconcile deployment/simple-app (apps/v1) namespace: default + 10:20:29PM: ^ Waiting for 1 unavailable replicas + 10:20:29PM: L ok: waiting on replicaset/simple-app-6fb57f844b (apps/v1) namespace: default + 10:20:29PM: L ongoing: waiting on pod/simple-app-6fb57f844b-jk7d8 (v1) namespace: default + 10:20:29PM: ^ Pending: ContainerCreating + 10:20:29PM: ---- waiting on 1 changes [1/2 done] ---- + 10:20:31PM: ok: reconcile deployment/simple-app (apps/v1) namespace: default + 10:20:31PM: ---- applying complete [2/2 done] ---- + 10:20:31PM: ---- waiting complete [2/2 done] ---- + Succeeded + updatedAt: "2019-12-02T22:20:31Z" + fetch: + exitCode: 0 + startedAt: "2019-12-02T22:20:27Z" + updatedAt: "2019-12-02T22:20:27Z" + inspect: + exitCode: 0 + stdout: |- + Resources in app 'simple-app-ctrl' + Namespace Name Kind Owner Conds. Rs Ri Age + default simple-app Deployment kapp 2/2 t ok - 4s + default L simple-app-6fb57f844b ReplicaSet cluster - ok - 4s + default L.. simple-app-6fb57f844b-jk7d8 Pod cluster 4/4 t ok - 4s + default simple-app Service kapp - ok - 4s + default L simple-app Endpoints cluster - ok - 4s + Rs: Reconcile state + Ri: Reconcile information + 5 resources + Succeeded + updatedAt: "2019-12-02T22:20:32Z" + observedGeneration: 2 + template: + exitCode: 0 + updatedAt: "2019-12-02T22:20:28Z" + + 1 resources + + Succeeded + ``` + +- Update simple-app App CR to reconfigure simple-app. In this example we are changing data values for ytt templates. + + ```bash-plain + $ kapp deploy -a simple-app -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/2.yml -c + # or... kubectl apply -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/2.yml + + --- update app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + ... + 23, 23 template: + 24 - - ytt: {} + 24 + - ytt: + 25 + inline: + 26 + pathsFrom: + 27 + - secretRef: + 28 + name: simple-app-values + 25, 29 status: + 26, 30 conditions: + --- create secret/simple-app-values (v1) namespace: default + 0 + apiVersion: v1 + 1 + kind: Secret + 2 + metadata: + 3 + labels: + 4 + kapp.k14s.io/app: "1575325198404867000" + 5 + kapp.k14s.io/association: v1.7a671029ad7db07aa797301eac59e9ad + 6 + name: simple-app-values + 7 + namespace: default + 8 + stringData: + 9 + values2.yml: | + 10 + #@data/values + 11 + --- + 12 + hello_msg: updated + 13 + + + Changes + + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app App 1/1 t 2m update reconcile ok - + ^ simple-app-values Secret - - create reconcile - - + + Op: 1 create, 0 delete, 1 update, 0 noop + Wait to: 2 reconcile, 0 delete, 0 noop + + Continue? [yN]: y + + 5:23:13PM: ---- applying 2 changes [0/2 done] ---- + 5:23:13PM: update app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:23:13PM: create secret/simple-app-values (v1) namespace: default + 5:23:14PM: ---- waiting on 2 changes [0/2 done] ---- + 5:23:14PM: ongoing: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:23:14PM: ok: reconcile secret/simple-app-values (v1) namespace: default + 5:23:14PM: ---- waiting on 1 changes [1/2 done] ---- + 5:23:17PM: ok: reconcile app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:23:17PM: ---- applying complete [2/2 done] ---- + 5:23:17PM: ---- waiting complete [2/2 done] ---- + + Succeeded + ``` + +- Delete simple-app App CR + + ```bash-plain + $ kapp delete -a simple-app + # or... kubectl delete -f https://raw.githubusercontent.com/k14s/kapp-controller/develop/examples/simple-app-git/2.yml + + Changes + + Namespace Name Kind Conds. Age Op Wait to Rs Ri + default simple-app App 1/1 t 6m delete delete ok - + ^ simple-app-values Secret - 3m delete delete ok - + + Op: 0 create, 2 delete, 0 update, 0 noop + Wait to: 0 reconcile, 2 delete, 0 noop + + Continue? [yN]: y + + 5:26:25PM: ---- applying 2 changes [0/2 done] ---- + 5:26:25PM: delete secret/simple-app-values (v1) namespace: default + 5:26:25PM: delete app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:26:26PM: ---- waiting on 2 changes [0/2 done] ---- + 5:26:26PM: ok: delete secret/simple-app-values (v1) namespace: default + 5:26:26PM: ongoing: delete app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:26:26PM: ---- waiting on 1 changes [1/2 done] ---- + 5:26:30PM: ok: delete app/simple-app (kappctrl.k14s.io/v1alpha1) namespace: default + 5:26:30PM: ---- applying complete [2/2 done] ---- + 5:26:30PM: ---- waiting complete [2/2 done] ---- + + Succeeded + ``` diff --git a/site/content/kapp/docs/latest/_index.md b/site/content/kapp/docs/develop/_index.md similarity index 99% rename from site/content/kapp/docs/latest/_index.md rename to site/content/kapp/docs/develop/_index.md index 1886e018f..36b1a3b91 100644 --- a/site/content/kapp/docs/latest/_index.md +++ b/site/content/kapp/docs/develop/_index.md @@ -2,7 +2,7 @@ title: "About kapp" toc: "false" cascade: - version: latest + version: develop toc: "true" type: docs layout: docs diff --git a/site/content/kapp/docs/latest/apply-ordering.md b/site/content/kapp/docs/develop/apply-ordering.md similarity index 100% rename from site/content/kapp/docs/latest/apply-ordering.md rename to site/content/kapp/docs/develop/apply-ordering.md diff --git a/site/content/kapp/docs/latest/apply-waiting.md b/site/content/kapp/docs/develop/apply-waiting.md similarity index 100% rename from site/content/kapp/docs/latest/apply-waiting.md rename to site/content/kapp/docs/develop/apply-waiting.md diff --git a/site/content/kapp/docs/latest/apply.md b/site/content/kapp/docs/develop/apply.md similarity index 100% rename from site/content/kapp/docs/latest/apply.md rename to site/content/kapp/docs/develop/apply.md diff --git a/site/content/kapp/docs/latest/apps.md b/site/content/kapp/docs/develop/apps.md similarity index 100% rename from site/content/kapp/docs/latest/apps.md rename to site/content/kapp/docs/develop/apps.md diff --git a/site/content/kapp/docs/latest/cheatsheet.md b/site/content/kapp/docs/develop/cheatsheet.md similarity index 100% rename from site/content/kapp/docs/latest/cheatsheet.md rename to site/content/kapp/docs/develop/cheatsheet.md diff --git a/site/content/kapp/docs/latest/config.md b/site/content/kapp/docs/develop/config.md similarity index 100% rename from site/content/kapp/docs/latest/config.md rename to site/content/kapp/docs/develop/config.md diff --git a/site/content/kapp/docs/latest/dangerous-flags.md b/site/content/kapp/docs/develop/dangerous-flags.md similarity index 100% rename from site/content/kapp/docs/latest/dangerous-flags.md rename to site/content/kapp/docs/develop/dangerous-flags.md diff --git a/site/content/kapp/docs/latest/diff.md b/site/content/kapp/docs/develop/diff.md similarity index 100% rename from site/content/kapp/docs/latest/diff.md rename to site/content/kapp/docs/develop/diff.md diff --git a/site/content/kapp/docs/latest/faq.md b/site/content/kapp/docs/develop/faq.md similarity index 100% rename from site/content/kapp/docs/latest/faq.md rename to site/content/kapp/docs/develop/faq.md diff --git a/site/content/kapp/docs/latest/gitops.md b/site/content/kapp/docs/develop/gitops.md similarity index 100% rename from site/content/kapp/docs/latest/gitops.md rename to site/content/kapp/docs/develop/gitops.md diff --git a/site/content/kapp/docs/develop/hpa-deployment-rebase.md b/site/content/kapp/docs/develop/hpa-deployment-rebase.md new file mode 100644 index 000000000..60d73473a --- /dev/null +++ b/site/content/kapp/docs/develop/hpa-deployment-rebase.md @@ -0,0 +1,41 @@ +--- +title: HPA and Deployment rebase +--- +## HPA and Deployment rebase + +Here is an example on how to use custom `rebaseRules` to "prefer" server chosen value for `spec.replicas` field for a particular Deployment. + +```yaml +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config +rebaseRules: +- path: [spec, replicas] + type: copy + sources: [existing, new] + resourceMatchers: + - kindNamespaceNameMatcher: + kind: Deployment + namespace: my-ns + name: my-app +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: my-app + namespace: my-ns +... +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: my-app + namespace: my-ns +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: my-app + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 50 +``` diff --git a/site/content/kapp/docs/latest/install.md b/site/content/kapp/docs/develop/install.md similarity index 100% rename from site/content/kapp/docs/latest/install.md rename to site/content/kapp/docs/develop/install.md diff --git a/site/content/kapp/docs/latest/integrating-with-other-tools.md b/site/content/kapp/docs/develop/integrating-with-other-tools.md similarity index 100% rename from site/content/kapp/docs/latest/integrating-with-other-tools.md rename to site/content/kapp/docs/develop/integrating-with-other-tools.md diff --git a/site/content/kapp/docs/latest/merge-method.md b/site/content/kapp/docs/develop/merge-method.md similarity index 100% rename from site/content/kapp/docs/latest/merge-method.md rename to site/content/kapp/docs/develop/merge-method.md diff --git a/site/content/kapp/docs/latest/rbac.md b/site/content/kapp/docs/develop/rbac.md similarity index 100% rename from site/content/kapp/docs/latest/rbac.md rename to site/content/kapp/docs/develop/rbac.md diff --git a/site/content/kapp/docs/develop/rebase-pvc.md b/site/content/kapp/docs/develop/rebase-pvc.md new file mode 100644 index 000000000..c60e2a554 --- /dev/null +++ b/site/content/kapp/docs/develop/rebase-pvc.md @@ -0,0 +1,101 @@ +--- +title: PersistentVolumeClaim rebase +--- +## PersistentVolumeClaim rebase + +Here is an example on how to use custom `rebaseRules` to "prefer" server chosen value for several annotations added by PVC controller (in other words, cluster owned fields), instead of removing them based on given configuration. + +Let's deploy via `kapp deploy -a test -f config.yml -c` with following configuration `config.yml`: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysqlclaim +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +``` + +Without additional rebase rules following diff will be presented upon next deploy, stating that several annotations will be removed (since they were not present in the initial configuration): + +```bash +$ kapp deploy -a test -f config.yml -c + +Target cluster 'https://x.x.x.x' (nodes: gke-dk-jan-9-default-pool-a218b1c9-55sl, 3+) + +--- update persistentvolumeclaim/mysqlclaim (v1) namespace: default + ... + 2, 2 metadata: + 3 - annotations: + 4 - pv.kubernetes.io/bind-completed: "yes" + 5 - pv.kubernetes.io/bound-by-controller: "yes" + 6 - volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/gce-pd + 7, 3 creationTimestamp: "2020-03-08T22:17:29Z" + 8, 4 finalizers: + ... + 24, 20 storageClassName: standard + 25 - volumeMode: Filesystem + 26, 21 volumeName: pvc-1be63b2b-20de-429c-863a-9e7eb062f5d3 + 27, 22 status: + +Changes + +Namespace Name Kind Conds. Age Op Wait to Rs Ri +default mysqlclaim PersistentVolumeClaim - 43s update reconcile ok - + +Op: 0 create, 0 delete, 1 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +Continue? [yN]: +``` + +To let kapp know that these annotations should be copied from the live resource copy, we can augment deploys with following configuration `kapp-config.yml`: + +```yaml +--- +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config + +rebaseRules: +- path: [metadata, annotations, pv.kubernetes.io/bind-completed] + type: copy + sources: [new, existing] + resourceMatchers: &pvcs + - apiVersionKindMatcher: + apiVersion: v1 + kind: PersistentVolumeClaim + +- path: [metadata, annotations, pv.kubernetes.io/bound-by-controller] + type: copy + sources: [new, existing] + resourceMatchers: *pvcs + +- path: [metadata, annotations, volume.beta.kubernetes.io/storage-provisioner] + type: copy + sources: [new, existing] + resourceMatchers: *pvcs + +- path: [spec, volumeMode] + type: copy + sources: [new, existing] + resourceMatchers: *pvcs +``` + +```bash +$ kapp deploy -a test -f config.yml -f rules.yml -c + +Target cluster 'https://x.x.x.x' (nodes: gke-dk-jan-9-default-pool-a218b1c9-55sl, 3+) + +Changes + +Namespace Name Kind Conds. Age Op Wait to Rs Ri + +Op: 0 create, 0 delete, 0 update, 0 noop +Wait to: 0 reconcile, 0 delete, 0 noop + +Succeeded +``` diff --git a/site/content/kapp/docs/latest/security.md b/site/content/kapp/docs/develop/security.md similarity index 100% rename from site/content/kapp/docs/latest/security.md rename to site/content/kapp/docs/develop/security.md diff --git a/site/content/kapp/docs/latest/state-namespace.md b/site/content/kapp/docs/develop/state-namespace.md similarity index 100% rename from site/content/kapp/docs/latest/state-namespace.md rename to site/content/kapp/docs/develop/state-namespace.md diff --git a/site/content/kapp/docs/v0.44.0/_index.md b/site/content/kapp/docs/v0.44.0/_index.md new file mode 100644 index 000000000..50cf60a35 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/_index.md @@ -0,0 +1,43 @@ +--- +title: "About kapp" +toc: "false" +cascade: + version: v0.44.0 + toc: "true" + type: docs + layout: docs +--- + +`kapp` (pronounced: `kap`) CLI encourages Kubernetes users to manage resources in bulk by working with "Kubernetes applications" (sets of resources with the same label). It focuses on resource diffing, labeling, deployment and deletion. Unlike tools like Helm, `kapp` considers YAML templating and management of packages outside of its scope, though it works great with tools that generate Kubernetes configuration. + +![Kapp Deploy](/images/kapp/kapp-deploy-screenshot.png) + +Features: + +- Works with standard Kubernetes YAMLs +- Focuses exclusively on deployment workflow, not packaging or templating + - but plays well with tools (such as [ytt](/ytt)) that produce Kubernetes configuration +- Converges application resources (creates, updates and/or deletes resources) in each deploy + - based on comparison between provided files and live objects in the cluster +- Separates calculation of changes ([diff stage](diff.md)) from application of changes ([apply stage](apply.md)) +- [Waits for resources](apply-waiting.md) to be "ready" +- Creates CRDs and Namespaces first and supports [custom change ordering](apply-ordering.md) +- Works [without admin privileges](rbac.md) and does not use custom CRDs + - making it possible to use kapp as a regular user in a single namespace +- Records application deployment history +- Opt-in resource version management + - for example, to trigger Deployment rollout when ConfigMap changes +- Optionally streams Pod logs during deploy +- Works with any group of labeled resources (`kapp -a label:tier=web inspect -t`) +- Works without server side components +- GitOps friendly (`kapp app-group deploy -g all-apps --directory .`) + +## Blog posts + +- [Deploying Kubernetes Applications with ytt, kbld, and kapp](/blog/deploying-apps-with-ytt-kbld-kapp) + +## Talks + +- [ytt and kapp @ TGI Kubernetes 079](https://www.youtube.com/watch?v=CSglwNTQiYg) with Joe Beda +- [Managing Applications in Production: Helm vs ytt & kapp @ Kubecon 2020](https://www.youtube.com/watch?v=WJw1MDFMVuk) +- [Introduction to Carvel @ Rawkode Live](https://www.youtube.com/watch?v=LBCmMTofNxw) diff --git a/site/content/kapp/docs/v0.44.0/apply-ordering.md b/site/content/kapp/docs/v0.44.0/apply-ordering.md new file mode 100644 index 000000000..225c74e94 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/apply-ordering.md @@ -0,0 +1,93 @@ +--- +title: Apply Ordering +--- + +## Overview + +kapp includes builtin rules to make sure certain changes are applied in particular order: + +- Creates/updates + - CRDs are created/updated before custom resources + - Namespaces are created/updated before namespaced resources + - Pod related resources (ServiceAccount, ConfigMap, Secret, etc.) are created/updated before other resources (v0.25.0+) + - RBAC related resources (Role, RoleBinding, etc.) are created/updated before other resources (v0.25.0+) +- Deletions (below is order as of v0.29.0+) + - Custom resources are deleted first + - CRDs are deleted next + - Rest of resoures are deleted + +As of v0.25.0+, builtin rules are specified via [changeGroupBindings and changeRuleBindings](config.md#changegroupbindings) configurations. Custom rules can be added via same mechanism. + +Additionally kapp allows to customize order of changes via following resource annotations: + +- `kapp.k14s.io/change-group` annotation to group one or more resource changes into arbitrarily named group. Example: `apps.big.co/db-migrations`. You can specify multiple change groups by suffixing each annotation with a `.x` where `x` is unique identifier (e.g. `kapp.k14s.io/change-group.istio-sidecar-order`). +- `kapp.k14s.io/change-rule` annotation to control when resource change should be applied (created, updated, or deleted) relative to other changes. You can specify multiple change rules by suffixing each annotation with a `.x` where `x` is unique identifier (e.g. `kapp.k14s.io/change-rule.istio-sidecar-order`). + +`kapp.k14s.io/change-rule` annotation value format is as follows: `(upsert|delete) (after|before) (upserting|deleting) `. For example: + +- `kapp.k14s.io/change-rule: "upsert after upserting apps.big.co/db-migrations"` +- `kapp.k14s.io/change-rule: "delete before upserting apps.big.co/service"` + +As of v0.41.0+, kapp provides change group placeholders, which can be used in change-group and change-rule annotation values and are later replaced by values from the resource manifest of the resource they are associated with. For example: + +- `kapp.k14s.io/change-group: apps.co/db-migrations-{name}` - Here `{name}` would later be replaced by the name of the resource. +- `kapp.k14s.io/change-rule: upsert after upserting apps.co/namespaces-{namespace}` - Here `{namespace}` would later be replaced by the namespace of the resource. + +kapp provides the following placeholders: + +- `{api-group}` - apiGroup +- `{kind}` - kind +- `{name}` - name +- `{namespace}` - namespace +- `{crd-kind}` - spec.names.kind (available for CRDs only) +- `{crd-group}` - spec.group (available for CRDs only) + +These placeholders can also be used in changeGroupBindings and changeRuleBindings. By default, they are used for CRDs, CRs, namespaces and namespaced resources. Due to this, CRs now wait for their respective CRDs only and namespaced resources now wait for their respective namespaces only. + +## Example + +Following example shows how to run `job/migrations`, start and wait for `deployment/app`, and finally `job/app-health-check`. + +```yaml +kind: ConfigMap +metadata: + name: app-config + annotations: {} +#... +--- +kind: Job +metadata: + name: migrations + annotations: + kapp.k14s.io/change-group: "apps.big.co/db-migrations" +#... +--- +kind: Service +metadata: + name: app + annotations: + kapp.k14s.io/change-group: "apps.big.co/deployment" +#... +--- +kind: Ingress +metadata: + name: app + annotations: + kapp.k14s.io/change-group: "apps.big.co/deployment" +#... +--- +kind: Deployment +metadata: + name: app + annotations: + kapp.k14s.io/change-group: "apps.big.co/deployment" + kapp.k14s.io/change-rule: "upsert after upserting apps.big.co/db-migrations" +#... +--- +kind: Job +metadata: + name: app-health-check + annotations: + kapp.k14s.io/change-rule: "upsert after upserting apps.big.co/deployment" +#... +``` diff --git a/site/content/kapp/docs/v0.44.0/apply-waiting.md b/site/content/kapp/docs/v0.44.0/apply-waiting.md new file mode 100644 index 000000000..4dcf7a615 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/apply-waiting.md @@ -0,0 +1,46 @@ +--- +title: Apply Waiting +--- + +## Overview + +kapp includes builtin rules on how to wait for the following resource types: + +- any resource with `metadata.deletionTimestamp`: wait for resource to be fully removed +- any resource matching Config's waitRules: [see "Custom waiting behaviour" below](#custom-waiting-behaviour) +- [`apiextensions.k8s.io//CustomResourceDefinition`](https://github.com/vmware-tanzu/carvel-kapp/blob/develop/pkg/kapp/resourcesmisc/api_extensions_vx_crd.go): wait for Established and NamesAccepted conditions to be `True` (note that this is wait rule for CustomResourceDefinition resource itself, not CRs) +- `apps/v1/DaemonSet`: wait for `status.numberUnavailable` to be 0 +- `apps/v1/Deployment`: [see "apps/v1/Deployment resource" below](#apps-v1-deployment-resource) +- `apps/v1/ReplicaSet`: wait for `status.replicas == status.availableReplicas` +- `batch/v1/Job`: wait for `Complete` or `Failed` conditions to appear +- `batch//CronJob`: immediately considered done +- `/v1/Pod`: looks at `status.phase` +- `/v1/Service`: wait for `spec.clusterIP` and/or `status.loadBalancer.ingress` to become set +- `apps/v1/StatefulSet`: [see "apps/v1/StatefulSet resource" below](#appsv1statefulset-resource) + +If resource is not affected by the above rules, its waiting behaviour depends on aggregate of waiting states of its associated resources (associated resources are resources that share same `kapp.k14s.io/association` label value). + +## Controlling waiting via resource annotations + +- `kapp.k14s.io/disable-wait` annotation controls whether waiting will happen at all. Possible values: "". +- `kapp.k14s.io/disable-associated-resources-wait` annotation controls whether associated resources impact resource's waiting state. Possible values: "". + +## apps/v1/Deployment resource + +kapp by default waits for `apps/v1/Deployment` resource to have `status.unavailableReplicas` equal to zero. Additionally waiting behaviour can be controlled via following annotations: + +- `kapp.k14s.io/apps-v1-deployment-wait-minimum-replicas-available` annotation controls how many new available replicas are enough to consider waiting successful. Example values: `"10"`, `"5%"`. + +## apps/v1/StatefulSet resource + +Available in v0.32.0+. + +kapp will wait for any pods created from the updated template to be ready based on StatefulSet's status. This behaviour depends on the [update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies) used. + +Note: kapp does not do anything special when `OnDelete` strategy is used. It will wait for StatefulSet to report it's reconciled (expecting some actor in the system to delete Pods per `OnDelete` requirements). + +## Custom waiting behaviour + +Available in v0.29.0+. + +kapp can be extended with custom waiting behaviour by specifying [wait rules as additional config](config.md#wait-rules). (If this functionality is not enough to wait for resources in your use case, please reach out on Slack to discuss further.) diff --git a/site/content/kapp/docs/v0.44.0/apply.md b/site/content/kapp/docs/v0.44.0/apply.md new file mode 100644 index 000000000..bf9f2e587 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/apply.md @@ -0,0 +1,120 @@ +--- +title: Apply stage +--- + +## Overview + +Once change set is calculated (see [Diff](diff.md) section for details), kapp asks for user confirmation (unless `--yes` flag is specified) to proceed with changes. + +Changes are applied in particular order as described in [Apply ordering](apply-ordering.md). + +All created resources are labeled with several labels: + +- `kapp.k14s.io/app` to track which application "owns" resource +- `kapp.k14s.io/identity` to identify preferred API version used when creating resource +- `kapp.k14s.io/association` to track (best effort) parent-child relationships between resources + +Every time application is deployed, new application change record is saved. They can be viewed via `kapp app-change ls -a app-name`. + +Related: [ownership label rules](config.md#ownershiplabelrules) and [label scoping rules](config.md#labelscopingrules). + +## Controlling apply via resource annotations + +### kapp.k14s.io/create-strategy + +`kapp.k14s.io/create-strategy` annotation controls create behaviour (rarely necessary) + +Possible values: "" (default), `fallback-on-update`. + +In some cases creation of a resource may conflict with that resource being created in the cluster by other means (often automated). An example of that is creation of default ServiceAccount by kapp racing with Kubernetes service accounts controller doing the same thing. By specifying `fallback-on-update` value, kapp will catch resource creation conflicts and apply resource as an update. + +### kapp.k14s.io/update-strategy + +`kapp.k14s.io/update-strategy` annotation controls update behaviour + +Possible values: "" (default), `fallback-on-replace`, `always-replace`, `skip`. + +In some cases entire resources or subset resource fields are immutable which forces kapp users to specify how to apply wanted update. + +- "" means to issue plain update call +- `fallback-on-replace` causes kapp to fallback to resource replacement if update call results in `Invalid` error. Note that if resource is replaced (= delete + create), it may be negatively affected (loss of persistent data, loss of availability, etc.). For example, if Deployment or DaemonSet is first deleted and then created then associated Pods will be recreated as well, but all at the same time (even if rolling update is enabled), which likely causes an availability gap. +- `always-replace` causes kapp to always delete and then create resource (See note above as well.) +- `skip` causes kapp to not apply update (it will show up in a diff next time). Available in v0.33.0+. + +### kapp.k14s.io/delete-strategy + +`kapp.k14s.io/delete-strategy` annotation controls deletion behaviour + +Possible values: "" (default), `orphan`. + +By default resource is deleted, however; choosing `orphan` value will make kapp forget about this resource. Note that if this resource is owned by a different resource that's being deleted, it might still get deleted. Orphaned resources are labeled with `kapp.k14s.io/orphaned` label. As of v0.31.0+, resource is also disassociated from owning app so that it can be owned by future apps. + +### kapp.k14s.io/owned-for-deletion + +`kapp.k14s.io/owned-for-deletion` annotation controls resource deletion during `kapp delete` command + +Possible values: "". + +By default non-kapp owned resources are not explicitly deleted by kapp, but expected to be deleted by the cluster (for example Endpoints resource for each Service). In some cases it's desired to annotate non-kapp owned resource so that it does get explicitly deleted, possibly because cluster does not plan to delete it (e.g. PVCs created by StatefulSet are not deleted by StatefulSet controller; [https://github.com/vmware-tanzu/carvel-kapp/issues/36](https://github.com/vmware-tanzu/carvel-kapp/issues/36)). + +### kapp.k14s.io/nonce + +`kapp.k14s.io/nonce` annotation allows to inject unique ID + +Possible values: "" (default). + +Annotation value will be replaced with a unique ID on each deploy. This allows to force resource update as value changes every time. + +### kapp.k14s.io/deploy-logs + +`kapp.k14s.io/deploy-logs` annotation indicates which Pods' log output to show during deploy + +Possible values: + +- "" (default; equivalent to `for-new`) +- `for-new` (only newly created Pods are tailed) +- `for-existing` (only existing Pods are tailed) +- `for-new-or-existing` (both newly created and existing Pods are tailed) + +Especially useful when added to Jobs. For example, see [examples/resource-ordering/sync-check.yml](https://github.com/vmware-tanzu/carvel-kapp/blob/develop/examples/resource-ordering/sync-check.yml) + +### kapp.k14s.io/deploy-logs-container-names + +`kapp.k14s.io/deploy-logs-container-names` annotation indicates which containers' log output to show during deploy + +Possible values: "" (default), `containerName1`, `containerName1,containerName2` + +### kapp.k14s.io/exists + +Available in v0.43.0+ + +`kapp.k14s.io/exists` will ensure that resource exists in Kubernetes. It will not be considered to be part of the app (not labeled). + +If the resource is not present already, then kapp uses the `exists` operation and ensures that the resource exists in Kubernetes. + +If the resource already exists, kapp does not perform any operation on it (the `noop` operation is used). + +Possible values: "". + +Especially useful in scenarios where an external agency such as a controller might be creating a resource that we want to wait for. + +### kapp.k14s.io/noop + +Available in v0.43.0+ + +`kapp.k14s.io/noop` ensures that kapp is aware of the resource. It will not be considered to be part of the app (not labeled). + +kapp always uses the `noop` operation for these resources. + +Possible values: "". + +--- +## Controlling apply via deploy flags + +- `--apply-ignored=bool` explicitly applies ignored changes; this is useful in cases when controllers lose track of some resources instead of for example deleting them +- `--apply-default-update-strategy=string` controls default strategy for all resources (see `kapp.k14s.io/update-strategy` annotation above) +- `--apply-exit-status=bool` (default `false`) controls exit status (`0`: unused, `1`: any error, `2`: no changes applied, `3`: at least one change applied) +- `--wait=bool` (default `true`) controls whether kapp will wait for resource to "stabilize". See [Apply waiting](apply-waiting.md) +- `--wait-ignored=bool` controls whether kapp will wait for ignored changes (regardless whether they were initiated by kapp or by controllers) +- `--logs=bool` (default `true`) controls whether to show logs as part of deploy output for Pods annotated with `kapp.k14s.io/deploy-logs: ""` +- `--logs-all=bool` (deafult `false`) controls whether to show all logs as part of deploy output for all Pods diff --git a/site/content/kapp/docs/v0.44.0/apps.md b/site/content/kapp/docs/v0.44.0/apps.md new file mode 100644 index 000000000..fa885154f --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/apps.md @@ -0,0 +1,40 @@ +--- +title: Applications +--- + +## Overview + +kapp considers a set of resources with the same label as an application. These resources could span any number of namespaces or could be cluster-wide (e.g. CRDs). + +kapp has two methods of finding resources: + +1. via unique-to-Namespace application name (via `-a my-name` flag), or +2. via user provided label (via `-a label:my-label=val` flag) + +First approach is most common as kapp generates a unique label for each tracked application and associates that with an application name. + +## List + +Applications can be listed via `ls` command: + +```bash +$ kapp ls +``` + +## Deploy + +To create or update an application use `deploy` command: + +```bash +$ kapp deploy -a my-name -f my-app-config/ +``` + +Deploy command consists of two stages: [resource "diff" stage](diff.md), and [resource "apply" stage](apply.md). + +## Delete + +To delete an application use `delete` command: + +```bash +$ kapp delete -a my-name +``` diff --git a/site/content/kapp/docs/v0.44.0/cheatsheet.md b/site/content/kapp/docs/v0.44.0/cheatsheet.md new file mode 100644 index 000000000..ceea27867 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/cheatsheet.md @@ -0,0 +1,128 @@ +--- +title: Cheatsheet +--- + + +## List + +```bash +kapp ls -A +``` + +List all app in the cluster (across all namespaces) + +## Deploy + +Deploy app named `app1` with configuration from `config/`: + +```bash +kapp deploy -a app1 -f config/ -c +``` + +Deploy app named `app1` with configuration piped in (see alternative that does not require `--yes` next): + +```bash +ytt -f config/ | kapp deploy -a app1 -f- -c -y +``` + +Deploy app named `app1` with configuration generated inline and with confirmation dialog: + +```bash +kapp deploy -a app1 -f <(ytt -f config/ ) +``` + +Show more diff context when reviewing changes during deploy: + +```bash +kapp deploy -a app1 -f config/ -c --diff-context=10 +``` + +Show diff and exit successfully (without applying any changes): + +```bash +kapp deploy -a app1 -f config/ --diff-run +``` + +Show logs from all app `Pods` throughout deploy: + +```bash +kapp deploy -a app1 -f config/ --logs-all +``` + +Rewrite all resources to specify `app1-ns` namespace: + +```bash +kapp deploy -a app1 -f config/ --into-ns app1-ns +``` + +## Inspect + +Show summary of all resources in app `app1`: + +```bash +kapp inspect -a app1 +``` + +Show summary organized as a tree of all resources in app `app1`: + +```bash +kapp inspect -a app1 --tree +``` + +Show status subresources for each resource in app `app1`: + +```bash +kapp inspect -a app1 --status +``` + +Show all resources in the cluster: + +```bash +kapp inspect -a 'label:' +``` + +Show all resources in particular namespace (note that it currently does namespace filtering client-side): + +```bash +kapp inspect -a 'label:' --filter-ns some-ns +``` + +Show all resources labeled `tier=web` in the cluster: + +```bash +kapp inspect -a 'label:tier=web' +``` + +Show all `Deployment` resources in the cluster **not** managed by kapp: + +```bash +kapp inspect -a 'label:!kapp.k14s.io/app' --filter-kind Deployment +``` + +## Delete + +Delete resources under particular label (in this example deleting resources associated with some app): + +```bash +kapp delete -a 'label:kapp.k14s.io/app=1578599579922603000' +``` + +## Misc + +See which labels are used in your cluster (add `--values` to see label values): + +```bash +kapp tools list-labels +``` + +Shows app labels that are still present in the cluster (could be combined with delete command below): + +```bash +kapp tools list-labels --values --tty=false | grep kapp.k14s.io/app +``` + +Delete all app changes older than 500h (v0.12.0+): + +```bash +kapp deploy -a label:kapp.k14s.io/is-app-change --filter-age 500h+ --dangerous-allow-empty-list-of-resources --apply-ignored +``` diff --git a/site/content/kapp/docs/v0.44.0/config.md b/site/content/kapp/docs/v0.44.0/config.md new file mode 100644 index 000000000..bf8990c19 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/config.md @@ -0,0 +1,334 @@ +--- +title: Configuration +--- + +## Overview + +kapp supports custom `Config` resource to specify its own configuration. It's expected to be included with your other Kubernetes configuration. Config resource is never applied to the cluster, though it follows general Kubernetes resource format. Multiple config resources are allowed. + +kapp comes with __built-in configuration__ (see it via `kapp deploy-config`) that includes rules for common resources. + +## Format + +```yaml +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config + +minimumRequiredVersion: 0.23.0 + +rebaseRules: +- path: [spec, clusterIP] + type: copy + sources: [new, existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} + +ownershipLabelRules: +- path: [metadata, labels] + resourceMatchers: + - allMatcher: {} + +labelScopingRules: +- path: [spec, selector] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} + +templateRules: +- resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: ConfigMap} + affectedResources: + objectReferences: + - path: [spec, template, spec, containers, {allIndexes: true}, env, {allIndexes: true}, valueFrom, configMapKeyRef] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + - path: [spec, template, spec, containers, {allIndexes: true}, envFrom, {allIndexes: true}, configMapRef] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + +additionalLabels: + department: marketing + cost-center: mar201 + +diffAgainstLastAppliedFieldExclusionRules: +- path: [metadata, annotations, "deployment.kubernetes.io/revision"] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + +diffMaskRules: +- path: [data] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Secret} +``` + +### minimumRequiredVersion + +`minimumRequiredVersion` forces kapp to exit with a validation error if kapp's version is below minimum required version. Available in v0.23.0+. + +### rebaseRules + +`rebaseRules` specify origin of field values. + +kapp rebase rules explicitly define how to merge resources during an update. To read more about why rebase rules are necessary, see [Resource Merge Method](merge-method.md). +For examples of rebase rules in use, see [HPA and Deployment rebase](hpa-deployment-rebase.md) or [PersistentVolumeClaim rebase](rebase-pvc.md). + +- `rebaseRules` (array) list of rebase rules + - `path` (array of strings) specifies location within a resource to rebase. Mutually exclusive with `paths`. Example: `[spec, clusterIP]` + - `paths` (array of `path`) specifies multiple locations within a resource to rebase. This is a convenience for specifying multiple rebase rules with only different paths. Mutually exclusive with `path`. Available in v0.27.0+. + - `type` (string) specifies strategy to modify field values. Allowed values: `copy` or `remove`. `copy` will update the field value; `remove` will delete the field. + - `sources` (array of `new` or `existing`) specifies a preference order for the source of the referenced field value being rebased. `new` refers to an updated resource from user input, where `existing` refers to a resource already in the cluster. If the field value being rebased is not found in any of the sources provided, kapp will error. Only used with `type: copy`. \ + Examples: + - `[existing, new]` – If field value is present in the `existing` resource on cluster, use that value, otherwise use the value in the `new` user input. + - `[existing]` – Only look for field values in resources already on cluster, corresponding value you provide in new resource will be overwritten. + - `resourceMatchers` (array) specifies rules to find matching resources. See various resource matchers below. + - `ytt` specifies choice as [ytt](https://carvel.dev/ytt/) for rebase rule. Available in v0.38.0+. + - `overlayContractV1` allows to use ytt overlay to modify provided resource based on existing resource. + - `overlay.yml` overlay YAML file. + +Rebase rule to `copy` the `clusterIP` field value to `Service`/`v1` resources; if `clusterIp` is present in the `new` user input, use that value, otherwise use the value in `existing` resource on cluster: +```yaml +rebaseRules: +- path: [spec, clusterIP] + type: copy + sources: [new, existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} +``` + +Rebase rule to `copy` the `clusterIP` and `healthCheckNodePort` field values from the `existing` resource on cluster, to `Service`/`v1` resources: +```yaml +rebaseRules: +- paths: + - [spec, clusterIP] + - [spec, healthCheckNodePort] + type: copy + sources: [existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} +``` + +See [ytt rebase rule](https://github.com/vmware-tanzu/carvel-kapp/blob/d3ee9a01b5f0d7d5632b6a157ea7d0338730d497/pkg/kapp/config/default.go#L123-L154) (included in default configuration) for retaining cluster added token secret in ServiceAccount's secrets array. + +### ownershipLabelRules + +`ownershipLabelRules` specify locations for inserting kapp generated labels. These labels allow kapp to track which resources belong to which application. For resources that describe creation of other resources (e.g. `Deployment` or `StatefulSet`), configuration may need to specify where to insert labels for child resources that will be created. `kapp.k14s.io/disable-default-ownership-label-rules: ""` (value must be empty) annotation can be be used to exclude an individual resource from default onwership label rules. + +### labelScopingRules + +`labelScopingRules` specify locations for inserting kapp generated labels that scope resources to resources within current application. `kapp.k14s.io/disable-default-label-scoping-rules: ""` (as of v0.33.0+, or use `kapp.k14s.io/disable-label-scoping: ""` in earlier versions) annotation can be used to exclude an individual resource from label scoping. + +### waitRules + +Available in v0.29.0+. + +`waitRules` specify how to wait for resources that kapp does not wait for by default. Each rule provides a way to specify which `status.conditions` indicate success or failure. Once any of the condition matchers successfully match against one of the resource's conditions, kapp will stop waiting for the matched resource and report any failures. (If this functionality is not enough to wait for resources in your use case, please reach out on Slack to discuss further.) + +```yaml +waitRules: +- supportsObservedGeneration: true + conditionMatchers: + - type: Failed + status: "True" + failure: true + - type: Deployed + status: "True" + success: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: corp.com/v1, kind: DatabaseInstance} +``` + +```yaml +waitRules: +- supportsObservedGeneration: true + conditionMatchers: + - type: Ready + status: "False" + failure: true + - type: Ready + status: "True" + success: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: corp.com/v1, kind: Application} +``` + +### templateRules + +`templateRules` specify how versioned resources affect other resources. In above example, versioned config maps are said to affect deployments. [Read more about versioned resources](diff.md#versioned-resources). + +### additionalLabels + +`additionalLabels` specify additional labels to apply to all resources for custom uses by the user (added based on `ownershipLabelRules`). + +### diffAgainstLastAppliedFieldExclusionRules + +`diffAgainstLastAppliedFieldExclusionRules` specify which fields should be removed before diff-ing against last applied resource. These rules are useful for fields are "owned" by the cluster/controllers, and are only later updated. For example `Deployment` resource has an annotation that gets set after a little bit of time after resource is created/updated (not during resource admission). It's typically not necessary to use this configuration. + +### diffMaskRules + +`diffMaskRules` specify which field values should be masked in diff. By default `v1/Secret`'s `data` fields are masked. Currently only applied to `deploy` command. + +### changeGroupBindings + +Available in v0.25.0+. + +`changeGroupBindings` bind specified change group to resources matched by resource matchers. This is an alternative to using `kapp.k14s.io/change-group` annotation to add change group to resources. See `kapp deploy-config` for default bindings. + +### changeRuleBindings + +Available in v0.25.0+. + +`changeRuleBindings` bind specified change rules to resources matched by resource matchers. This is an alternative to using `kapp.k14s.io/change-rule` annotation to add change rules to resources. See `kapp deploy-config` for default bindings. + +--- +## Resource matchers + +Resource matchers (as used by `rebaseRules`, `ownershipLabelRules`, `labelScopingRules`, `templateRules`, `diffAgainstLastAppliedFieldExclusionRules`, and `diffMaskRules`): + +### allMatcher + +Matches all resources + +```yaml +allMatcher: {} +``` + +### anyMatcher + +Matches resources that match one of matchers + +```yaml +anyMatcher: + matchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + - apiVersionKindMatcher: {apiVersion: extensions/v1alpha1, kind: Deployment} +``` + +### notMatcher + +Matches any resource that does not match given matcher + +```yaml +notMatcher: + matcher: + apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} +``` + +### andMatcher + +Matches any resource that matches all given matchers + +```yaml +andMatcher: + matchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + - hasNamespaceMatcher: {} +``` + +### apiGroupKindMatcher + +```yaml +apiGroupKindMatcher: {apiGroup: apps, kind: Deployment} +``` + +### apiVersionKindMatcher + +```yaml +apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} +``` + +### kindNamespaceNameMatcher + +```yaml +kindNamespaceNameMatcher: {kind: Deployment, namespace: mysql, name: mysql} +``` + +### hasAnnotationMatcher + +Matches resources that have particular annotation + +```yaml +hasAnnotationMatcher: + keys: + - kapp.k14s.io/change-group +``` + +### hasNamespaceMatcher + +Matches any resource that has a non-empty namespace + +```yaml +hasNamespaceMatcher: {} +``` + +Matches any resource with namespace that equals to one of the specified names + +```yaml +hasNamespaceMatcher: + names: [app1, app2] +``` + +### customResourceMatcher + +Matches any resource that is not part of builtin k8s API groups (e.g. apps, batch, etc.). It's likely that over time some builtin k8s resources would not be matched. + +```yaml +customResourceMatcher: {} +``` + +### emptyFieldMatcher + +Available in v0.34.0+. + +Matches any resource that has empty specified field + +```yaml +emptyFieldMatcher: + path: [aggregationRule] +``` + +--- +## Paths + +Path specifies location within a resource (as used `rebaseRules` and `ownershipLabelRules`): + +``` +[spec, clusterIP] +``` + +``` +[spec, volumeClaimTemplates, {allIndexes: true}, metadata, labels] +``` + +``` +[spec, volumeClaimTemplates, {index: 0}, metadata, labels] +``` + +--- +## Config wrapped in ConfigMap + +Available of v0.34.0+. + +Config resource could be wrapped in a ConfigMap to support same deployment configuration by tools that do not understand kapp's `Config` resource directly. ConfigMap carrying kapp config must to be labeled with `kapp.k14s.io/config` and have `config.yml` data key. Such config maps will be applied to the cluster, unlike config given as `Config` resource. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-kapp-config + labels: + kapp.k14s.io/config: "" +data: + config.yml: | + apiVersion: kapp.k14s.io/v1alpha1 + kind: Config + rebaseRules: + - path: [rules] + type: copy + sources: [existing, new] + resourceMatchers: + - notMatcher: + matcher: + emptyFieldMatcher: + path: [aggregationRule] +``` + +NOTE: `kapp` is _only_ affected by a `Config` (whether wrapped in a `ConfigMap` or not) when supplied as a direct input (i.e. as a `-f` argument). Any `ConfigMap` containing a `Config` is already present in the target cluster has _no affect whatsoever_ on `kapp`'s behavior. diff --git a/site/content/kapp/docs/v0.44.0/dangerous-flags.md b/site/content/kapp/docs/v0.44.0/dangerous-flags.md new file mode 100644 index 000000000..683898083 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/dangerous-flags.md @@ -0,0 +1,33 @@ +--- +title: Dangerous Flags +--- + +## Overview + +There are several flags in `kapp deploy/delete/etc.` commands that might be helpful in rare cases, but can cause problems if used improperly. These are their stories: + +## `--dangerous-allow-empty-list-of-resources` + +This flag allows `kapp deploy` to accept empty set of new resources. Given that kapp deploy converges set of resources, when empty set is provided, kapp will delete all existing resources. + +This commonly happens unintentionally. When configuration is piped into kapp (e.g. `ytt -f config/ | kapp deploy -f- ...`) and resource producing command fails (ytt in this example), kapp will not receive any resources by the time is closes. Since providing empty set of resources intentionally is pretty rare, this functionality is behind a flag. + +## `--dangerous-override-ownership-of-existing-resources` + +This flag allows `kapp deploy` to take ownership of resources that are already associated with another application (i.e. already has `kapp.k14s.io/app` label with a different value). + +Most commonly user may have _unintentionally_ included resource that is already deployed, hence by default we do not want to override that resource with a new copy. This may happen when multiple apps accidently specified same resource (i.e. same name under same namespace). In most cases this is not what user wants. + +This flag may be useful in cases when multiple applications (managed by kapp) need to be merged into one, or may be previously owning application have been deleted but its resources were kept. + +Note that by default if resource is given to kapp and it already exists in the cluster, and is not owned by another application, kapp will label it to belong to deploying app. + +## `--dangerous-ignore-failing-api-services` + +In some cases users may encounter that they have misbehaving `APIServices` within they cluster. Since `APIServices` affect how one finds existing resources within a cluster, by default kapp will show error similar to below and stop: + +``` +Error: ... unable to retrieve the complete list of server APIs: <...>: the server is currently unable to handle the request +``` + +In cases when APIService cannot be fixed, this flag can be used to let kapp know that it is okay to proceed even though it's not able to see resources under that `APIService`. Note when this flag is used, kapp will effectively think that resources under misbehaving `APIService` do not exist. diff --git a/site/content/kapp/docs/v0.44.0/diff.md b/site/content/kapp/docs/v0.44.0/diff.md new file mode 100644 index 000000000..c80469de6 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/diff.md @@ -0,0 +1,375 @@ +--- +title: Diff stage +--- +## Overview + +kapp compares resources specified in files against resources that exist in Kubernetes API. Once change set is calculated, it provides an option to apply it (see [Apply](apply.md) section for further details). + +There are five different types of operations: `create`, `update`, `delete`, `noop` (shown as empty), `exists` (added in v0.43.0). Seen in `Op` column of diff summary table. Additionally there is `Op strategy` column (shorted as `Op st.`), added in v0.31.0+, that shows supplemental information how operation will be performed (for example [`fallback on replace`](apply.md#kappk14sioupdate-strategy) for `update` operation). + +There are three different types of waiting: `reconcile` (waits until resource has converged to its desired state; see [apply waiting](apply-waiting.md) for waiting semantics), `delete` (waits until resource is gone), `noop` (shown as empty). Seen in `Wait to` column of diff summary table. + +## Diff strategies + +There are two diff strategies used by kapp: + +1. kapp compares against last applied resource content (previously applied by kapp; stored in annotation `kapp.k14s.io/original`) **if** there were no outside changes done to the resource (i.e. done outside of kapp, for example, by another team member or controller); kapp tries to use this strategy as much as possible to produce more user-friendly diffs. + +2. kapp compares against live resource content **if** it detects there were outside changes to the resource (hence, sometimes you may see a diff that shows several deleted fields even though these fields are not specified in the original file) + +Strategy is selected for each resource individually. You can control which strategy is used for all resources via `--diff-against-last-applied=bool` flag. + +Related: [rebase rules](config.md/#rebaserules). + +## Versioned Resources + +In some cases it's useful to represent an update to a resource as an entirely new resource. Common example is a workflow to update ConfigMap referenced by a Deployment. Deployments do not restart their Pods when ConfigMap changes making it tricky for wide variety of applications for pick up ConfigMap changes. kapp provides a solution for such scenarios, by offering a way to create uniquely named resources based on an original resource. + +Anytime there is a change to a resource marked as a versioned resource, entirely new resource will be created instead of updating an existing resource. + +To make resource versioned, add `kapp.k14s.io/versioned` annotation with an empty value. Created resource follow `{resource-name}-ver-{n}` naming pattern by incrementing `n` any time there is a change. + +Example: +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-sa-sample + annotations: + kapp.k14s.io/versioned: "" +``` +This will create versioned resource named `secret-sa-sample-ver-1` + +```bash +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default secret-sa-sample-ver-1 Secret - - create - reconcile - - + +Op: 1 create, 0 delete, 0 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop +``` + +Additionally kapp follows configuration rules (default ones, and ones that can be provided as part of application) to find and update object references (since new resource name is not something that configuration author knew about). + +{{< detail-tag "Example" >}} +Sample Config +```yaml +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config +templateRules: + - resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: ConfigMap} + affectedResources: + objectReferences: + - path: [spec, template, spec, containers, {allIndexes: true}, env, {allIndexes: true}, valueFrom, configMapKeyRef] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: special-config + annotations: + kapp.k14s.io/versioned: "" +data: + special.how: very-good +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + env: + - name: SPECIAL_LEVEL_KEY + valueFrom: + configMapKeyRef: + name: special-config + key: special.how +``` +Here we have specified the configuration rules that will update the ConfigMap object reference in resources of Kind Deployment. Here `ConfigMap` special-config is marked as versioned so anytime there is an update it will create a new resource with name `special-config-ver-{n}` and update the same name in resource of kind `Deployment` under `configMapKeyRef`. This example is part of [default configuration rule](https://github.com/vmware-tanzu/carvel-kapp/blob/28b17b775558ef4c64ce27a5655b81c00c8a2f59/pkg/kapp/config/default.go#L299) that kapp follows. +{{< /detail-tag >}} + +As of v0.38.0+, `kapp.k14s.io/versioned-keep-original` annotation can be used in conjunction with `kapp.k14s.io/versioned` to have the original resource (resource without `-ver-{n}` suffix in name) along with versioned resource. + +Example: +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-sa-sample + annotations: + kapp.k14s.io/versioned: "" + kapp.k14s.io/versioned-keep-original: "" +``` +This will create two resources one with original name `secret-sa-sample` and one with `-ver-{n}` suffix in name `secret-sa-sample-ver-1`. +```bash +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default secret-sa-sample Secret - - create - reconcile - - +^ secret-sa-sample-ver-1 Secret - - create - reconcile - - + +Op: 2 create, 0 delete, 0 update, 0 noop +Wait to: 2 reconcile, 0 delete, 0 noop +``` + +You can control number of kept resource versions via `kapp.k14s.io/num-versions=int` annotation. + +As of v0.41.0+, the `kapp.k14s.io/versioned-explicit-ref` can be used to explicitly refer to a versioned resource. This annotation allows a resource to be updated whenever a new version of the referred resource is created. + +Example: +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + annotations: + kapp.k14s.io/versioned: "" +data: + foo: bar +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-2 + annotations: + kapp.k14s.io/versioned-explicit-ref: | + apiVersion: v1 + kind: ConfigMap + name: config-1 +data: + foo: bar +``` +Here, `config-2` explicitly refers `config-1` and is updated with the latest versioned name when `config-1` is versioned. +```bash +@@ create configmap/config-1-ver-2 (v1) namespace: default @@ + ... + 1, 1 data: + 2 - foo: bar + 2 + foo: alpha + 3, 3 kind: ConfigMap + 4, 4 metadata: +@@ update configmap/config-2 (v1) namespace: default @@ + ... + 8, 8 kind: ConfigMap + 9 - name: config-1-ver-1 + 9 + name: config-1-ver-2 + 10, 10 creationTimestamp: "2021-09-29T17:27:34Z" + 11, 11 labels: + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default config-1-ver-2 ConfigMap - - create - reconcile - - +^ config-2 ConfigMap - 14s update - reconcile ok - +``` + +Try deploying [redis-with-configmap example](https://github.com/vmware-tanzu/carvel-kapp/tree/develop/examples/gitops/redis-with-configmap) and changing `ConfigMap` in a next deploy. + +--- +## Controlling diff via resource annotations + +### kapp.k14s.io/disable-original + +kapp, by default, records the resource copy into its annotation `kapp.k14s.io/original` while applying the resource to the cluster. + +{{< detail-tag "Example" >}} +Sample Config +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default +data: + foo: bar +``` +After deploying the resource, kapp added the annotation `kapp.k14s.io/original` with the content of the resource that was given to kapp: + +```bash +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default + annotations: + kapp.k14s.io/original: '{ "apiVersion": "v1", "kind": "ConfigMap", ...snip... }' +data: + foo: bar +``` +{{< /detail-tag >}} + +`kapp.k14s.io/disable-original` annotation controls whether to record provided resource copy (rarely wanted) + +Possible values: "" (empty). In some cases it's not possible or wanted to record applied resource copy into its annotation `kapp.k14s.io/original`. One such case might be when resource is extremely lengthy (e.g. long ConfigMap or CustomResourceDefinition) and will exceed annotation value max length of 262144 bytes. + +{{< detail-tag "Example" >}} +Sample Config +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default + annotations: + kapp.k14s.io/disable-original: "" +data: + foo: bar +``` +After deploying the resource, kapp didn't add the annotation `kapp.k14s.io/original` this time: + +```bash +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default + annotations: + kapp.k14s.io/disable-original: "" +data: + foo: bar +``` +{{< /detail-tag >}} + +--- +## Controlling diff via deploy flags + +Diff summary shows quick information about what's being changed: +- `--diff-summary=bool` (default `true`) shows diff summary, listing how resources have changed + {{< detail-tag "Example" >}} +Sample config +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample +stringData: + foo: bar +``` +```bash +$ kapp deploy -a sample-secret -f config.yaml --diff-summary=true +Target cluster 'https://127.0.0.1:56540' (nodes: kind-control-plane) + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default sample Secret - - create - reconcile - - + +Op: 1 create, 0 delete, 0 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +Continue? [yN]: +``` + {{< /detail-tag >}} + +Diff changes (line-by-line diffs) are useful for looking at actual changes, when app is re-deployed: +- `--diff-changes=bool` (`-c`) (default `false`) shows line-by-line diffs +- `--diff-context=int` (default `2`) controls number of lines to show around changed lines +- `--diff-mask=bool` (default `true`) controls whether to mask sensitive fields + {{< detail-tag "Example" >}} +Sample config +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample +stringData: + foo: bar +``` +```bash +# deploy sample-secre app +$ kapp deploy -a sample-secret -f config.yaml + +#update config +... +stringData: + foo: bars +... + +# re-deploy sample-secret app with required diff-changes flag to see line by line changes +$ kapp deploy -a sample-secret -f config.yaml --diff-changes=true --diff-context=4 +Target cluster 'https://127.0.0.1:56540' (nodes: kind-control-plane) + +@@ update secret/sample (v1) namespace: default @@ + ... + 30, 30 resourceVersion: "244751" + 31, 31 uid: b2453c2a-8dc8-4ed1-9b59-791547f78ea8 + 32, 32 stringData: + 33 - foo: <-- value not shown (#1) + 33 + foo: <-- value not shown (#2) + 34, 34 + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default sample Secret - 7m update - reconcile ok - + +Op: 0 create, 0 delete, 1 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +Continue? [yN]: + +# --diff-mask=true by default, note the masked value for secret data + +# try out kapp deploy -a sample-secret -f config.yaml --diff-mask=false --diff-changes=true --diff-context=2 +``` + {{< /detail-tag >}} + +Controlling how diffing is done: + +- `--diff-against-last-applied=bool` (default `true`) forces kapp to use particular diffing strategy (see above). +- `--diff-run=bool` (default `false`) set the flag to true, to stop after showing diff information. +- `--diff-exit-status=bool` (default `false`) controls exit status for diff runs (`0`: unused, `1`: any error, `2`: no changes, `3`: pending changes) + {{< detail-tag "Example" >}} + Sample config +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample +stringData: + foo: bar +``` +```bash +# deploy secret-sample app +$ kapp deploy -a secret-sample -f config.yaml --diff-run=true --diff-exit-status=true +Target cluster 'https://127.0.0.1:56540' (nodes: kind-control-plane) + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default sample Secret - - create - reconcile - - + +Op: 1 create, 0 delete, 0 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +kapp: Error: Exiting after diffing with pending changes (exit status 3) + +# note that kapp exits after diff and displays the exit status + +``` + {{< /detail-tag >}} + +Diff filter allows to filter changes based on operation (add/update/delete), newResource (configuration provided to kapp) and existingResource (resources in Kubernetes cluster) + +- `--diff-filter='{"and":[{"ops":["update"]},{"existingResource":{"kinds":["Deployment"]}]}'` will keep the resources which are getting updated and were of kind Deployment. diff --git a/site/content/kapp/docs/v0.44.0/faq.md b/site/content/kapp/docs/v0.44.0/faq.md new file mode 100644 index 000000000..20ef7b1a0 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/faq.md @@ -0,0 +1,127 @@ +--- +title: FAQ +--- + +## Migrating from `kubectl apply` to kapp + +Switching from `kubectl apply` to `kapp deploy` will allow kapp to adopt resources mentioned in a given config. +However, kapp will try to insert a few of its labels in bodies of some resources, like Deployments, which may fail due to those resources having immutable fields that kapp tries to update (spec.selector on Deployments). + +To prevent this failure, add the [`kapp.k14s.io/disable-default-label-scoping-rules: ""` annotation](config.md#labelscopingrules) as a [kapp configuration](config.md) to prevent kapp from touching the immutable fields when adopting a resource. + +Additional Resources: [GitHub Issue](https://github.com/vmware-tanzu/carvel-kapp/issues/204), [Slack Thread](https://kubernetes.slack.com/archives/CH8KCCKA5/p1606079730457700) + +## `Error: Asking for confirmation: EOF` + +This probably means you have piped configuration into kapp and did not specify `--yes` (`-y`) flag to continue. It's necessary because kapp can no longer ask for confirmation via stdin. Feel free to re-run the command with `--diff-changes` (`-c`) to make sure pending changes are correct. Instead of using a pipe you can also use an anonymous fifo keeping stdin free for the confirmation prompt, e.g. `kapp deploy -a app1 -f <(ytt -f config/)` + +--- +## Where to store app resources (i.e. in which namespace)? + +See [state namespace](state-namespace.md) doc page. + +--- +## `... Field is immutable` error + +> After changing the labels/selectors in one of my templates, I'm getting the `MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable (reason: Invalid)` errors on deployment resource. Is there a way to tell kapp to force the change? + +[via slack](https://kubernetes.slack.com/archives/CH8KCCKA5/p1565600090224400) + +Some fields on a resource are immutable. kapp provides a `kapp.k14s.io/update-strategy` annotation that controls how kapp will update resource. One of the strategies is `fallback-on-replace` which will have kapp recreate an object (delete, wait, then create) if initial update results in `Invalid` error. See [Controlling apply via resource annotations](apply.md#controlling-apply-via-resource-annotations) for details. + +--- +## `Job.batch is invalid: ... spec.selector: Required value` error + +`batch.Job` resource is augmented by the Job controller with unique labels upon its creation. When using kapp to subsequently update existing Job resource, API server will return `Invalid` error since given configuration does not include `spec.selector`, and `job-name` and `controller-uid` labels. kapp's [rebase rules](config.md#rebaserules) can be used to copy over necessary configuration from server side copy; however, since Job resource is mostly immutable, we recommend to use [`kapp.k14s.io/update-strategy` annotation](apply.md#kappk14sioupdate-strategy) set to `fallback-on-replace` to recreate Job resource with any updates. + +--- +## Updating Deployments when ConfigMap changes + +> Can kapp force update on ConfigMaps in Deployments/DaemonSets? Just noticed that it didn't do that and I somehow expected it to. + +[via slack](https://kubernetes.slack.com/archives/CH8KCCKA5/p1565624685226400) + +kapp has a feature called [versioned resources](diff.md#versioned-resources) that allows kapp to create uniquely named resources instead of updating resources with changes. Resources referencing versioned resources are forced to be updated with new names, and therefore are changed, thus solving a problem of how to propagate changes safely. + +--- +## Quick way to find common kapp command variations + +See [cheatsheet](cheatsheet.md). + +--- +## Limit number of ReplicaSets for Deployments + +> Everytime I do a new deploy w/ kapp I see a new replicaset, along with all of the previous ones. + +[via slack](https://kubernetes.slack.com/archives/CH8KCCKA5/p1565887856281400) + +`Deployment` resource has a field `.spec.revisionHistoryLimit` that controls how many previous `ReplicaSets` to keep. See [Deployment's clean up polciy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy) for more details. + +--- +## Changes detected immediately after successful deploy + +Sometimes Kubernetes API server will convert submitted field values into their canonical form server-side. This will be detected by kapp as a change during a next deploy. To avoid such changes in future, you will have to change your provided field values to what API server considers as canonical. + +``` +... +186 - cpu: "2" +187 - memory: 1Gi + 170 + cpu: 2000m + 171 + memory: 1024Mi +... +``` + +Consider using [ytt](/ytt) and [its overlay feature](/ytt/docs/latest/lang-ref-ytt-overlay/) to change values if you do not control source configuration. + +--- +## Changes detected after resource is modified server-side + +There might be cases where other system actors (various controllers) may modify resource outside of kapp. Common example is Deployment's `spec.replicas` field is modified by Horizontal Pod Autoscaler controller. To let kapp know of such external behaviour use custom `rebaseRules` configuration (see [HPA and Deployment rebase](hpa-deployment-rebase.md) for details). + +--- +## Colors are not showing up in my CI build, in my terminal, etc. + +Try setting `FORCE_COLOR=1` environment variable to force enabling color output. Available in v0.23.0+. + +--- +## How can I version apps deployed by kapp? + +kapp itself does not provide any notion of versioning, since it's just a tool to reconcile config. We recommend to include a ConfigMap in your deployment with application metadata e.g. git commit, release notes, etc. + +--- +## `Resource ... is associated with a different label value` + +Resource ownership is tracked by app labels. kapp expects that each resource is owned by exactly one app. + +If you are receiving this error and are using correct app name, it might be that you are targeting wrong namespace where app is located. Use `--namespace` to set correct namespace. + +Additional resources: [State Namespace](state-namespace.md), [Slack Thread](https://kubernetes.slack.com/archives/CH8KCCKA5/p1589264289257000) + +--- +## Why does kapp hang when trying to delete a resource? + +By default, kapp won't delete resources it didn't create. You can see which resources are owned by kapp in output of `kapp inspect -a app-name` in its `Owner` column. You can force kapp to apply this ignored change using `--apply-ignored` [flag](apply.md#controlling-apply-via-deploy-flags). Alternatively if you are able to set [kapp.k14s.io/owned-for-deletion](apply.md#kappk14sioowned-for-deletion) annotation on resource that will be created, kapp will take that as a request to "own it" for deletion. This comes in handy for example with PVCs created by StatefulSet. + +--- +## How does kapp handle merging? + +kapp explicitly decided against _basic_ 3 way merge, instead allowing the user to specify how to resolve conflicts via rebase rules. + +Resources: [merge method](merge-method.md), [rebase rules](config.md#rebaserules) + +--- +## Can I force an update for a change that does not produce a diff? + +If kapp does not detect changes, it won't perform an update. To force changes every time you can set [`kapp.k14s.io/nonce`](apply.md#kappk14siononce) annotation. That way, every time you deploy the resource will appear to have changes. + +--- +## How can I remove decorative headings from kapp inspect output? + +Use `--tty=false` flag which will disable decorative output. Example: `kapp inspect --raw --tty=false`. + +Additional resources: [tty flag in kapp code](https://github.com/vmware-tanzu/carvel-kapp/blob/3f3e207d7198cdedd6985761ecb0d9616a84e305/pkg/kapp/cmd/ui_flags.go#L20) + +--- +## How can I get kapp to skip waiting on some resources? + +kapp allows to control waiting behavior per resource via [resource annotations](apply-waiting.md#controlling-waiting-via-resource-annotations). When used, the resource will be applied to the cluster, but will not wait to reconcile. diff --git a/site/content/kapp/docs/v0.44.0/gitops.md b/site/content/kapp/docs/v0.44.0/gitops.md new file mode 100644 index 000000000..49db4dfc8 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/gitops.md @@ -0,0 +1,16 @@ +--- +title: GitOps +--- + +## Using kapp with GitOps workflow + +kapp provides a set of commands to make GitOps workflow very easy. Assuming that you have a CI environment or some other place where `kapp` can run based on a trigger (e.g. for every Git repo change) or continuously (e.g. every 5 mins), you can use following command: + +```bash +$ ls my-repo +. .. app1/ app2/ app3/ + +$ kapp app-group deploy -g my-env --directory my-repo +``` + +Above command will deploy an application for each subdirectory in `my-repo` directory (in this case `app1`, `app2` and `app3`). It will also remove old applications if subdirectories are deleted. diff --git a/site/content/kapp/docs/latest/hpa-deployment-rebase.md b/site/content/kapp/docs/v0.44.0/hpa-deployment-rebase.md similarity index 100% rename from site/content/kapp/docs/latest/hpa-deployment-rebase.md rename to site/content/kapp/docs/v0.44.0/hpa-deployment-rebase.md diff --git a/site/content/kapp/docs/v0.44.0/install.md b/site/content/kapp/docs/v0.44.0/install.md new file mode 100644 index 000000000..7ea772fa8 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/install.md @@ -0,0 +1,57 @@ +--- +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ kapp version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ kapp version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/vmware-tanzu/homebrew-carvel](https://github.com/vmware-tanzu/homebrew-carvel). + +```bash +$ brew tap vmware-tanzu/carvel +$ brew install kapp +$ kapp version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/vmware-tanzu/carvel-kapp/releases), for example for 'kapp-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/kapp-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/kapp-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/kapp-darwin-amd64 /usr/local/bin/kapp + +# Make binary executable +$ chmod +x /usr/local/bin/kapp + +# Check its version +$ kapp version +``` diff --git a/site/content/kapp/docs/v0.44.0/integrating-with-other-tools.md b/site/content/kapp/docs/v0.44.0/integrating-with-other-tools.md new file mode 100644 index 000000000..e74154e1e --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/integrating-with-other-tools.md @@ -0,0 +1,26 @@ +--- +title: Integrating with Other Tools +--- + +**Note:** This is a non-exhaustive list of integrations + +## ytt and kbld + +We recommend to use kapp with [ytt](/ytt) and [kbld](/kbld) to cover your configuration templating and image building needs. Typical workflow may look like this: + +```bash +ytt -f config/ | kbld -f - | kapp deploy -a app1 -f- -c -y +``` + +## Helm + +If you want to take advantage of both Helm templating and kapp deployment mechanisms, you can use `helm template` command to build configuration, and have kapp apply to the cluster: + +```bash +helm template ... | kapp deploy -a app1 -f- -c -y +``` + +## PV labeling controller + +If you want to have better visibility into which persistent volumes (PVs) are associated with persistent volume claims (PVCs), you can install [https://github.com/k14s/pv-labeling-controller](https://github.com/k14s/pv-labeling-controller) so that it copies several kapp applied labels to associated PVs. Once that's done you will see PVs in `kapp inspect` output. + diff --git a/site/content/kapp/docs/v0.44.0/merge-method.md b/site/content/kapp/docs/v0.44.0/merge-method.md new file mode 100644 index 000000000..c86d85aab --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/merge-method.md @@ -0,0 +1,26 @@ +--- +title: "Resource Merge Method" +--- + +## Why not basic 3 way merge? + +kapp explicitly decided to _not_ do basic 3 way merge, and instead allow the user to specify how to resolve "conflicts". Here is our thinking: + +- you as an operator have a set of files (input files given to kapp via -f) which describe desired configuration +- cluster has resources that need to be converged to whatever input files specify, with one exception: in some cases, cluster is the source of truth for certain information (but not most) and should keep that state on resources (common examples: some annotation on Deployment, clusterIP on Service, etc.) + +Given information above there are multiple ways to converge: + +- make assumptions about how to merge things (basic 3 way merge, what kubectl and helm does afaik) +- be explicit about how to merge things (kapp with rebase rules) +- or, just override + +Overriding is not really an option as it removes potentially important cluster changes (e.g. removes replicas value as scaled by HPA). + +Regarding explicit vs implicit: we decided to go with the explicit option. kapp allows users to add [rebase rules](config.md#rebaserules) to specify exactly which information to retain from existing resources. That gives control to the user to decide what's important to be kept based on cluster state and what's not. This method ensures that there are no _surprising_ changes left in the cluster (if basic 3 way merge was used, then user cannot confidently know how final resource will look like; ... imagine if you had a field `allowUnauthenticatedRequests: true` in some resource that someone flipped on in your cluster, and your configs never specified it; it would not be removed unless you decide to also specify this field in your configs). + +kapp comes with some common k8s rebase rules. you can see them via `kapp deploy-config`. + +tldr: kapp takes user provided config as the only source of truth, but also allows to explicitly specify that certain fields are cluster controlled. This method guarantees that clusters don't drift, which is better than what basic 3 way merge provides. + +Originally answered [here](https://github.com/vmware-tanzu/carvel-kapp/issues/58#issuecomment-559214883). diff --git a/site/content/kapp/docs/v0.44.0/rbac.md b/site/content/kapp/docs/v0.44.0/rbac.md new file mode 100644 index 000000000..f5fae1c95 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/rbac.md @@ -0,0 +1,69 @@ +--- +title: Permissions +--- + +## Running kapp under restricted permissions + +In a multi-tenant Kubernetes cluster, user's actions may be limited to one or more namespaces via `Role` and `RoleBinding` configuration. + +Following setup is currently expected by kapp (v0.10.0+): + +- [required] kapp requires list/get/create/update/delete for `v1/ConfigMap` in [state namespace](state-namespace.md) so that it can store record of application and deployment history. +- [optional] kapp requires one `ClusterRole` rule: listing of namespaces. This requirement is necessary for kapp to find all namespaces so that it can search in each namespace resources that belong to a particular app (via a label). As of v0.11.0+, kapp will fallback to only [state namespace](state-namespace.md) if it is forbidden to list all namespaces. +- otherwise, kapp does _not_ require permissions to resource types that are not used in deployed configuration. In other words, if you are not deploying `Job` resource then kapp does not need any permissions for `Job`. Note that some resources are "cluster" created (e.g. `Pods` are created by k8s deployment controller when `Deployment` resource is created) hence users may not see all app associated resources in `kapp inspect` command if they are restricted (this could be advantageous and disadvantegeous in different setups). + +Please reach out to us in #carvel channel in k8s slack (linked at the bottom of the page) if current kapp permissions model isn't compatible with your use cases. We are eager to learn about your setup and potentially improve kapp. + +Example of `Namespace` listing permission needed by kapp: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kapp-restricted-cr +rules: +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kapp-restricted-cr-binding +subjects: +- kind: ServiceAccount + name: # ??? + namespace: # ??? (some tenant ns) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kapp-restricted-cr +``` + +Example of `ConfigMap` permissions needed by kapp: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kapp-restricted-role + namespace: # ??? (some tenant ns) +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["list", "get", "create", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kapp-restricted-role-binding + namespace: # ??? (some tenant ns) +subjects: +- kind: ServiceAccount + name: # ??? + namespace: # ??? (some tenant ns) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kapp-restricted-role +``` diff --git a/site/content/kapp/docs/latest/rebase-pvc.md b/site/content/kapp/docs/v0.44.0/rebase-pvc.md similarity index 100% rename from site/content/kapp/docs/latest/rebase-pvc.md rename to site/content/kapp/docs/v0.44.0/rebase-pvc.md diff --git a/site/content/kapp/docs/v0.44.0/security.md b/site/content/kapp/docs/v0.44.0/security.md new file mode 100644 index 000000000..bae369dd0 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/security.md @@ -0,0 +1,7 @@ +--- +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `kapp`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/kapp/docs/v0.44.0/state-namespace.md b/site/content/kapp/docs/v0.44.0/state-namespace.md new file mode 100644 index 000000000..ff945a9b4 --- /dev/null +++ b/site/content/kapp/docs/v0.44.0/state-namespace.md @@ -0,0 +1,52 @@ +--- +title: Namespace for State Storage +--- + +## Overview + +To show list of deployed applications (via `kapp ls`), kapp manages metadata `ConfigMap` for each saved application. Each metadata `ConfigMap` contains generated label used to label all application resources. Additionally kapp creates `ConfigMap` per each deploy to record deployment history (seen via `kapp app-change list -a app1`). + +`-n` (`--namespace`) flag allows to control which namespace is used for finding and storing metadata `ConfigMaps`. If namespace is not explicitly specified your current namespace is selected from kube config (typically `~/.kube/config`). + +There are currently two approaches to deciding which namespace to use for storing metadata `ConfigMaps`: + +- for each application, keep metadata `ConfigMap` and app resources themselves in the same namespace. That namespace will have to be created before running `kapp deploy` since kapp will first want to create a `ConfigMap` representing application. + + ```bash + $ kubectl create ns app1 + $ kapp deploy -n app1 -f config.yml + $ kapp ls -n app1 + ``` + +- create a dedicated namespace to store metadata `ConfigMaps` representing apps, and have kapp create `Namespace` resources for applications from their config. With this approach namespace management (creation and deletion) is tied to a particular app configuration which makes it a bit easier to track `Namespaces` via configuration. + + ```bash + $ kubectl create ns apps + $ kapp deploy -n apps -f app1/config.yml + $ kapp deploy -n apps -f app2/config.yml + $ kapp ls -n apps + ``` + + for example, `app1/config.yml` may look like this: + + ```yaml + apiVersion: v1 + kind: Namespace + metadata: + name: app1 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: dep + namespace: app1 + ... + ``` + +Note: It's currently not possible to have kapp place app `ConfigMap` resource into `Namespace` that kapp creates for that application. + +## App Changes + +As mentioned above, app changes (stored as `ConfigMap`) are stored in state namespace. App changes do not store any information necessary for kapp to operate, but rather act as informational records. There is currently no cap on how many app changes are kept per app. + +To remove older app changes, use `kapp app-change gc -a app1` which by default will keep 200 most recent changes (as of v0.12.0). Alternatively use `--app-changes-max-to-keep` flag on the `deploy` command to control number of changes kept at the time of deploy. diff --git a/site/content/kapp/docs/v0.45.0/_index.md b/site/content/kapp/docs/v0.45.0/_index.md new file mode 100644 index 000000000..cb396384c --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/_index.md @@ -0,0 +1,44 @@ +--- +aliases: [/kapp/docs/latest/] +title: "About kapp" +toc: "false" +cascade: + version: v0.45.0 + toc: "true" + type: docs + layout: docs +--- + +`kapp` (pronounced: `kap`) CLI encourages Kubernetes users to manage resources in bulk by working with "Kubernetes applications" (sets of resources with the same label). It focuses on resource diffing, labeling, deployment and deletion. Unlike tools like Helm, `kapp` considers YAML templating and management of packages outside of its scope, though it works great with tools that generate Kubernetes configuration. + +![Kapp Deploy](/images/kapp/kapp-deploy-screenshot.png) + +Features: + +- Works with standard Kubernetes YAMLs +- Focuses exclusively on deployment workflow, not packaging or templating + - but plays well with tools (such as [ytt](/ytt)) that produce Kubernetes configuration +- Converges application resources (creates, updates and/or deletes resources) in each deploy + - based on comparison between provided files and live objects in the cluster +- Separates calculation of changes ([diff stage](diff.md)) from application of changes ([apply stage](apply.md)) +- [Waits for resources](apply-waiting.md) to be "ready" +- Creates CRDs and Namespaces first and supports [custom change ordering](apply-ordering.md) +- Works [without admin privileges](rbac.md) and does not use custom CRDs + - making it possible to use kapp as a regular user in a single namespace +- Records application deployment history +- Opt-in resource version management + - for example, to trigger Deployment rollout when ConfigMap changes +- Optionally streams Pod logs during deploy +- Works with any group of labeled resources (`kapp -a label:tier=web inspect -t`) +- Works without server side components +- GitOps friendly (`kapp app-group deploy -g all-apps --directory .`) + +## Blog posts + +- [Deploying Kubernetes Applications with ytt, kbld, and kapp](/blog/deploying-apps-with-ytt-kbld-kapp) + +## Talks + +- [ytt and kapp @ TGI Kubernetes 079](https://www.youtube.com/watch?v=CSglwNTQiYg) with Joe Beda +- [Managing Applications in Production: Helm vs ytt & kapp @ Kubecon 2020](https://www.youtube.com/watch?v=WJw1MDFMVuk) +- [Introduction to Carvel @ Rawkode Live](https://www.youtube.com/watch?v=LBCmMTofNxw) diff --git a/site/content/kapp/docs/v0.45.0/apply-ordering.md b/site/content/kapp/docs/v0.45.0/apply-ordering.md new file mode 100644 index 000000000..6a60e1044 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/apply-ordering.md @@ -0,0 +1,94 @@ +--- +aliases: [/kapp/docs/latest/apply-ordering] +title: Apply Ordering +--- + +## Overview + +kapp includes builtin rules to make sure certain changes are applied in particular order: + +- Creates/updates + - CRDs are created/updated before custom resources + - Namespaces are created/updated before namespaced resources + - Pod related resources (ServiceAccount, ConfigMap, Secret, etc.) are created/updated before other resources (v0.25.0+) + - RBAC related resources (Role, RoleBinding, etc.) are created/updated before other resources (v0.25.0+) +- Deletions (below is order as of v0.29.0+) + - Custom resources are deleted first + - CRDs are deleted next + - Rest of resoures are deleted + +As of v0.25.0+, builtin rules are specified via [changeGroupBindings and changeRuleBindings](config.md#changegroupbindings) configurations. Custom rules can be added via same mechanism. + +Additionally kapp allows to customize order of changes via following resource annotations: + +- `kapp.k14s.io/change-group` annotation to group one or more resource changes into arbitrarily named group. Example: `apps.big.co/db-migrations`. You can specify multiple change groups by suffixing each annotation with a `.x` where `x` is unique identifier (e.g. `kapp.k14s.io/change-group.istio-sidecar-order`). +- `kapp.k14s.io/change-rule` annotation to control when resource change should be applied (created, updated, or deleted) relative to other changes. You can specify multiple change rules by suffixing each annotation with a `.x` where `x` is unique identifier (e.g. `kapp.k14s.io/change-rule.istio-sidecar-order`). + +`kapp.k14s.io/change-rule` annotation value format is as follows: `(upsert|delete) (after|before) (upserting|deleting) `. For example: + +- `kapp.k14s.io/change-rule: "upsert after upserting apps.big.co/db-migrations"` +- `kapp.k14s.io/change-rule: "delete before upserting apps.big.co/service"` + +As of v0.41.0+, kapp provides change group placeholders, which can be used in change-group and change-rule annotation values and are later replaced by values from the resource manifest of the resource they are associated with. For example: + +- `kapp.k14s.io/change-group: apps.co/db-migrations-{name}` - Here `{name}` would later be replaced by the name of the resource. +- `kapp.k14s.io/change-rule: upsert after upserting apps.co/namespaces-{namespace}` - Here `{namespace}` would later be replaced by the namespace of the resource. + +kapp provides the following placeholders: + +- `{api-group}` - apiGroup +- `{kind}` - kind +- `{name}` - name +- `{namespace}` - namespace +- `{crd-kind}` - spec.names.kind (available for CRDs only) +- `{crd-group}` - spec.group (available for CRDs only) + +These placeholders can also be used in changeGroupBindings and changeRuleBindings. By default, they are used for CRDs, CRs, namespaces and namespaced resources. Due to this, CRs now wait for their respective CRDs only and namespaced resources now wait for their respective namespaces only. + +## Example + +Following example shows how to run `job/migrations`, start and wait for `deployment/app`, and finally `job/app-health-check`. + +```yaml +kind: ConfigMap +metadata: + name: app-config + annotations: {} +#... +--- +kind: Job +metadata: + name: migrations + annotations: + kapp.k14s.io/change-group: "apps.big.co/db-migrations" +#... +--- +kind: Service +metadata: + name: app + annotations: + kapp.k14s.io/change-group: "apps.big.co/deployment" +#... +--- +kind: Ingress +metadata: + name: app + annotations: + kapp.k14s.io/change-group: "apps.big.co/deployment" +#... +--- +kind: Deployment +metadata: + name: app + annotations: + kapp.k14s.io/change-group: "apps.big.co/deployment" + kapp.k14s.io/change-rule: "upsert after upserting apps.big.co/db-migrations" +#... +--- +kind: Job +metadata: + name: app-health-check + annotations: + kapp.k14s.io/change-rule: "upsert after upserting apps.big.co/deployment" +#... +``` diff --git a/site/content/kapp/docs/v0.45.0/apply-waiting.md b/site/content/kapp/docs/v0.45.0/apply-waiting.md new file mode 100644 index 000000000..b33e6638e --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/apply-waiting.md @@ -0,0 +1,47 @@ +--- +aliases: [/kapp/docs/latest/apply-waiting] +title: Apply Waiting +--- + +## Overview + +kapp includes builtin rules on how to wait for the following resource types: + +- any resource with `metadata.deletionTimestamp`: wait for resource to be fully removed +- any resource matching Config's waitRules: [see "Custom waiting behaviour" below](#custom-waiting-behaviour) +- [`apiextensions.k8s.io//CustomResourceDefinition`](https://github.com/vmware-tanzu/carvel-kapp/blob/develop/pkg/kapp/resourcesmisc/api_extensions_vx_crd.go): wait for Established and NamesAccepted conditions to be `True` (note that this is wait rule for CustomResourceDefinition resource itself, not CRs) +- `apps/v1/DaemonSet`: wait for `status.numberUnavailable` to be 0 +- `apps/v1/Deployment`: [see "apps/v1/Deployment resource" below](#apps-v1-deployment-resource) +- `apps/v1/ReplicaSet`: wait for `status.replicas == status.availableReplicas` +- `batch/v1/Job`: wait for `Complete` or `Failed` conditions to appear +- `batch//CronJob`: immediately considered done +- `/v1/Pod`: looks at `status.phase` +- `/v1/Service`: wait for `spec.clusterIP` and/or `status.loadBalancer.ingress` to become set +- `apps/v1/StatefulSet`: [see "apps/v1/StatefulSet resource" below](#appsv1statefulset-resource) + +If resource is not affected by the above rules, its waiting behaviour depends on aggregate of waiting states of its associated resources (associated resources are resources that share same `kapp.k14s.io/association` label value). + +## Controlling waiting via resource annotations + +- `kapp.k14s.io/disable-wait` annotation controls whether waiting will happen at all. Possible values: "". +- `kapp.k14s.io/disable-associated-resources-wait` annotation controls whether associated resources impact resource's waiting state. Possible values: "". + +## apps/v1/Deployment resource + +kapp by default waits for `apps/v1/Deployment` resource to have `status.unavailableReplicas` equal to zero. Additionally waiting behaviour can be controlled via following annotations: + +- `kapp.k14s.io/apps-v1-deployment-wait-minimum-replicas-available` annotation controls how many new available replicas are enough to consider waiting successful. Example values: `"10"`, `"5%"`. + +## apps/v1/StatefulSet resource + +Available in v0.32.0+. + +kapp will wait for any pods created from the updated template to be ready based on StatefulSet's status. This behaviour depends on the [update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies) used. + +Note: kapp does not do anything special when `OnDelete` strategy is used. It will wait for StatefulSet to report it's reconciled (expecting some actor in the system to delete Pods per `OnDelete` requirements). + +## Custom waiting behaviour + +Available in v0.29.0+. + +kapp can be extended with custom waiting behaviour by specifying [wait rules as additional config](config.md#wait-rules). (If this functionality is not enough to wait for resources in your use case, please reach out on Slack to discuss further.) diff --git a/site/content/kapp/docs/v0.45.0/apply.md b/site/content/kapp/docs/v0.45.0/apply.md new file mode 100644 index 000000000..7e8e57e27 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/apply.md @@ -0,0 +1,121 @@ +--- +aliases: [/kapp/docs/latest/apply] +title: Apply stage +--- + +## Overview + +Once change set is calculated (see [Diff](diff.md) section for details), kapp asks for user confirmation (unless `--yes` flag is specified) to proceed with changes. + +Changes are applied in particular order as described in [Apply ordering](apply-ordering.md). + +All created resources are labeled with several labels: + +- `kapp.k14s.io/app` to track which application "owns" resource +- `kapp.k14s.io/identity` to identify preferred API version used when creating resource +- `kapp.k14s.io/association` to track (best effort) parent-child relationships between resources + +Every time application is deployed, new application change record is saved. They can be viewed via `kapp app-change ls -a app-name`. + +Related: [ownership label rules](config.md#ownershiplabelrules) and [label scoping rules](config.md#labelscopingrules). + +## Controlling apply via resource annotations + +### kapp.k14s.io/create-strategy + +`kapp.k14s.io/create-strategy` annotation controls create behaviour (rarely necessary) + +Possible values: "" (default), `fallback-on-update`. + +In some cases creation of a resource may conflict with that resource being created in the cluster by other means (often automated). An example of that is creation of default ServiceAccount by kapp racing with Kubernetes service accounts controller doing the same thing. By specifying `fallback-on-update` value, kapp will catch resource creation conflicts and apply resource as an update. + +### kapp.k14s.io/update-strategy + +`kapp.k14s.io/update-strategy` annotation controls update behaviour + +Possible values: "" (default), `fallback-on-replace`, `always-replace`, `skip`. + +In some cases entire resources or subset resource fields are immutable which forces kapp users to specify how to apply wanted update. + +- "" means to issue plain update call +- `fallback-on-replace` causes kapp to fallback to resource replacement if update call results in `Invalid` error. Note that if resource is replaced (= delete + create), it may be negatively affected (loss of persistent data, loss of availability, etc.). For example, if Deployment or DaemonSet is first deleted and then created then associated Pods will be recreated as well, but all at the same time (even if rolling update is enabled), which likely causes an availability gap. +- `always-replace` causes kapp to always delete and then create resource (See note above as well.) +- `skip` causes kapp to not apply update (it will show up in a diff next time). Available in v0.33.0+. + +### kapp.k14s.io/delete-strategy + +`kapp.k14s.io/delete-strategy` annotation controls deletion behaviour + +Possible values: "" (default), `orphan`. + +By default resource is deleted, however; choosing `orphan` value will make kapp forget about this resource. Note that if this resource is owned by a different resource that's being deleted, it might still get deleted. Orphaned resources are labeled with `kapp.k14s.io/orphaned` label. As of v0.31.0+, resource is also disassociated from owning app so that it can be owned by future apps. + +### kapp.k14s.io/owned-for-deletion + +`kapp.k14s.io/owned-for-deletion` annotation controls resource deletion during `kapp delete` command + +Possible values: "". + +By default non-kapp owned resources are not explicitly deleted by kapp, but expected to be deleted by the cluster (for example Endpoints resource for each Service). In some cases it's desired to annotate non-kapp owned resource so that it does get explicitly deleted, possibly because cluster does not plan to delete it (e.g. PVCs created by StatefulSet are not deleted by StatefulSet controller; [https://github.com/vmware-tanzu/carvel-kapp/issues/36](https://github.com/vmware-tanzu/carvel-kapp/issues/36)). + +### kapp.k14s.io/nonce + +`kapp.k14s.io/nonce` annotation allows to inject unique ID + +Possible values: "" (default). + +Annotation value will be replaced with a unique ID on each deploy. This allows to force resource update as value changes every time. + +### kapp.k14s.io/deploy-logs + +`kapp.k14s.io/deploy-logs` annotation indicates which Pods' log output to show during deploy + +Possible values: + +- "" (default; equivalent to `for-new`) +- `for-new` (only newly created Pods are tailed) +- `for-existing` (only existing Pods are tailed) +- `for-new-or-existing` (both newly created and existing Pods are tailed) + +Especially useful when added to Jobs. For example, see [examples/resource-ordering/sync-check.yml](https://github.com/vmware-tanzu/carvel-kapp/blob/develop/examples/resource-ordering/sync-check.yml) + +### kapp.k14s.io/deploy-logs-container-names + +`kapp.k14s.io/deploy-logs-container-names` annotation indicates which containers' log output to show during deploy + +Possible values: "" (default), `containerName1`, `containerName1,containerName2` + +### kapp.k14s.io/exists + +Available in v0.43.0+ + +`kapp.k14s.io/exists` will ensure that resource exists in Kubernetes. It will not be considered to be part of the app (not labeled). + +If the resource is not present already, then kapp uses the `exists` operation and ensures that the resource exists in Kubernetes. + +If the resource already exists, kapp does not perform any operation on it (the `noop` operation is used). + +Possible values: "". + +Especially useful in scenarios where an external agency such as a controller might be creating a resource that we want to wait for. + +### kapp.k14s.io/noop + +Available in v0.43.0+ + +`kapp.k14s.io/noop` ensures that kapp is aware of the resource. It will not be considered to be part of the app (not labeled). + +kapp always uses the `noop` operation for these resources. + +Possible values: "". + +--- +## Controlling apply via deploy flags + +- `--apply-ignored=bool` explicitly applies ignored changes; this is useful in cases when controllers lose track of some resources instead of for example deleting them +- `--apply-default-update-strategy=string` controls default strategy for all resources (see `kapp.k14s.io/update-strategy` annotation above) +- `--apply-exit-status=bool` (default `false`) controls exit status (`0`: unused, `1`: any error, `2`: no changes applied, `3`: at least one change applied) +- `--wait=bool` (default `true`) controls whether kapp will wait for resource to "stabilize". See [Apply waiting](apply-waiting.md) +- `--wait-ignored=bool` controls whether kapp will wait for ignored changes (regardless whether they were initiated by kapp or by controllers) +- `--logs=bool` (default `true`) controls whether to show logs as part of deploy output for Pods annotated with `kapp.k14s.io/deploy-logs: ""` +- `--logs-all=bool` (deafult `false`) controls whether to show all logs as part of deploy output for all Pods diff --git a/site/content/kapp/docs/v0.45.0/apps.md b/site/content/kapp/docs/v0.45.0/apps.md new file mode 100644 index 000000000..ebaf13fab --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/apps.md @@ -0,0 +1,41 @@ +--- +aliases: [/kapp/docs/latest/apps] +title: Applications +--- + +## Overview + +kapp considers a set of resources with the same label as an application. These resources could span any number of namespaces or could be cluster-wide (e.g. CRDs). + +kapp has two methods of finding resources: + +1. via unique-to-Namespace application name (via `-a my-name` flag), or +2. via user provided label (via `-a label:my-label=val` flag) + +First approach is most common as kapp generates a unique label for each tracked application and associates that with an application name. + +## List + +Applications can be listed via `ls` command: + +```bash +$ kapp ls +``` + +## Deploy + +To create or update an application use `deploy` command: + +```bash +$ kapp deploy -a my-name -f my-app-config/ +``` + +Deploy command consists of two stages: [resource "diff" stage](diff.md), and [resource "apply" stage](apply.md). + +## Delete + +To delete an application use `delete` command: + +```bash +$ kapp delete -a my-name +``` diff --git a/site/content/kapp/docs/v0.45.0/cheatsheet.md b/site/content/kapp/docs/v0.45.0/cheatsheet.md new file mode 100644 index 000000000..167300a8f --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/cheatsheet.md @@ -0,0 +1,129 @@ +--- +aliases: [/kapp/docs/latest/cheatsheet] +title: Cheatsheet +--- + + +## List + +```bash +kapp ls -A +``` + +List all app in the cluster (across all namespaces) + +## Deploy + +Deploy app named `app1` with configuration from `config/`: + +```bash +kapp deploy -a app1 -f config/ -c +``` + +Deploy app named `app1` with configuration piped in (see alternative that does not require `--yes` next): + +```bash +ytt -f config/ | kapp deploy -a app1 -f- -c -y +``` + +Deploy app named `app1` with configuration generated inline and with confirmation dialog: + +```bash +kapp deploy -a app1 -f <(ytt -f config/ ) +``` + +Show more diff context when reviewing changes during deploy: + +```bash +kapp deploy -a app1 -f config/ -c --diff-context=10 +``` + +Show diff and exit successfully (without applying any changes): + +```bash +kapp deploy -a app1 -f config/ --diff-run +``` + +Show logs from all app `Pods` throughout deploy: + +```bash +kapp deploy -a app1 -f config/ --logs-all +``` + +Rewrite all resources to specify `app1-ns` namespace: + +```bash +kapp deploy -a app1 -f config/ --into-ns app1-ns +``` + +## Inspect + +Show summary of all resources in app `app1`: + +```bash +kapp inspect -a app1 +``` + +Show summary organized as a tree of all resources in app `app1`: + +```bash +kapp inspect -a app1 --tree +``` + +Show status subresources for each resource in app `app1`: + +```bash +kapp inspect -a app1 --status +``` + +Show all resources in the cluster: + +```bash +kapp inspect -a 'label:' +``` + +Show all resources in particular namespace (note that it currently does namespace filtering client-side): + +```bash +kapp inspect -a 'label:' --filter-ns some-ns +``` + +Show all resources labeled `tier=web` in the cluster: + +```bash +kapp inspect -a 'label:tier=web' +``` + +Show all `Deployment` resources in the cluster **not** managed by kapp: + +```bash +kapp inspect -a 'label:!kapp.k14s.io/app' --filter-kind Deployment +``` + +## Delete + +Delete resources under particular label (in this example deleting resources associated with some app): + +```bash +kapp delete -a 'label:kapp.k14s.io/app=1578599579922603000' +``` + +## Misc + +See which labels are used in your cluster (add `--values` to see label values): + +```bash +kapp tools list-labels +``` + +Shows app labels that are still present in the cluster (could be combined with delete command below): + +```bash +kapp tools list-labels --values --tty=false | grep kapp.k14s.io/app +``` + +Delete all app changes older than 500h (v0.12.0+): + +```bash +kapp deploy -a label:kapp.k14s.io/is-app-change --filter-age 500h+ --dangerous-allow-empty-list-of-resources --apply-ignored +``` diff --git a/site/content/kapp/docs/v0.45.0/config.md b/site/content/kapp/docs/v0.45.0/config.md new file mode 100644 index 000000000..b6c0f486d --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/config.md @@ -0,0 +1,335 @@ +--- +aliases: [/kapp/docs/latest/config] +title: Configuration +--- + +## Overview + +kapp supports custom `Config` resource to specify its own configuration. It's expected to be included with your other Kubernetes configuration. Config resource is never applied to the cluster, though it follows general Kubernetes resource format. Multiple config resources are allowed. + +kapp comes with __built-in configuration__ (see it via `kapp deploy-config`) that includes rules for common resources. + +## Format + +```yaml +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config + +minimumRequiredVersion: 0.23.0 + +rebaseRules: +- path: [spec, clusterIP] + type: copy + sources: [new, existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} + +ownershipLabelRules: +- path: [metadata, labels] + resourceMatchers: + - allMatcher: {} + +labelScopingRules: +- path: [spec, selector] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} + +templateRules: +- resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: ConfigMap} + affectedResources: + objectReferences: + - path: [spec, template, spec, containers, {allIndexes: true}, env, {allIndexes: true}, valueFrom, configMapKeyRef] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + - path: [spec, template, spec, containers, {allIndexes: true}, envFrom, {allIndexes: true}, configMapRef] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + +additionalLabels: + department: marketing + cost-center: mar201 + +diffAgainstLastAppliedFieldExclusionRules: +- path: [metadata, annotations, "deployment.kubernetes.io/revision"] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + +diffMaskRules: +- path: [data] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Secret} +``` + +### minimumRequiredVersion + +`minimumRequiredVersion` forces kapp to exit with a validation error if kapp's version is below minimum required version. Available in v0.23.0+. + +### rebaseRules + +`rebaseRules` specify origin of field values. + +kapp rebase rules explicitly define how to merge resources during an update. To read more about why rebase rules are necessary, see [Resource Merge Method](merge-method.md). +For examples of rebase rules in use, see [HPA and Deployment rebase](hpa-deployment-rebase.md) or [PersistentVolumeClaim rebase](rebase-pvc.md). + +- `rebaseRules` (array) list of rebase rules + - `path` (array of strings) specifies location within a resource to rebase. Mutually exclusive with `paths`. Example: `[spec, clusterIP]` + - `paths` (array of `path`) specifies multiple locations within a resource to rebase. This is a convenience for specifying multiple rebase rules with only different paths. Mutually exclusive with `path`. Available in v0.27.0+. + - `type` (string) specifies strategy to modify field values. Allowed values: `copy` or `remove`. `copy` will update the field value; `remove` will delete the field. + - `sources` (array of `new` or `existing`) specifies a preference order for the source of the referenced field value being rebased. `new` refers to an updated resource from user input, where `existing` refers to a resource already in the cluster. If the field value being rebased is not found in any of the sources provided, kapp will error. Only used with `type: copy`. \ + Examples: + - `[existing, new]` – If field value is present in the `existing` resource on cluster, use that value, otherwise use the value in the `new` user input. + - `[existing]` – Only look for field values in resources already on cluster, corresponding value you provide in new resource will be overwritten. + - `resourceMatchers` (array) specifies rules to find matching resources. See various resource matchers below. + - `ytt` specifies choice as [ytt](https://carvel.dev/ytt/) for rebase rule. Available in v0.38.0+. + - `overlayContractV1` allows to use ytt overlay to modify provided resource based on existing resource. + - `overlay.yml` overlay YAML file. + +Rebase rule to `copy` the `clusterIP` field value to `Service`/`v1` resources; if `clusterIp` is present in the `new` user input, use that value, otherwise use the value in `existing` resource on cluster: +```yaml +rebaseRules: +- path: [spec, clusterIP] + type: copy + sources: [new, existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} +``` + +Rebase rule to `copy` the `clusterIP` and `healthCheckNodePort` field values from the `existing` resource on cluster, to `Service`/`v1` resources: +```yaml +rebaseRules: +- paths: + - [spec, clusterIP] + - [spec, healthCheckNodePort] + type: copy + sources: [existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: Service} +``` + +See [ytt rebase rule](https://github.com/vmware-tanzu/carvel-kapp/blob/d3ee9a01b5f0d7d5632b6a157ea7d0338730d497/pkg/kapp/config/default.go#L123-L154) (included in default configuration) for retaining cluster added token secret in ServiceAccount's secrets array. + +### ownershipLabelRules + +`ownershipLabelRules` specify locations for inserting kapp generated labels. These labels allow kapp to track which resources belong to which application. For resources that describe creation of other resources (e.g. `Deployment` or `StatefulSet`), configuration may need to specify where to insert labels for child resources that will be created. `kapp.k14s.io/disable-default-ownership-label-rules: ""` (value must be empty) annotation can be be used to exclude an individual resource from default onwership label rules. + +### labelScopingRules + +`labelScopingRules` specify locations for inserting kapp generated labels that scope resources to resources within current application. `kapp.k14s.io/disable-default-label-scoping-rules: ""` (as of v0.33.0+, or use `kapp.k14s.io/disable-label-scoping: ""` in earlier versions) annotation can be used to exclude an individual resource from label scoping. + +### waitRules + +Available in v0.29.0+. + +`waitRules` specify how to wait for resources that kapp does not wait for by default. Each rule provides a way to specify which `status.conditions` indicate success or failure. Once any of the condition matchers successfully match against one of the resource's conditions, kapp will stop waiting for the matched resource and report any failures. (If this functionality is not enough to wait for resources in your use case, please reach out on Slack to discuss further.) + +```yaml +waitRules: +- supportsObservedGeneration: true + conditionMatchers: + - type: Failed + status: "True" + failure: true + - type: Deployed + status: "True" + success: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: corp.com/v1, kind: DatabaseInstance} +``` + +```yaml +waitRules: +- supportsObservedGeneration: true + conditionMatchers: + - type: Ready + status: "False" + failure: true + - type: Ready + status: "True" + success: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: corp.com/v1, kind: Application} +``` + +### templateRules + +`templateRules` specify how versioned resources affect other resources. In above example, versioned config maps are said to affect deployments. [Read more about versioned resources](diff.md#versioned-resources). + +### additionalLabels + +`additionalLabels` specify additional labels to apply to all resources for custom uses by the user (added based on `ownershipLabelRules`). + +### diffAgainstLastAppliedFieldExclusionRules + +`diffAgainstLastAppliedFieldExclusionRules` specify which fields should be removed before diff-ing against last applied resource. These rules are useful for fields are "owned" by the cluster/controllers, and are only later updated. For example `Deployment` resource has an annotation that gets set after a little bit of time after resource is created/updated (not during resource admission). It's typically not necessary to use this configuration. + +### diffMaskRules + +`diffMaskRules` specify which field values should be masked in diff. By default `v1/Secret`'s `data` fields are masked. Currently only applied to `deploy` command. + +### changeGroupBindings + +Available in v0.25.0+. + +`changeGroupBindings` bind specified change group to resources matched by resource matchers. This is an alternative to using `kapp.k14s.io/change-group` annotation to add change group to resources. See `kapp deploy-config` for default bindings. + +### changeRuleBindings + +Available in v0.25.0+. + +`changeRuleBindings` bind specified change rules to resources matched by resource matchers. This is an alternative to using `kapp.k14s.io/change-rule` annotation to add change rules to resources. See `kapp deploy-config` for default bindings. + +--- +## Resource matchers + +Resource matchers (as used by `rebaseRules`, `ownershipLabelRules`, `labelScopingRules`, `templateRules`, `diffAgainstLastAppliedFieldExclusionRules`, and `diffMaskRules`): + +### allMatcher + +Matches all resources + +```yaml +allMatcher: {} +``` + +### anyMatcher + +Matches resources that match one of matchers + +```yaml +anyMatcher: + matchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + - apiVersionKindMatcher: {apiVersion: extensions/v1alpha1, kind: Deployment} +``` + +### notMatcher + +Matches any resource that does not match given matcher + +```yaml +notMatcher: + matcher: + apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} +``` + +### andMatcher + +Matches any resource that matches all given matchers + +```yaml +andMatcher: + matchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} + - hasNamespaceMatcher: {} +``` + +### apiGroupKindMatcher + +```yaml +apiGroupKindMatcher: {apiGroup: apps, kind: Deployment} +``` + +### apiVersionKindMatcher + +```yaml +apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} +``` + +### kindNamespaceNameMatcher + +```yaml +kindNamespaceNameMatcher: {kind: Deployment, namespace: mysql, name: mysql} +``` + +### hasAnnotationMatcher + +Matches resources that have particular annotation + +```yaml +hasAnnotationMatcher: + keys: + - kapp.k14s.io/change-group +``` + +### hasNamespaceMatcher + +Matches any resource that has a non-empty namespace + +```yaml +hasNamespaceMatcher: {} +``` + +Matches any resource with namespace that equals to one of the specified names + +```yaml +hasNamespaceMatcher: + names: [app1, app2] +``` + +### customResourceMatcher + +Matches any resource that is not part of builtin k8s API groups (e.g. apps, batch, etc.). It's likely that over time some builtin k8s resources would not be matched. + +```yaml +customResourceMatcher: {} +``` + +### emptyFieldMatcher + +Available in v0.34.0+. + +Matches any resource that has empty specified field + +```yaml +emptyFieldMatcher: + path: [aggregationRule] +``` + +--- +## Paths + +Path specifies location within a resource (as used `rebaseRules` and `ownershipLabelRules`): + +``` +[spec, clusterIP] +``` + +``` +[spec, volumeClaimTemplates, {allIndexes: true}, metadata, labels] +``` + +``` +[spec, volumeClaimTemplates, {index: 0}, metadata, labels] +``` + +--- +## Config wrapped in ConfigMap + +Available of v0.34.0+. + +Config resource could be wrapped in a ConfigMap to support same deployment configuration by tools that do not understand kapp's `Config` resource directly. ConfigMap carrying kapp config must to be labeled with `kapp.k14s.io/config` and have `config.yml` data key. Such config maps will be applied to the cluster, unlike config given as `Config` resource. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-kapp-config + labels: + kapp.k14s.io/config: "" +data: + config.yml: | + apiVersion: kapp.k14s.io/v1alpha1 + kind: Config + rebaseRules: + - path: [rules] + type: copy + sources: [existing, new] + resourceMatchers: + - notMatcher: + matcher: + emptyFieldMatcher: + path: [aggregationRule] +``` + +NOTE: `kapp` is _only_ affected by a `Config` (whether wrapped in a `ConfigMap` or not) when supplied as a direct input (i.e. as a `-f` argument). Any `ConfigMap` containing a `Config` is already present in the target cluster has _no affect whatsoever_ on `kapp`'s behavior. diff --git a/site/content/kapp/docs/v0.45.0/dangerous-flags.md b/site/content/kapp/docs/v0.45.0/dangerous-flags.md new file mode 100644 index 000000000..25969c7d6 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/dangerous-flags.md @@ -0,0 +1,34 @@ +--- +aliases: [/kapp/docs/latest/dangerous-flags] +title: Dangerous Flags +--- + +## Overview + +There are several flags in `kapp deploy/delete/etc.` commands that might be helpful in rare cases, but can cause problems if used improperly. These are their stories: + +## `--dangerous-allow-empty-list-of-resources` + +This flag allows `kapp deploy` to accept empty set of new resources. Given that kapp deploy converges set of resources, when empty set is provided, kapp will delete all existing resources. + +This commonly happens unintentionally. When configuration is piped into kapp (e.g. `ytt -f config/ | kapp deploy -f- ...`) and resource producing command fails (ytt in this example), kapp will not receive any resources by the time is closes. Since providing empty set of resources intentionally is pretty rare, this functionality is behind a flag. + +## `--dangerous-override-ownership-of-existing-resources` + +This flag allows `kapp deploy` to take ownership of resources that are already associated with another application (i.e. already has `kapp.k14s.io/app` label with a different value). + +Most commonly user may have _unintentionally_ included resource that is already deployed, hence by default we do not want to override that resource with a new copy. This may happen when multiple apps accidently specified same resource (i.e. same name under same namespace). In most cases this is not what user wants. + +This flag may be useful in cases when multiple applications (managed by kapp) need to be merged into one, or may be previously owning application have been deleted but its resources were kept. + +Note that by default if resource is given to kapp and it already exists in the cluster, and is not owned by another application, kapp will label it to belong to deploying app. + +## `--dangerous-ignore-failing-api-services` + +In some cases users may encounter that they have misbehaving `APIServices` within they cluster. Since `APIServices` affect how one finds existing resources within a cluster, by default kapp will show error similar to below and stop: + +``` +Error: ... unable to retrieve the complete list of server APIs: <...>: the server is currently unable to handle the request +``` + +In cases when APIService cannot be fixed, this flag can be used to let kapp know that it is okay to proceed even though it's not able to see resources under that `APIService`. Note when this flag is used, kapp will effectively think that resources under misbehaving `APIService` do not exist. diff --git a/site/content/kapp/docs/v0.45.0/diff.md b/site/content/kapp/docs/v0.45.0/diff.md new file mode 100644 index 000000000..3e1190028 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/diff.md @@ -0,0 +1,376 @@ +--- +aliases: [/kapp/docs/latest/diff] +title: Diff stage +--- +## Overview + +kapp compares resources specified in files against resources that exist in Kubernetes API. Once change set is calculated, it provides an option to apply it (see [Apply](apply.md) section for further details). + +There are five different types of operations: `create`, `update`, `delete`, `noop` (shown as empty), `exists` (added in v0.43.0). Seen in `Op` column of diff summary table. Additionally there is `Op strategy` column (shorted as `Op st.`), added in v0.31.0+, that shows supplemental information how operation will be performed (for example [`fallback on replace`](apply.md#kappk14sioupdate-strategy) for `update` operation). + +There are three different types of waiting: `reconcile` (waits until resource has converged to its desired state; see [apply waiting](apply-waiting.md) for waiting semantics), `delete` (waits until resource is gone), `noop` (shown as empty). Seen in `Wait to` column of diff summary table. + +## Diff strategies + +There are two diff strategies used by kapp: + +1. kapp compares against last applied resource content (previously applied by kapp; stored in annotation `kapp.k14s.io/original`) **if** there were no outside changes done to the resource (i.e. done outside of kapp, for example, by another team member or controller); kapp tries to use this strategy as much as possible to produce more user-friendly diffs. + +2. kapp compares against live resource content **if** it detects there were outside changes to the resource (hence, sometimes you may see a diff that shows several deleted fields even though these fields are not specified in the original file) + +Strategy is selected for each resource individually. You can control which strategy is used for all resources via `--diff-against-last-applied=bool` flag. + +Related: [rebase rules](config.md/#rebaserules). + +## Versioned Resources + +In some cases it's useful to represent an update to a resource as an entirely new resource. Common example is a workflow to update ConfigMap referenced by a Deployment. Deployments do not restart their Pods when ConfigMap changes making it tricky for wide variety of applications for pick up ConfigMap changes. kapp provides a solution for such scenarios, by offering a way to create uniquely named resources based on an original resource. + +Anytime there is a change to a resource marked as a versioned resource, entirely new resource will be created instead of updating an existing resource. + +To make resource versioned, add `kapp.k14s.io/versioned` annotation with an empty value. Created resource follow `{resource-name}-ver-{n}` naming pattern by incrementing `n` any time there is a change. + +Example: +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-sa-sample + annotations: + kapp.k14s.io/versioned: "" +``` +This will create versioned resource named `secret-sa-sample-ver-1` + +```bash +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default secret-sa-sample-ver-1 Secret - - create - reconcile - - + +Op: 1 create, 0 delete, 0 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop +``` + +Additionally kapp follows configuration rules (default ones, and ones that can be provided as part of application) to find and update object references (since new resource name is not something that configuration author knew about). + +{{< detail-tag "Example" >}} +Sample Config +```yaml +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config +templateRules: + - resourceMatchers: + - apiVersionKindMatcher: {apiVersion: v1, kind: ConfigMap} + affectedResources: + objectReferences: + - path: [spec, template, spec, containers, {allIndexes: true}, env, {allIndexes: true}, valueFrom, configMapKeyRef] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: apps/v1, kind: Deployment} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: special-config + annotations: + kapp.k14s.io/versioned: "" +data: + special.how: very-good +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + env: + - name: SPECIAL_LEVEL_KEY + valueFrom: + configMapKeyRef: + name: special-config + key: special.how +``` +Here we have specified the configuration rules that will update the ConfigMap object reference in resources of Kind Deployment. Here `ConfigMap` special-config is marked as versioned so anytime there is an update it will create a new resource with name `special-config-ver-{n}` and update the same name in resource of kind `Deployment` under `configMapKeyRef`. This example is part of [default configuration rule](https://github.com/vmware-tanzu/carvel-kapp/blob/28b17b775558ef4c64ce27a5655b81c00c8a2f59/pkg/kapp/config/default.go#L299) that kapp follows. +{{< /detail-tag >}} + +As of v0.38.0+, `kapp.k14s.io/versioned-keep-original` annotation can be used in conjunction with `kapp.k14s.io/versioned` to have the original resource (resource without `-ver-{n}` suffix in name) along with versioned resource. + +Example: +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-sa-sample + annotations: + kapp.k14s.io/versioned: "" + kapp.k14s.io/versioned-keep-original: "" +``` +This will create two resources one with original name `secret-sa-sample` and one with `-ver-{n}` suffix in name `secret-sa-sample-ver-1`. +```bash +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default secret-sa-sample Secret - - create - reconcile - - +^ secret-sa-sample-ver-1 Secret - - create - reconcile - - + +Op: 2 create, 0 delete, 0 update, 0 noop +Wait to: 2 reconcile, 0 delete, 0 noop +``` + +You can control number of kept resource versions via `kapp.k14s.io/num-versions=int` annotation. + +As of v0.41.0+, the `kapp.k14s.io/versioned-explicit-ref` can be used to explicitly refer to a versioned resource. This annotation allows a resource to be updated whenever a new version of the referred resource is created. + +Example: +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + annotations: + kapp.k14s.io/versioned: "" +data: + foo: bar +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-2 + annotations: + kapp.k14s.io/versioned-explicit-ref: | + apiVersion: v1 + kind: ConfigMap + name: config-1 +data: + foo: bar +``` +Here, `config-2` explicitly refers `config-1` and is updated with the latest versioned name when `config-1` is versioned. +```bash +@@ create configmap/config-1-ver-2 (v1) namespace: default @@ + ... + 1, 1 data: + 2 - foo: bar + 2 + foo: alpha + 3, 3 kind: ConfigMap + 4, 4 metadata: +@@ update configmap/config-2 (v1) namespace: default @@ + ... + 8, 8 kind: ConfigMap + 9 - name: config-1-ver-1 + 9 + name: config-1-ver-2 + 10, 10 creationTimestamp: "2021-09-29T17:27:34Z" + 11, 11 labels: + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default config-1-ver-2 ConfigMap - - create - reconcile - - +^ config-2 ConfigMap - 14s update - reconcile ok - +``` + +Try deploying [redis-with-configmap example](https://github.com/vmware-tanzu/carvel-kapp/tree/develop/examples/gitops/redis-with-configmap) and changing `ConfigMap` in a next deploy. + +--- +## Controlling diff via resource annotations + +### kapp.k14s.io/disable-original + +kapp, by default, records the resource copy into its annotation `kapp.k14s.io/original` while applying the resource to the cluster. + +{{< detail-tag "Example" >}} +Sample Config +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default +data: + foo: bar +``` +After deploying the resource, kapp added the annotation `kapp.k14s.io/original` with the content of the resource that was given to kapp: + +```bash +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default + annotations: + kapp.k14s.io/original: '{ "apiVersion": "v1", "kind": "ConfigMap", ...snip... }' +data: + foo: bar +``` +{{< /detail-tag >}} + +`kapp.k14s.io/disable-original` annotation controls whether to record provided resource copy (rarely wanted) + +Possible values: "" (empty). In some cases it's not possible or wanted to record applied resource copy into its annotation `kapp.k14s.io/original`. One such case might be when resource is extremely lengthy (e.g. long ConfigMap or CustomResourceDefinition) and will exceed annotation value max length of 262144 bytes. + +{{< detail-tag "Example" >}} +Sample Config +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default + annotations: + kapp.k14s.io/disable-original: "" +data: + foo: bar +``` +After deploying the resource, kapp didn't add the annotation `kapp.k14s.io/original` this time: + +```bash +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-1 + namespace: default + annotations: + kapp.k14s.io/disable-original: "" +data: + foo: bar +``` +{{< /detail-tag >}} + +--- +## Controlling diff via deploy flags + +Diff summary shows quick information about what's being changed: +- `--diff-summary=bool` (default `true`) shows diff summary, listing how resources have changed + {{< detail-tag "Example" >}} +Sample config +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample +stringData: + foo: bar +``` +```bash +$ kapp deploy -a sample-secret -f config.yaml --diff-summary=true +Target cluster 'https://127.0.0.1:56540' (nodes: kind-control-plane) + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default sample Secret - - create - reconcile - - + +Op: 1 create, 0 delete, 0 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +Continue? [yN]: +``` + {{< /detail-tag >}} + +Diff changes (line-by-line diffs) are useful for looking at actual changes, when app is re-deployed: +- `--diff-changes=bool` (`-c`) (default `false`) shows line-by-line diffs +- `--diff-context=int` (default `2`) controls number of lines to show around changed lines +- `--diff-mask=bool` (default `true`) controls whether to mask sensitive fields + {{< detail-tag "Example" >}} +Sample config +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample +stringData: + foo: bar +``` +```bash +# deploy sample-secre app +$ kapp deploy -a sample-secret -f config.yaml + +#update config +... +stringData: + foo: bars +... + +# re-deploy sample-secret app with required diff-changes flag to see line by line changes +$ kapp deploy -a sample-secret -f config.yaml --diff-changes=true --diff-context=4 +Target cluster 'https://127.0.0.1:56540' (nodes: kind-control-plane) + +@@ update secret/sample (v1) namespace: default @@ + ... + 30, 30 resourceVersion: "244751" + 31, 31 uid: b2453c2a-8dc8-4ed1-9b59-791547f78ea8 + 32, 32 stringData: + 33 - foo: <-- value not shown (#1) + 33 + foo: <-- value not shown (#2) + 34, 34 + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default sample Secret - 7m update - reconcile ok - + +Op: 0 create, 0 delete, 1 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +Continue? [yN]: + +# --diff-mask=true by default, note the masked value for secret data + +# try out kapp deploy -a sample-secret -f config.yaml --diff-mask=false --diff-changes=true --diff-context=2 +``` + {{< /detail-tag >}} + +Controlling how diffing is done: + +- `--diff-against-last-applied=bool` (default `true`) forces kapp to use particular diffing strategy (see above). +- `--diff-run=bool` (default `false`) set the flag to true, to stop after showing diff information. +- `--diff-exit-status=bool` (default `false`) controls exit status for diff runs (`0`: unused, `1`: any error, `2`: no changes, `3`: pending changes) + {{< detail-tag "Example" >}} + Sample config +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: sample +stringData: + foo: bar +``` +```bash +# deploy secret-sample app +$ kapp deploy -a secret-sample -f config.yaml --diff-run=true --diff-exit-status=true +Target cluster 'https://127.0.0.1:56540' (nodes: kind-control-plane) + +Changes + +Namespace Name Kind Conds. Age Op Op st. Wait to Rs Ri +default sample Secret - - create - reconcile - - + +Op: 1 create, 0 delete, 0 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +kapp: Error: Exiting after diffing with pending changes (exit status 3) + +# note that kapp exits after diff and displays the exit status + +``` + {{< /detail-tag >}} + +Diff filter allows to filter changes based on operation (add/update/delete), newResource (configuration provided to kapp) and existingResource (resources in Kubernetes cluster) + +- `--diff-filter='{"and":[{"ops":["update"]},{"existingResource":{"kinds":["Deployment"]}]}'` will keep the resources which are getting updated and were of kind Deployment. diff --git a/site/content/kapp/docs/v0.45.0/faq.md b/site/content/kapp/docs/v0.45.0/faq.md new file mode 100644 index 000000000..a75b85f77 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/faq.md @@ -0,0 +1,128 @@ +--- +aliases: [/kapp/docs/latest/faq] +title: FAQ +--- + +## Migrating from `kubectl apply` to kapp + +Switching from `kubectl apply` to `kapp deploy` will allow kapp to adopt resources mentioned in a given config. +However, kapp will try to insert a few of its labels in bodies of some resources, like Deployments, which may fail due to those resources having immutable fields that kapp tries to update (spec.selector on Deployments). + +To prevent this failure, add the [`kapp.k14s.io/disable-default-label-scoping-rules: ""` annotation](config.md#labelscopingrules) as a [kapp configuration](config.md) to prevent kapp from touching the immutable fields when adopting a resource. + +Additional Resources: [GitHub Issue](https://github.com/vmware-tanzu/carvel-kapp/issues/204), [Slack Thread](https://kubernetes.slack.com/archives/CH8KCCKA5/p1606079730457700) + +## `Error: Asking for confirmation: EOF` + +This probably means you have piped configuration into kapp and did not specify `--yes` (`-y`) flag to continue. It's necessary because kapp can no longer ask for confirmation via stdin. Feel free to re-run the command with `--diff-changes` (`-c`) to make sure pending changes are correct. Instead of using a pipe you can also use an anonymous fifo keeping stdin free for the confirmation prompt, e.g. `kapp deploy -a app1 -f <(ytt -f config/)` + +--- +## Where to store app resources (i.e. in which namespace)? + +See [state namespace](state-namespace.md) doc page. + +--- +## `... Field is immutable` error + +> After changing the labels/selectors in one of my templates, I'm getting the `MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable (reason: Invalid)` errors on deployment resource. Is there a way to tell kapp to force the change? + +[via slack](https://kubernetes.slack.com/archives/CH8KCCKA5/p1565600090224400) + +Some fields on a resource are immutable. kapp provides a `kapp.k14s.io/update-strategy` annotation that controls how kapp will update resource. One of the strategies is `fallback-on-replace` which will have kapp recreate an object (delete, wait, then create) if initial update results in `Invalid` error. See [Controlling apply via resource annotations](apply.md#controlling-apply-via-resource-annotations) for details. + +--- +## `Job.batch is invalid: ... spec.selector: Required value` error + +`batch.Job` resource is augmented by the Job controller with unique labels upon its creation. When using kapp to subsequently update existing Job resource, API server will return `Invalid` error since given configuration does not include `spec.selector`, and `job-name` and `controller-uid` labels. kapp's [rebase rules](config.md#rebaserules) can be used to copy over necessary configuration from server side copy; however, since Job resource is mostly immutable, we recommend to use [`kapp.k14s.io/update-strategy` annotation](apply.md#kappk14sioupdate-strategy) set to `fallback-on-replace` to recreate Job resource with any updates. + +--- +## Updating Deployments when ConfigMap changes + +> Can kapp force update on ConfigMaps in Deployments/DaemonSets? Just noticed that it didn't do that and I somehow expected it to. + +[via slack](https://kubernetes.slack.com/archives/CH8KCCKA5/p1565624685226400) + +kapp has a feature called [versioned resources](diff.md#versioned-resources) that allows kapp to create uniquely named resources instead of updating resources with changes. Resources referencing versioned resources are forced to be updated with new names, and therefore are changed, thus solving a problem of how to propagate changes safely. + +--- +## Quick way to find common kapp command variations + +See [cheatsheet](cheatsheet.md). + +--- +## Limit number of ReplicaSets for Deployments + +> Everytime I do a new deploy w/ kapp I see a new replicaset, along with all of the previous ones. + +[via slack](https://kubernetes.slack.com/archives/CH8KCCKA5/p1565887856281400) + +`Deployment` resource has a field `.spec.revisionHistoryLimit` that controls how many previous `ReplicaSets` to keep. See [Deployment's clean up polciy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy) for more details. + +--- +## Changes detected immediately after successful deploy + +Sometimes Kubernetes API server will convert submitted field values into their canonical form server-side. This will be detected by kapp as a change during a next deploy. To avoid such changes in future, you will have to change your provided field values to what API server considers as canonical. + +``` +... +186 - cpu: "2" +187 - memory: 1Gi + 170 + cpu: 2000m + 171 + memory: 1024Mi +... +``` + +Consider using [ytt](/ytt) and [its overlay feature](/ytt/docs/latest/lang-ref-ytt-overlay/) to change values if you do not control source configuration. + +--- +## Changes detected after resource is modified server-side + +There might be cases where other system actors (various controllers) may modify resource outside of kapp. Common example is Deployment's `spec.replicas` field is modified by Horizontal Pod Autoscaler controller. To let kapp know of such external behaviour use custom `rebaseRules` configuration (see [HPA and Deployment rebase](hpa-deployment-rebase.md) for details). + +--- +## Colors are not showing up in my CI build, in my terminal, etc. + +Try setting `FORCE_COLOR=1` environment variable to force enabling color output. Available in v0.23.0+. + +--- +## How can I version apps deployed by kapp? + +kapp itself does not provide any notion of versioning, since it's just a tool to reconcile config. We recommend to include a ConfigMap in your deployment with application metadata e.g. git commit, release notes, etc. + +--- +## `Resource ... is associated with a different label value` + +Resource ownership is tracked by app labels. kapp expects that each resource is owned by exactly one app. + +If you are receiving this error and are using correct app name, it might be that you are targeting wrong namespace where app is located. Use `--namespace` to set correct namespace. + +Additional resources: [State Namespace](state-namespace.md), [Slack Thread](https://kubernetes.slack.com/archives/CH8KCCKA5/p1589264289257000) + +--- +## Why does kapp hang when trying to delete a resource? + +By default, kapp won't delete resources it didn't create. You can see which resources are owned by kapp in output of `kapp inspect -a app-name` in its `Owner` column. You can force kapp to apply this ignored change using `--apply-ignored` [flag](apply.md#controlling-apply-via-deploy-flags). Alternatively if you are able to set [kapp.k14s.io/owned-for-deletion](apply.md#kappk14sioowned-for-deletion) annotation on resource that will be created, kapp will take that as a request to "own it" for deletion. This comes in handy for example with PVCs created by StatefulSet. + +--- +## How does kapp handle merging? + +kapp explicitly decided against _basic_ 3 way merge, instead allowing the user to specify how to resolve conflicts via rebase rules. + +Resources: [merge method](merge-method.md), [rebase rules](config.md#rebaserules) + +--- +## Can I force an update for a change that does not produce a diff? + +If kapp does not detect changes, it won't perform an update. To force changes every time you can set [`kapp.k14s.io/nonce`](apply.md#kappk14siononce) annotation. That way, every time you deploy the resource will appear to have changes. + +--- +## How can I remove decorative headings from kapp inspect output? + +Use `--tty=false` flag which will disable decorative output. Example: `kapp inspect --raw --tty=false`. + +Additional resources: [tty flag in kapp code](https://github.com/vmware-tanzu/carvel-kapp/blob/3f3e207d7198cdedd6985761ecb0d9616a84e305/pkg/kapp/cmd/ui_flags.go#L20) + +--- +## How can I get kapp to skip waiting on some resources? + +kapp allows to control waiting behavior per resource via [resource annotations](apply-waiting.md#controlling-waiting-via-resource-annotations). When used, the resource will be applied to the cluster, but will not wait to reconcile. diff --git a/site/content/kapp/docs/v0.45.0/gitops.md b/site/content/kapp/docs/v0.45.0/gitops.md new file mode 100644 index 000000000..156a06c19 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/gitops.md @@ -0,0 +1,17 @@ +--- +aliases: [/kapp/docs/latest/gitops] +title: GitOps +--- + +## Using kapp with GitOps workflow + +kapp provides a set of commands to make GitOps workflow very easy. Assuming that you have a CI environment or some other place where `kapp` can run based on a trigger (e.g. for every Git repo change) or continuously (e.g. every 5 mins), you can use following command: + +```bash +$ ls my-repo +. .. app1/ app2/ app3/ + +$ kapp app-group deploy -g my-env --directory my-repo +``` + +Above command will deploy an application for each subdirectory in `my-repo` directory (in this case `app1`, `app2` and `app3`). It will also remove old applications if subdirectories are deleted. diff --git a/site/content/kapp/docs/v0.45.0/hpa-deployment-rebase.md b/site/content/kapp/docs/v0.45.0/hpa-deployment-rebase.md new file mode 100644 index 000000000..a1f8d5df1 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/hpa-deployment-rebase.md @@ -0,0 +1,42 @@ +--- +aliases: [/kapp/docs/latest/gitops] +title: HPA and Deployment rebase +--- +## HPA and Deployment rebase + +Here is an example on how to use custom `rebaseRules` to "prefer" server chosen value for `spec.replicas` field for a particular Deployment. + +```yaml +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config +rebaseRules: +- path: [spec, replicas] + type: copy + sources: [existing, new] + resourceMatchers: + - kindNamespaceNameMatcher: + kind: Deployment + namespace: my-ns + name: my-app +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: my-app + namespace: my-ns +... +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: my-app + namespace: my-ns +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: my-app + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 50 +``` diff --git a/site/content/kapp/docs/v0.45.0/install.md b/site/content/kapp/docs/v0.45.0/install.md new file mode 100644 index 000000000..e2bda94da --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/install.md @@ -0,0 +1,58 @@ +--- +aliases: [/kapp/docs/latest/install] +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ kapp version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ kapp version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/vmware-tanzu/homebrew-carvel](https://github.com/vmware-tanzu/homebrew-carvel). + +```bash +$ brew tap vmware-tanzu/carvel +$ brew install kapp +$ kapp version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/vmware-tanzu/carvel-kapp/releases), for example for 'kapp-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/kapp-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/kapp-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/kapp-darwin-amd64 /usr/local/bin/kapp + +# Make binary executable +$ chmod +x /usr/local/bin/kapp + +# Check its version +$ kapp version +``` diff --git a/site/content/kapp/docs/v0.45.0/integrating-with-other-tools.md b/site/content/kapp/docs/v0.45.0/integrating-with-other-tools.md new file mode 100644 index 000000000..aa84510bf --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/integrating-with-other-tools.md @@ -0,0 +1,27 @@ +--- +aliases: [/kapp/docs/latest/integrating-with-other-tools] +title: Integrating with Other Tools +--- + +**Note:** This is a non-exhaustive list of integrations + +## ytt and kbld + +We recommend to use kapp with [ytt](/ytt) and [kbld](/kbld) to cover your configuration templating and image building needs. Typical workflow may look like this: + +```bash +ytt -f config/ | kbld -f - | kapp deploy -a app1 -f- -c -y +``` + +## Helm + +If you want to take advantage of both Helm templating and kapp deployment mechanisms, you can use `helm template` command to build configuration, and have kapp apply to the cluster: + +```bash +helm template ... | kapp deploy -a app1 -f- -c -y +``` + +## PV labeling controller + +If you want to have better visibility into which persistent volumes (PVs) are associated with persistent volume claims (PVCs), you can install [https://github.com/k14s/pv-labeling-controller](https://github.com/k14s/pv-labeling-controller) so that it copies several kapp applied labels to associated PVs. Once that's done you will see PVs in `kapp inspect` output. + diff --git a/site/content/kapp/docs/v0.45.0/merge-method.md b/site/content/kapp/docs/v0.45.0/merge-method.md new file mode 100644 index 000000000..b19c51b09 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/merge-method.md @@ -0,0 +1,27 @@ +--- +aliases: [/kapp/docs/latest/merge-method] +title: "Resource Merge Method" +--- + +## Why not basic 3 way merge? + +kapp explicitly decided to _not_ do basic 3 way merge, and instead allow the user to specify how to resolve "conflicts". Here is our thinking: + +- you as an operator have a set of files (input files given to kapp via -f) which describe desired configuration +- cluster has resources that need to be converged to whatever input files specify, with one exception: in some cases, cluster is the source of truth for certain information (but not most) and should keep that state on resources (common examples: some annotation on Deployment, clusterIP on Service, etc.) + +Given information above there are multiple ways to converge: + +- make assumptions about how to merge things (basic 3 way merge, what kubectl and helm does afaik) +- be explicit about how to merge things (kapp with rebase rules) +- or, just override + +Overriding is not really an option as it removes potentially important cluster changes (e.g. removes replicas value as scaled by HPA). + +Regarding explicit vs implicit: we decided to go with the explicit option. kapp allows users to add [rebase rules](config.md#rebaserules) to specify exactly which information to retain from existing resources. That gives control to the user to decide what's important to be kept based on cluster state and what's not. This method ensures that there are no _surprising_ changes left in the cluster (if basic 3 way merge was used, then user cannot confidently know how final resource will look like; ... imagine if you had a field `allowUnauthenticatedRequests: true` in some resource that someone flipped on in your cluster, and your configs never specified it; it would not be removed unless you decide to also specify this field in your configs). + +kapp comes with some common k8s rebase rules. you can see them via `kapp deploy-config`. + +tldr: kapp takes user provided config as the only source of truth, but also allows to explicitly specify that certain fields are cluster controlled. This method guarantees that clusters don't drift, which is better than what basic 3 way merge provides. + +Originally answered [here](https://github.com/vmware-tanzu/carvel-kapp/issues/58#issuecomment-559214883). diff --git a/site/content/kapp/docs/v0.45.0/rbac.md b/site/content/kapp/docs/v0.45.0/rbac.md new file mode 100644 index 000000000..7eb89bd96 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/rbac.md @@ -0,0 +1,70 @@ +--- +aliases: [/kapp/docs/latest/rbac] +title: Permissions +--- + +## Running kapp under restricted permissions + +In a multi-tenant Kubernetes cluster, user's actions may be limited to one or more namespaces via `Role` and `RoleBinding` configuration. + +Following setup is currently expected by kapp (v0.10.0+): + +- [required] kapp requires list/get/create/update/delete for `v1/ConfigMap` in [state namespace](state-namespace.md) so that it can store record of application and deployment history. +- [optional] kapp requires one `ClusterRole` rule: listing of namespaces. This requirement is necessary for kapp to find all namespaces so that it can search in each namespace resources that belong to a particular app (via a label). As of v0.11.0+, kapp will fallback to only [state namespace](state-namespace.md) if it is forbidden to list all namespaces. +- otherwise, kapp does _not_ require permissions to resource types that are not used in deployed configuration. In other words, if you are not deploying `Job` resource then kapp does not need any permissions for `Job`. Note that some resources are "cluster" created (e.g. `Pods` are created by k8s deployment controller when `Deployment` resource is created) hence users may not see all app associated resources in `kapp inspect` command if they are restricted (this could be advantageous and disadvantegeous in different setups). + +Please reach out to us in #carvel channel in k8s slack (linked at the bottom of the page) if current kapp permissions model isn't compatible with your use cases. We are eager to learn about your setup and potentially improve kapp. + +Example of `Namespace` listing permission needed by kapp: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kapp-restricted-cr +rules: +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kapp-restricted-cr-binding +subjects: +- kind: ServiceAccount + name: # ??? + namespace: # ??? (some tenant ns) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kapp-restricted-cr +``` + +Example of `ConfigMap` permissions needed by kapp: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kapp-restricted-role + namespace: # ??? (some tenant ns) +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["list", "get", "create", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kapp-restricted-role-binding + namespace: # ??? (some tenant ns) +subjects: +- kind: ServiceAccount + name: # ??? + namespace: # ??? (some tenant ns) +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kapp-restricted-role +``` diff --git a/site/content/kapp/docs/v0.45.0/rebase-pvc.md b/site/content/kapp/docs/v0.45.0/rebase-pvc.md new file mode 100644 index 000000000..b690668d8 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/rebase-pvc.md @@ -0,0 +1,102 @@ +--- +aliases: [/kapp/docs/latest/rebase-pvc] +title: PersistentVolumeClaim rebase +--- +## PersistentVolumeClaim rebase + +Here is an example on how to use custom `rebaseRules` to "prefer" server chosen value for several annotations added by PVC controller (in other words, cluster owned fields), instead of removing them based on given configuration. + +Let's deploy via `kapp deploy -a test -f config.yml -c` with following configuration `config.yml`: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysqlclaim +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +``` + +Without additional rebase rules following diff will be presented upon next deploy, stating that several annotations will be removed (since they were not present in the initial configuration): + +```bash +$ kapp deploy -a test -f config.yml -c + +Target cluster 'https://x.x.x.x' (nodes: gke-dk-jan-9-default-pool-a218b1c9-55sl, 3+) + +--- update persistentvolumeclaim/mysqlclaim (v1) namespace: default + ... + 2, 2 metadata: + 3 - annotations: + 4 - pv.kubernetes.io/bind-completed: "yes" + 5 - pv.kubernetes.io/bound-by-controller: "yes" + 6 - volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/gce-pd + 7, 3 creationTimestamp: "2020-03-08T22:17:29Z" + 8, 4 finalizers: + ... + 24, 20 storageClassName: standard + 25 - volumeMode: Filesystem + 26, 21 volumeName: pvc-1be63b2b-20de-429c-863a-9e7eb062f5d3 + 27, 22 status: + +Changes + +Namespace Name Kind Conds. Age Op Wait to Rs Ri +default mysqlclaim PersistentVolumeClaim - 43s update reconcile ok - + +Op: 0 create, 0 delete, 1 update, 0 noop +Wait to: 1 reconcile, 0 delete, 0 noop + +Continue? [yN]: +``` + +To let kapp know that these annotations should be copied from the live resource copy, we can augment deploys with following configuration `kapp-config.yml`: + +```yaml +--- +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config + +rebaseRules: +- path: [metadata, annotations, pv.kubernetes.io/bind-completed] + type: copy + sources: [new, existing] + resourceMatchers: &pvcs + - apiVersionKindMatcher: + apiVersion: v1 + kind: PersistentVolumeClaim + +- path: [metadata, annotations, pv.kubernetes.io/bound-by-controller] + type: copy + sources: [new, existing] + resourceMatchers: *pvcs + +- path: [metadata, annotations, volume.beta.kubernetes.io/storage-provisioner] + type: copy + sources: [new, existing] + resourceMatchers: *pvcs + +- path: [spec, volumeMode] + type: copy + sources: [new, existing] + resourceMatchers: *pvcs +``` + +```bash +$ kapp deploy -a test -f config.yml -f rules.yml -c + +Target cluster 'https://x.x.x.x' (nodes: gke-dk-jan-9-default-pool-a218b1c9-55sl, 3+) + +Changes + +Namespace Name Kind Conds. Age Op Wait to Rs Ri + +Op: 0 create, 0 delete, 0 update, 0 noop +Wait to: 0 reconcile, 0 delete, 0 noop + +Succeeded +``` diff --git a/site/content/kapp/docs/v0.45.0/security.md b/site/content/kapp/docs/v0.45.0/security.md new file mode 100644 index 000000000..6b16ba957 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/security.md @@ -0,0 +1,8 @@ +--- +aliases: [/kapp/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `kapp`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/kapp/docs/v0.45.0/state-namespace.md b/site/content/kapp/docs/v0.45.0/state-namespace.md new file mode 100644 index 000000000..1c278ca29 --- /dev/null +++ b/site/content/kapp/docs/v0.45.0/state-namespace.md @@ -0,0 +1,53 @@ +--- +aliases: [/kapp/docs/latest/state-namespace] +title: Namespace for State Storage +--- + +## Overview + +To show list of deployed applications (via `kapp ls`), kapp manages metadata `ConfigMap` for each saved application. Each metadata `ConfigMap` contains generated label used to label all application resources. Additionally kapp creates `ConfigMap` per each deploy to record deployment history (seen via `kapp app-change list -a app1`). + +`-n` (`--namespace`) flag allows to control which namespace is used for finding and storing metadata `ConfigMaps`. If namespace is not explicitly specified your current namespace is selected from kube config (typically `~/.kube/config`). + +There are currently two approaches to deciding which namespace to use for storing metadata `ConfigMaps`: + +- for each application, keep metadata `ConfigMap` and app resources themselves in the same namespace. That namespace will have to be created before running `kapp deploy` since kapp will first want to create a `ConfigMap` representing application. + + ```bash + $ kubectl create ns app1 + $ kapp deploy -n app1 -f config.yml + $ kapp ls -n app1 + ``` + +- create a dedicated namespace to store metadata `ConfigMaps` representing apps, and have kapp create `Namespace` resources for applications from their config. With this approach namespace management (creation and deletion) is tied to a particular app configuration which makes it a bit easier to track `Namespaces` via configuration. + + ```bash + $ kubectl create ns apps + $ kapp deploy -n apps -f app1/config.yml + $ kapp deploy -n apps -f app2/config.yml + $ kapp ls -n apps + ``` + + for example, `app1/config.yml` may look like this: + + ```yaml + apiVersion: v1 + kind: Namespace + metadata: + name: app1 + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: dep + namespace: app1 + ... + ``` + +Note: It's currently not possible to have kapp place app `ConfigMap` resource into `Namespace` that kapp creates for that application. + +## App Changes + +As mentioned above, app changes (stored as `ConfigMap`) are stored in state namespace. App changes do not store any information necessary for kapp to operate, but rather act as informational records. There is currently no cap on how many app changes are kept per app. + +To remove older app changes, use `kapp app-change gc -a app1` which by default will keep 200 most recent changes (as of v0.12.0). Alternatively use `--app-changes-max-to-keep` flag on the `deploy` command to control number of changes kept at the time of deploy. diff --git a/site/content/kbld/docs/latest/_index.md b/site/content/kbld/docs/develop/_index.md similarity index 98% rename from site/content/kbld/docs/latest/_index.md rename to site/content/kbld/docs/develop/_index.md index 2928b490b..523edc81f 100644 --- a/site/content/kbld/docs/latest/_index.md +++ b/site/content/kbld/docs/develop/_index.md @@ -2,7 +2,7 @@ title: "About kbld" toc: "false" cascade: - version: latest + version: develop toc: "true" type: docs layout: docs diff --git a/site/content/kbld/docs/latest/auth.md b/site/content/kbld/docs/develop/auth.md similarity index 100% rename from site/content/kbld/docs/latest/auth.md rename to site/content/kbld/docs/develop/auth.md diff --git a/site/content/kbld/docs/latest/building.md b/site/content/kbld/docs/develop/building.md similarity index 100% rename from site/content/kbld/docs/latest/building.md rename to site/content/kbld/docs/develop/building.md diff --git a/site/content/kbld/docs/latest/cnab-image-relocation.md b/site/content/kbld/docs/develop/cnab-image-relocation.md similarity index 100% rename from site/content/kbld/docs/latest/cnab-image-relocation.md rename to site/content/kbld/docs/develop/cnab-image-relocation.md diff --git a/site/content/kbld/docs/develop/config.md b/site/content/kbld/docs/develop/config.md new file mode 100644 index 000000000..cc4d9e830 --- /dev/null +++ b/site/content/kbld/docs/develop/config.md @@ -0,0 +1,493 @@ +--- +title: Configuration +--- + +## Overview + +Customize how `kbld`: +- searches for image references, +- resolves image names, +- builds images from source, and +- pushes built images to container registries. + +This is done by supplying one or more configuration files. These files are in the Kubernetes resource format (i.e. has `apiVersion` and `kind`). `kbld` consumes and removes such files from its output. + +A `kbld` configuration file is structured like so: + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +minimumRequiredVersion: 0.31.0 + +searchRules: ... +overrides: ... +sources: ... +destinations: ... +``` + +where: +- `minimumRequiredVersion` (optional; semver) — specifies the minimum required version of `kbld` needed to work with this configuration. + +and any combination of the follow four sections: +- `searchRules` — a list of [Search Rules](#search-rules) that `kbld` follows to locate container image references within the input files. +- `overrides` — a list of [Overrides](#overrides) that `kbld` applies to found container image references before it attempts to resolve or build the actual image. +- `sources` — a list of [Sources](#sources), each of which describes where to find the source that makes up the contents of a given image _and_ which build tool to use to construct it. +- `destinations` — a list of [Destinations](#destinations), each of which describes where (i.e. to which container registry) to publish a given image. + +_(Note: prior to v0.28.0, different types of configuration were supplied in different `kind` of files: `kind: Sources`, `kind: ImageDestinations`, `kind: ImageOverrides`, `kind: ImageKeys`. Since v0.28.0, all such configuration can be specified in one kind of file: `kind: Config`; this is recommended.)_ + +--- +## Search Rules + +`kbld` scans input files for references to container images. Search rules describe how `kbld` should identify those references and how to process them. + +A search rule has two parts: +- conditions for matching (either by key, by value, or both), and +- a strategy for how to parse and update a matched item. + +A search rule is expressed like this: + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +searchRules: +- keyMatcher: ... + valueMatcher: ... + updateStrategy: ... +``` + +where a rule consists of one or both of the following... +- `keyMatcher` a [Key Matcher](#key-matcher) — identifies container image references based on an item's key. +- `valueMatcher` a [Value Matcher](#value-matcher) — identifies container image references based on an item's value. + +_(If both are specified, their results are "and'ed" together.)_ + +...and optionally, +- `updateStrategy` an [Update Strategy](#update-strategy) — what `kbld` should do with the matched container image reference. + +**Multiple Matching search rules** \ +If multiple search rules match the same item, the rule that was defined first is applied. + +Note: `kbld` includes its own [Default Search Rules](#default-search-rules) + +### Key Matcher + +Specifies whether a given key indicates that its corresponding value is an image reference. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +searchRules: +- keyMatcher: + name: + path: +``` + +where: +- `name` (string) specifies the key name (e.g. `sidecarImage`) +- `path` (array) specifies key path from the root of the YAML document.\ + Each path part can be one of: + - the literal key name. (e.g. `[data, sidecarImage]` maps to `data.sidecarImage`) + - an array indexing tactic (choose one): + - `index:` (int) — search one element in the array at the (zero-based) index given. \ + (example: `[spec, template, spec, containers, {index: 0}, image]`) + - `allIndexes:` (bool) — search all elements in the array. \ + (example: `[spec, images, {allIndexes: true}]`) + +### Value Matcher + +Specifies whether a given value contains an image reference. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +searchRules: +- valueMatcher: + ( image | imageRepo ): +``` +where (choose one): +- `image` (string) value to match exactly +- `imageRepo` (string) of values in the format (`[registry]repo[:tag]\[@sha256:...]`), the value of the `repo` portion. \ + (example: `imageRepo: gcr.io/project/app` \ + matches `gcr.io/projects/app:v1.1` and `gcr.io/projects/app@sha256:f33e111...` \ + but not `gcr.io/projects/app_v1`) + +### Update Strategy + +Given a container image reference identified by one or more matchers, an Update Strategy describes how to update that value (v0.21.0+). + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +searchRules: +- ( keyMatcher | valueMatcher ): ... + updateStrategy: + yaml: + searchRules: ... + json: + searchRules: ... + none: {} + entireValue: {} +``` + +where (choose one): +- `yaml` parses YAML and identifies image refs by specified search rules + - `searchRules` — one or more [Search Rules](#search-rules), scoped to the parsed content. +- `json` parses JSON and identifies image refs by specified search rules + - `searchRules` — one or more [Search Rules](#search-rules), scoped to the parsed content. +- `none` (empty) excludes value from processing (v0.22.0+) +- `entireValue` (empty; default) uses the exact value as the image reference. + +#### Example for `updateStrategy` that parses YAML + +```yaml +--- +kind: ConfigMap +metadata: + name: config +data: + data.yml: | + name: nginx + image: nginx # <-- below config finds and updates this image +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +minimumRequiredVersion: 0.15.0 +searchRules: +- keyMatcher: + name: data.yml + updateStrategy: + yaml: + searchRules: + - keyMatcher: + name: image +``` + +Note that in the `ConfigMap`, that `/data/data.yml` holds a multi-line string. That string happens to also be in YAML format. When _that_ YAML is parsed, the `image` key holds a container image reference. + + +### Default Search Rules + +After custom search rules have been processed, `kbld` appends the following search rule: + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +searchRules: +- keyMatcher: + name: image + updateStrategy: + entireValue: {} +``` + +--- +## Overrides + +After `kbld` searches for container image references, it applies a set of "overrides" which effectively rewrite those references. It does this _before_ attempting to build the corresponding image or resolve it to a digest reference. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +overrides: +- image: + newImage: + preresolved: + tagSelection: ... +``` + +where: +- `image` (required; string) exact value found while searching for container image references. +- `newImage` (required; string) value with which to replace/override. This ought to be an image reference in the format `[registry]repo[:tag]\[@sha256:...]`. \ + Examples: + - `nginx` + - `quay.io/bitnami/nginx` + - `nginx:1.21.1` + - `nginx@sha256:a05b0cd...` + - `index.docker.io/library/nginx@sha256:a05b0cd...` +- `preresolved` (optional; bool) specifies if `newImage` should be used as is (rather than be [re]resolved to a digest reference). +- `tagSelection` (optional; [VersionSelection](/vendir/docs/latest/versions/#versionselection-type)) when `newImage` _is_ being resolved, specifies how to select the tag part of the reference before resolving to a digest reference. (Available as of v0.28.0+) + - In this case, `newImage` must not have a tag or digest part (e.g. `gcr.io/my-corp/app`). + +**Example: Static Rewrite** + +With the following configuration, any container image reference of the value `"unknown"` will get rewritten to `"docker.io/library/nginx:1.14.2"` before being resolved to a digest reference. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +overrides: +- image: unknown + newImage: docker.io/library/nginx:1.14.2 +``` + +**Example: Preresolved** + +This configuration replaces references of `"unknown"` with `"docker.io/library/nginx:1.14.2"`. It short-circuits any building, resolution, or even connecting to a registry to obtain metadata about the image reference. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +overrides: +- image: unknown + newImage: docker.io/library/nginx:1.14.2 + preresolved: true +``` + +**Example: Dynamic Tag Selection** + +This configuration first rewrites references of `"unknown"` with `"docker.io/library/nginx"`. It then queries the registry to locate the latest version just prior to 1.15.0 (which turns out to be 1.14.2). + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +overrides: +- image: unknown + newImage: docker.io/library/nginx + tagSelection: + semver: + constraints: "<1.15.0" +``` + + +--- +## Sources + +Rather than resolve an image reference, `kbld` can build the image from source. More precisely, `kbld` can be configured to use one of the many image building tools it integrates with to construct the image from source. For an overview, see [Building Images](building.md). + +Such integration is enabled by configuring a "Source": + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: +- image: image1 + path: src/ + ( docker | pack | kubectlBuildkit | ko | bazel ): ... +``` + +where: +- `image` (required; string) exact value found while searching for container image references. +- `path` (required; string) filesystem path to the source for the to-be-built image +- a builder configuration (optional; choose one) — name/configure a specific image builder tool: + - `docker:` (default) use [Docker](#docker) to build from source. + - `pack:` use [Pack](#pack) to build the image via buildpacks from source. + - `kubectlBuildkit:` use [Buildkit CLI for kubectl](#buildkit-cli-for-kubectl) to build the image via a Kubernetes cluster form source. + - `ko:` use [ko](#ko) to build the image from Go source. + - `bazel:` use [Bazel](#bazel) to build the image via Bazel Container Image Rules. + + +### Docker + +Using this integration requires: +- Docker — https://docs.docker.com/get-docker + +The [Docker CLI](https://docs.docker.com/engine/reference/commandline/cli/) must be on the `$PATH`. + +To configure an image to be built via Docker, include a `docker:`-flavored [Source](#sources): +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: +- image: image1 + path: src/ + docker: + build: + target: some-target + pull: true + noCache: true + file: hack/Dockerfile.dev + buildkit: true + rawOptions: ["--squash", "--build-arg", "ARG_IN_DOCKERFILE=value"] +``` +where: +- `target` (string): the target build stage to build (no default) +- `pull` (bool): always attempt to pull a newer version of the image (default is false) +- `noCache` (bool): do not use cache when building the image (default is false) +- `file` (string): name of the Dockerfile (default is Dockerfile) +- `buildkit` (bool): enable [Builtkit](https://docs.docker.com/develop/develop-images/build_enhancements) for this build. +- `rawOptions` ([]string): Refer to [`docker build` reference](https://docs.docker.com/engine/reference/commandline/build/) for all available options + +### Pack + +Using this integration requires: +- Docker — https://docs.docker.com/get-docker +- Pack — https://buildpacks.io/docs/tools/pack/ + +The [Pack CLI](https://buildpacks.io/docs/tools/pack/cli/pack/) must be on the `$PATH`. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: +- image: image1 + path: src/ + pack: + build: + builder: cloudfoundry/cnb:bionic +``` +where (all options are optional): +- `builder` (string): Set builder image (required) +- `buildpacks` ([]string): Set list of buildpacks to be used (no default) +- `clearCache` (bool): Clear cache before building image (default is false) +- `rawOptions` ([]string): Refer to `pack build -h` for all available flags + +### BuildKit CLI for kubectl + +Available as of v0.28.0+ + +Using this integration requires: +- `kubectl` — https://kubernetes.io/docs/tasks/tools/ +- Buildkit for kubectl — https://github.com/vmware-tanzu/buildkit-cli-for-kubectl#installing-the-tarball \ + _(`kbld` v0.28.0+ is tested with buildkit-for-kubectl v0.1.0, but may work well with other versions.)_ + +The [`kubectl` CLI](https://kubernetes.io/docs/reference/kubectl/kubectl/) must be on the `$PATH`. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: +- image: image1 + path: src/ + kubectlBuildkit: + build: + target: "some-target" + pull: true + noCache: true + file: "hack/Dockefile.dev" + rawOptions: ["--platform=..."] +``` + +where (all options are optional): +- `target` (string): Set the target build stage to build (no default) +- `pull` (bool): Always attempt to pull a newer version of the image (default is false) +- `noCache` (bool): Do not use cache when building the image (default is false) +- `file` (string): Name of the Dockerfile (default is Dockerfile) +- `rawOptions` ([]string): Refer to `kubectl buildkit build -h` for all available options + +#### Authenticating to Registry for Pushing Images + +To provide registry credentials to the builder, create a Kubernetes `docker-registry` secret: + +``` +kubectl create secret docker-registry buildkit --docker-server=https://index.docker.io/v1/ --docker-username=my-user --docker-password=my-password +``` + +See project site for details: [buildkit-cli-for-kubectl](https://github.com/vmware-tanzu/buildkit-cli-for-kubectl). + +### ko + +Available as of v0.28.0+ + +Using this integration requires: +- `ko` — https://github.com/google/ko \ + (`kbld` v0.28.0+ is tested with `ko` v0.8.0, but may work well with other versions.) + +The `ko` CLI must be on the `$PATH`. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: +- image: image1 + path: ./src/ + ko: + build: + rawOptions: ["--disable-optimizations"] +``` + +where: +- `rawOptions` (optional; []string): Refer to `ko publish -h` for all available options. + +By default `kbld` provides the `--local` flag + +### Bazel + +Available as of v0.31.0+ + +Using this integration requires: +- Docker — https://docs.docker.com/get-docker +- Bazel — https://docs.bazel.build/versions/main/install.html \ + _(`kbld` v0.31.0+ is tested with Bazel v4.2.0 and [Container Image Rules](https://github.com/bazelbuild/rules_docker/tree/v0.18.0) v0.18.0, but may work well with other versions.)_ + +The `bazel` CLI must be on the `$PATH`. + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: + - image: image1 + path: ./src/ + bazel: + run: + target: :image1-go-container + rawOptions: ["--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64"] +``` + +where: +- `target` (string): bazel build target; when passed to `bazel run` will build and load the desired image. +- `rawOptions` ([]string): Refer to https://docs.bazel.build/versions/main/user-manual.html for all available options. + +#### Using Language-specific Container Rules + +This integration invokes the `bazel run` command (as opposed to the `bazel build` command). + +Typically, when configuring the Bazel integration with `kbld`, the specified target is one of the [Container Image Rules](https://github.com/bazelbuild/rules_docker/tree/master#bazel-container-image-rules). + +With such rules, the `bazel build` command _produces_ the artifacts that make up all images described in the `BUILD`, but stops short of loading any image into the Docker daemon. In order for an image to be pushed to a registry (i.e. published), it must first be loaded into the local Docker daemon. The `bazel run` command takes that additional step to execute the script that performs that `docker load` _for the named target, only_. + +For so-called [`lang_image` rules](https://github.com/bazelbuild/rules_docker/tree/master#language-rules), this also results in launching a container from that image (i.e. `docker run` the built image). For our purposes, this is undesirable because it effectively halts the build. + +To skip this that launching step, append the args `-- --norun` via the `rawOptions:` key: + +```yaml +... + bazel: + run: + target: :image1-go-container + rawOptions: ["--", "--norun"] +``` + +See also https://github.com/bazelbuild/rules_docker#using-with-docker-locally. + +--- +## Destinations + +Once `kbld` has processed the found container image references (either resolved or built them), it can publish those images to a different registry (on an image-by-image basis). + +Pushing images to registries through this configuration requires: +- Docker — https://docs.docker.com/get-docker + +(In the case where all published images are built through [BuildKit CLI for kubectl](#buildkit-cli-for-kubectl), Docker is not required; publishing happens from within the Kubernetes cluster). + +Do this by configuring one or more "destinations": + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +destinations: +- image: adservice + newImage: docker.io/dkalinin/microservices-demo-adservice + tags: [latest, tag2] +``` + +where: +- `image` (required; string) exact value found while searching for container image references. +- `newImage` (required; string) image destination (i.e. fully qualified registry location for the image) +- `tags` (optional; array of strings) tags to apply to pushed images (v0.26.0+) diff --git a/site/content/kbld/docs/latest/install.md b/site/content/kbld/docs/develop/install.md similarity index 100% rename from site/content/kbld/docs/latest/install.md rename to site/content/kbld/docs/develop/install.md diff --git a/site/content/kbld/docs/latest/packaging.md b/site/content/kbld/docs/develop/packaging.md similarity index 100% rename from site/content/kbld/docs/latest/packaging.md rename to site/content/kbld/docs/develop/packaging.md diff --git a/site/content/kbld/docs/latest/resolving.md b/site/content/kbld/docs/develop/resolving.md similarity index 100% rename from site/content/kbld/docs/latest/resolving.md rename to site/content/kbld/docs/develop/resolving.md diff --git a/site/content/kbld/docs/latest/security.md b/site/content/kbld/docs/develop/security.md similarity index 100% rename from site/content/kbld/docs/latest/security.md rename to site/content/kbld/docs/develop/security.md diff --git a/site/content/kbld/docs/v0.32.0/_index.md b/site/content/kbld/docs/v0.32.0/_index.md new file mode 100644 index 000000000..9b603e872 --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/_index.md @@ -0,0 +1,30 @@ +--- +aliases: [/kbld/docs/latest/] +title: "About kbld" +toc: "false" +cascade: + version: v0.32.0 + toc: "true" + type: docs + layout: docs +--- + +`kbld` (pronounced: `kei·bild`) seamlessly incorporates image building and image pushing into your development and deployment workflows. + +Features: + +- Orchestrates image builds (delegates to tools like Docker, pack, kubectl-buildkit) and registry pushes +- Works with local Docker daemon and remote registries, for development and production cases +- Records metadata about image sources in annotation on Kubernetes resources (see examples below) +- Resolves image references to their digest (immutable) form +- Not specific to Kubernetes, but works really well with Kubernetes configuration files + +See [building and deploying simple Go application to Kubernetes example](/blog/deploying-apps-with-ytt-kbld-kapp/#building-container-images-locally) that uses kbld. + +## Why digest references? + +Docker images can be referenced by their name (`nginx`), name-tag pair (`nginx:1.14`), or a digest (`nginx@sha256:c398dc3f2...`). One can avoid potential deployment inconsistencies by using digest references as they are immutable, and therefore always points to an exact image. kbld helps Kubernetes users convert image references to their digest form to make sure exact image is used during deploys. + +## Blog posts + +- [Deploying Kubernetes Applications with ytt, kbld, and kapp](/blog/deploying-apps-with-ytt-kbld-kapp) diff --git a/site/content/kbld/docs/v0.32.0/auth.md b/site/content/kbld/docs/v0.32.0/auth.md new file mode 100644 index 000000000..f1df9f1b5 --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/auth.md @@ -0,0 +1,88 @@ +--- +aliases: [/kbld/docs/latest/auth] +title: Authentication +--- + +## Via Docker config + +Even though `kbld` commands use registry APIs directly, by default it uses credentials stored in `~/.docker/config.json` which are typically generated via `docker login` command. + +Example generated `~/.docker/config.json`: + +```json +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "dXNlcjpwYXNzd29yZA==" + }, + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.09.6 (darwin)" + } +} +``` + +where `dXNlcjpwYXNzd29yZA==` is `base64("username:password")`. + +## Via Environment Variables + +As of v0.23.0+, kbld can also use following environment variables: + +- `KBLD_REGISTRY_HOSTNAME` to specify registry hostname (e.g. gcr.io, docker.io) +- `KBLD_REGISTRY_USERNAME` to specify registry username +- `KBLD_REGISTRY_PASSWORD` to specify registry password + +Since you may need to provide multiple registry credentials, above environment variables multiple times with a suffix like so `KBLD_REGISTRY_HOSTNAME_0` (suffix can be 1+ alphanumeric characters). Use same suffix for hostname, username and password. + +Currently credentials provided via environment variables do not apply when building images with Docker. Continue using `docker login` to authenticate Docker daemon. + +## gcr.io + +- Create service account with "Storage Admin" for push access + - See [Permissions and Roles](https://cloud.google.com/container-registry/docs/access-control#permissions_and_roles) +- Download JSON service account key and place it somewhere on filesystem (e.g. `/tmp/key`) + - See [Advanced authentication](https://cloud.google.com/container-registry/docs/advanced-authentication#json_key_file) +- Run `cat /tmp/key | docker login -u _json_key --password-stdin https://gcr.io` to authenticate + +## AWS ECR + +- Create ECR repository +- Create IAM user with ECR policy that allows to read/write + - See [Amazon ECR Policies](https://docs.aws.amazon.com/AmazonECR/latest/userguide/ecr_managed_policies.html) +- Run `aws configure` and specify access key ID, secret access key and region + - To install on Ubuntu, run `apt-get install pip3` and `pip3 install awscli` + - See [Installing the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) +- Run `eval $(aws ecr get-login --no-include-email)` to authenticate + - See [get-login command](https://docs.aws.amazon.com/cli/latest/reference/ecr/get-login.html) + +Example ECR policy from https://docs.aws.amazon.com/AmazonECR/latest/userguide/ecr_managed_policies.html + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage" + ], + "Resource": "*" + } + ] +} +``` + +## Harbor + +You may have to provide `--registry-ca-cert-path` flag with a path to a CA certificate file for Harbor Registry API. diff --git a/site/content/kbld/docs/v0.32.0/building.md b/site/content/kbld/docs/v0.32.0/building.md new file mode 100644 index 000000000..afd16045c --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/building.md @@ -0,0 +1,152 @@ +--- +aliases: [/kbld/docs/latest/building] +title: Building images +--- + +## Building images from source + +kbld can be used to orchestrate build tools such as [Docker](https://docs.docker.com/engine/reference/commandline/cli/) and [pack](https://github.com/buildpacks/pack) to build images from source and record resulting image reference in a YAML file. This is especially convenient during local development when working with one or more changing applications. + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: simple-app #! <-- unresolved image ref +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test2 +spec: + selector: + matchLabels: + app: kbld-test2 + template: + metadata: + labels: + app: kbld-test2 + spec: + containers: + - name: my-app + image: another-simple-app #! <-- unresolved image ref +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +sources: +- image: simple-app + path: src/simple-app +- image: another-simple-app + path: src/another-simple-app +``` + +(See [Configuration](config.md) for more details about `Sources`.) + +Running above example via `kbld -f file.yml` will start two `docker build` processes and produce following output + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: kbld:1556053998479026000-simple-app #! <-- resolved image ref +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test2 +spec: + selector: + matchLabels: + app: kbld-test2 + template: + metadata: + labels: + app: kbld-test2 + spec: + containers: + - name: my-app + image: kbld:1556053998479039000-another-simple-app #! <--resolved image ref +``` + +Note that because we are using Docker daemon for local images and are not pushing them into a remote registry we cannot unfortunately use digest reference form (a limitation of Docker daemon); however, tags generated by kbld uniquely identify produced images. As soon as images are pushed to a remote registry, tags are converted into digest references. + +**Hint**: [Minikube](https://kubernetes.io/docs/setup/minikube/) comes with Docker daemon inside its VM. You can expose by running `eval $(minikube docker-env)` before executing kbld. + +## Pushing images + +As long as building tool has proper push access (run `docker login` for Docker), kbld can push out built images to specified repositories. Just add following configuration: + +```yaml +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +destinations: +- image: simple-app + newImage: docker.io/dkalinin/simple-app +- image: another-simple-app + newImage: docker.io/dkalinin/another-simple-app +``` + +With addition of above configuration, kbld will produce following YAML: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: index.docker.io/dkalinin/simple-app@sha256:f7988fb6c02e... #! <-- pushed image ref +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test2 +spec: + selector: + matchLabels: + app: kbld-test2 + template: + metadata: + labels: + app: kbld-test2 + spec: + containers: + - name: my-app + image: index.docker.io/dkalinin/another-simple-app@sha256:a7355fb1007e... #! <-- pushed image ref +``` diff --git a/site/content/kbld/docs/v0.32.0/cnab-image-relocation.md b/site/content/kbld/docs/v0.32.0/cnab-image-relocation.md new file mode 100644 index 000000000..152de7a64 --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/cnab-image-relocation.md @@ -0,0 +1,65 @@ +--- +aliases: [/kbld/docs/latest/cnab-image-relocation] +title: CNAB Image Maps +--- + +CNAB spec mentions [Image Relocation](https://github.com/cnabio/cnab-spec/blob/master/103-bundle-runtime.md#image-relocation) as part of bundle runtime. + +kbld supports applying `relocation-mapping.json` on top of YAML configuration via `kbld --image-map-file /cnab/app/relocation-mapping.json ...`. For example: + +/cnab/app/relocation-mapping.json: + +```json +{ + "gabrtv/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687": "my.registry/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687", + "technosophos/helloworld:0.1.0": "my.registry/helloworld:0.1.0" +} +``` + +and kbld input: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: gabrtv/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687 + - name: my-app2 + image: technosophos/helloworld:0.1.0 +``` + +would result in: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: my.registry/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687 + - name: my-app2 + image: my.registry/helloworld:0.1.0 +``` diff --git a/site/content/kbld/docs/latest/config.md b/site/content/kbld/docs/v0.32.0/config.md similarity index 99% rename from site/content/kbld/docs/latest/config.md rename to site/content/kbld/docs/v0.32.0/config.md index c6b3db7ee..79ed9336d 100644 --- a/site/content/kbld/docs/latest/config.md +++ b/site/content/kbld/docs/v0.32.0/config.md @@ -1,4 +1,5 @@ --- +aliases: [/kbld/docs/latest/config] title: Configuration --- diff --git a/site/content/kbld/docs/v0.32.0/install.md b/site/content/kbld/docs/v0.32.0/install.md new file mode 100644 index 000000000..4b15fb2bd --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/install.md @@ -0,0 +1,58 @@ +--- +aliases: [/kbld/docs/latest/install] +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ kbld version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ kbld version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/vmware-tanzu/homebrew-carvel](https://github.com/vmware-tanzu/homebrew-carvel). + +```bash +$ brew tap vmware-tanzu/carvel +$ brew install kbld +$ kbld version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/vmware-tanzu/carvel-kbld/releases), for example for 'kbld-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/kbld-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/kbld-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/kbld-darwin-amd64 /usr/local/bin/kbld + +# Make binary executable +$ chmod +x /usr/local/bin/kbld + +# Check its version +$ kbld version +``` diff --git a/site/content/kbld/docs/v0.32.0/packaging.md b/site/content/kbld/docs/v0.32.0/packaging.md new file mode 100644 index 000000000..95d65a0bd --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/packaging.md @@ -0,0 +1,190 @@ +--- +aliases: [/kbld/docs/latest/packaging] +title: Packaging and Relocation +--- + +### Deprecation +This functionality was deprecated in `kbld` starting from version v0.30.0 and will be removed soon. +These features are now available on [imgpkg](https://carvel.dev/imgpkg). + +## Overview + +kbld provides a way to relocate (i.e. copy) images between multiple registries. Two approaches are available: + +- `kbld relocate` (available v0.23.0+) allows to efficiently copy images between registries as long as running `relocate` command has connectivity to both registries. +- `kbld package` and `kbld unpackage` allows to export images into a single tarball, and later import them from given tarball into a different (or same) registry. This approach does _not_ require connectivity to source registry during the `pkg unpackage` time. + +Use cases: + +- packaging applications for fully offline environments with a private registry +- copying images from one registry to another +- backing up images + +There are two approaches to do this: + +- With lock file (recommended) +- Directly against configuration + +## With lock file + +For example, to package referenced images into a single tarball: + +1. Produce lock file for referenced images in configuration + + ```bash + $ cat /tmp/manifest + images: + - image: nginx + - image: haproxy + + $ kbld -f /tmp/manifest --lock-output /tmp/manifest.lock + ... + + $ cat /tmp/manifest.lock + apiVersion: kbld.k14s.io/v1alpha1 + kind: Config + minimumRequiredVersion: 0.21.0 + overrides: + - image: haproxy + newImage: index.docker.io/library/haproxy@sha256:e6f9faf0c2a0cf2d2d5a53307351fa896d90ca9ccd62817c24026460d97dde92 + preresolved: true + - image: nginx + newImage: index.docker.io/library/nginx@sha256:86ae264c3f4acb99b2dee4d0098c40cb8c46dcf9e1148f05d3a51c4df6758c12 + preresolved: true + ``` + +1. Feed `/tmp/manifest.lock` to `kbld pkg` command to download images and pack them into a single tarball `/tmp/packaged-images.tar`: + + ```bash + $ kbld pkg -f /tmp/manifest.lock --output /tmp/packaged-images.tar + package | exporting 2 images... + package | will export index.docker.io/library/nginx@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + package | will export index.docker.io/library/haproxy@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + package | exported 2 images + + $ ls -lah /tmp/packaged-images.tar + -rw-r--r-- 1 root root 314M May 3 18:59 /tmp/packaged-images.tar + ``` + + Note: Depending on your internet connection this may be slow. + +To import packaged images from a tarball: + +1. Specify new repository location `docker.io/dkalinin/app1` and provide tarball: + + ```bash + $ kbld unpkg -f /tmp/manifest.lock --input /tmp/packaged-images.tar --repository docker.io/dkalinin/app1 --lock-output /tmp/manifest.lock.copied + unpackage | importing 2 images... + unpackage | importing index.docker.io/library/nginx@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c -> docker.io/dkalinin/app1@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c... + unpackage | importing index.docker.io/library/haproxy@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 -> docker.io/dkalinin/app1@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5... + unpackage | imported 2 images + ``` + + Images will be imported under a single new repository `docker.io/dkalinin/app1`. **You are guaranteed that images are exactly same as they are referenced by the same digests in produced YAML configuration (though under a different repository name)**. + +1. **Alternatively**, using `relocate` command against a lock file: + + ```bash + $ kbld relocate -f /tmp/manifest.lock --repository docker.io/dkalinin/app1 --lock-output /tmp/manifest.lock.copied + relocate | ... + ``` + +1. Use newely generated lock file with configuration to get updated results + + ```bash + $ kbld -f /tmp/manifest -f /tmp/manifest.lock.copied + images: + - image: docker.io/dkalinin/app1@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + - image: docker.io/dkalinin/app1@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + ``` + +--- +## Directly against configuration + +For example, to package referenced images into a single tarball: + +1. First make sure all image references are in their digest form + + ```bash + $ cat /tmp/manifest + images: + - image: nginx + - image: haproxy + + $ kbld -f /tmp/manifest > /tmp/resolved-manifest + resolve | final: haproxy -> index.docker.io/library/haproxy@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + resolve | final: nginx -> index.docker.io/library/nginx@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + + $ cat /tmp/resolved-manifest + images: + - image: index.docker.io/library/nginx@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + - image: index.docker.io/library/haproxy@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + ``` + +1. Feed `/tmp/resolved-manifest` to `kbld pkg` command to download images and pack them into a single tarball `/tmp/packaged-images.tar`: + + ```bash + $ kbld pkg -f /tmp/resolved-manifest --output /tmp/packaged-images.tar + package | exporting 2 images... + package | will export index.docker.io/library/nginx@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + package | will export index.docker.io/library/haproxy@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + package | exported 2 images + + $ ls -lah /tmp/packaged-images.tar + -rw-r--r-- 1 root root 314M May 3 18:59 /tmp/packaged-images.tar + ``` + + Note: Depending on your internet connection this may be slow. + +To import packaged images from a tarball: + +1. Specify new repository location `docker.io/dkalinin/app1` and provide tarball: + + ```bash + $ kbld unpkg -f /tmp/resolved-manifest --input /tmp/packaged-images.tar --repository docker.io/dkalinin/app1 + unpackage | importing 2 images... + unpackage | importing index.docker.io/library/nginx@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c -> docker.io/dkalinin/app1@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c... + unpackage | importing index.docker.io/library/haproxy@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 -> docker.io/dkalinin/app1@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5... + unpackage | imported 2 images + images: + - image: docker.io/dkalinin/app1@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + - image: docker.io/dkalinin/app1@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + ``` + + Images will be imported under a single new repository `docker.io/dkalinin/app1`. **You are guaranteed that images are exactly same as they are referenced by the same digests in produced YAML configuration (though under a different repository name)**. + +1. **Alternatively**, using `relocate` command: + + ```bash + $ kbld relocate -f /tmp/resolved-manifest --repository docker.io/dkalinin/app1 + relocate | ... + images: + - image: docker.io/dkalinin/app1@sha256:e71b1bf4281f25533cf15e6e5f9be4dac74d2328152edf7ecde23abc54e16c1c + - image: docker.io/dkalinin/app1@sha256:6dae9c8674e2e5f418c3dd040041a05f6b490597315139c0bcacadf65a46cfd5 + ``` + +--- +## Authentication + +See general authentication steps in [Authentication doc](auth.md). + +## Using with gcr.io + +- Run `kbld unpkg -f /tmp/resolved-manifest --input /tmp/packaged-images.tar --repository gcr.io/{project-id}/app1` to import images (e.g. project id is `dkalinin`) + +## Using with AWS ECR + +**Note**: AWS ECR does not support manifest list media types from Docker Registry v2 API. Manifest lists are used for images that are built against multiple architectures and platforms and are referenced through a single digest or tag. Most user-built images do not use manifest lists (as it's a single image); however, common Docker Hub library images are represented by manifest lists and will fail upon import into AWS ECR. You will see following error: `Writing image index: Retried 5 times: uploading manifest2: UNSUPPORTED: Invalid parameter at 'imageTag' failed to satisfy constraint: 'must satisfy regular expression '[a-zA-Z0-9-_.]+'`. Related [AWS feature request](https://forums.aws.amazon.com/thread.jspa?threadID=292294)). + +- Run `kbld unpkg -f /tmp/resolved-manifest --input /tmp/packaged-images.tar --repository {uri}` to import images (e.g. uri is `823869848626.dkr.ecr.us-east-1.amazonaws.com/k14s/kbld-test`) + +## Using with Harbor + +You may have to provide `--registry-ca-cert-path` flag with a path to a CA certificate file for Harbor Registry API. + +## Notes + +- Produced tarball does not have duplicate image layers, as they are named by their digest (see `tar tvf /tmp/packaged-images.tar`). +- If digest reference points to an image index, all children (images and other image indexes) will be included in the export. Saving only a portion of contents would of course change the digest. +- Only Docker v2 and OCI images and indexes are supported. Docker v1 format is not supported, hence, not all images out there could be exported and only registries supporting v2 format can be used for imports. +- Images that once were in different repositories are imported into the same repository to make it easier to manage them in bulk. diff --git a/site/content/kbld/docs/v0.32.0/resolving.md b/site/content/kbld/docs/v0.32.0/resolving.md new file mode 100644 index 000000000..556ba6c66 --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/resolving.md @@ -0,0 +1,191 @@ +--- +aliases: [/kbld/docs/latest/resolving] +title: Resolving images +--- + +## Resolving image references to digests + +kbld looks for `image` keys within YAML documents and tries to resolve image reference to its full digest form. + +For example, following + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: nginx:1.14.2 + #! ^-- image reference in its tag form +``` + +will be transformed to + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: index.docker.io/library/nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d + #! ^-- resolved image reference to its digest form +``` + +via + +```bash +kbld -f file.yml +``` + +Few other variations + +```bash +pbpaste | kbld -f- +kbld -f . +kbld -f file.yml -f config2.yml +``` + +### Generating resolution `imgpkg` lock output + +Available in 0.28.0+ + +Using the `--imgpkg-lock-output` flag, users are able to create an [ImagesLock](https://github.com/vmware-tanzu/carvel-imgpkg/blob/develop/docs/resources.md#imageslock) file that can be used as input for the packaging and distribution tool: [`imgpkg`](https://github.com/vmware-tanzu/carvel-imgpkg) + +For example, the command `kbld -f input.yml --imgpkg-lock-output /tmp/imgpkg.lock.yml` with `input.yml`: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: nginx:1.14.2 + #! ^-- image reference in its tag form +``` + +will produce `/tmp/imgpkg.lock.yml`: + +```yaml +apiVersion: imgpkg.carvel.dev/v1alpha1 +kind: ImagesLock +images: +- image: index.docker.io/library/nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d + annotations: + kbld.carvel.dev/id: nginx:1.14.2 +``` + +An ImagesLock can be included with configuration via `-f` to produce same resolved configuration, for example, `kbld -f input.yml -f /tmp/imgpkg.lock.yml` produces: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: index.docker.io/library/nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d +``` + +### Generating resolution lock output + +In some cases recording resolution results may be useful. To do so add `--lock-output /path-to-file` to the `kbld` command. + +For example, command `kbld -f input.yml --lock-output /tmp/kbld.lock.yml` with `input.yml`: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: nginx:1.14.2 + #! ^-- image reference in its tag form +``` + +will produce `/tmp/kbld.lock.yml`: + +```yaml +apiVersion: kbld.k14s.io/v1alpha1 +kind: Config +minimumRequiredVersion: 0.17.0 +overrides: +- image: nginx:1.14.2 + newImage: index.docker.io/library/nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d + preresolved: true +``` + +Lock content can be included with configuration via `-f` to produce same resolved configuration, for example, `kbld -f input.yml -f /tmp/kbld.lock.yml` produces: + +```yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbld-test1 +spec: + selector: + matchLabels: + app: kbld-test1 + template: + metadata: + labels: + app: kbld-test1 + spec: + containers: + - name: my-app + image: index.docker.io/library/nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d +``` diff --git a/site/content/kbld/docs/v0.32.0/security.md b/site/content/kbld/docs/v0.32.0/security.md new file mode 100644 index 000000000..0888a2254 --- /dev/null +++ b/site/content/kbld/docs/v0.32.0/security.md @@ -0,0 +1,8 @@ +--- +aliases: [/kbld/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `kbld`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/vendir/docs/latest/_index.md b/site/content/vendir/docs/develop/_index.md similarity index 97% rename from site/content/vendir/docs/latest/_index.md rename to site/content/vendir/docs/develop/_index.md index b93397919..c9a8386bb 100644 --- a/site/content/vendir/docs/latest/_index.md +++ b/site/content/vendir/docs/develop/_index.md @@ -2,7 +2,7 @@ title: "About vendir" toc: "false" cascade: - version: latest + version: develop toc: "true" type: docs layout: docs diff --git a/site/content/vendir/docs/latest/github-release.md b/site/content/vendir/docs/develop/github-release.md similarity index 100% rename from site/content/vendir/docs/latest/github-release.md rename to site/content/vendir/docs/develop/github-release.md diff --git a/site/content/vendir/docs/latest/install.md b/site/content/vendir/docs/develop/install.md similarity index 100% rename from site/content/vendir/docs/latest/install.md rename to site/content/vendir/docs/develop/install.md diff --git a/site/content/vendir/docs/latest/security.md b/site/content/vendir/docs/develop/security.md similarity index 100% rename from site/content/vendir/docs/latest/security.md rename to site/content/vendir/docs/develop/security.md diff --git a/site/content/vendir/docs/latest/sync.md b/site/content/vendir/docs/develop/sync.md similarity index 100% rename from site/content/vendir/docs/latest/sync.md rename to site/content/vendir/docs/develop/sync.md diff --git a/site/content/vendir/docs/latest/vendir-lock-spec.md b/site/content/vendir/docs/develop/vendir-lock-spec.md similarity index 100% rename from site/content/vendir/docs/latest/vendir-lock-spec.md rename to site/content/vendir/docs/develop/vendir-lock-spec.md diff --git a/site/content/vendir/docs/latest/vendir-spec.md b/site/content/vendir/docs/develop/vendir-spec.md similarity index 100% rename from site/content/vendir/docs/latest/vendir-spec.md rename to site/content/vendir/docs/develop/vendir-spec.md diff --git a/site/content/vendir/docs/latest/versions.md b/site/content/vendir/docs/develop/versions.md similarity index 100% rename from site/content/vendir/docs/latest/versions.md rename to site/content/vendir/docs/develop/versions.md diff --git a/site/content/vendir/docs/v0.24.0/_index.md b/site/content/vendir/docs/v0.24.0/_index.md new file mode 100644 index 000000000..30d3f6caf --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/_index.md @@ -0,0 +1,25 @@ +--- +aliases: [/vendir/docs/latest/] +title: "About vendir" +toc: "false" +cascade: + version: v0.24.0 + toc: "true" + type: docs + layout: docs +--- + +vendir allows to declaratively state what should be in a directory. It was designed to easily manage libraries for [ytt](/ytt); however, it is a generic tool and does not care how files within managed directories are used. + +Supported sources for fetching: + +- git +- hg (Mercurial) +- http +- image (image from OCI registry) +- imgpkgBundle (bundle from OCI registry) +- githubRelease +- helmChart +- directory + +Examples could be found in [carvel-vendir's `examples/` directory](https://github.com/vmware-tanzu/carvel-vendir/tree/develop/examples). diff --git a/site/content/vendir/docs/v0.24.0/github-release.md b/site/content/vendir/docs/v0.24.0/github-release.md new file mode 100644 index 000000000..3d6bbf0d4 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/github-release.md @@ -0,0 +1,17 @@ +--- +aliases: [/vendir/docs/latest/github-release] +title: Github Release +--- + +vendir supports downloading software stored as a Github release. See [`vendir.yml` spec](vendir-spec.md) for how to configure. + +## Github API Rate Limiting + +If your public IP address is shared by multiple machines (e.g. workstations in an office), you may run into [Github rate limiting errors](https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#rate-limiting). vendir as of v0.8.0 supports providing "Personal access token" to increase Github API rate limits. You can specify it via an environment variable: + +```bash +$ export VENDIR_GITHUB_API_TOKEN=8c0a3... +$ vendir sync +``` + +To obtain personal access token go to [Github.com: Settings / Developer Settings / Personal access tokens](https://github.com/settings/tokens). During token creation, you will be prompted for selection of scopes, and in most cases there is no need to select any scopes because this token only used to identify API usage. For organizations that enable SSO, you will need to "Enable SSO" for created token. diff --git a/site/content/vendir/docs/v0.24.0/install.md b/site/content/vendir/docs/v0.24.0/install.md new file mode 100644 index 000000000..33d2a1ee6 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/install.md @@ -0,0 +1,58 @@ +--- +aliases: [/vendir/docs/latest/install] +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ vendir version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ vendir version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/vmware-tanzu/homebrew-carvel](https://github.com/vmware-tanzu/homebrew-carvel). + +```bash +$ brew tap vmware-tanzu/carvel +$ brew install vendir +$ vendir version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/vmware-tanzu/carvel-vendir/releases), for example for 'vendir-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/vendir-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/vendir-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/vendir-darwin-amd64 /usr/local/bin/vendir + +# Make binary executable +$ chmod +x /usr/local/bin/vendir + +# Check its version +$ vendir version +``` diff --git a/site/content/vendir/docs/v0.24.0/security.md b/site/content/vendir/docs/v0.24.0/security.md new file mode 100644 index 000000000..0e42ae4c1 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/security.md @@ -0,0 +1,8 @@ +--- +aliases: [/vendir/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `vendir`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). diff --git a/site/content/vendir/docs/v0.24.0/sync.md b/site/content/vendir/docs/v0.24.0/sync.md new file mode 100644 index 000000000..3cfaf1f85 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/sync.md @@ -0,0 +1,42 @@ +--- +aliases: [/vendir/docs/latest/sync] +title: Sync command +--- + +## Overview + +`vendir sync` command looks for [`vendir.yml`](vendir-spec.md) file in current directory for its configuration. `vendir.yml` specifies source of files for each managed directory. + +``` +# Run to sync directory contents as specified by vendir.yml +$ vendir sync +``` + +See [`vendir.yml` spec](vendir-spec.md) for its schema. + +## Sync with local changes override + +As of v0.7.0 you can use `--directory` flag to override contents of particular directories by pointing them to local directories. When this flag is specified other directories will not be synced (hence lock config is not going to be updated). + +``` +$ vendir sync --directory vendor/local-dir=local-dir-dev +``` + +## Sync with locks + +`vendir sync` writes [`vendir.lock.yml`](vendir-lock-spec.md) (next to `vendir.yml`) that contains resolved references: + +- for `git`, resolved SHAs are recorded +- for `hg`, resolved SHAs are recorded +- for `http`, nothing is recorded +- for `image`, resolved URL as a digest reference +- for `githubRelease`, permanent links are recorded +- for `helmChart`, resolved version +- for `directory`, nothing is recorded +- for `manual`, nothing is recorded + +To use these resolved references on top of `vendir.yml`, use `vendir sync -l`. + +## Syncing from different directory + +As of v0.22.0, you can use `--chdir` flag with `vendir sync` command to change current working directory of vendir before any syncing occurs. All other paths provided to `vendir sync` should be relative to the changed directory. diff --git a/site/content/vendir/docs/v0.24.0/vendir-lock-spec.md b/site/content/vendir/docs/v0.24.0/vendir-lock-spec.md new file mode 100644 index 000000000..027838675 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/vendir-lock-spec.md @@ -0,0 +1,69 @@ +--- +aliases: [/vendir/docs/latest/vendir-lock-spec] +title: vendir.lock.yml spec +--- + +Lock file is generated by `vendir sync` and is placed next to related `vendir.yml`. + +```yaml +apiVersion: vendir.k14s.io/v1alpha1 +kind: LockConfig + +directories: +- path: config/_ytt_lib + contents: + - path: github.com/cloudfoundry/cf-k8s-networking + + # present if this is managed manually + manual: {} + + # present if git + git: + # resolved checked out commit SHA + sha: 2b009b61fa8afb330a4302c694ee61b11104c54c + # resolved checked out commit title + commitTitle: 'feat: add /metrics prometheus scrapable endpoint...' + # resolved to a set of tags pointing to sha (v0.11.0+) + tags: + - "4.0.0" + + # present if hg (v0.22.0+) + hg: + # resolved checked out change SHA + sha: 180c776fe29448afa8c756ab572bab7a1cf17a06 + # resolved checked out change title + changeSetTitle: 'Prevent wrapping filenames to preserve whitespace' + + # present if github release + githubRelease: + # resolved release url + url: https://api.github.com/repos/pivotal/kpack/releases/22747441 + + # present if helm chart (v0.11.0+) + helmChart: + appVersion: "5.0.7" + version: "10.5.7" + + # present if http + http: {} + + # present if image (v0.11.0+) + image: + # fully resolve image URL with digest + url: index.docker.io/dkalinin/consul-helm@sha256:d1cdbd46561a144332f0744302d45f27583fc0d75002cba473d840f46630c9f7 + # included if image URL included a tag (v0.22.0+) + tag: "some-tag" + + # present if imgpkgBundle (v0.16.0+) + imgpkgBundle: + # fully resolve image URL with digest + image: index.docker.io/dkalinin/consul-helm@sha256:d1cdbd46561a144332f0744302d45f27583fc0d75002cba473d840f46630c9f7 + # included if image URL included a tag (v0.22.0+) + tag: "some-tag" + + # present if inline (v0.11.0+) + inline: {} + + # present if this was sourced from local directory + directory: {} +``` diff --git a/site/content/vendir/docs/v0.24.0/vendir-spec.md b/site/content/vendir/docs/v0.24.0/vendir-spec.md new file mode 100644 index 000000000..8560e7441 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/vendir-spec.md @@ -0,0 +1,238 @@ +--- +aliases: [/vendir/docs/latest/vendir-spec] +title: vendir.yml spec +--- + +```yaml +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config + +# declaration of minimum required vendir binary version (optional) +minimumRequiredVersion: 0.8.0 + +# one or more directories to manage with vendir +directories: +- # path is relative to vendir.yml location + path: config/_ytt_lib + + contents: + - # path lives relative to directory path # (required) + path: github.com/cloudfoundry/cf-k8s-networking + + # uses git to clone Git repository (optional) + git: + # http or ssh urls are supported (required) + url: https://github.com/cloudfoundry/cf-k8s-networking + # branch, tag, commit; origin is the name of the remote (required) + # optional if refSelection is specified (available in v0.11.0+) + ref: origin/master + # specifies a strategy to resolve to an explicit ref (optional; v0.11.0+) + refSelection: + semver: + # list of semver constraints (see versions.md for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.12.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.12.0+) + identifiers: [beta, rc] + # skip downloading lfs files (optional) + lfsSkipSmudge: false + # verify gpg signatures on commits or tags (optional; v0.12.0+) + verification: + publicKeysSecretRef: + name: my-git-gpg-auth + # specifies name of a secret with auth details; + # secret may include 'ssh-privatekey', 'ssh-knownhosts', + # 'username', 'password' keys (optional) + secretRef: + # (required) + name: my-git-auth + + # uses hg to clone Mercurial repository (optional; v0.22.0+) + hg: + # http or ssh urls are supported (required) + url: https://hg.sr.ht/~sircmpwn/hg.sr.ht + # branch, tag, commit (required) + ref: 180c776fe29448afa8c756ab572bab7a1cf17a06 + # specifies name of a secret with auth details; + # secret may include 'ssh-privatekey', 'ssh-knownhosts', + # 'username', 'password' keys (optional) + secretRef: + # (required) + name: my-hg-auth + + # fetches asset over HTTP (optional) + http: + # asset URL (required) + url: + # verification checksum (optional) + sha256: "" + # specifies name of a secret with basic auth details; + # secret may include 'username', 'password' keys (optional) + secretRef: + # (required) + name: my-http-auth + + # fetches asset from an image registry (optional; v0.11.0+) + image: + # image URL; could be plain, tagged or digest reference (required) + url: gcr.io/repo/image:v1.0.0 + # specifies a strategy to choose a tag (optional; v0.22.0+) + # if specified, do not include a tag in url key + tagSelection: + semver: + # list of semver constraints (see versions.md for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.12.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.12.0+) + identifiers: [beta, rc] + # specifies name of a secret with registry auth details; + # secret may include 'username', 'password' and/or 'token' keys; + # as of v0.19.0+, dockerconfigjson secrets are also supported (optional) + # of of v0.22.0+, multiple registry credentials are supported and + # are passed to imgpkg via env. registry hostname must match url + # for auth information to be chosen by imgpkg. + # (https://carvel.dev/imgpkg/docs/latest/auth/#via-environment-variables) + secretRef: + # (required) + name: my-image-auth + # specify wether to skip TLS verification; defaults to false (optional;v0.18.0+) + dangerouSkipTLSVerify: false + + # fetches imgpkg bundle from an image registry (optional; v0.16.0+) + imgpkgBundle: + # could be plain, tagged or digest reference (required) + image: gcr.io/repo/bundle:v1.0.0 + # specifies a strategy to choose a tag (optional; v0.22.0+) + # if specified, do not include a tag in image key + tagSelection: + semver: + # list of semver constraints (see versions.md for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.12.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.12.0+) + identifiers: [beta, rc] + # specifies name of a secret with registry auth details; + # secret may include 'username', 'password' and/or 'token' keys; + # as of v0.19.0+, dockerconfigjson secrets are also supported (optional) + # of of v0.22.0+, multiple registry credentials are supported and + # are passed to imgpkg via env. registry hostname must match image + # for auth information to be chosen by imgpkg. + # (https://carvel.dev/imgpkg/docs/latest/auth/#via-environment-variables) + secretRef: + # (required) + name: my-image-auth + # specify wether to skip TLS verification; defaults to false (optional;v0.18.0+) + dangerouSkipTLSVerify: false + + # fetches assets from a github release (optional) + githubRelease: + # slug for repository (org/repo) (required) + slug: k14s/kapp-controller + # use release tag (optional) + # optional if tagSelection is specified (available in v0.22.0+) + tag: v0.1.0 + # specifies a strategy to choose a tag (optional; v0.22.0+) + tagSelection: + semver: + # list of semver constraints (see versions.md for details) (required) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional; v0.12.0+) + prereleases: + # select prerelease versions that include given identifiers (optional; v0.12.0+) + identifiers: [beta, rc] + # use latest published version (optional) + latest: true + # use exact release URL (optional) + url: https://api.github.com/repos/k14s/kapp-controller/releases/21912613 + # only download specific assets (optional; v0.12.0+) + assetNames: ["release*.yml"] + # checksums for downloaded files (optional) + # (if release text body contains checksums, it's not necessary + # to manually specify them here) + checksums: + release.yml: 26bf09c42d72ae448af3d1ee9f6a933c87c4ec81d04d37b30e1b6a339f5983a7 + # disables checking auto-found checksums for downloaded files (optional) + # (checksums are extracted from release's text body + # based on following format ` `) + disableAutoChecksumValidation: true + # specifies which archive to unpack for contents (optional) + unpackArchive: + # (required) + path: release.tgz + # specifies name of a secret with github auth details; + # secret may include 'token' key (optional) + secretRef: + # (required) + name: my-gh-auth + + # fetch Helm chart contents (optional; v0.11.0+) + helmChart: + # chart name (required) + name: stable/redis + # use specific chart version (string; optional) + version: "1.2.1" + # specifies Helm repository to fetch from (optional) + repository: + # repository url; supports exprimental oci helm fetch via + # oci:// scheme (required) + url: https://... + # specifies name of a secret with helm repo auth details; + # secret may include 'username', 'password'; + # as of v0.19.0+, dockerconfigjson secrets are also supported (optional) + # as of v0.22.0+, 0 or 1 auth credential is expected within dockerconfigjson secret + # if >1 auth creds found, error will be returned. (currently registry hostname + # is not used when found in provide auth credential.) + secretRef: + # (required) + name: my-helm-auth + # specify helm binary version to use; + # '3' means binary 'helm3' needs to be on the path (optional) + helmVersion: "3" + + # copy contents from local directory (optional) + directory: + # local file system path relative to vendir.yml + path: some-path + + # states that directory specified by above path + # is managed by hand; nothing to do for vendir (optional) + manual: {} + + # specify contents inline within this file (optional; v0.11.0+) + inline: + # specifies mapping of paths to their content (optional) + paths: + dir/file.ext: file-content + # specifies content via secrets and config maps (optional) + pathsFrom: + - secretRef: + # (required) + name: secret-name + # specifies where to place files found in secret (optional) + directoryPath: dir + - configMapRef: + # (required) + name: cfgmap-name + # specifies where to place files found in config map (optional) + directoryPath: dir + + # includes paths specify what should be included. by default + # all paths are included (optional) + includePaths: + - cfroutesync/crds/**/* + - install/ytt/networking/**/* + + # exclude paths are "placed" on top of include paths (optional) + excludePaths: [] + + # specifies paths to files that need to be includes for + # legal reasons such as LICENSE file. Defaults to few + # LICENSE, NOTICE and COPYRIGHT variations (optional) + legalPaths: [] + + # make subdirectory to be new root path within this asset (optional; v0.11.0+) + newRootPath: cfroutesync +``` diff --git a/site/content/vendir/docs/v0.24.0/versions.md b/site/content/vendir/docs/v0.24.0/versions.md new file mode 100644 index 000000000..346558e28 --- /dev/null +++ b/site/content/vendir/docs/v0.24.0/versions.md @@ -0,0 +1,111 @@ +--- +aliases: [/vendir/docs/latest/versions] +title: Versions +--- + +Available in v0.12.0+. + +Vendir uses version selection in following places: + +- git source type for selection of `ref` based on Git tags +- image source type for selection of `tag` based on registry tags +- imgpkgBundle source type for selection of `tag` based on registry tags +- githubRelease source type for selection of `tag` based on tags + +--- +## VersionSelection type + +`VersionSelection` type may be used by other projects (such as kbld) for selection of versions in different contexts. All usage follows same spec: + +```yaml +# interpret versions according to semantic version spec. +# see semver section below for further details (required) +semver: + # list of semver constraints (optional) + constraints: ">0.4.0" + # by default prerelease versions are not included (optional) + prereleases: + # select prerelease versions that include given identifiers (optional) + identifiers: [beta, rc] +``` + +--- +## Semver + +[github.com/blang/semver/v4 package](https://github.com/blang/semver) is used for parsing "semver" versions. + +For valid semver syntax refer to . (Commonly-used `v` prefix will be ignored during parsing) + +For constraints syntax refer to [blang/semver's Ranges section](https://github.com/blang/semver#ranges). + +By default prerelease versions are not included in selection. See examples for details. + +### Examples + +Any version greater than 0.4.0 _without_ prereleases. + +```yaml +semver: + constraints: ">0.4.0" +``` + +Any version greater than 0.4.0 _with_ all prereleases. + +```yaml +semver: + constraints: ">0.4.0" + prereleases: {} +``` + +Any version greater than 0.4.0 _with_ only beta or rc prereleases. + +```yaml +semver: + constraints: ">0.4.0" + prereleases: + identifiers: [beta, rc] +``` + +### sort-semver command + +`vendir tools sort-semver` command is included to showcase how vendir parses versions. + +- `--version` (`-v`) specifies one or more versions +- `--constraint` (`-c`) specifies zero or more constraints +- `--prerelease` specifies to include prereleases +- `--prerelease-identifier` specifies zero or more identifiers to match prereleases + +```bash +$ vendir tools sort-semver -v "v0.0.1 v0.1.0 v0.2.0-pre.20 v0.2.0+build.1 v0.2.1 v0.2.0 v0.3.0" +Versions + +Version +v0.0.1 +v0.1.0 +v0.2.0-pre.20 +v0.2.0+build.1 +v0.2.0 +v0.2.1 +v0.3.0 + +Highest version: v0.3.0 + +Succeeded +``` + +Note that by default prerelease versions are not included. Use configuration or flag to include them. + +```bash +$ vendir tools sort-semver -v "v0.0.1 v0.1.0 v0.2.0-pre.20 v0.2.0+build.1 v0.2.0 v0.3.0" -c ">=0.1.0" +Versions + +Version +v0.1.0 +v0.2.0+build.1 +v0.2.0 +v0.3.0 + +Highest version: v0.3.0 + +Succeeded +``` diff --git a/site/content/ytt/docs/latest/_index.md b/site/content/ytt/docs/develop/_index.md similarity index 99% rename from site/content/ytt/docs/latest/_index.md rename to site/content/ytt/docs/develop/_index.md index 42af37dd4..0749d19ff 100644 --- a/site/content/ytt/docs/latest/_index.md +++ b/site/content/ytt/docs/develop/_index.md @@ -1,7 +1,7 @@ --- title: "About ytt" cascade: - version: latest + version: develop toc: "true" type: docs layout: docs diff --git a/site/content/ytt/docs/latest/data-values-schema-migration-guide.md b/site/content/ytt/docs/develop/data-values-schema-migration-guide.md similarity index 100% rename from site/content/ytt/docs/latest/data-values-schema-migration-guide.md rename to site/content/ytt/docs/develop/data-values-schema-migration-guide.md diff --git a/site/content/ytt/docs/latest/data-values-vs-overlays.md b/site/content/ytt/docs/develop/data-values-vs-overlays.md similarity index 100% rename from site/content/ytt/docs/latest/data-values-vs-overlays.md rename to site/content/ytt/docs/develop/data-values-vs-overlays.md diff --git a/site/content/ytt/docs/latest/faq.md b/site/content/ytt/docs/develop/faq.md similarity index 100% rename from site/content/ytt/docs/latest/faq.md rename to site/content/ytt/docs/develop/faq.md diff --git a/site/content/ytt/docs/latest/file-marks.md b/site/content/ytt/docs/develop/file-marks.md similarity index 100% rename from site/content/ytt/docs/latest/file-marks.md rename to site/content/ytt/docs/develop/file-marks.md diff --git a/site/content/ytt/docs/latest/how-it-works.md b/site/content/ytt/docs/develop/how-it-works.md similarity index 100% rename from site/content/ytt/docs/latest/how-it-works.md rename to site/content/ytt/docs/develop/how-it-works.md diff --git a/site/content/ytt/docs/latest/how-to-export-schema.md b/site/content/ytt/docs/develop/how-to-export-schema.md similarity index 96% rename from site/content/ytt/docs/latest/how-to-export-schema.md rename to site/content/ytt/docs/develop/how-to-export-schema.md index 02f61319a..5531fea28 100644 --- a/site/content/ytt/docs/latest/how-to-export-schema.md +++ b/site/content/ytt/docs/develop/how-to-export-schema.md @@ -2,11 +2,9 @@ title: Export Schema in OpenAPI Format --- -Available in v0.38.0+ unless otherwise noted. - The primary use of schema is to declare (and type-check) Data Values. -`ytt` _also_ supports the ability to export schema definitions in [OpenAPI v3.0.x](https://swagger.io/specification/) format. This is useful for tools that require schema definition in this industry-standard format. For example, `kapp-controller`, requires [Package](https://carvel.dev/kapp-controller/docs/latest/packaging/#package) 's `spec.valuesSchema.openAPIv3` to contain an [OpenAPI Schema map](https://swagger.io/specification/#schema-object). +`ytt` _also_ supports the ability to export schema definitions in [OpenAPI v3.0.x](https://swagger.io/specification/) format. This is useful for tools that require schema definition in this industry-standard format. For example, `kapp-controller`, requires [Package](/kapp-controller/docs/latest/packaging/#package) 's `spec.valuesSchema.openAPIv3` to contain an [OpenAPI Schema map](https://swagger.io/specification/#schema-object). In this way, one can write their schema once and reuse it in situations that require the OpenAPI v3 format. diff --git a/site/content/ytt/docs/latest/how-to-use-data-values.md b/site/content/ytt/docs/develop/how-to-use-data-values.md similarity index 100% rename from site/content/ytt/docs/latest/how-to-use-data-values.md rename to site/content/ytt/docs/develop/how-to-use-data-values.md diff --git a/site/content/ytt/docs/latest/how-to-write-schema.md b/site/content/ytt/docs/develop/how-to-write-schema.md similarity index 100% rename from site/content/ytt/docs/latest/how-to-write-schema.md rename to site/content/ytt/docs/develop/how-to-write-schema.md diff --git a/site/content/ytt/docs/latest/injecting-secrets.md b/site/content/ytt/docs/develop/injecting-secrets.md similarity index 100% rename from site/content/ytt/docs/latest/injecting-secrets.md rename to site/content/ytt/docs/develop/injecting-secrets.md diff --git a/site/content/ytt/docs/latest/install.md b/site/content/ytt/docs/develop/install.md similarity index 100% rename from site/content/ytt/docs/latest/install.md rename to site/content/ytt/docs/develop/install.md diff --git a/site/content/ytt/docs/latest/known-limitations.md b/site/content/ytt/docs/develop/known-limitations.md similarity index 100% rename from site/content/ytt/docs/latest/known-limitations.md rename to site/content/ytt/docs/develop/known-limitations.md diff --git a/site/content/ytt/docs/latest/lang-ref-annotation.md b/site/content/ytt/docs/develop/lang-ref-annotation.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-annotation.md rename to site/content/ytt/docs/develop/lang-ref-annotation.md diff --git a/site/content/ytt/docs/latest/lang-ref-def.md b/site/content/ytt/docs/develop/lang-ref-def.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-def.md rename to site/content/ytt/docs/develop/lang-ref-def.md diff --git a/site/content/ytt/docs/latest/lang-ref-dict.md b/site/content/ytt/docs/develop/lang-ref-dict.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-dict.md rename to site/content/ytt/docs/develop/lang-ref-dict.md diff --git a/site/content/ytt/docs/latest/lang-ref-for.md b/site/content/ytt/docs/develop/lang-ref-for.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-for.md rename to site/content/ytt/docs/develop/lang-ref-for.md diff --git a/site/content/ytt/docs/latest/lang-ref-if.md b/site/content/ytt/docs/develop/lang-ref-if.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-if.md rename to site/content/ytt/docs/develop/lang-ref-if.md diff --git a/site/content/ytt/docs/latest/lang-ref-list.md b/site/content/ytt/docs/develop/lang-ref-list.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-list.md rename to site/content/ytt/docs/develop/lang-ref-list.md diff --git a/site/content/ytt/docs/latest/lang-ref-load.md b/site/content/ytt/docs/develop/lang-ref-load.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-load.md rename to site/content/ytt/docs/develop/lang-ref-load.md diff --git a/site/content/ytt/docs/latest/lang-ref-string.md b/site/content/ytt/docs/develop/lang-ref-string.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-string.md rename to site/content/ytt/docs/develop/lang-ref-string.md diff --git a/site/content/ytt/docs/latest/lang-ref-structs.md b/site/content/ytt/docs/develop/lang-ref-structs.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-structs.md rename to site/content/ytt/docs/develop/lang-ref-structs.md diff --git a/site/content/ytt/docs/latest/lang-ref-yaml-fragment.md b/site/content/ytt/docs/develop/lang-ref-yaml-fragment.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-yaml-fragment.md rename to site/content/ytt/docs/develop/lang-ref-yaml-fragment.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt-library.md b/site/content/ytt/docs/develop/lang-ref-ytt-library.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt-library.md rename to site/content/ytt/docs/develop/lang-ref-ytt-library.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt-overlay.md b/site/content/ytt/docs/develop/lang-ref-ytt-overlay.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt-overlay.md rename to site/content/ytt/docs/develop/lang-ref-ytt-overlay.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt-schema.md b/site/content/ytt/docs/develop/lang-ref-ytt-schema.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt-schema.md rename to site/content/ytt/docs/develop/lang-ref-ytt-schema.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt-struct.md b/site/content/ytt/docs/develop/lang-ref-ytt-struct.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt-struct.md rename to site/content/ytt/docs/develop/lang-ref-ytt-struct.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt-template.md b/site/content/ytt/docs/develop/lang-ref-ytt-template.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt-template.md rename to site/content/ytt/docs/develop/lang-ref-ytt-template.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt-version.md b/site/content/ytt/docs/develop/lang-ref-ytt-version.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt-version.md rename to site/content/ytt/docs/develop/lang-ref-ytt-version.md diff --git a/site/content/ytt/docs/latest/lang-ref-ytt.md b/site/content/ytt/docs/develop/lang-ref-ytt.md similarity index 100% rename from site/content/ytt/docs/latest/lang-ref-ytt.md rename to site/content/ytt/docs/develop/lang-ref-ytt.md diff --git a/site/content/ytt/docs/latest/lang.md b/site/content/ytt/docs/develop/lang.md similarity index 100% rename from site/content/ytt/docs/latest/lang.md rename to site/content/ytt/docs/develop/lang.md diff --git a/site/content/ytt/docs/latest/outputs.md b/site/content/ytt/docs/develop/outputs.md similarity index 100% rename from site/content/ytt/docs/latest/outputs.md rename to site/content/ytt/docs/develop/outputs.md diff --git a/site/content/ytt/docs/latest/security.md b/site/content/ytt/docs/develop/security.md similarity index 100% rename from site/content/ytt/docs/latest/security.md rename to site/content/ytt/docs/develop/security.md diff --git a/site/content/ytt/docs/latest/strict.md b/site/content/ytt/docs/develop/strict.md similarity index 100% rename from site/content/ytt/docs/latest/strict.md rename to site/content/ytt/docs/develop/strict.md diff --git a/site/content/ytt/docs/latest/yaml-primer.md b/site/content/ytt/docs/develop/yaml-primer.md similarity index 100% rename from site/content/ytt/docs/latest/yaml-primer.md rename to site/content/ytt/docs/develop/yaml-primer.md diff --git a/site/content/ytt/docs/latest/ytt-data-values.md b/site/content/ytt/docs/develop/ytt-data-values.md similarity index 100% rename from site/content/ytt/docs/latest/ytt-data-values.md rename to site/content/ytt/docs/develop/ytt-data-values.md diff --git a/site/content/ytt/docs/latest/ytt-overlays.md b/site/content/ytt/docs/develop/ytt-overlays.md similarity index 100% rename from site/content/ytt/docs/latest/ytt-overlays.md rename to site/content/ytt/docs/develop/ytt-overlays.md diff --git a/site/content/ytt/docs/latest/ytt-playground-screenshot.png b/site/content/ytt/docs/develop/ytt-playground-screenshot.png similarity index 100% rename from site/content/ytt/docs/latest/ytt-playground-screenshot.png rename to site/content/ytt/docs/develop/ytt-playground-screenshot.png diff --git a/site/content/ytt/docs/latest/ytt-text-templating.md b/site/content/ytt/docs/develop/ytt-text-templating.md similarity index 100% rename from site/content/ytt/docs/latest/ytt-text-templating.md rename to site/content/ytt/docs/develop/ytt-text-templating.md diff --git a/site/content/ytt/docs/latest/ytt-vs-x.md b/site/content/ytt/docs/develop/ytt-vs-x.md similarity index 100% rename from site/content/ytt/docs/latest/ytt-vs-x.md rename to site/content/ytt/docs/develop/ytt-vs-x.md diff --git a/site/content/ytt/docs/v0.38.0/_index.md b/site/content/ytt/docs/v0.38.0/_index.md new file mode 100644 index 000000000..82ed938a3 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/_index.md @@ -0,0 +1,87 @@ +--- +aliases: [/ytt/docs/latest/] +title: "About ytt" +cascade: + version: v0.38.0 + toc: "true" + type: docs + layout: docs +--- + +## Overview + +`ytt` is a command-line tool used to template and patch YAML files. It also provides the means to collect fragments and piles of YAML into modular chunks for easy re-use. + +In practice, these YAML files are [Kubernetes configuration](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/), [Concourse Pipeline](https://concourse-ci.org/pipelines.html#schema.pipeline), [Docker Compose](https://github.com/compose-spec/compose-spec/blob/master/spec.md#compose-file), [GitHub Action workflow](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions) files..., really anything that is in YAML format. + +`ytt` is most useful when manually maintaining these files has or will become too much work. + +## Templating + +A plain YAML file is turned into a `ytt` template by adding specially formatted comments, that is: annotating. Through these annotations, one can inject input values, logic like conditionals and looping, and perform transformations on the content. + +Those "input values" are called "Data Values" in `ytt`. Such inputs are included in a separate file. + +For more: +* see it in action in the [load data files](/ytt/#example:example-load-data-values) +example on the playground +* check out the [Using Data Values](how-to-use-data-values.md) guide +- look up reference details of the programming language used in templates in [Language](lang.md). + +## Patching (aka Overlaying) + +`ytt` can also be used to patch YAML files. These edits are called "Overlays" and are themselves written in YAML. + +For more around overlaying... +- see overlaying in action through a progressive set of examples in [the `ytt` Playground](/ytt/#example:example-match-all-docs); +- learn more about Overlays in [ytt Overlays overview](ytt-overlays.md); +- for a reference of all Overlay functionality, see [Overlay module](lang-ref-ytt-overlay.md). + +## Modularizing + +`ytt` provides powerful techniques for extracting and reusing chunks of YAML and logic. + +Functions in `ytt` capture either a calculation or fragment of YAML. Functions can be used in the same templates that define them, or — if defined in a module file — loaded into any template. Entire sets of templates, overlays, and library code can be encapsulated in a `ytt` library. + +For more about modular code... +- see live examples in the `ytt` Playground around [functions](ytt/#example:example-function) and [`ytt` libraries](/ytt/#example:example-ytt-library-module); +- read further about [functions](lang-ref-def.md), [YAML Fragments](lang-ref-yaml-fragment.md), and [loading reusable modules and libraries](lang-ref-load.md); +- catch-up on a particularly relevant [discussion about using modules and libraries in `ytt`](https://github.com/vmware-tanzu/carvel-ytt/discussions/392#discussioncomment-766445). + +## Further Reading + +Hopefully, the pointers above helped get you started. If you're looking to go either deeper or broader, here are some resources we can recommend. + +### Documentation + +- [How it Works](how-it-works.md) \ + _a more detailed look of how all these parts fit together._ + +- [ytt vs. X](ytt-vs-x.md) \ + _how `ytt` differs from similar tools._ + +### Articles + +- [ytt: The YAML Templating Tool that simplifies complex configuration management](https://developer.ibm.com/blogs/yaml-templating-tool-to-simplify-complex-configuration-management/) \ + _a complete introduction of `ytt` including motivations and contrasts with other tools._ + +- [Deploying Kubernetes Applications with ytt, kbld, and kapp](/blog/deploying-apps-with-ytt-kbld-kapp) \ + _how `ytt` works well with other Carvel tools._ + +### Presentations and Interviews + +- IBM Developer podcast: [Introducing the YAML Templating Tool (ytt)](https://www.youtube.com/watch?v=KbB5tI_g3bo) \ + _a thorough introduction to `ytt` especially contrasted with Helm._ + +- TGI Kubernetes: [#079: YTT and Kapp](https://www.youtube.com/watch?v=CSglwNTQiYg) \ + _Joe Beda puts both `ytt` and sister tool `kapp` through their paces._ + +- Helm Summit 2019: [ytt: An Alternative to Text Templating of YAML Configuration in Helm](https://www.youtube.com/watch?v=7-PqgpkxC7E) + ([slides](https://github.com/k14s/meetups/blob/develop/ytt-2019-sep-helm-summit.pdf)) \ + _in-depth review of the philosophy behind `ytt` and how it differs from other YAML templating approaches._ + +- Kubecon 2020: [Managing Applications in Production: Helm vs ytt & kapp @ Kubecon 2020](https://www.youtube.com/watch?v=WJw1MDFMVuk) \ + _how `ytt` and `kapp` can provide an alternative way to manage the lifecycle of application on Kubernetes._ + +- Rawkode Live: [Introduction to Carvel](https://www.youtube.com/watch?v=LBCmMTofNxw) \ + _Dmitriy joins David McKay to talk through many tools in the Carvel suite, including `ytt`._ diff --git a/site/content/ytt/docs/v0.38.0/data-values-schema-migration-guide.md b/site/content/ytt/docs/v0.38.0/data-values-schema-migration-guide.md new file mode 100644 index 000000000..f272646f9 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/data-values-schema-migration-guide.md @@ -0,0 +1,207 @@ +--- +aliases: [/ytt/docs/latest/data-values-schema-migration-guide] +title: Schema Migration Guide +toc: "true" +--- + +Schema documents provide a way to declare Data Values with their types, and default values. Without Schema, validating the presence of Data Values requires additional `ytt` configuration containing Starlark assertions. + +- Learn more about [writing Schema](how-to-write-schema.md) +- Read the detailed [Data Vaues Schema Reference](lang-ref-ytt-schema.md) + +## How do I, a configuration author, migrate my `ytt` library to use Schemas? + +To make use of the Schema feature, your `ytt` invocation must first contain files using the [Data Values feature](ytt-data-values.md). Migrating to Schemas involves converting your Data Values files into a Schema file. + +### Single Data Values file + +Starting with a single Data Values file, `values.yml`: +```yaml +#@data/values +--- +key1: myVal +key2: 8080 +``` + +Convert this Data Values file into Schema by changing the top level annotation in the document to say `#@data/values-schema`, and (optional) rename `values.yml` to `values-schema.yml`: +```yaml +#@data/values-schema +--- +key1: myVal +key2: 8080 +``` +Now simply include this Schema file in your `ytt` invocation to receive the benefits of `ytt` Schemas. + + +**Note: If your Data Values file contains arrays (ie. `["example"]`, `- example`), be sure to [provide default values for arrays](#how-do-i-provide-default-values-for-an-array).** + +### Multiple Data Values files +Sometimes, it makes sense to [split Data Values into multiple files](ytt-data-values.md#splitting-data-values-overlays-into-multiple-files). +If this is your situation, there are a few things to note. + +Given a `ytt` configuration with two Data Values files: +```bash +$ tree . +. +├── config.yml +├── values-1.yml +└── values-2.yml +``` + +`values-1.yml`: +```yaml +#@data/values +--- +key1: myVal +key2: 8080 +``` + +`values-2.yml`: +```yaml +#@data/values +--- +key2: 8088 +#@overlay/match missing_ok=True +key3: + host: registry.dev.io + port: 8080 +``` +You can convert each Data Values document into its own Schema document by [following the steps to convert a single Data Values file](#single-data-values-file). + +[Multiple Schemas combine exactly like Data Values, via overlays](lang-ref-ytt-schema.md#multiple-schema-documents): +the first Schema establishes the base set of "data value" declarations, and subsequent Schema files are overlays on top of that base. + +`values-1-schema.yml`: +```yaml +#@data/values-schema +--- +key1: myVal +key2: 8080 +``` + +`values-2-schema.yml`: +```yaml +#@data/values-schema +--- +key2: 8088 +#@overlay/match missing_ok=True +key3: + host: registry.dev.io + port: 8080 +``` + +Now just include these Schema files in your `ytt` invocation instead of the Data Values files. + +### Multiple Data Values files + Private Libraries +If your configuration depends on a ytt library — outlined in the [library module docs](lang-ref-ytt-library.md) — there are a few points to note. + +```bash +$ tree . +. +└── config + ├── config.yml + ├── values-1.yml + ├── values-2.yml + └── _ytt_lib + └── lib + ├── service.yml + └── values.yml +``` +`config.yml`: +```yaml +#@ load("@ytt:data", "data") +#@ load("@ytt:library", "library") +#@ load("@ytt:template", "template") + +#@ lib = library.get("lib").with_data_values(data.values) +--- #@ template.replace(lib.eval()) +``` +Using the same `values-1.yml` and `values-2.yml` files from the [multiple Data Values files Schema migration example above](#multiple-data-values-files). + +Migrating to Schema happens one library at a time. Let's start with the root library, which includes everything at and below the file system level where the `ytt` invocation was called, not including the `_ytt_lib` folder: +```bash +. +└── config + ├── config.yml + ├── values-1.yml + └── values-2.yml +``` +As seen in the [previous example](#multiple-data-values-files), migrating this library to Schemas simply involves converting each `values-1.yml` and `values-2.yml`into a Schema file. + +Now we have migrated the root library to use Schemas, and the `ytt` invocation will succeed as the same as before. Each library can independently opt-in to using Schemas. +```bash +$ tree . +. +└── config + ├── config.yml + ├── values-1-schema.yml + ├── values-2-schema.yml + └── _ytt_lib + └── lib + ├── service.yml + └── values.yml +``` +Migrating a private library to use Schemas involves the same process as the root library. You can narrow the context to just the children of the `_ytt_lib` directory: +```bash +└── _ytt_lib + └── lib + ├── service.yml + └── values.yml +``` +Now simply follow the steps in either of the previous examples to migrate the private library to use Schemas. + +--- + +## How do I provide default values for an array? +Arrays in Schemas are [handled differently](lang-ref-ytt-schema.md#defaults-for-arrays) than other types: +exactly one element is specified in the array, and that value is _only_ used to infer the type of that array's elements — +the default value, by default, is an empty list (i.e. `[]`). + +The example below shows how to define an array in a Schema and then provide default values via the `@schema/default` annotation. + +`values-schema.yml`: +```yaml +#@ def default_conns(): +- host: registry.dev.io + port: 8080 + transport: tcp +#@ end + +#@data/values-schema +--- +#@schema/default default_conns() +key: +- host: "" + port: 0 + transport: "" + insecure_disable_tls_validation: false +``` + +Given that schema, if a template file were to use the `key` data value: + +```yaml +key: #@ data.values.key +``` + +this would output + +```yaml +key: +- host: registry.dev.io + port: 8080 + transport: tcp + insecure_disable_tls_validation: false +``` + + +## How do I mark a section of Data Values as "optional"? +Sometimes your configuration includes a section of Data Values that are not typically used or in some way optional. + +If this the case, consider the guidance in [Writing Schema: Marking a Data Value as Optional](how-to-write-schema.md#marking-a-data-value-as-optional), use the +[@schema/nullalble](lang-ref-ytt-schema.md#schemanullable) annotation to default such a section to `null`. + +## How do I mark a Data Value as containing any kind of YAML? +For those looking to relax the typing that Schema applies to Data Values, the [@schema/type any=True](lang-ref-ytt-schema.md#schematype) annotation +can be used to override the inferred typing on the node it annotates and its children. + +Situations like this are covered in detail in [Writing Schema: Specific Use-Cases](how-to-write-schema.md#specific-use-cases) diff --git a/site/content/ytt/docs/v0.38.0/data-values-vs-overlays.md b/site/content/ytt/docs/v0.38.0/data-values-vs-overlays.md new file mode 100644 index 000000000..78353da0c --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/data-values-vs-overlays.md @@ -0,0 +1,221 @@ +--- +aliases: [/ytt/docs/latest/data-values-vs-overlays] +title: "Data Values vs Overlays" +--- + +## Overview + +As folks get started with `ytt`, a common question that arises is, “when should +I use data values versus overlays?” While these features do address a similar +problem space, we recommend using one feature versus the other depending on the +use case. We will detail our guidance below. + +## Data Values + +[Data values](ytt-data-values.md) provide a way to inject input data into a +template. If you think about a ytt template as a function, then data values are +the varying parameters. The configuration author will expose values that are +likely to change often, such as with every new environment, as data values. +Authors can also set data values to reasonable defaults or leave them empty and +require the consumers to input their own. + +Common use cases: + +1. Set default values (for generic or specific use-cases) +1. Provide visibility to values that could be updated by the consumer +1. Enable simple conditional behavior (such as requiring a value to be set) + +Consider this simple example provided by the configuration author: + +`config.yml` +```yaml +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ data.values.deployment_name or assert.fail("missing deployment_name") + labels: + app: nginx +spec: + replicas: #@ data.values.replicas + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: #@ data.values.nginx_image + ports: + - containerPort: 80 +``` + +`values.yml` +```yaml +#@data/values +--- +deployment_name: "" +replicas: 1 +nginx_image: nginx:1.14.2 +``` + +The configuration consumer can provide their own data values files that override the ones +provided. + +This example portrays how to: + +1. Set default values. The configuration author has set appropriate default + values for `replicas` and `nginx_image`. +1. Provide visibility to values that could be updated by the consumer. + `deployment_name`, `replicas`, and `nginx_image` are all configurable by the + consumer. +1. Enable simple conditional behavior. `deployment_name` must be set by the + consumer, otherwise an error will be returned. + +Pros: + +- Simple (easy to use, easy to understand) + +Cons: + +- Limited in capability (by design) + +--- +## Overlays + +When consumers would like to configure fields beyond what the original +author has exposed as data values, they should turn to +[Overlays](lang-ref-ytt-overlay.md). These documents provide a way to specify +locations within configuration and either add to, remove from, or replace within +that existing configuration. With basic usage, overlays can act as an extension +of data values, but as situations inevitably become more complex, overlays +provide many more capabilities. + +Common use cases: +1. To replace, delete, or append configuration +1. Situation-specific values + +Consider this extension of our previous example: + +`config.yml` +```yaml +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ data.values.deployment_name or assert.fail("missing deployment_name") + labels: + app: nginx +spec: + replicas: #@ data.values.replicas + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: #@ data.values.nginx_image + ports: + - containerPort: 80 + - name: to-be-removed + image: image:1.2.3 + ports: + - containerPort: 80 +``` + +`values.yml` +```yaml +#@data/values +--- +deployment_name: "" +replicas: 1 +nginx_image: nginx:1.14.2 +``` + +`add-namespace.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + #@overlay/match missing_ok=True + namespace: my-namespace +``` + +`remove-container.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by="name" + #@overlay/remove + - name: to-be-removed +``` + +`add-container.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +spec: + template: + spec: + containers: + - name: appended-container + image: image:1.2.3 + ports: + - containerPort: #@ data.values.appended_container_port +``` +Prior to v0.32.0 append array items with [`#@overlay/append`](lang-ref-ytt-overlay.md#overlayappend) + +`prod-values.yml` +```yaml +#@data/values +--- +deployment_name: 3 +replicas: 3 +#@overlay/match missing_ok=True +appended_container_port: 8080 +``` + +This example demonstrates a number of overlay capabilities: + +1. Appending configuration via `add-namespace.yml` and `add-container.yml` +1. Removing configuration via `remove-container.yml` +1. Extending and updating data values via `prod-values.yml` + +Use the [ytt playground to play with this +example](/ytt/#gist:https://gist.github.com/aaronshurley/b6868b76e25fcb24aedde42f522734af). + +Pros: +- Has more functionality +- Doesn’t change the source +- Programmatic capabilities (this wasn't demonstrated here, see + [example](lang-ref-ytt-overlay.md#programmatic-access)) + +Cons: +- Added complexity + +# If you want to learn more... + +Check out the [Getting Started +tutorial](/ytt/#example:example-hello-world) on the ytt website +for a detailed introduction to ytt. diff --git a/site/content/ytt/docs/v0.38.0/faq.md b/site/content/ytt/docs/v0.38.0/faq.md new file mode 100644 index 000000000..86ab15340 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/faq.md @@ -0,0 +1,184 @@ +--- +aliases: [/ytt/docs/latest/faq] +title: FAQ +--- + +## Data Values + +[Data values doc](ytt-data-values.md) + +## Is it possible to add a new key to my values via the `--data-value` command line argument? +No. As with all data values, those passed through `--data-value` must be overrides, not new values. Instead, overlays are the intended way to provide new keys. +See the [data values vs overlays doc](data-values-vs-overlays.md) for more information. + +## How can I dynamically set or replace map key as a data value in my template? +You can use data.values value as a key in a map by using [Text Templating](ytt-text-templating.md) feature. +That way, you can dynamically set keys in a map using data values. +```yaml +#@yaml/text-templated-strings +(@= data.values.some_key @): some-value +``` +Additionally, see this [playground example](/ytt/#example:example-text-template) which illustrates the use of text templating to set key of a map item. + + +## How do I load json for use as a data value? +An important note here is that json is valid yaml. yaml syntax is a superset of json syntax.\ +ytt can naturally parse json by passing it through `--data-value-yaml`, or json can be loaded by passing the file as a `--data-value-file`. + +Additional resources: [json is valid yaml](https://gist.github.com/pivotaljohn/debe4596df5b4158c7c09f6f1841dd47), [loading by file](https://gist.github.com/pivotaljohn/d3468c3239f79fea7e232751757e779a) + +## How do I check if a key is set? +You can check for the existence of a key by using `hasattr`.\ +For example, to check if struct `foo` has attribute `bar`, use `hasattr(foo, "bar")`. + +## How do I provide a default for a data value when it may not be defined? +When a value may be null, you can use `or` to specify a default. +```yaml +#@ data.values.foo or "bar" +``` + +## How do I error if a data value is not provided? + +ytt library's `assert` package is useful for such situations. It can be used like so: + +```yaml +password: #@ data.values.env.mysql_password if data.values.env.mysql_password else assert.fail("missing env.mysql_password") +``` + +or even more compactly, + +```yaml +password: #@ data.values.env.mysql_password or assert.fail("missing env.mysql_password") +``` + +Note that empty strings are falsy in Starlark. + +--- + +## Overlays + +[Overlays doc](lang-ref-ytt-overlay.md) + +## How do I remove a document subset? + +`#@overlay/remove` in conjunction with `#@overlay/match by=overlay.subset()` annotations are useful for removing a subset of a document. + +Additional resources: [overlay remove docs](lang-ref-ytt-overlay.md#overlayremove), [overlay subset docs](lang-ref-ytt-overlay.md#overlaysubset) + +## How do I add items to an existing array?   + +Using v0.32.0 or later, the default behavior of overlays is to append array items. Simply put your array item in an overlay. + +Prior to v0.32.0, To add an item, either provide the matching annotation (eg. `#@overlay/match by="field_name"`), or use the `#@overlay/append` annotation to add to the end of the list. Note that the append annotation must be applied to each item you want to insert. + +Additional resources: [overlay append docs](lang-ref-ytt-overlay.md#overlayappend), [example gist on playground](/ytt/#gist:https://gist.github.com/pivotaljohn/8c7f48e183158ce12107f576eeab937c), [replace-list gist](/ytt/#gist:https://gist.github.com/pivotaljohn/2b3a9b3367137079195971e1409d539e), [edit-list gist](/ytt/#gist:https://gist.github.com/pivotaljohn/217e8232dc080bb764bfd064ffa9c115) + +## Why am I getting an exception when trying to append to an array? + +A common append issue is incorrectly setting the `#@overlay/match missing_ok=True` annotation on the key which gets replaced by new key-values. Instead, it should be applied to each child (made convenient with the `#@overlay/match-child-defaults missing_ok=True` annotation). See this [illustrative gist](https://gist.github.com/cppforlife/bf42f2d3d23dacf07affcd4150370cb9) for an example. + +## How do I rename a key without changing the value? + +An `#@overlay/replace` annotation with a lambda `via`. For example, to replace the key `bad_name` with `better_name` while retaining the value, you can use: +```yaml +#@overlay/replace via=lambda a,b: {"better_name": a["bad_name"]} +``` +See [this gist](/ytt/#gist:https://gist.github.com/gcheadle-vmware/3c41645a80201caaeefa878e84fff958) for the full example. + +## How do I add or replace a value in a dictionary? + +A `#@ template.replace()` annotation can be used for these purposes. See [this example](/ytt/#example:example-replace). You can also use overlays to edit a dictionary, an example can be found on [this gist playground](/ytt/#gist:https://gist.github.com/gcheadle-vmware/af8aeb3120386e58922c816d76f47ab6). + +## How do I match a field.name that starts with a string? + +```yaml +overlay/match by=lambda a,_: a["field"]["name"].startswith("string") +``` + +## How do I match a struct based on the presence of a key? + +To match a dictionary from a list of dictionaries if the `foo` key is present, you can use +```#@overlay/match by=lambda idx,old,new: "foo" in old, expects="1+"```. + +## How do I modify only part of a multi-line string? + +An `#@overlay/replace` annotation with a lambda `via` function can modify part of a string. See this [modify-string gist](/ytt/#gist:https://gist.github.com/cppforlife/7633c2ed0560e5c8005e05c8448a74d2) for an example. + +## How can I match a regex pattern in the subset matcher? + +The subset matcher does not directly support regex patterns. Instead, a custom matcher can be written. See this [playground gist](/ytt/#gist:https://gist.github.com/ewrenn8/3409e44252f93497a9b447900f3fb5b7) for an example. + +--- + +## When should I include a space in my ytt comment? Does it matter? Is it `#@load` or `#@ load`? `#@overlay/match` or `#@ overlay/match` +The space is subtly meaningful, and directly, load needs a space, while overlay/match does not – but why? + +ytt wraps two concepts in its comment syntax: +1. Annotations on a node +1. Directives for ytt + +Annotations do not have a space, and they refer to a given node in the tree. These comments attach metadata to the annotated node, which can be used during templating. +Some examples of annotations are: +- When inserting a node via an [overlay](#ytt-overlays.md), we would annotate that node with `#@overlay/insert`. +- When we want to mark a document as containing [data values](#ytt-data-values.md), we annotate the document start marker with `#@data/values`. + +Directives, on the other hand, do include a space, and are used to _direct_ ytt to execute the arguments. +Some examples of directives are: +- [Loading](#lang-ref-load.md) a library, we add the `#@ load` directive to the doc, unattached to any particular node. +- To begin and end a [for loop](#lang-ref-for.md) or [conditional](lang-ref-if.md), we use the `#@ for`, `#@ if`, and `#@ end` directives. + +For further exploration, investigate what happens when you move an annotation comment and compare it with moving a directive comment! + +## Why is `ytt` complaining about "Unknown comment syntax"; can't I write standard YAML comments (#)? + +You can, but it is discouraged to avoid tricky errors that can go unchecked. + +The recommended approach is when writing `ytt` templated files, use `ytt` comments: + +``` +#! this is a ytt comment; it's like she-bang! +``` + +If for some reason you need to disable this "linting" check (e.g. you're gradually migrating an existing YAML file to become a `ytt` template and it's onerous to convert all those comments at once), include the `--ignore-unknown-comments` flag. + +For a detailed explanation of how `ytt` detects and processes YAML files, see [File Marks > type detection for YAML files](file-marks.md#type-detection-for-yaml-files). + + +## Why is my anchor reference null despite my anchor's successful template? + +This is a [known limitation](known-limitations.md) of ytt. + +## Can I generate random strings with ytt? +No. A design goal of ytt is determinism, which keeps randomness out of scope. + +If you want to generate secrets, see the [injecting secrets doc](injecting-secrets.md) or the [kubernetes secretgen-controller](https://github.com/vmware-tanzu/carvel-secretgen-controller) + +## Can I load multiple functions without having to name each one? + +Yes! Functions can be stored in a struct which can be imported all together. You can then call individual functions from within that struct. Note that because Starlark does not provide forward references, you must declare the struct that collects the functions to export at the end of the file. + +Storing functions in struct: + +```yaml +#@ load("@ytt:struct", "struct") +#@ mod = struct.make(func1=func1, func2=func2) +``` + +Loading and calling functions in template: + +```yaml +#@ load("helpers.lib.yml", "mod") +something: #@ mod.func1() +``` + +Additional resources: [Load Statement doc](lang-ref-load.md) + +## How do I inject secrets? +See the [injecting secrets doc](injecting-secrets.md). + +## How do I template values within text? +See the [text templating doc](ytt-text-templating.md). Additionally, see this [playground example](/ytt/#example:example-text-template) which illustrates some ways text templating can be done. + +## What templating language does ytt use? + +ytt uses a fork of [Starlark](https://github.com/bazelbuild/starlark), with a few changes. See the [Language reference](lang.md#Language) for more information. diff --git a/site/content/ytt/docs/v0.38.0/file-marks.md b/site/content/ytt/docs/v0.38.0/file-marks.md new file mode 100644 index 000000000..4f87f2e7f --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/file-marks.md @@ -0,0 +1,138 @@ +--- +aliases: [/ytt/docs/latest/file-marks] +title: File Marks +--- + +## Overview + +ytt allows to control certain metadata about files via `--file-mark` flag. + +```bash +$ ytt ... --file-mark := +``` + +where: +- `path` — location to the file(s) being marked + - exact path (use `--files-inspect` to see paths as seen by ytt) + - path with `*` to match files in a directory + - path with `**/*` to match files and directories recursively +- `mark` — metadata to modify on the file(s) selected by `path` +- `value` — the value for the mark + +Note that this flag can be specified multiple times. + +Example: + +```bash +ytt -f . \ + --file-mark 'alt-example**/*:type=data' \ + --file-mark 'example**/*:type=data' \ + --file-mark 'generated.go.txt:exclusive-for-output=true' \ + --output-files ../../tmp/ +``` + +--- +## Available Marks + +### path + +Changes the relative path. + +``` +--file-mark ':path=' +``` + +Example: + +``` +--file-mark 'generated.go.txt:path=gen.go.txt' +``` + +renames `generated.go.txt` to `gen.go.txt` + +### exclude + +Exclude file from any kind of processing. + +``` +--file-mark ':exclude=true' +``` + +### type + +Change type of file, affecting how `ytt` processes it. + +``` +--file-mark ':type=' +``` + +By default `file-type` is determined based on file extension and — as of v0.32.0 — content. + +`file-type` can be: +- `yaml-template` (default for: `.yml` or `.yaml`) — parsed as a YAML document and evaluated for templating. +- `yaml-plain` — parsed as a simple YAML document (_not_ evaluated for templating). +- `text-template` (default for: `.txt`) — parsed as a text document containing text templating. +- `text-plain` — included in output, as is. +- `starlark` (default for: `.star`) — a Starlark source file (executed) +- `data` (default for all other files) — a text file that can be loaded by [`data.read()`](lang-ref-ytt.md#data) + +Example: + +``` +--file-mark 'config.yml:type=data' +``` + +indicates that `config.yml` is _not_ a `yaml-template`, but is `data`. This file will _not_ be parsed, evaluated, or included in the output, but _can_ be loaded using [`data.read()`](lang-ref-ytt.md#data). + +#### type detection for YAML files +_(as of v0.32.0)_ + +First, `ytt` determines the type of each YAML file: +1. if it has a `.yml` or `.yaml` extension, it's assumed to be a `yaml-template` +2. if it is `yaml-template` but does not contain any ytt templating, it is treated as if it were `yaml-plain` +3. if it is marked with `type=yaml-plain`, it is treated as `yaml-plain` + +and then processes each: +- `yaml-template` files are: + 1. parsed as YAML, + 2. evaluated (i.e. executes `ytt` the template), and + 3. linted (i.e. detects when non-ytt comments are included) + - this catches errors where the `@` is accidentally omitted, + - if the `--ignore-unknown-comments` flag is included, this "linting" is disabled. +- `yaml-plain` files are simply parsed as YAML (with no evaluation or linting). + + +### for-output + +Marks a file to be included in the output. + +Files of [type](#type) `yaml-template`, `yaml-plain`, `text-template`, and `text-plain` are part of the output by default. + +``` +--file-mark ':for-output=true' +``` + +Example +``` +--file-mark 'config.lib.yml:for-output=true' +``` + +By default, `.lib.yml` files are not included in the rendered output (they are loaded +by other templates). With this file mark, `config.lib.yml` _is_ included in the output. + +### exclusive-for-output + +Limits output to only marked files. + +If there is at least one file marked this way, only these files will be used in output. + +``` +--file-mark ':exclusive-for-output=true' +``` + +Example: +``` +--file-mark 'config.lib.yml:exclusive-for-output=true' +``` + +Causes output to only include `config.lib.yml`. diff --git a/site/content/ytt/docs/v0.38.0/how-it-works.md b/site/content/ytt/docs/v0.38.0/how-it-works.md new file mode 100644 index 000000000..c9ad769de --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/how-it-works.md @@ -0,0 +1,214 @@ +--- +aliases: [/ytt/docs/latest/how-it-works] +title: "How it works" +--- + +## Overview + +Let's get an idea of how `ytt` works by looking at the high-level concepts and flow. + +## The `ytt` Pipeline + +When you invoke `ytt` ... + +```console +$ ytt -f values/ -f config/ +``` + +... you can think of it as a pipeline in four stages, looking something like this: + +![ytt pipeline overview](/images/ytt/ytt-pipeline-overview.jpg) + + +_(Input documents (grey, pale yellow and blue) flow through four pipeline steps (black), into evaluated intermediary documents (bright yellow and blue), and combined ultimately into plain YAML output (green).)_ + +### The Input Files + +The top-left section of the diagram shows the input files. + +Let's explore each, after a general note about files and documents. + +#### About YAML files and their documents + +If a given "input file" contains one or more `ytt` annotations (i.e. lines that contain `#@`), it's a `ytt` template. + +This is a `ytt` template... +```yaml +--- +foo: 14 +bar: +#@ for/end name in ["Alice", "Bob", "world"]: +- #@ "Hello, " + name +``` +... and, this is plain YAML ... +```yaml +--- +foo: 14 +bar: +- Hello, Alice +- Hello, Bob +- Hello, world +``` + +Each file can have one or more YAML documents in them (separated by `---`). _(from here on, we'll refer to YAML documents as just "documents".)_ +```yaml +foo: here's the first document (the initial `---` is optional) +--- +bar: this is a separate document +--- +qux: third document's a charm! +``` + +Now, let's look at each type of document, in turn. + +#### Data Values Schema Documents +If a document begins with the [`@data/values-schema`](how-to-write-schema.md#starting-a-schema-document) annotation, we call it a "Data Values Schema Document" (the light grey dashed box in the illustration, [above](#the-ytt-pipeline)). +```yaml +#@data/values-schema +--- +instances: 1 +... +``` + +These files declare variables (Data Values) by setting their name, default value, and type. + +#### Data Values Documents + +When a document starts with the [`@data/values`](how-to-use-data-values.md) annotation, it's called a "Data Values Document" (the light grey dashed box in the illustration, [above](#the-ytt-pipeline)). +```yaml +#@data/values +--- +instances: 8 +... +``` + +These contain the variables that provide values for templates (explained in more detail, below). + +#### Plain Documents + +If a document has _no_ `ytt` annotations, we'll call those "Plain Documents" (like the bright yellow item in "Input Files", [above](#the-ytt-pipeline)). +```yaml +--- +notes: +- this will be part of the output +- it does get parsed as YAML, but that's about it +``` +These documents need no processing (outside of being parsed as YAML), and are included as part of the output of the pipeline. + +#### Templated Documents + +If a document _does_ contain templating (i.e. lines containing `#@`) it's known as a "Templated Document" (the _pale_ yellow one, [above](#the-ytt-pipeline)). + ```yaml + #@ load("@ytt:data", "data") + + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: my-app + spec: + replicas: #@ data.values.instances + ... + ``` +These documents are — after being processed — also included as part of the output of the pipeline. + +#### Overlay Documents + +When a document _starts_ with the [`@overlay/match...`](lang-ref-ytt-overlay.md#overlays-as-files) annotation (i.e. above the `---`), it's referred to as an "Overlay Document" (denoted as a pale blue item, [above](#the-ytt-pipeline)): + ```yaml + #@overlay/match by=overlay.all + --- + metadata: + #@overlay/match missing_ok=True + namespace: staging + ... + ``` +These documents describe edits to apply just before generating the output (described in detail, below). + +#### Kinds of Documents in Input Files + +Note that an "input file" can contain any combination of "Plain Documents", "Templated Documents", and "Overlay Documents". In contrast, "Data Values Schema Documents" and "Data Values Documents" must be in their own "Input File", as illustrated [above](#the-ytt-pipeline). + +### The Pipeline Steps + +Now that we have a sense of the four kinds of inputs, let's explore what happens at each step in the pipeline. + +#### Step 1: Calculate Data Values + +As the first black pipeline box shows, [above](#the-ytt-pipeline): + +1. process all the "Data Values Schema" documents (light grey input) — evaluating any templating in them; +1. merge those documents, [in order](lang-ref-ytt-schema.md#multiple-schema-documents), generating a "Data Values" document populated with default values and type information. +1. process all the "Data Values" documents (light grey input) — evaluating any templating in them; +1. merge those documents, [in order](ytt-data-values.md#splitting-data-values-overlays-into-multiple-files). That is, start with the first document and then overlay the second one onto it; then overlay the third document on top of _that_, and so on... + +The result of all this is the final set of values that will be available to templates: the dark grey "final Data Values". + +#### Step 2: Evaluate Templates + +Next, evaluate the remaining templates (all the other kinds of "Input File" documents): +1. "evaluate" means executing all of the [Starlark code](lang.md): loops are run, conditionals decided, expressions evaluated. +1. one notable exception is overlay annotations (i.e. those that start with `@overlay/...`), these are deferred until the next step. +1. a template accesses input variables (i.e. the Data Values calculated in the previous step) via the [`@ytt:data` module](lang-ref-ytt.md#data); + +The result of all this evaluation is a set of YAML documents, configured with the Data Values (shown as "Evaluated Document Set" in the diagram, [above](#the-ytt-pipeline)). + +#### Step 3: Apply Overlays + +Note that the "Evaluated Document Set" (see the output from the second step in [the diagram](#the-ytt-pipeline)) contains two groups: +- "Evaluated Documents" — these are the pile of "Plain Documents" and evaluated "Template Documents" from the previous step. These are a bright yellow. +- "Overlay Documents" — these are the input file "Overlay Documents" wherein everything except the `@overlay/...` annotations have been evaluated. These are a bright blue. + +With these in hand: +1. apply each "Overlay Document" on top of the full set of "Evaluated Documents".\ + You can think of each overlay as like a SQL `update` command: + - the value of it's `by` argument is like a `where` clause that selects over the whole collection of "Evaluated Documents". For example, + ```yaml + #@overlay/match by=overlay.subset({"kind": "Deployment"}), ... + --- + ``` + selects all of the documents which contain a key `"kind"` whose value is `"Deployment"` + - for each of the documents selected, apply the overlay on top of it. This is like a series of `set` clauses, each updating a portion of the document. For example, + ```yaml + #@overlay/match by=overlay.subset({"kind": "Deployment"}), ... + --- + #@overlay/match-child-defaults missing_ok=True + metadata: + labels: + app: frontend + ``` + sets each "Deployment"'s `metadata.labels.app` to be `"frontend"`. +1. repeat that process for each "Overlay Document", [in order](lang-ref-ytt-overlay.md#overlay-order). + +The result is (shown as "Output Document Set" in the diagram, [above](#the-ytt-pipeline)) — the finalized set of YAML documents, in memory. Which leaves one last step... + +#### Step 4: Serialize + +This is simply iterating over the "Output Document Set", rendering each YAML Document ("Output Files", [above](#the-ytt-pipeline)). + +The result is sent to standard out (suitable for piping into other tools). If desired, the output can be sent instead to disk using the [`--output...` flags](outputs.md). + +## Further Reading + +We've scratched the surface: an end-to-end flow from pre-processing inputs, processing templates, post-processing overlays, and finally rendering the resulting output. + +To learn more about... +- **Data Values Schema** + - learn about [writing Schema](how-to-write-schema.md) for Data Values + - read-up on the details in the "[Data Values Schema Referce](lang-ref-ytt-schema.md)" material + - work with a complete example from the source: \ + [vmware-tanzu/carvel-ytt/../examples/schema](https://github.com/vmware-tanzu/carvel-ytt/tree/develop/examples/schema) +- **Data Values**... + - poke at a working example in the ytt Playground: [Load Data Values](/ytt/#example:example-load-data-values) example + - read-up on the details in "[Using Data Values](how-to-use-data-values.md)" + - work with a complete example from the source: \ + [vmware-tanzu/carvel-ytt/../examples/data-values](https://github.com/vmware-tanzu/carvel-ytt/tree/develop/examples/data-values) +- **Templates**... + - learn Starlark by example in the first bunch of ["Basics" examples](/ytt/#example:example-plain-yaml). + - read-up on [`ytt`'s built-in libraries](lang-ref-ytt.md) to encode/decode, hash, regex match over data. + - get an helpfully abridged tour of Starlark in "[Language](lang.md)" +- **Overlays**... + - walk through the "Overlays" group of examples: + 1. go to the [ytt Playground](/ytt/#example:example-match-all-docs) + 2. find the "Overlays" group header, click on it to reveal the group _(you can close the "Basics" group by clicking on it)_. + - read an introduction at "[Using Overlays](ytt-overlays.md)" diff --git a/site/content/ytt/docs/v0.38.0/how-to-export-schema.md b/site/content/ytt/docs/v0.38.0/how-to-export-schema.md new file mode 100644 index 000000000..00c37c261 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/how-to-export-schema.md @@ -0,0 +1,221 @@ +--- +aliases: [/ytt/docs/latest/how-to-export-schema] +title: Export Schema in OpenAPI Format +--- + +The primary use of schema is to declare (and type-check) Data Values. + +`ytt` _also_ supports the ability to export schema definitions in [OpenAPI v3.0.x](https://swagger.io/specification/) format. This is useful for tools that require schema definition in this industry-standard format. For example, `kapp-controller`, requires [Package](/kapp-controller/docs/latest/packaging/#package) 's `spec.valuesSchema.openAPIv3` to contain an [OpenAPI Schema map](https://swagger.io/specification/#schema-object). + +In this way, one can write their schema once and reuse it in situations that require the OpenAPI v3 format. + +## Working Example + +Given a `ytt` schema: + +```yaml +#! schema.yml + +#@data/values-schema +--- +#@schema/title "LoadBalancer type service" +#@schema/desc "Whether to include a LoadBalancer type service and if so, what its IP address is." +load_balancer: + enabled: true + #@schema/nullable + static_ip: "" + +#@schema/title "DNS domains" +#@schema/desc "DNS domains to accept traffic for." +#@schema/default ["apps.example.com", "mobile.example.com"] +app_domains: + #@schema/examples ("Example app domain", "web.myapp.com"), ("","localhost:8080") + - "" + +#@schema/title "Database connections" +#@schema/desc "Connection information for databases used by the system." +#@schema/examples ("Example for local db", [{"name": "default", "adapter": "postgresql", "host": "localhost", "port": 8080}]) +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + +#@schema/title "Additional configuration" +#@schema/desc "Configuration for experimental/optional components; see documentation for more details." +#@schema/examples ("Example of additional config", {"username": "default", "password": "password", "insecureFlag": True}) +#@schema/type any=True +additional_config: {} +``` +One can generate the corresponding OpenAPI Document: + +```bash +$ ytt -f schema.yml --data-values-schema-inspect --output openapi-v3 +``` +where: +- `--data-values-schema-inspect` — after all Data Value Schema files have been processed, print the effective schema (instead of rendering any templates). +- `--ouput=openapi-v3` — the schema format to use when rendering is OpenAPI v3.0.x (as opposed to `ytt` formatted schema). + +... which yields: + +```yaml +openapi: 3.0.0 +info: + version: 0.1.0 + title: Schema for data values, generated by ytt +paths: {} +components: + schemas: + dataValues: + type: object + additionalProperties: false + properties: + load_balancer: + title: LoadBalancer type service + type: object + additionalProperties: false + description: Whether to include a LoadBalancer type service and if so, what its IP address is. + properties: + enabled: + type: boolean + default: true + static_ip: + type: string + nullable: true + default: null + app_domains: + title: DNS domains + type: array + description: DNS domains to accept traffic for. + items: + type: string + x-example-description: Example app domain + example: web.myapp.com + default: "" + default: + - apps.example.com + - mobile.example.com + databases: + title: Database connections + type: array + description: Connection information for databases used by the system. + x-example-description: Example for local db + example: + - name: default + adapter: postgresql + host: localhost + port: 8080 + items: + type: object + additionalProperties: false + properties: + name: + type: string + default: "" + adapter: + type: string + default: postgresql + host: + type: string + default: "" + port: + type: integer + default: 5432 + default: [] + additional_config: + title: Additional configuration + nullable: true + description: Configuration for experimental/optional components; see documentation for more details. + x-example-description: Example of additional config + example: + username: default + password: password + insecureFlag: true + default: {} +``` + +## Exported Properties + +Of the properties declared in the [OpenAPI Schema specification](https://swagger.io/specification/#schema-object), the following are generated. + +### `title` + +(As of v0.39.0+) + +Sets the user-friendly name or title of the node + +- when a data value is annotated `@schema/title`, the value of that title is the value of this property, verbatim; +- otherwise, this property is omitted. + +### `type` + +Names the type of the schema. + +The value of this property depends on the type of data value implied in the `ytt` schema: +- map ==> `object` +- array ==> `array` +- boolean ==> `bool` +- string ==> `string` +- integer ==> `integer` +- float ==> `type: number; format: float` + +### `additionalProperties` + +Indicates whether other keys are allowed in a mapping/object. + +- whenever inspecting a data value that is a map, `ytt` includes this property, setting it to `false`; +- when a data value is annotated `@schema/type any=True`, this property is omitted. + +### `nullable` + +Indicates whether `null` is also allowed. + +- when a data value is annotated `@schema/nullable` or `@schema/type any=True`, this property is included and set to `true`; +- otherwise, this property is omitted. + +### `description` + +Explains the contents and/or consequences of certain values of the property. + +- when a data value is annotated `@schema/desc`, the value of that description is the value of this property, verbatim; +- otherwise, this property is omitted. + +### `x-example-description` + +(As of v0.39.0+) + +Explains the contents of the `example` property. + +- when a data value is annotated `@schema/examples`, examples are provided via 2-tuples, only the first tuple will be exported to the OpenAPI Document. +- the example description is the first argument of 2-tuple, and must be a string. +- otherwise, this property is omitted. + +### `example` + +(As of v0.39.0+) + +Presents an example value for the data value. + +- when a data value is annotated `@schema/examples`, examples are provided via 2-tuples, only the first tuple will be exported to the OpenAPI Document. +- a 2-tuple contains string description, and an example value: `("first example", exampleYAML())`. +- example values should conform to the type of the same property. +- otherwise, this property is omitted. + +### `default` + +Declares a default value. + +The value of this property: +- is always included; +- is typically the value of the data value in the `ytt` schema; +- if the data value is annotated `@schema/default`, _that_ value is used instead. + +## Known Limitations + +The following are not yet supported in `ytt`: +- `ytt` schema does not yet support declarative validations and thus does not produce such validations in an OpenAPI export. +- there is no means yet to modify the [`info` section of the OpenAPI document](https://swagger.io/specification/#info-object) from within a `--data-values-schema-inspect`. There are defaults supplied, however we recommend that these fields are updated manually. +- inspecting output in `ytt` Schema format is not yet supported. + + + diff --git a/site/content/ytt/docs/v0.38.0/how-to-use-data-values.md b/site/content/ytt/docs/v0.38.0/how-to-use-data-values.md new file mode 100644 index 000000000..e5f1245a1 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/how-to-use-data-values.md @@ -0,0 +1,99 @@ +--- +aliases: [/ytt/docs/latest/how-to-use-data-values] +title: Using Data Values +--- + +## Overview + +The way to introduce a variable in `ytt` (i.e. to externalize a configuration value) is to: +1. declare it as a "Data Value" by naming it in a schema file, +2. reference it in templates, and +3. configure it through Data Value inputs. + +This guide shows how to do this. + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + + +### Declaring Data Values + +In `ytt`, before a Data Value can be used in a template, it must be declared. This is typically done in a schema file. + +For example: + +`schema.yml` +```yaml +#@data/values-schema +--- +load_balancer: + enabled: true + external_ip: "" +``` + +declares three Data Values: +- `load_balancer` is a map that contains two map items: `enabled` and `external_ip`. +- `load_balancer.enabled` is a boolean; by default it is `true`. +- `load_balancer.external_ip` is a string; by default it is an empty string. + + +_(see the [How To Write Schema](how-to-write-schema.md) guide, for details.)_ + + +### Referencing Data Values + +Those Data Values can then be used in a template via the `@ytt:data` module. + +`config.yml` +```yaml +#@ load("@ytt:data", "data") +--- +service: #@ data.values.load_balancer +``` + +_(For details on using the Data module, refer to [`@ytt:data`](lang-ref-ytt.md#data).)_ + +### Configuring Data Values + +Further, those Data Values can be customized by a Consumer by providing their own data values: + +`values.yml` +```yaml +#@data/values +--- +load_balancer: + external_ip: 172.120.12.232 +``` + +When Data Values are supplied, their values are checked against the schema to ensure they are of the right type and shape. If there are any errors, `ytt` stops processing and reports them. + +_(For details on how to configure Data Values, consult the [Data Values](ytt-data-values.md) reference.)_ + +... + +Using the example files from above, `ytt` produces this output: + +```console +$ ytt -f schema.yml -f values.yml -f config.yml +service: + load_balancer: + enabled: true + external_ip: 172.120.12.232 +``` + +Note: +- `load_balancer.enabled` is the default as set in `schema.yml` +- `load_balancer.external_ip` is the configured value from `values.yml` + +## Next Steps + +More examples: +- simple and complete example of declaring and using Data Values through schema: \ + https://github.com/vmware-tanzu/carvel-ytt/tree/develop/examples/schema +- example declaring and configuring an array: \ + https://github.com/vmware-tanzu/carvel-ytt/tree/develop/examples/schema-arrays + +Related documentation: +- [How To Write Schema](how-to-write-schema.md) guide +- [Data Values](ytt-data-values.md) reference +- [Data Values Schema Reference](lang-ref-ytt-schema.md) +- [Schema Migration Guide](data-values-schema-migration-guide.md) diff --git a/site/content/ytt/docs/v0.38.0/how-to-write-schema.md b/site/content/ytt/docs/v0.38.0/how-to-write-schema.md new file mode 100644 index 000000000..4c60e752e --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/how-to-write-schema.md @@ -0,0 +1,274 @@ +--- +aliases: [/ytt/docs/latest/how-to-write-schema] +title: Writing Schema +--- + +## Overview + +In `ytt`, before a Data Value can be used in a template, it must be declared. This is typically done via Data Values Schema. + +This guide shows how to write such schema. + +_(For a broader overview of Data Values, see [Using Data Values](how-to-use-data-values.md))._ + +--- +## Starting a Schema Document + +One writes Data Values Schema in a YAML document annotated with `#@data/values-schema`: + +```yaml +#@data/values-schema +--- +#! Schema contents +``` + +Files containing Schema documents are included via the `--file`/`-f` flag. + +```bash +$ ytt ... -f schema.yml ... +``` + +The contents of such a document _are_ the Data Values being declared. + +## Declaring Data Values + +Each item in the schema document declares a Data Value. + +A Data Value declaration has three (3) parts: a **name**, a **default value**, and a **type**. + +For example, +```yaml +#@data/values-schema +--- +system_domain: "" +``` + +declares the Data Value, with the name `system_domain` to be of type **string** with the default value of an **empty string** (i.e. `""`). + +--- +## Implying Types + +In `ytt` Schema types of Data Values are _inferred_ from the values given. So, when one writes schema, they are _implying_ the type of each Data Value through the default value they give. + +For example: +```yaml +#@data/values-schema +--- +system_domain: "" + +load_balancer: + enabled: true + static_ip: "" + +app_domains: +- "" + +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + user: admin + secretRef: + name: "" +``` + +effectively declares the following data values: +- `system_domain` — a string +- `load_balancer` — a map containing two items: + - `enabled` — a boolean + - `static_ip` — a string +- `app_domains` — an array of strings (and only strings) +- `databases` — an array where each element is a map. Each map has exactly six items: + - `name` — a string + - `adapter` — a string + - `host` — a string + - `port` — an integer + - `user` — a string + - `secretRef` — a map having exactly one item: + - `name` — a string + +_(see [Data Values Schema Reference: Types](lang-ref-ytt-schema.md#types) for details.)_ + +--- +## Setting Default Values + +In `ytt` Schema, the default value for a Data Value is almost always simply the value specified. + +From the example, [above](#implying-types), the corresponding Data Values have the following defaults: +- `system_domain` is empty string (i.e. `[]`), +- `load_balancer.enabled` is `true`, and +- `load_balancer.static_ip` initializes to empty string. + +For details on how to set individual default values, see [Data Values Schema Reference: Default Values](lang-ref-ytt-schema.md#default-values). + +**Special Case: Arrays** + +There is one exception: arrays. As described in [Data Values Schema Reference: Defaults for Arrays](lang-ref-ytt-schema.md#defaults-for-arrays), the default value for arrays, by default, is an empty list (i.e. `[]`). That said, when an item is added to the array, _that item's_ value is defaulted as defined in the schema. + +In the example, [above](#implying-types), the definition of `databases` is an array. Each item in _that_ array is a map with six keys including `adapter`, `port`, etc. + +`database` starts out as an empty list. Then, as each item added to it, they will be defaulted with the values given in the schema. + +Continuing with our example schema, [above](#implying-types), if a Data Values overlay gives an actual value for the array: + +```yaml +#@data/values +--- +databases: +- name: core + host: localhost +``` + +That item will be filled-in with defaults: + +```yaml +name: core +adapter: postgresql +host: localhost +port: 5432 +user: admin +secretRef: + name: "" +``` + +In order to override the default of the array, _itself_, within schema see [Setting a Default Value for Arrays](#setting-a-default-value-for-an-array). + + +--- +## Specific Use-Cases + +A few less common, but real-world scenarios: + +- [setting a default value for arrays](#setting-a-default-value-for-an-array) +- [marking a Data Value as optional](#marking-a-data-value-as-optional) +- [allowing multiple types of maps or arrays](#allowing-multiple-types-of-maps-or-arrays) +- [declaring "pass-through" Data Values](#declaring-pass-through-data-values) + +### Setting a Default Value for an Array + +As explained in [Data Values Schema Reference: Defaults for Arrays](lang-ref-ytt-schema.md#defaults-for-arrays), unlike all other types, the default value for an array is an empty list (i.e. `[]`). + +In some cases, it is useful to provide a non-empty default value for an array. To do so, one uses the `@schema/default` annotation. + +For example, with this schema: + +```yaml +#@data/values-schema +--- +#@schema/default ["apps.cf-apps.io", "mobile.cf-apps.io"] +app_domains: +- "" +``` + +The default value for `app_domains` will be `["apps.cf-apps.io", "mobile.cf-apps.io"]`. + +See also: [@schema/default](lang-ref-ytt-schema.md#schemadefault). + +### Marking a Data Value as Optional + +Sometimes, it can be useful to define a section of Data Values as optional. This typically means that templates that rely on those values conditionally include output that use the contained value. + +For example, the following template: +```yaml +#@ load("@ytt:data", "data") +--- +... +spec: + #@ if data.values.load_balancer: + loadBalancerIP: #@ data.values.load_balancer.static_ip + #@ end +... +``` +will only include `spec.loadBalancerIP` if a value is provided for the `load_balancer` Data Value. + +One notes this in Schema using the `@schema/nullable` annotation: + +```yaml +#@data/values-schema +--- +#@schema/nullable +load_balancer: + static_ip: "" +``` + +which indicates that `load_balancer` is `null`, by default. However, if a value _is_ provided for `load_balancer`, it must be the `static_ip` and have a value that is a **string**. + +For more details see [Data Values Schema Reference: `@schema/nullable`](lang-ref-ytt-schema.md#schemanullable). + +### Allowing Multiple Types of Maps or Arrays + +In rare cases, a given Data Value needs allow more than one type. + +Currently, `ytt` Schema does not explicitly support specifying more than one type for a Data Value. + +In the meantime, one can mark such Data Values as having [`any` Type](lang-ref-ytt-schema.md#any-type): + +```yaml +#@schema/type any=True +int_or_string: "" +``` +so that: +- `int_or_string` is, by default, an empty string +- it can accept an **integer** or a **string** ... or any other type, +- and the value of `int_or_string` and its children is not checked by schema. + +If it is critical to ensure that the type of `int_or_string` to be _only_ an integer or string, one can include a validating library that does so explicitly: + +```python +load("@ytt:assert", "assert") +load("@ytt:data", "data") + +if type(data.value.int_or_string) not in ("int", "string"): + assert.fail("'int_or_string' must be either an integer or a string.") +end +``` + +### Declaring "Pass-through" Data Values + +In certain cases, one designs a Data Value to carry a chunk of YAML whose exact type or shape is unimportant to templates. In these situations, it is undesirable for that chunk of YAML to be type-checked. + +One can effectively disable schema type checking and defaulting by marking the Data Value as of type "any": + +```yaml +#@data/values-schema +--- +honeycomb: + enabled: false + api_key: so124me14v4al1i5da5p5i180key + #@schema/type any=True + optional_config: null +``` + +Here, `additional_config` can contain any valid YAML. + +For example: + +```yaml +#@data/values +--- +honeycomb: + optional_config: + default_series: + id: 1001 + description: Administrative Actions +``` + +In a template, the Data Value can be referenced and its contents will be inserted: + +```yaml +#@ load("@ytt:data", "data") +--- +#@ if/end data.values.honeycomb.enabled: +honeycomb: + config: + api_key: #@ data.values.honeycomb.api_key + #@ if/end data.values.honeycomb.optional_config: + optional: #@ data.values.honeycomb.optional_config +``` + + +--- +## Next Steps + +Once you've declared your Data Values, they can be referenced in `ytt` templates as described in [Using Data Values > Referencing Data Values](how-to-use-data-values.md#referencing-data-values) diff --git a/site/content/ytt/docs/v0.38.0/injecting-secrets.md b/site/content/ytt/docs/v0.38.0/injecting-secrets.md new file mode 100644 index 000000000..9e0d095ff --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/injecting-secrets.md @@ -0,0 +1,160 @@ +--- +aliases: [/ytt/docs/latest/injecting-secrets] +title: Injecting Secrets +--- + +## Overview + +**This document is work in progress.** + +Unlike most configuration, many organizations disallow storing of plain secret values next to other code/configuration. + +This document: +- presents several common approaches used to make secrets available to your templates +- does _not_ cover injection of secrets directly into an application at runtime (overall may be the best approach) +- does _not_ recommend one approach over another (though it does state pros and cons) +- does _not_ talk about where resulting templates are forwarded + +One common question that's asked is why not to extend ytt to allow it to shell out to other programs or why not include builtin library that can fetch secrets from outside (e.g. imagine having builtin `vault` function). We are _not_ planning to support either of these features because we want to maintain strong sandbox boundary between templating language and ytt's external environment. Our expectation is that if ytt user wants to provide specific data to templates, it should be injected into ytt rather than templates (or ytt itself) fetching it. We believe this is a good security posture and as a side effect makes templates more portable (some users may store secrets in Vault and some other in Lastpass). + +- [Via command line args](#via-command-line-args) +- [Via environment variables](#via-environment-variables) +- [Via secret references, before templating](#via-secret-references-before-templating) +- [Via encrypted secrets, before templating](#via-encrypted-secrets-before-templating) +- [Via secret references, after templating](#via-secret-references-after-templating) +- [Via encrypted resources, later decrypted inside the k8s cluster](#via-encrypted-resources-later-decrypted-inside-the-k8s-cluster) + +There might be other ways to inject secrets into ytt that are not listed here. + +--- +## Via command line args + +```bash +ytt -f . --data-value username=user --data-value password=pass +``` + +Cons: + +- depending on a computing environment secret values are visible in a process tree (as part of full command line) **which typically is considered unwanted** + +--- +## Via environment variables + +```bash +export CFG_secrets__username=user +export CFG_secrets__password=pass +ytt -f . --data-values-env CFG +``` + +Cons: + +- depending on a computing environment secret values are visible in process metadata (e.g. typically accessible to root user under `/proc//environ`) + +--- +## Via secret references, before templating + +Given following two files: + +`config.yml`: + +```yaml +#@ load("@ytt:data", "data") + +users: +- username: #@ data.values.username + password: #@ data.values.password +``` + +`secrets`: + +```yaml +username: (( vault "secret/my/credentials/admin:username" )) +password: (( vault "secret/my/credentials/admin:password" )) +``` + +(Note that `secrets` file does not have `yaml` extension to make ytt exclude it from template result. Alternatively `--file-mark` flag can be used to exclude that file.) + +One can compose ytt with external tooling that knows how to resolve referenced values. For example, [spruce](https://starkandwayne.com/blog/simple-secure-credentials-into-yaml-with-vault-and-spruce/) can resolve secret references and then forward that information onto ytt: + +```bash +echo -e "#@data/values\n---\n$(spruce merge ./secrets)" | ytt -f . -f - +``` + +Pros: + +- able to commit mapping of data values to secrets in your preferred secret storage (in this example Vault) +- actual secret values are available to templates, hence, can be inserted in the middle of a string or base64 encoded (as sometimes required by k8s) + +--- +## Via encrypted secrets, before templating + +Similar to above "Via secret references, before templating" approach, instead of storing secret references in `secrets` file one can store encrypted secret values in `secrets` next to other configuration. Various tools like [sops](https://github.com/mozilla/sops) make encrypting configuration files fairly easy: + +```bash +echo -e "#@data/values\n---\n$(sops -d ./secrets)" | ytt -f . -f - +``` + +Pros: + +- provides a way to version (backup, etc) secrets next to other configuration +- actual secret values are available to templates, hence, can be inserted in the middle of a string or base64 encoded (as sometimes required by k8s) + +Cons: + +- depending on organization requirements even encrypted secrets may not be allowed to be stored next to other configuration + +--- +## Via secret references, after templating + +Given following two files: + +`config.yml`: + +```yaml +#@ load("@ytt:data", "data") + +users: +- username: #@ data.values.username + password: #@ data.values.password +``` + +`values.yml`: + +```yaml +#@data/values +--- +username: (( vault "secret/my/credentials/admin:username" )) +password: (( vault "secret/my/credentials/admin:password" )) +``` + +One can compose ytt with external tooling that knows how to resolve referenced values. Such tools typically look for a particular patterned map or array items. One such tool is [spruce](https://starkandwayne.com/blog/simple-secure-credentials-into-yaml-with-vault-and-spruce/): + +```bash +ytt -f . | spruce merge - +``` + +Pros: + +- provides a way to commit mapping of data values and their associated secret locations (e.g. `username` comes from `vault` under `secret/my/credentials/admin:username`) + +Cons: + +- secret resolution tools may not be able to find secret references inside strings (e.g. `http://(( ... )):(( ... ))@website`) +- secret resolution tools will not understand if secret reference was encoded (for example with `base64` encoding as wanted sometimes by k8s) + +--- +## Via encrypted resources, later decrypted inside the k8s cluster + +This approach is Kubernetes specific. + +[Bitnami's `sealed-secrets` project](https://github.com/bitnami-labs/sealed-secrets) provides a way to encrypt secrets hence allowing to commit them next to your other configuration. With this approach, secret values are not accessible to templates directly; however, they can be referenced by k8s resources (for example via `secretRef` in `Pod` spec). At the time of the deploy of such resources, sealed secrets controller will create appropriate `Secret` objects with decrypted secret values. + +Pros: + +- provides a way to version (backup, etc) secrets next to other configuration +- no way to access decrypted secret values in templates directly (though, this may not be necessary in many cases) + +Cons: + +- Kubernetes specific +- depending on organization requirements even encrypted secrets may not be allowed to be stored next to other configuration diff --git a/site/content/ytt/docs/v0.38.0/install.md b/site/content/ytt/docs/v0.38.0/install.md new file mode 100644 index 000000000..2d5e4908b --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/install.md @@ -0,0 +1,66 @@ +--- +aliases: [/ytt/docs/latest/install] +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ ytt version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ ytt version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/vmware-tanzu/homebrew-carvel](https://github.com/vmware-tanzu/homebrew-carvel). + +```bash +$ brew tap vmware-tanzu/carvel +$ brew install ytt +$ ytt version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/vmware-tanzu/carvel-ytt/releases), for example for 'ytt-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/ytt-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/ytt-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/ytt-darwin-amd64 /usr/local/bin/ytt + +# Make binary executable +$ chmod +x /usr/local/bin/ytt + +# Check its version +$ ytt version +``` +## Shell Completion + +The `ytt completion` command generates an autocompletion script for the specified shell. + +See `ytt completion --help` for information and instructions. + +For detailed instructions on enabling shell completion, specify the type of shell in the help command. For example: +`ytt completion zsh --help`. diff --git a/site/content/ytt/docs/v0.38.0/known-limitations.md b/site/content/ytt/docs/v0.38.0/known-limitations.md new file mode 100644 index 000000000..a02a65476 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/known-limitations.md @@ -0,0 +1,12 @@ +--- +aliases: [/ytt/docs/latest/known-limitations] +title: Known Limitations +--- + +- YAML anchors and templating directive for the same YAML node are not supported. + + ```yaml + first: &content #@ 123 + second: *content + ``` + `second` key-value pair will _not_ contain 123 since YAML anchors are resolved before ytt evaluates templating directives. diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-annotation.md b/site/content/ytt/docs/v0.38.0/lang-ref-annotation.md new file mode 100644 index 000000000..bfe6a4a12 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-annotation.md @@ -0,0 +1,54 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-annotation] +title: Annotations +--- + +## Format + +``` +@ann1-name [ann-arg1, ann-arg2, ..., keyword-ann-arg1=val1] +``` + +- content between brackets is optional. + +- annotation names are typically namespaced, for example, `overlay/merge` is an annotation within an `overlay` namespace. Annotation namespaces are there for general organization, they are not associated with loaded packages (from `load` keyword). + +- annotation arguments (positional and keyword) is just plain code + +## Shared templating annotations + +- `@template/code [code]` or just `@ [code]` (on its own line) represents plain code line + +```yaml +#@ a = calculate(100) +key: value +``` + +- `@template/value [code]` or just `@ [code]` (at the end of line) represents a value associated structure + +```yaml +key: #@ value +array: +- #@ value +``` + +## YAML templating annotations + +- `@yaml/map-key-override` (no args) + - necessary to indicate that map key is being intentionally overriden + +- `@yaml/text-templated-strings` (no args) + - necessary to indicate that node contents (including map key and map value) should be text templated (ie `(@` indicates start of text templating) (see [text templating](ytt-text-templating.md) for more details.) + +## Text templating annotations + +- `@text/trim-left` trims space to the left of code node +- `@text/trim-right` trims space to the right of code node + +## Data module annotations + +- `@data/values` (no args) specifies values accessible via `data.values` from `@ytt:data` package (see [ytt @data/values](ytt-data-values.md) for more details) + +## Overlay module annotations + +See [Overlay module](lang-ref-ytt-overlay.md) for list of annotations. diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-def.md b/site/content/ytt/docs/v0.38.0/lang-ref-def.md new file mode 100644 index 000000000..7b418f365 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-def.md @@ -0,0 +1,73 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-def] +title: Functions +--- + +Refer to [Starlark function specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#functions) for details about various types of function arguments. Note that ytt's Starlark use requires functions to be closed with an `end`. + +- function definition within YAML + +Labels returns map with two keys: test1, and test2: + +```yaml +#@ def my_labels(): +test1: 123 +test2: 124 +#@ end +``` + +Above is _almost_ equivalent to (differnce is that return type in one case is a YAML fragment and in another it's a dict): + +```yaml +#@ def my_labels(): +#@ return {"test1": 123, "test2": 124} +#@ end +``` + +- function definition within Starlark (.star files) + +```python +def my_labels(): + return {"test1": 123, "test2": 124} +end +``` + +- function arguments (positional and keyword arguments) + +```yaml +#@ def my_deployment(name, replicas=1, labels={}): +kind: Deployment +metadata: + name: #@ name + labels: #@ labels +spec: + replicas: #@ replicas +#@ end + +--- +kind: List +items: +- #@ my_deployment("dep1", replicas=3) +``` + +- common function usage + +To set `labels` key to return value of `my_labels()`: + +```yaml +labels: #@ my_labels() +labels_as_array: +- #@ my_labels() +``` + +To merge return value of `my_labels()` into `labels` map: + +```yaml +#@ load("@ytt:template", "template") + +labels: + another-label: true + _: #@ template.replace(my_labels()) +``` + +Note that in most cases `template.replace` is not necessary since it's only helps replacing one item (array item, map item or document) with multiple items of that type. diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-dict.md b/site/content/ytt/docs/v0.38.0/lang-ref-dict.md new file mode 100644 index 000000000..e04eaa843 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-dict.md @@ -0,0 +1,80 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-dict] +title: Dictionaries +--- + +```yaml +#@ color = {"red": 123, "yellow": 100, "blue": "245"} +red: #@ color["red"] +``` + +Copied here for convenience from [Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#dictclear). + +- [dict·clear](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·clear) (`D.clear()`) +```python +x = {"one": 1, "two": 2} +x.clear() # None +print(x) # {} +``` + +- [dict·get](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·get) (`D.get(key[, default])`) +```python +x = {"one": 1, "two": 2} +x.get("one") # 1 +x.get("three") # None +x.get("three", 0) # 0 +``` + +- [dict·items](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·items) (`D.items()`) +```python +x = {"one": 1, "two": 2} +x.items() # [("one", 1), ("two", 2)] +``` + +- [dict·keys](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·keys) (`D.keys()`) +```python +x = {"one": 1, "two": 2} +x.keys() # ["one", "two"] +``` + +- [dict·pop](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·pop) (`D.pop(key[, default])`) +```python +x = {"one": 1, "two": 2} +x.pop("one") # 1 +x # {"two": 2} +x.pop("three", 0) # 0 +x.pop("four") # error: missing key +``` + +- [dict·popitem](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·popitem) (`D.popitem()`) +```python +x = {"one": 1, "two": 2} +x.popitem() # ("one", 1) +x.popitem() # ("two", 2) +x.popitem() # error: empty dict +``` + +- [dict·setdefault](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·setdefault) (`D.setdefault(key[, default])`) +```python +x = {"one": 1, "two": 2} +x.setdefault("one") # 1 +x.setdefault("three", 0) # 0 +x # {"one": 1, "two": 2, "three": 0} +x.setdefault("four") # None +x # {"one": 1, "two": 2, "three": None} +``` + +- [dict·update](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·update) (`D.update([pairs][, name=value[, ...])`) +```python +x = {} +x.update([("a", 1), ("b", 2)], c=3) +x.update({"d": 4}) +x.update(e=5) +x # {"a": 1, "b": "2", "c": 3, "d": 4, "e": 5} +``` + +- [dict·values](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·values) (`D.values()`) +```python +x = {"one": 1, "two": 2} +x.values() # [1, 2] +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-for.md b/site/content/ytt/docs/v0.38.0/lang-ref-for.md new file mode 100644 index 000000000..d73222ef5 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-for.md @@ -0,0 +1,40 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-for] +title: For loop +--- + +Refer to [Starlark for loop specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#for-loops) for details. + +- iterating with values + +```yaml +array: +#@ for i in range(0,3): +- #@ i +- #@ i+1 +#@ end +``` + +- iterating with index + +```yaml +array: +#@ arr = [1,5,{"key":"val"}] +#@ for i in range(len(arr)): +- val: #@ arr[i] + index: #@ i +#@ end +``` + +- use of `continue/break` + +```yaml +array: +#@ for i in range(0,3): +#@ if i == 1: +#@ continue +#@ end +- #@ i +- #@ i+1 +#@ end +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-if.md b/site/content/ytt/docs/v0.38.0/lang-ref-if.md new file mode 100644 index 000000000..914115d3a --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-if.md @@ -0,0 +1,82 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-if] +title: If Statements +--- + +Refer to [Starlark if statement specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#if-statements) for details. + +- if + +```yaml +#@ if True: +test1: 123 +test2: 124 +#@ end +``` + +- if (negative) + +```yaml +#@ if not True: +test1: 123 +#@ end +``` + +- single-node if + +```yaml +#@ if/end True: +test1: 123 +``` + +- if-else conditional + +```yaml +#@ if True: +test1: 123 +#@ else: +test2: 124 +#@ end +``` + +- if-elif-else conditional + +```yaml +#@ if True: +test2: 123 +#@ elif False: +test2: 124 +#@ else: +test2: 125 +#@ end +``` + +- if-elif-else conditional boolean (and/or) \ + See [Starlark or/and operators](https://github.com/google/starlark-go/blob/master/doc/spec.md#or-and-and) for more details. + + +```yaml +#@ test = 123 +#@ if test > 100 and test < 200: +test1: 123 +#@ elif test == 100 or test == 200: +test2: 124 +#@ else: +test3: 125 +#@ end +``` + + +- single line if + +```yaml +#@ passwd = "..." +test1: #@ passwd if passwd else assert.fail("password must be set") +``` + +- implicit if + +```yaml +#@ passwd = "..." +test1: #@ passwd or assert.fail("password must be set") +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-list.md b/site/content/ytt/docs/v0.38.0/lang-ref-list.md new file mode 100644 index 000000000..33661f57b --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-list.md @@ -0,0 +1,60 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-list] +title: Lists +--- + +```yaml +#@ nums = [123, 374, 490] +first: #@ nums[0] +``` + +Copied here for convenience from [Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#listappend). + +- [list·append](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·append) (`L.append(x)`) +```python +x = [] +x.append(1) # None +x.append(2) # None +x.append(3) # None +x # [1, 2, 3] +``` + +- [list·clear](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·clear) (`L.clear()`) +```python +x = [1, 2, 3] +x.clear() # None +x # [] +``` + +- [list·extend](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·extend) (`L.extend(x)`) +```python +x = [] +x.extend([1, 2, 3]) # None +x.extend(["foo"]) # None +x # [1, 2, 3, "foo"] +``` + +- [list·index](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·index) (`L.index(x[, start[, end]])`) +```python +x = list("banana".codepoints()) +x.index("a") # 1 (bAnana) +x.index("a", 2) # 3 (banAna) +x.index("a", -2) # 5 (bananA) +``` + +- [list·insert](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·insert) (`L.insert(i, x)`) +```python +x = ["b", "c", "e"] +x.insert(0, "a") # None +x.insert(-1, "d") # None +x # ["a", "b", "c", "d", "e"] +``` + +- [list·pop](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·pop) (`L.pop([index])`) +- [list·remove](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·remove) (`L.remove(x)`) +```python +x = [1, 2, 3, 2] +x.remove(2) # None (x == [1, 3, 2]) +x.remove(2) # None (x == [1, 3]) +x.remove(2) # error: element not found +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-load.md b/site/content/ytt/docs/v0.38.0/lang-ref-load.md new file mode 100644 index 000000000..31c746323 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-load.md @@ -0,0 +1,115 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-load] +title: Load Statements +--- + +## Terminology + +- `module`: single file; can export variables, functions, or be templated => some type of result e.g. yaml structure, or string, or None) +- `package`: single directory; contains modules +- `library`: collection of packages + +## Usage + +Load statement allows to load functions from other modules (such as ones from [builtin `ytt` library](lang-ref-ytt.md)). + +- [load](https://github.com/google/starlark-go/blob/master/doc/spec.md#load-statements) +```python +load("@ytt:overlay", "overlay") # load overlay module from builtin ytt library +load("@ytt:overlay", ov="overlay") # load overlay symbol under a different alias +load("helpers.star", "func1", "func2") # load func1, func2 from Starlark file +load("helpers.lib.yml", "func1", "func2") # load func1, func2 from YAML file +load("helpers.lib.txt", "func1", "func2") # load func1, func2 from text file +load("/dir/helpers.lib.yml", "func1") # load func1 from file relative to root of library +load("sub-dir/helpers.lib.txt", "func1") # load func1 from a sub-directory +load("@project:dir/helpers.lib.txt", "func1") # load func1 from a project located under _ytt_lib +``` + +`load` arguments are as follows: + +1. location which takes following shape `[@[library]:][package/]{0,n}module`, where, + - `library` could be `ytt` or local path under `_ytt_lib` directory + - examples: [`ytt`](lang-ref-ytt.md), `github.com/k14s/k8s-lib`, `common` + - `package` could be a directory path + - examples: `overlay`, `regexp`, `app/`, `/app/something` + - `module` is a file name or predefined name (included in `ytt` library) + - examples: `module.lib.yml` +1. one or more symbols to import with optional aliases + - examples: `func1`, `func1="as_func1"` + +Files can be loaded from current or child directories. As of ytt v0.24.0, `/` package prefix can be used to load files relative to the root of the current library. + +Note that there is a distinction between using `load("@project:dir/helpers.lib.txt", "func1")` or `load("@project/dir:helpers.lib.txt", "func1")`, in that, `:` signifies what ytt considers a self-contained library (i.e. is `dir` simply a package in `project` or `project/dir` a standalone library). This allows files to load other files relative to the library root. + +To load a set of functions from a single file, you can create a `struct` that contains references to the functions. For example: +`funcs.star`: +``` +load("@ytt:struct", "struct") + +def testfunc(): + return 123 +end + +def otherfunc(): + return 456 +end + +mod = struct.make(testfunc=testfunc, otherfunc=otherfunc) +``` + +`config.yml`: +``` +#@ load("funcs.star", "mod") + +result: #@ mod.testfunc() +other_result: #@ mod.otherfunc() +``` + +## _ytt_lib directory + +`_ytt_lib` directory allows to keep private dependencies from consumers of libraries. + +For example given following directory structure: + +``` +app1.yml +_ytt_lib/big-corp/sre.lib.yml +_ytt_lib/big-corp/_ytt_lib/big-corp/common/deployments.lib.yml +_ytt_lib/big-corp/_ytt_lib/big-corp/common/services.lib.yml +``` + +- `app1.yml` _can_ load `big-corp/sre.lib.yml` via `@big-corp:sre.lib.yml` +- `app1.yml` _cannot_ load `big-corp/_ytt_lib/big-corp/common/services.lib.yml` as it is a private dependency of anything inside `_ytt_lib/big-corp/` directory (e.g. `sre.lib.yml`) + +hence making it possible for `big-corp/sre.lib.yml` module to keep its `big-corp/common` library dependency private. +## Files + +To make files available to `load` statement they have to be given to ytt CLI via `--file` (`-f`) option. The argument of that option can be a path to either of: + +- a **file**: in which case the file can be loaded by its name. +- a **directory**: in which case all the files found can be loaded by using paths relative to the directory. If the directory contains a `_ytt_lib` folder, then libraries in it can also be loaded. + +For example, given following directory structure: + +``` +app1.yml +helpers.lib.yml +_ytt_lib/apps/apps.lib.yml +sub-dir/more-helpers.lib.yml +sub-dir/_ytt_lib/weird-lib/funcs.lib.yml +``` + +- `ytt -f .` will make it possible for `app1.yml` to load: + - `helpers.lib.yml` + - `@apps:apps.lib.yml` + - `sub-dir/more-helpers.lib.yml` +- `ytt -f helpers.lib.yml -f sub-dir -f app1.yml` will make it possible for `app1.yml` to load: + - `helpers.lib.yml` + - `more-helpers.lib.yml` (not `sub-dir/more-helpers.lib.yml`) + - `@weird-lib:funcs.lib.yml` + +## Examples + +- [Load](/ytt/#example:example-load) +- [Load ytt library](/ytt/#example:example-load-ytt-library) +- [Load custom library](/ytt/#example:example-load-custom-library) diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-string.md b/site/content/ytt/docs/v0.38.0/lang-ref-string.md new file mode 100644 index 000000000..4be6bbf5b --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-string.md @@ -0,0 +1,130 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-string] +title: Strings +--- + +```yaml +name1: #@ name + "-deployment" +name2: #@ "{}-deployment".format("name") +``` + +Copied here for convenience from [Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#stringelem_ords). + +- [string·elem_ords](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·elem_ords) +- [string·capitalize](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·capitalize) (`S.capitalize()`) +```python +"hello, world!".capitalize() # "Hello, world!"` +``` + +- [string·codepoint_ords](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·codepoint_ords) +- [string·count](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·count) (`S.count(sub[, start[, end]])`) +```python +"hello, world!".count("o") # 2 +"hello, world!".count("o", 7, 12) # 1 (in "world") +``` + +- [string·endswith](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·endswith) (`S.endswith(suffix[, start[, end]])`) +```python +"filename.star".endswith(".star") # True +'foo.cc'.endswith(('.cc', '.h')) # True +``` + +- [string·find](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·find) (`S.find(sub[, start[, end]])`) +```python +"bonbon".find("on") # 1 +"bonbon".find("on", 2) # 4 +"bonbon".find("on", 2, 5) # -1 +``` + +- [string·format](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·format) (`S.format(*args, **kwargs)`) +```python +"a{x}b{y}c{}".format(1, x=2, y=3) # "a2b3c1" +"a{}b{}c".format(1, 2) # "a1b2c" +"({1}, {0})".format("zero", "one") # "(one, zero)" +``` + +- [string·index](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·index) (`S.index(sub[, start[, end]])`) +```python +"bonbon".index("on") # 1 +"bonbon".index("on", 2) # 4 +"bonbon".index("on", 2, 5) # error: substring not found (in "nbo") +``` + +- [string·isalnum](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isalnum) +```python +"base64".isalnum() # True +"Catch-22".isalnum() # False +``` + +- [string·isalpha](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isalpha) +- [string·isdigit](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isdigit) +- [string·islower](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·islower) +- [string·isspace](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isspace) +- [string·istitle](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·istitle) +- [string·isupper](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isupper) +- [string·join](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·join) (`S.join(iterable)`) +```python +", ".join(["one", "two", "three"]) # "one, two, three" +"a".join("ctmrn".codepoints()) # "catamaran" +``` + +- [string·lower](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·lower) +- [string·lstrip](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·lstrip) +```python +" hello ".lstrip() # "hello " +``` + +- [string·partition](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·partition) +- [string·replace](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·replace) (`S.replace(old, new[, count])`) +```python +"banana".replace("a", "o") # "bonono" +"banana".replace("a", "o", 2) # "bonona" +``` + +- [string·rfind](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rfind) (`S.rfind(sub[, start[, end]])`) +```python +"bonbon".rfind("on") # 4 +"bonbon".rfind("on", None, 5) # 1 +"bonbon".rfind("on", 2, 5) # -1 +``` + +- [string·rindex](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rindex) (`S.rindex(sub[, start[, end]])`) +- [string·rpartition](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rpartition) +- [string·rsplit](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rsplit) +- [string·rstrip](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rstrip) (`S.rstrip()`) +```python +" hello ".rstrip() # " hello" +``` + +- [string·split](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·split) (`S.split([sep [, maxsplit]])`) +```python +"one two three".split() # ["one", "two", "three"] +"one two three".split(" ") # ["one", "two", "", "three"] +"one two three".split(None, 1) # ["one", "two three"] +"banana".split("n") # ["ba", "a", "a"] +"banana".split("n", 1) # ["ba", "ana"] +``` + +- [string·elems](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·elems) +- [string·codepoints](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·codepoints) +- [string·splitlines](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·splitlines) (`S.splitlines([keepends])`) +```python +"one\n\ntwo".splitlines() # ["one", "", "two"] +"one\n\ntwo".splitlines(True) # ["one\n", "\n", "two"] +``` + +- [string·startswith](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·startswith) (`S.startswith(prefix[, start[, end]])`) +```python +"filename.star".startswith("filename") # True` +``` + +- [string·strip](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·strip) (`S.strip()`) +```python +" hello ".strip() # "hello" +``` + +- [string·title](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·title) +- [string·upper](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·upper) (`S.upper()`) +```python +"Hello, World!".upper() # "HELLO, WORLD!" +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-structs.md b/site/content/ytt/docs/v0.38.0/lang-ref-structs.md new file mode 100644 index 000000000..aa4ca708c --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-structs.md @@ -0,0 +1,94 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-structs] +title: Structs +--- + +## Overview + +Structs are well-defined data objects, comprised of key/value pairs known as "attributes". They are a way to store and refer to data of a known structure. + +The most commonly used `struct` is `data.values`, supplied by the [`@ytt:data`](ytt-data-values.md) module. For example, a data values defined by: + +```yaml +#@data/values +--- +db_conn: + host: acme.example.com +``` + +is automatically processed into a `struct` (named `values`): the keys in the `@data/values` file are defined one-for-one as attributes on the `struct`. + +Those attribues can be referenced by name: + +```yaml +#@ load("@ytt:data", "data") +--- +persistence: + db_url: #@ data.values.db_conn.host +``` + +--- +## Attributes + +Attributes are a key/value pair, where the key is a `string` and the value can be of any type. + +Attributes can be referenced: +- by field (using "[dot notation](https://github.com/google/starlark-go/blob/master/doc/spec.md#dot-expressions)") + ```yaml + db_url: #@ data.values.db_conn.host + ``` +- by string (using "[index notation](https://github.com/google/starlark-go/blob/master/doc/spec.md#index-expressions)") _(as of v0.31.0)_: + ```yaml + db_url: #@ data.values["db_conn"]["host"] + ``` + useful when the name of the attribute is not known statically. + +Referencing an attribute that is not defined on the `struct` results in an error: +```yaml +db_url: #@ data.values.bd_conn # <== struct has no .bd_conn field or method +db_host: #@ data.values["db_conn"]["hast"] # <== key "hast" not in struct +``` + +--- +## Built-in Functions + +The following built-ins can be useful with `struct` values: + +- [`dir()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#dir) enumerates all its attributes. + ``` + load("@ytt:struct", "struct") + + foo = struct.encode({"a": 1, "b": 2, "c": 3}) + keys = dir(foo) # <== ["a", "b", "c"] + ``` + +- [`getattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#getattr) can be used to select an attribute. + ```yaml + #@ for vpc_config in getattr(getattr(data.values.accounts, foundation_name), vpc_name): + ... + #@ end + ``` + - as of v0.31.0, `struct`s support [index notation](https://github.com/google/starlark-go/blob/master/doc/spec.md#index-expressions) behaving identically, more succinctly/readably: + ```yaml + #@ for vpc_config in data.values.accounts[foundation_name][vpc_name]: + ... + #@ end + ``` + +- [`hasattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#hasattr) reports whether a value has a specific attribute + ```python + # `value` is a struct that _might_ have a field, "additional_ports" + + def ports(value): + if hasattr(value, "additional_ports"): + ... + end + ``` + +--- +## Creating structs + +`struct` instances can be made using the `@ytt:struct` module. + +See [ytt Library: struct module](lang-ref-ytt-struct.md) for details. + diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-yaml-fragment.md b/site/content/ytt/docs/v0.38.0/lang-ref-yaml-fragment.md new file mode 100644 index 000000000..547c8a7b8 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-yaml-fragment.md @@ -0,0 +1,104 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-yaml-fragment] +title: YAMLFragments +--- + +## Overview + +YAMLFragment is a type of value that is defined directly in YAML (instead of plain Starlark). For example, function `val()` returns a value of type `yamlfragment`. + +```yaml +#@ def vals(): +key1: val1 +key2: + subkey: val2 +#@ end +``` + +YAMLFragment may contain: + +- YAML document set (array of YAML documents) +- YAML array +- YAML map +- null + +Given various contents it wraps, YAMLFragment currently exposes limited ways of accessing its contents directly. Following accessors are available in v0.26.0+. + +## YAML Document Set + +```yaml +#@ def docs(): +--- +doc1 +--- +doc2 +--- +doc3 +#@ end +``` + +- access contents of a document at particular index +```python +docs()[1] # returns "doc2" +``` + +- loop over each document, setting `val` to its contents +```python +for val in docs(): + val # ... +end +``` + +## YAML Array + +```yaml +#@ def vals(): +- val1 +- val2 +#@ end +``` + +- access contents of an array item at particular index +```python +vals()[1] # returns "val2" +``` + +- loop over each array item, setting `val` to its contents +```python +for val in vals(): + val # ... +end +``` + +## YAML Map + +```yaml +#@ def vals(): +key1: val1 +key2: + subkey: val2 +#@ end +``` + +- access contents of a map item with particular key +```python +vals()["key1"] # returns "val1" +``` + +- check if map contains particular key +```python +"key1" in vals() # returns True +"key6" in vals() # returns False +``` + +- loop over each map item, setting `val` to its contents +```python +for key in vals(): + val = vals()[key] # ... +end +``` + +- convert to a dictionary +```python +dict(**vals()) # returns {"key1": "val1", "key2": yamlfragment({"subkey": "val2"})} +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt-library.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-library.md new file mode 100644 index 000000000..e33355ead --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-library.md @@ -0,0 +1,389 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-library] +title: Library Module +--- + +## Overview + +You can extract a whole set of input files (i.e. templates, overlays, data values, etc.) into a "Library". + +For example: + +```yaml +config/ +├── _ytt_lib/ +│ └── frontend/ +│ ├── schema.yml +│ └── store.yml +└── config.yml +``` +where: +- `config/_ytt_lib/frontend/` and its contents is a library named `"frontend"` + +Libraries are _not_ automatically included in `ytt` output; one must programmatically load, configure, evaluate, and insert those results into a template that is part of the output. + +```yaml +#! config/config.yml -- example of using a library + +#@ load("@ytt:library", "library") +#@ load("@ytt:template", "template") + +#! 1. Load an instance of the library +#@ app = library.get("frontend") + +#! 2. Create a configured copy of the library (does not mutate original) +#@ app_with_vals = app.with_data_values({"apiDomain": "gateway.example.com"}) + +#! 3. Evaluate the library and include results (a document set) in the output +--- #@ template.replace(app_with_vals.eval()) +``` + +For a complete working example, see [ytt-library-module example](/ytt/#example:example-ytt-library-module). + +## What is a Library? + +A `ytt` library is a directory tree contained within a specially-named directory: [`_ytt_lib/`](lang-ref-load.md#_ytt_lib-directory). +- The library's _name_ is the path relative from the `_ytt_lib/` directory. +- The library's _contents_ are those of the directory along with subdirectories, recursively. +- A library may contain libraries as well, if one of its subdirectories is `_ytt_lib/`. + +The root directory of a `ytt` invocation is itself a library known as the "root library". + +Libraries are evaluated in isolation: each a separate execution of the pipeline described in [How it works](how-it-works.md). +- Each library has its own data values schema. +- Overlays within a library only apply over _its_ evaluated document set. +- The final evaluated result is returned as a [YAML Fragment wrapping a document set.](lang-ref-yaml-fragment.md#yaml-document-set). + +--- + +## Functions + +There's but one function in the `@ytt:library` module: [`library.get()`](#libraryget) + +### library.get() + +Contructs a new [`@ytt:library.instance`](#library-instances) based on the contents from the named library. + +```python +instance = library.get(name, []) +``` + +- **`name`** (`string`) — path to the base directory of the desired library: `./_ytt_lib/`. Can contain slashes `/` for sub-directories (e.g. `github.com/vmware-tanzu/carvel-ytt-library-for-kubernetes/app`) +- keyword arguments (optional): + - **`alias=`** (`string`) — unique name for this library instance. See [Aliases](#aliases), below. + - **`ignore_unknown_comments=`** (`bool`) — equivalent to `ytt --ignore-unknown-comments`; see [File Marks > type detection for YAML files](file-marks.md#type-detection-for-yaml-files) for more details (default: `False`). (as of v0.31.0) + - **`implicit_map_key_overrides=`** (`bool`) — equivalent to `ytt --implicit-map-key-overrides`; see [@yaml/map-key-override](lang-ref-annotation.md#yaml-templating-annotations) for more details. (default: `False`). (as of v0.31.0) + - **`strict=`** (`bool`) — equivalent to `ytt --strict` (default: `False`). (as of v0.31.0) +- **`instance`** ([`@ytt:library.instance`](#library-instances)) — a new library instance backed by the contents of the named library. + +The file containing this method invocation must be a sibling of the [`_ytt_lib` directory](lang-ref-load.md#_ytt_lib-directory). + +--- + +## Library Instances + +Each library returned from a function within this module is a copy: a separate instance. + +A library instance (a value of type `@ytt:library.instance`) is created from source with [`library.get()`](#libraryget). + +With a library instance: +- create configured copies using: + - [`instance.with_data_values_schema()`](#instancewith_data_values_schema) + - [`instance.with_data_values()`](#instancewith_data_values) +- evaluate its contents via [`instance.eval()`](#instanceeval) +- fetch values from it using: + - [`instance.data_values()`](#instancedata_values) for the final data values for the library + - [`instance.export()`](#instanceexport) to access its functions and variables + +### instance.data_values() + +Calculates and returns just the Data Values configured on this library instance. + +```python +dvs = instance.data_values() +``` + +- **`dvs`** ([`struct`](lang-ref-structs.md)) — the final data values (i.e. the net result of [all configured data values](ytt-data-values.md#library-data-values)). + +### instance.eval() + +Calculates the library's final data values (i.e. the net result of [all configured data values](ytt-data-values.md#library-data-values)), evaluates its templates into a document set, and applies its overlays on that document set (i.e. executes the pipeline described in [How it works](how-it-works.md) for this library instance's inputs and contents). + +```python +document_set = instance.eval() +``` + +- **`document_set`** ([`yamlfragment`](lang-ref-yaml-fragment.md#yaml-document-set)) — the YAML document set resulting from the evaluation of this instance. + +### instance.export() + +(As of v0.28.0) + +Returns the value of an identifier declared within the library instance. + +```python +value = instance.export(name, [path=]) +``` + +- **`name`** (`string`) — the name of a function or a variable declared within some [module](lang-ref-load.md#terminology)/file in the library. (i.e. a file with the extension `.lib.yml` or `.star`). +- **`path=`** (`string`) — the path to the module/file that contains the declaration. Only required when `name` is not unique within the library. +- **`value`** (any) — a copy of the specified value. + - if `value` is a function, it is executed within the context of _its_ library instance. For example, if the function depends on values from the [`@ytt:data`](lang-ref-ytt.md#data) module, the values provided are those of this library instance. + +**Examples:** + +_Example 1: Exporting a function from a library._ + +Assuming some module/file in the "helpers" library contains the definition: + +```python +... +def wrap_name(name): + ... +end +... +``` + +Can be exported and used from another library: + +```python +helpers = library.get("helpers") +wrap_name = helpers.export("wrap_name") + +full_name = wrap_name("app") +``` + +_Example 2: Disambiguating between multiple declarations of function._ + +Assuming two modules/files in the "helpers" library have the same name: + +```python +# main/funcs.star +def wrap_name(name): ... +``` +and +```python +# lib/funcs.star +def wrap_name(name): ... +``` + +One of which can be unambiguously referenced: + +```python +helpers = library.get("helpers") +wrap_name = helpers.export("wrap_name", path="lib/funcs.star") + +full_name = wrap_name("app") +``` + +Note: without the `path=` keyword argument, `helpers.export()` would report an error. + +### instance.with_data_values() + +Returns a copy of the library instance with data values overlayed with those given. + +```python +new_instance = instance.with_data_values(dvs, [plain=]) +``` + +- **`dvs`** (`struct` | [`yamlfragment`](lang-ref-yaml-fragment.md)) — data values with which to overlay (or set, if none exist). + - only `yamlfragment`s wrapping a map or an array are supported (i.e. `yamlfragment`s wrapping document sets are not supported). + - `yamlfragment` values _can_ contain [overlay annotations](lang-ref-ytt-overlay.md#overlay-annotations) for fine-grained overlay control. +- **`plain=`** (`bool`) — when `True` indicates that `dvs` should be "plain merged" over existing data values (i.e. the exact same behavior as [`--data-values-file`](ytt-data-values.md#configuring-data-values-via-command-line-flags)). + - `dvs` must be plain YAML (i.e. a `struct` or a `yamlfragment` with no annotations). +- **`new_instance`** ([`@ytt:library.instance`](#library-instances)) — a copy of `instance` with `dvs` overlayed on its data values; `instance` remains unchanged. + +### instance.with_data_values_schema() + +(As of v0.35.0) + +Returns a copy of the library instance with data values schema overlayed with that given. + +```python +new_instance = instance.with_data_values_schema(schema) +``` + +- **`schema`** (`struct` | [`yamlfragment`](lang-ref-yaml-fragment.md)) — schema for data values with which to overlay on existing schema (or set if none exist). + - only `yamlfragment`s wrapping a map or an array are supported (i.e. `yamlfragment`s wrapping document sets are not supported) + - `yamlfragment` values _can_ contain [overlay annotations](lang-ref-ytt-overlay.md#overlay-annotations) for fine-grained overlay control. +- **`new_instance`** ([`@ytt:library.instance`](#library-instances)) — a copy of `instance` with a schema updated with `schema`; `instance` remains unchanged. + +**Examples:** + +_Example 1: Declaring a new data value (and setting it)._ + +```yaml +#@ def app_schema(): +name: "" +#@overlay/match missing_ok=True +env_vars: + custom_key: "" +#@ end + +#@ app1_with_schema = app1.with_data_values_schema(app_schema()) +--- +#@ def app_vals(): +name: app1 +env_vars: + custom_key: some_val +#@ end + +#@ app1_with_vals = app1.with_data_values(app_vals()) +``` + +--- + +## Annotations + +### @library/ref + +(As of v0.28.0) + +Attaches a YAML document to the specified library. When the library is evaluated, the annotated document is included. +Only supported on documents annotated with `@data/values` and `@data/values-schema`. + +``` +@library/ref library_name +``` +- **`library_name`** (`string`) — `@`-prefixed path to the base directory of the desired library: `./_ytt_lib/`. Can contain slashes `/` for sub-directories (e.g. `github.com/vmware-tanzu/carvel-ytt-library-for-kubernetes/app`). Can also be an [alias](#aliases) for specific library instance(s). + +**Examples:** + +_Example 1: Change schema default for a data value in a library._ + +```yaml +#@data/values-schema +#@library/ref "@frontend" +--- +name: "custom" +``` + +Overlays the default value for `name` in the "frontend" library to be "custom". + +_Example 2: Target a data value overlay to a library._ + +```yaml +#@data/values +#@library/ref "@backend" +--- +#@overlay/replace +domains: +- internal.example.com +- internal-backup.example.com +``` + +Sets the "backend" library's `domains` data value to be exactly the values given. + +See also: [Data Values > Setting Library Values via Files](ytt-data-values.md#setting-library-values-via-files). + +Note: data values may also be attached to libraries via [command line flags](ytt-data-values.md#setting-library-values-via-command-line-flags). + +--- + +## Aliases + +To facilitate configuring specific library instances, one can mark them with an alias. + +An alias: +- is defined in a [`library.get()`](#libraryget) call, using the optional `alias=` keyword argument. +- is added to a library reference by prefixing it with a tilde, `~`: + - `@~` refers to _any_ library instance with the alias. + - `@~` refers to any instance of the named library that _also_ has the alias. + +For example, given a library known as "fruit": + +``` +├── apple-values.yml +├── config.yml +├── orange-values.yml +└── _ytt_lib + └── fruit + ├── doc.yml + └── values.yml +``` + +where: +```yaml +#! _ytt_lib/fruit/doc.yml + +#@ load("@ytt:data", "data") +--- #@ data.values +``` +the template in the library simply returns its data values as a document, and ... +```yaml +#! _ytt_lib/fruit/values.yml + +#@data/values +--- +variety: ordinary +poisoned: false +``` +... those are the data values in the library. + +The root library can assign aliases to library instances: + +```yaml +#! ./config.yml + +#@ load("@ytt:library", "library") + +#@ apple1 = library.get("fruit", alias="apple") +#@ apple2 = apple1.with_data_values({"variety": "jonamac"}) + +#@ orange = library.get("fruit", alias="orange") + +--- +apple: + 1: #@ apple1.eval()[0] + 2: #@ apple2.eval()[0] +orange: #@ orange.eval()[0] +``` +where: +- `apple1` has the alias "apple" +- `apple2` also has the alias "apple" (part of being a copy of `apple1`) +- `orange` has the alias "orange" + +These aliases can be used to target changes to specific library instance(s). + +For example, our root library has these two data values overlays: + +```yaml +#! ./apple-values.yml + +#@data/values +#@library/ref "@~apple" +--- +variety: red delicious +poisoned: true +``` +... which will affect all library instances with the alias "apple", and ... + +```yaml +#! ./orange-values.yml + +#@data/values +#@library/ref "@~orange" +--- +variety: valencia +``` + +... overlays on top of library instance with the alias "orange". + +When the whole fileset is evaluated, the result is: + +```yaml +apple: + 1: + variety: red delicious + poisoned: true + 2: + variety: jonamac + poisoned: true +orange: + variety: valencia + poisoned: false +``` + +notice: +- only the "@~orange" instance has the variety = "valencia" +- both "@~apple" library instances are poisoned; while the "orange" instance is not. diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt-overlay.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-overlay.md new file mode 100644 index 000000000..8f3224825 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-overlay.md @@ -0,0 +1,733 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-overlay] +title: Overlay module +--- + +## Overview + +`ytt`'s Overlay feature provides a way to combine YAML structures together with the help of annotations. + +There are two (2) structures involved in an overlay operation: +- the "left" — the YAML document(s) (and/or contained maps and arrays) being modified, and +- the "right" — the YAML document (and/or contained maps and arrays) that is the overlay, describing the modification. + +Each modification is composed of: +- a matcher (via an [`@overlay/(match)`](#matching-annotations) annotation), identifying which node(s) on the "left" are the target(s) of the edit, and +- an action (via an [`@overlay/(action)`](#action-annotations) annotation), describing the edit. + +Once written, an overlay can be applied in one of two ways: + +- on all rendered templates, [declaratively, via YAML documents annotated with `@overlay/match`](#overlays-as-files); this is the most common approach. +- on selected documents, [programmatically, via `overlay.apply()`](#programmatic-access). + +--- +## Overlays as files + +As `ytt` scans input files, it pulls aside any YAML Document that is annotated with `@overlay/match`, and considers it an overlay. + +After YAML templates are rendered, the collection of identified overlays are applied. Each overlay executes, one-at-a-time over the entire set of the rendered YAML documents. + +Order matters: modifications from earlier overlays are seen by later overlays. Overlays are applied in the order detailed in [Overlay order](#overlay-order), below. + +In the example below, the last YAML document is an overlay (it has the `@overlay/match` annotation). +That overlay matcher's selects the first YAML document *only*: it's the only one that has a `metadata.name` of `example-ingress`. + +```yaml +#@ load("@ytt:overlay", "overlay") + +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: example-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: another-example-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / + +#@overlay/match by=overlay.subset({"metadata":{"name":"example-ingress"}}) +--- +metadata: + annotations: + #@overlay/remove + ingress.kubernetes.io/rewrite-target: +``` + +yields: + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: example-ingress + annotations: {} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: another-example-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / +``` + +See also: [Overlay files example](/ytt/#example:example-overlay-files) in online playground. + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +### Overlay order + +(as of v0.13.0) + +Overlays are applied, in sequence, by: + +1. left-to-right for file flags + - e.g. in `-f overlay1.yml -f overlay2.yml`, `overlay1.yml` will be applied first +1. if file flag is set to a directory, files are alphanumerically sorted + - e.g. in `aaa/z.yml xxx/c.yml d.yml`, will be applied in following order `aaa/z.yml d.yml xxx/c.yml` +1. top-to-bottom order for overlay YAML documents within a single file + +### Next Steps + +Familiarize yourself with the [overlay annotations](#overlay-annotations). + +--- +## Programmatic access + +Overlays need not apply to the entire set of rendered YAML documents (as is the case with [the declarative approach](#overlays-as-files)). + +Instead, the declared modifications can be captured in a function and applied to a specific set of documents via [`overlay.apply()`](#overlayapply) in Starlark code. + +In this example we have `left()` function that returns target structure and `right()` that returns the overlay, specifying the modifications. + +`overlay.apply(...)` will execute the execute the overlay and return a new structure. + +```yaml +#@ load("@ytt:overlay", "overlay") + +#@ def left(): +key1: val1 +key2: + key3: + key4: val4 + key5: + - name: item1 + key6: val6 + - name: item2 + key7: val7 +#@ end + +#@ def right(): +#@overlay/remove +key1: val1 +key2: + key3: + key4: val4 + key5: + #@overlay/match by="name" + - name: item2 + #@overlay/match missing_ok=True + key8: new-val8 +#@ end + +result: #@ overlay.apply(left(), right()) +``` + +yields: + +```yaml +result: + key2: + key3: + key4: val4 + key5: + - name: item1 + key6: val6 + - name: item2 + key7: val7 + key8: new-val8 +``` + +### Next Steps + +Familiarize yourself with the two kinds of overlay annotations: +- matchers (via an [`@overlay/(match)`](#matching-annotations) annotation), and +- actions (via an [`@overlay/(action)`](#action-annotations) annotation). + +--- +## `@overlay` Annotations + +There are two groups of overlay annotations: + +- [Matching Annotations](#matching-annotations) +- [Action Annotations](#action-annotations) + +--- +## Matching Annotations + +These annotations are used to select which structure(s) will be modified: + +- [@overlay/match](#overlaymatch) +- [@overlay/match-child-defaults](#overlaymatch-child-defaults) + +### @overlay/match + +Specifies which nodes on the "left" to modify. + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/match [by=Function|String, expects=Int|String|List|Function, missing_ok=Bool, when=Int|String|List] +``` +- **`by=`**`Function|String` — criteria for matching nodes on the "left" + - `Function` — predicate of whether a given node is a match + - provided matcher functions (supplied by the `@ytt:overlay` module): + - [`overlay.all()`](#overlayall) + - [`overlay.subset()`](#overlaysubset) + - [`overlay.index()`](#overlayindex) + - [`overlay.map_key()`](#overlaymap_key) + - [Custom matcher function](#custom-overlay-matcher-functions) can also be used + - `String` — short-hand for [`overlay.map_key()`](#overlaymap_key) with the same argument + - Defaults (depends on the type of the annotated node): + - document or array item: none (i.e. `by` is required) + - map item: key equality (i.e. [`overlay.map_key()`](#overlaymap_key)) +- **`expects=`**`Int|String|List|Function` — (optional) expected number of nodes to be found in the "left." If not satisfied, raises an error. + - `Int` — must match this number, exactly + - `String` (e.g. `"1+"`) — must match _at least_ the number + - `Function(found):Bool` — predicate of whether the expected number of matches were found + - `found` (`Int`) — number of actual matches + - `List[Int|String|Function]` — must match one of the given criteria + - Default: `1` (`Int`) (i.e. expecting to match exactly one (1) node on the "left"). +- **`missing_ok=`**`Bool` (optional) shorthand syntax for `expects=[0,1]` +- **`when=`**`Int|String|List` (optional) criteria for when the overlay should apply. If the criteria is met, the overlay applies; otherwise, nothing happens. + - `Int` — must equal this number, exactly + - `String` (e.g. `"1+"`) — must match _at least_ the number + - `List[Int|String]` — must match one of the given criteria + +**Notes:** +- `expects`, `missing_ok`, and `when` are mutually-exclusive parameters. +- take care when `expects` includes zero (0); matching none is indistinguishable from a mistakenly written match (e.g. a misspelling of a key name) + +**Examples:** + +- `#@overlay/match by="id"`: expects to find one (1) node on the "left" that has the key `id` and value matching the same-named item on the "right." +- `#@overlay/match by=`[`overlay.map_key("name")`](#overlaymap_key): (same as above) +- `#@overlay/match by=`[`overlay.all`](#overlayall)`,expects="0+"`: has no effective matching expectations +- `#@overlay/match missing_ok=True`: expects to find 0 or 1 matching nodes on the "left" +- `#@overlay/match expects=2`: expects to find exactly two (2) matching nodes on the "left" +- `#@overlay/match expects="2+"`: expects to find two (2) or more matching nodes on the "left" +- `#@overlay/match expects=[0,1,4]`: expects to find 0, 1 or 4 matching nodes on the "left" +- `#@overlay/match expects=lambda x: return x < 10`: expects 9 or fewer matching nodes on the "left" +- `#@overlay/match when=2`: applies changes only if two (2) nodes are found on the "left" and will not error otherwise +- `#@overlay/match when=2+`: applies changes only if two (2) or more nodes are found on the "left" and will not error otherwise +- `#@overlay/match when=[0,1,4]`: applies changes only if there were exactly 0, 1 or 4 nodes found on the "left" and will not error otherwise + +**History:** +- v0.28.0+ — added `when` keyword argument. + +#### Custom Overlay Matcher Functions + +The matcher functions from `@ytt:overlay` cover many use-cases. From time-to-time, more precise matching is required. + +A matcher function has the following signature: + +`Function(indexOrKey,left,right):Boolean` + - `indexOrKey` + - when `left` is an array item: (`Int`) — the potential match's _zero-based index_ in the list that is the left-side + - when `left` is a map item: (`any`) — the potential match's _key_ in the map that is the left-side + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the potential match/target node + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node in the overlay + - returns `True` if `left` should be considered a match; `False` otherwise. + +Most custom matchers can be written as a Lambda expression. + +Lambda expressions start with the keyword `lambda`, followed by a parameter list, then a `:`, and a single expression that is the body of the function. + +**Examples:** + +_Example 1: Key presence or partial string match_ + +`left` contains `right`: +```python +lambda indexOrKey, left, right: right in left +``` +Returns `True` when `right` is "a member of" `left` + +(see also: [Starlark Spec: Membership tests](https://github.com/google/starlark-go/blob/master/doc/spec.md#membership-tests) for more details) + +_Example 2: Precise string matching_ + +`left` contains a key of the same name as the value of `right`: +```python +lambda indexOrKey, left, right: left["metadata"]["name"].endswith("server") +``` +See also: +- [Language: String](lang-ref-string.md) for more built-in functions on strings. +- [@ytt:regexp Library](lang-ref-ytt.md#regexp) for regular expression matching. + +_Example 3: Match on both key and value_ + +Is the map item with key/value of: `type: LoadBalancer`. + +```python +lambda indexOrKey, left, right: indexOrKey == "type" and left == "LoadBalancer" +``` + +### @overlay/match-child-defaults + +Sets default values for `expects`, `missing_ok`, or `when` for the children of the annotated node. +Does not set these values for the annotated node, itself. + +Commonly used to avoid repeating `@overlay/match missing_ok=True` on each child node in maps. + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/match-child-defaults [expects=Int|String|List|Function, missing_ok=Bool, when=Int|String|List] +``` + +_(see [@overlay/match](#overlaymatch) for parameter specifications.)_ + +**Examples:** + +Without the `#@overlay/match-child-defaults`, _each_ of the four new annotation would have needed an `@overlay/match missing_ok=True` to apply successfully: +```yaml +--- +metadata: + annotations: + ingress.kubernetes.io/rewrite-target: / + +#@overlay/match by=overlay.all +--- +metadata: + #@overlay/match-child-defaults missing_ok=True + annotations: + nginx.ingress.kubernetes.io/limit-rps: 2000 + nginx.ingress.kubernetes.io/enable-access-log: "true" + nginx.ingress.kubernetes.io/canary: "true" + nginx.ingress.kubernetes.io/client-body-buffer-size: 1M +``` + +--- +## Action Annotations + +The following annotations describe how to modify the matched "left" node. + +They are: +- [`@overlay/merge`](#overlaymerge) — (default) combine left and right nodes +- [`@overlay/remove`](#overlayremove) — delete nodes from left +- [`@overlay/replace`](#overlayreplace) — replace the left node +- [`@overlay/insert`](#overlayinsert) — insert right node into left +- [`@overlay/append`](#overlayappend) — add right node at end of collection on left +- [`@overlay/assert`](#overlayassert) — declare an invariant on the left node + +### @overlay/merge + +Merge the value of "right" node with the corresponding "left" node. + +**Valid on:** Map Item, Array Item. + +``` +@overlay/merge +``` +_(this annotation has no parameters.)_ + +**Note:** This is the default action; for each node in an overlay, either the action is explicitly specified or it is `merge`. + +### @overlay/remove + +Deletes the matched "left" node. + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/remove +``` +_(this annotation has no parameters.)_ + +### @overlay/replace + +Substitutes matched "left" node with the value of the "right" node (or by that of a provided function). + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/replace [via=Function] +``` + +- **`via=`**`Function(left, right): (any)` _(optional)_ determines the value to substitute in. If omitted, the value is `right`. + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the matched node's value + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node + +**History:** +- v0.26.0 — works with [`yamlfragment`](lang-ref-yaml-fragment.md) values. + +**Examples:** + +_Example 1: Use value from "right"_ + +Replaces the corresponding "left" with the value `"v1"` +```yaml +#@overlay/replace +apiVersion: v1 +``` + +_Example 2: Edit string value_ + +```yaml +#@overlay/replace via=lambda left, right: "prefix-"+left +``` + +See also: +- `ytt` modules that export functions useful for manipulating values: + - [base64 module](lang-ref-ytt.md#base64) + - [json module](lang-ref-ytt.md#json) + - [md5 module](lang-ref-ytt.md#md5) + - [sha256 module](lang-ref-ytt.md#sha256) + - [url module](lang-ref-ytt.md#url) + - [yaml module](lang-ref-ytt.md#yaml) +- [Language: String](lang-ref-string.md) for built-in string functions. +- Other Starlark language features that manipulate values: + - [string interpolation](https://github.com/google/starlark-go/blob/master/doc/spec.md#string-interpolation) + - [conditional expressions](https://github.com/google/starlark-go/blob/master/doc/spec.md#conditional-expressions) + - [index expressions](https://github.com/google/starlark-go/blob/master/doc/spec.md#index-expressions) + - [slice expressions](https://github.com/google/starlark-go/blob/master/doc/spec.md#slice-expressions) + +### @overlay/insert + +Inserts "right" node before/after the matched "left" node. + +**Valid on:** Document, Array Item. + +``` +@overlay/insert [before=Bool, after=Bool] +``` +- **`before=`**`Bool` whether to insert the "right" node immediately in front of the matched "left" node. +- **`after=`**`Bool` whether to insert the "right" node immediately following the matched "left" node. + +### @overlay/append + +Inserts the "right" node after the last "left" node. + +**Valid on:** Document, Array Item. + +``` +@overlay/append +``` +_(this annotation has no parameters.)_ + +**Note:** This action implies an `@overlay/match` selecting the last node. Any other `@overlay/match` annotation is ignored. + +**History:** +- v0.32.0 — omitting `@overlay/append` annotation implies a merge operation and 0 matches, defaulting to insert after the last item. + +### @overlay/assert + +Checks assertion that value of "left" matched node equals that of the annotated "right" node (_or_ a provided predicate). + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/assert [via=Function] +``` + +- Default: checks that the value of the matched "left" node equals the value of the annotated "right" node. +- **`via`**`=Function(left, right):(Bool|Tuple(Bool|String)|None)` _(optional)_ predicate indicating whether "left" passes the check + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the matched node's value + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node + - Return types: + - `Bool` — if `False`, the assertion fails; otherwise, nothing happens. + - `Tuple(Bool|String)` — if `False`, the assertion fails and specified string is appended to the resulting error message; otherwise nothing happens. + - `None` — the assertion assumes to succeed. In these situations, the function makes use of the [`@ytt:assert`](lang-ref-ytt.md#assert) module to effect the assertion. + +**History:** +- v0.26.0 — works with [`yamlfragment`](lang-ref-yaml-fragment.md) values. +- v0.24.0 — introduced + +**Examples:** + +_Example 1: Range check_ + +Fails the execution if `left` not between 0 and 1000, exclusively. + +```yaml +#@overlay/assert via=lambda left, right: left > 0 and left < 1000 +``` + +_Example 2: Well-formedness check_ + +Fails the execution if `left` contains anything other than lowercase letters or numbers. + +```yaml +#@overlay/assert via=lambda left, right: regexp.match("[a-z0-9]+", left) +``` + +See also: +- `ytt` hashing functions from: + - [md5 module](lang-ref-ytt.md#md5) + - [sha256 module](lang-ref-ytt.md#sha256) +- Boolean expression operators and built-in functions, including: + - [`in`](https://github.com/google/starlark-go/blob/master/doc/spec.md#membership-tests) (aka "membership test") + - [`and` and `or`](https://github.com/google/starlark-go/blob/master/doc/spec.md#or-and-and) + - [`any()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#any) or [`all()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#all) + - [`hasattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#hasattr) + - [`len()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#len) + - [`type()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#type) +- [Language: String](lang-ref-string.md) functions + +--- +## Functions + +The `@ytt:overlay` module provides several functions that support overlay use. + +To use these functions, include the `@ytt:overlay` module: + +```python +#@ load("@ytt:overlay", "overlay") +``` + +The functions exported by this module are: +- [overlay.apply()](#overlayapply) +- [overlay.map_key()](#overlaymap_key) +- [overlay.index()](#overlayindex) +- [overlay.all()](#overlayall) +- [overlay.subset()](#overlaysubset) +- [overlay.and_op()](#overlayand_op) +- [overlay.or_op()](#overlayor_op) +- [overlay.not_op()](#overlaynot_op) + +### overlay.apply() + +Executes the supplied overlays on top of the given structure. + +```python +overlay.apply(left, right1[, rightN...]) +``` + +- `left` ([`yamlfragment`](lang-ref-yaml-fragment.md)) — the target of the overlays +- `right1` ([`yamlfragment`](lang-ref-yaml-fragment.md) annotated with [`@overlay/(action)`](#action-annotations)) — the (first) overlay to apply on `left`. +- `rightN` ([`yamlfragment`](lang-ref-yaml-fragment.md) annotated with [`@overlay/(action)`](#action-annotations)) — the Nth overlay to apply on the result so far (which reflects the changes made by prior overlays) + +**Notes:** +- For details on how to use `apply()`, see [Programmatic access](#programmatic-access). + +**Examples:** + +```python +overlay.apply(left(), right()) +overlay.apply(left(), one(), two()) +``` + +See also: [Overlay example](/ytt/#example:example-overlay) in the ytt Playground. + +### overlay.map_key() + +An [Overlay matcher function](#overlaymatch) that matches when the collection (i.e. Map or Array) in the "left" contains a map item with the key of `name` and value equal to the corresponding map item from the "right." + +```python +overlay.map_key(name) +``` +- `name` (`String`) — the key of the contained map item on which to match + +**Note:** this matcher requires that _all_ items in the target collection have a map item with the key `name`; if this requirement cannot be guaranteed, consider using [`overlay.subset()`](#overlaysubset), instead. + +**Examples:** + +_Example 1: Over an Array_ + +With "left" similar to: +```yaml +clients: +- id: 1 +- id: 2 +``` +the following matches on the second array item: +```yaml +clients: +#@overlay/match by=overlay.map_key("id") +- id: 2 +``` + +_Example 2: Over a Map_ + +(as of v0.26.0+) + +With "left" similar to: +```yaml +clients: + clientA: + id: 1 + clientB: + id: 2 +``` +the following matches on the second map item: +```yaml +--- +clients: + #@overlay/match by=overlay.map_key("id") + _: + id: 2 +``` +(note: the key name `_` is arbitrary and ignored). + +### overlay.index() + +An [Overlay matcher function](#overlaymatch) that matches the array item at the given index + +```python +overlay.index(i) +``` +- `i` (`Int`) — the ordinal of the item in the array on the "left" to match (zero-based index) + +**Example:** + +```yaml +#@overlay/match by=overlay.index(0) +- item10 +``` + +### overlay.all() + +An [Overlay matcher function](#overlaymatch) that matches all contained nodes from the "left", unconditionally. + +```python +overlay.all() +``` +_(this function has no parameters.)_ + +**Examples:** + +_Example 1: Documents_ + +Matches each and every document: +```yaml +#@overlay/match by=overlay.all +--- +metadata: + annotations: ... +``` + +_Example 2: Array Items_ + +Matches each and every item in the array contained in `items` on the "left": +```yaml +items: +#@overlay/match by=overlay.all +- item10 +``` + +_Example 3: Map Items_ + +(as of v0.26.0+) + +Matches each and every item in the map contained in `items` on the "left": +```yaml +items: + #@overlay/match by=overlay.all + _: + name: item10 +``` +(note: the key name `_` is arbitrary and ignored) + +### overlay.subset() + +An [Overlay matcher function](#overlaymatch) that matches when the "left" node's structure and value equals the given `target`. + +```python +overlay.subset(target) +``` +- `target` (`any`) — value that the "left" node must equal. + +**Examples** + +_Example 1: Scalar_ + +To match, scalar values must be equal. + +```yaml +#@overlay/match by=overlay.subset(1) +#@overlay/match by=overlay.subset("Entire string must match") +#@overlay/match by=overlay.subset(True) +``` +(if a partial match is required, consider writing a [Custom Overlay Matcher function](#custom-overlay-matcher-functions)) + +_Example 2: Dict (aka "map")_ + +To match, dictionary literals must match the structure and value of `left`. + +```yaml +#@overlay/match by=overlay.subset({"kind": "Deployment"}) +#@overlay/match by=overlay.subset({"metadata": {"name": "istio-system"}}) +``` + +_Example 3: YAML Fragment_ + +To match, [`yamlfragment`](lang-ref-yaml-fragment.md)'s must match structure and value of `left`. + +```yaml +#@ def resource(kind, name): +kind: #@ kind +metadata: + name: #@ name +#@ end + +#@overlay/match by=overlay.subset(resource("Deployment", "istio-system")) +``` + +### overlay.and_op() + +(as of v0.26.0+) + +An [Overlay matcher function](#overlaymatch) that matches when all given matchers return `True`. + +```python +overlay.and_op(matcher1, matcher2, ...) +``` +- `matcher1`, `matcher2`, ... — one or more other [Overlay matcher function](#overlaymatch)s. + +**Examples:** + +```yaml +#@ not_sa = overlay.not_op(overlay.subset({"kind": "ServiceAccount"})) +#@ inside_ns = overlay.subset({"metadata": {"namespace": "some-ns"}}) + +#@overlay/match by=overlay.and_op(not_sa, inside_ns),expects="1+" +``` + +### overlay.or_op() + +(as of v0.26.0+) + +An [Overlay matcher function](#overlaymatch) that matches when at least one of the given matchers return `True`. + +```python +overlay.or_op(matcher1, matcher2, ...) +``` +- `matcher1`, `matcher2`, ... — one or more other [Overlay matcher function](#overlaymatch)s. + +**Examples:** + +```yaml +#@ config_maps = overlay.subset({"kind": "ConfigMap"}) +#@ secrets = overlay.subset({"kind": "Secret"}) + +#@overlay/match by=overlay.or_op(config_maps, secrets) +``` + +### overlay.not_op() + +(as of v0.26.0+) + +An [Overlay matcher function](#overlaymatch) that matches when the given matcher does not. + +```pythons +not_op(matcher) +``` +- `matcher` — another [Overlay matcher function](#overlaymatch). + +```yaml +#@overlay/match by=overlay.not_op(overlay.subset({"metadata": {"namespace": "app"}})) +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt-schema.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-schema.md new file mode 100644 index 000000000..2b7d21d4e --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-schema.md @@ -0,0 +1,394 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-schema] +title: Data Values Schema Reference +--- + +## Overview + +This reference covers details of Data Values Schema: supported types and annotations. + +For an introduction of Data Values, see [Using Data Values](how-to-use-data-values.md). \ +For details on writing Schema, see [Writing Schema](how-to-write-schema.md). + +## The Schema Document + +Schema is written in YAML. + +```yaml +#@data/values-schema +--- +... +``` +where: +- the document must be annotated as `@data/values-schema` +- each item in the document [declares a Data Value](#data-value-declarations) (i.e. an item in the [data.values](lang-ref-ytt.md#data) struct). +- (optionally) types and default values can be explicitly specified through [annotations](#annotations). +- a file containing a Schema document must not contain any other kind of document. + +### Multiple Schema Documents + +In some cases, it is useful to separate Schema into multiple YAML documents (typically in separate files). + +When doing so, it becomes relevant to know that Schema Documents are [`ytt` Overlays](ytt-overlays.md): +- they merge in [the same order as overlays](lang-ref-ytt-overlay.md#overlay-order), and +- one controls that merge via [overlay annotations](lang-ref-ytt-overlay.md#overlay-annotations). + +--- +## Data Value Declarations + +Each item in a Schema Document declares a Data Value. + +A Data Value declaration has three (3) parts: +- the **name** of the data value, +- its **default** value, and +- the **type** of the value. + +--- +### Names + +A Data Value is referred to by its name (aka "key" or "attribute"). \ +A Data Value name must be a string. + +When using multi-word names, it is recommended to employ snake-case (e.g. `database_connection`). This principally because: +- the underlying programming language in `ytt` — Starlark — is Pythonic in which identifiers are snake-cased, by convention +- as in most modern languages, the dash (i.e. `-`) is not allowed for identifier names in Starlark (allow characters are: Unicode letters, decimal digits, and underscores `_`, per the [Starlark spec](https://github.com/google/starlark-go/blob/master/doc/spec.md#lexical-elements)). + +Where disallowed characters in names cannot be avoided, references will need to employ either: +- the Starlark built-in [`getattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#getattr) + ```python + secure: #@ getattr(data.values, "db-conn").secure + ``` +- _(as of v0.31.0)_ or [index notation](lang-ref-structs.md#attributes): + ```python + secure: #@ data.values["db-conn"].secure + ``` + +--- +### Default Values + +The default value for a Data Value is specified in schema, directly: + +- the default value for a **scalar** is the value given; +- the default value for a **map** are all of the items specified in the schema (with _their_ defaults, recursively); +- the default value for an **array** is an empty list (i.e. `[]`). + - the default value for _an item_ in an array are the contents of the item specified in schema (with _their_ defaults, recursively). + +Default values can be overridden using the [@schema/default](#schemadefault) annotation. + +#### Defaults for Scalars + +When a [scalar value](#types-of-scalars) is specified, the default is merely that value. + +For example, +```yaml +#@data/values-schema +--- +system_domain: "" + +load_balancer: + enabled: true + +databases: +- name: "" + adapter: postgresql +``` + +- `system_domain` is `""` (i.e. an empty string), by default. +- `load_balancer.enabled` is `true` by default. +- `databases[].adapter` is the string `"postgres"`, by default. + +#### Defaults for Mappings + +The set of items of a map are its default: any missing item will be automatically added with _its_ defaults. + +For example, +```yaml +#@data/values-schema +--- +load_balancer: + enabled: true + static_ip: "" +``` + +if `load_balancer` were omitted from supplied Data Values, entirely, it would default to: +- `load_balancer.enabled` is `true` +- `load_balancer.static_ip` is `""` + +if `load_balancer` is _partially_ specified... + +```yaml +#@data/values +--- +load_balancer: + static_ip: 10.0.101.1 +``` + +the missing item (here, `enabled`) is defaulted: +- `load_balancer.static_ip` is `"10.0.101.1"` +- `load_balancer.enabled` is `true` + + +#### Defaults for Arrays + +The default value for all arrays is, by default, an empty array. + +This is different from all other types where the default value is literally what is specified in schema. For arrays, it is `[]` (i.e. an empty array). + +This means that the value given for the element is _only_ used to infer the type of the array's _elements_. + +```yaml +#@data/values-schema +--- +app_domains: +- "" + +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + user: admin + secretRef: + name: "" +``` + +- `app_domains` is `[]` by default. Each item in the array must be a **string**. +- `databases` is `[]` by default. Each item in the array must be a **map**. When an item is added to the array: + - its key must be one of those named in this schema: `name`, `adapter`, `host`, etc. + - if it lacks any of the six keys, they will be added with their default values. + +The default value of an array _itself_ can be overridden using the [@schema/default](#schemadefault) annotation. + +--- +### Types + +`ytt` Schema can infer the type of the Data Value from the following... + +#### Types of Scalars + +`ytt` recognizes the following scalar types: + +- **strings** — e.g. `""`, `"ConfigMap"`, `"0xbeadcafe"` +- **integers** — e.g. `42` +- **booleans** — `true`, `false` (and when not strict, `yes`, `no`, `Y`, `N`, etc.) +- **floats** — e.g. `0.4` + +#### Types of Maps + +A map is a collection of map items, each a key/value pair. + +The schema of a map is inferred from that collection. Each item declares a nested Data Value of the type inferred from the given map item's value. + +For example, +```yaml +load_balancer: + enabled: true + static_ip: "" +``` + +where: +- `load_balancer` has a type of a map that has two items: + - one item has a key `enabled` whose type is a **boolean**. + - the other item has a key of `static_ip` and is a **string**. + +#### Types of Arrays + +An array is a sequence of array items. + +The schema of an array must contain _exactly_ one (1) item. The type inferred from that item becomes the type of _all_ items in that array. That is, arrays in `ytt` are homogenous. + +For example, +```yaml +app_domains: +- "" +``` +where: +- `app_domains` has a type of an array. Each element in that array will be a **string**. +- note that the default value for `app_domains` is an empty list as explained in [Defaults for Arrays](#defaults-for-arrays), above. + +#### `null` Type + +The `null` value means the absence of a value. + +In `ytt` schema, a default value is not permitted to be `null` (with one exception described in [`any` Type](#any-type), below). This is because no useful type can be inferred from the value `null`. + +Instead, one provides a non-null default value and annotates the Data Value as "nullable". + +This results in a Data Value whose default value is `null`, but when set to a non-null value has an explicit type. See [`@schema/nullable`](#schemanullable) for details. + +#### `any` Type + +In certain cases, it may be necessary to relax all restrictions on the type or shape of a Data Value: + +- the Data Value is a pass-through, where template(s) using it merely insert its value, but care not about the actual contents; +- a heterogeneous array is required; +- there are multiple possible allowed types for a given Data Value. + +This is done by annotating the Data Value as having "any" type. See [`@schema/type`](#schematype) for details. + +--- +## Annotations + +`ytt` determines the type of each Data Value by _inferring_ it from the value specified in the schema file (as described in [Types](#types), above). Currently, there is no way to _explicitly_ set the type of a Data Value. + +Configuration Authors can explicit specify the type of a Data Value in two cases that are **not** inferrable: +- overriding the default value (via the [@schema/default](#schemadefault) annotation); +- also allowing null (via the [@schema/nullable](#schemanullable) annotation); +- allowing any type (via the [@schema/type](#schematype) annotation). + +### @schema/default + +Overrides the default value for the annotated node. + +```yaml +#@schema/default default_value +``` +- `default_value` — the value to set as the default for the annotated node. + - this value must be of the same type as the value given on the node. + +_(as of v0.38.0)_ + +_Example 1: Default value for an array of scalars_ + +```yaml +#@data/values-schema +--- +#@schema/default ["apps.example.com", "gateway.example.com"] +app_domains: +- "" +``` + +... yields the default: + +```yaml +app_domains: +- apps.example.com +- gateway.example.com +``` + +_Example 2: Default value for an array of maps_ + +When specifying values for an array of maps, it can quickly become unwieldy to keep on a single line. + +To handle these situations, enclose those values in a [Fragment function](lang-ref-yaml-fragment.md) and invoke that function as the value for `@schema/default`: + +```yaml +#! For best results, declare functions *before* the schema document. +#@ def default_dbs(): +- name: core + host: coredb + user: app1 +- name: audit + host: metrics.svc.local + user: observer +#@ end + +#@data/values-schema +--- +#@schema/default default_dbs() +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + user: admin + secretRef: + name: "" +``` + +Yields the default: + +```yaml +databases: +- name: core + adapter: postgresql + host: coredb + port: 5432 + user: app1 + secretRef: + name: "" +- name: audit + adapter: postgresql + host: metrics.svc.local + port: 5432 + user: observer + secretRef: + name: "" +``` + +Note: as the comment in the example schema indicates, it is best to declare the function prior to starting the schema document itself (see https://github.com/vmware-tanzu/carvel-ytt/issues/526 for details). + +### @schema/nullable + +Extends the type of the Data Value to also allow `null` _and_ sets the default value to be `null`. + +```yaml +#@schema/nullable +``` + +**Unset value for strings** + +The preferred way to express "unset value" for a string-type is _not_ to mark it as "nullable" but to provide the empty value: `""`. Empty values in Starlark are falsey (details in the [Starlark Spec > Booleans](https://github.com/google/starlark-go/blob/master/doc/spec.md#booleans)). + +When empty string is a useful/valid value for a given Data Value, _then_ marking it as "nullable" is appropriate. In this case, one must take care to explicitly check if that Data Value is not [`None`](lang.md#types). + +_Example: Nullable map_ + +```yaml +#@data/values-schema +--- +#@schema/nullable +aws: + username: admin + password: "1234" + +name: "" +``` + +Without other Data Value settings, `aws` is `null` by default: +```yaml +aws: null +name: "" +``` + +However, if a Data Value is set: + +```bash +$ ytt ... --data-value aws.username=sa ... +``` + +That effectively sets `aws` to be non-null: `username` is set to the custom value and `password` is defaulted. + +```yaml +aws: + username: sa + password: "1234" + +name: "" +``` + +### @schema/type + +Explicitly configures the type of the annotated node. Currently, the only supported configuration is whether to allow the "any" type or not. + +```yaml +#@schema/type any=True +``` + +where: +- `any` (`bool`) — whether or not any and all types are permitted on this node and its children. + +The annotated node and its nested children are not checked by schema, and has no schema default behavior. +However, the annotated node and its children are simply passed-through as a data value. +All nested `@schema` annotations are ignored. + +_Example: Using any=True to avoid schema restrictions on an array_ + +```yaml +#@data/values-schema +--- +#@schema/type any=True +app_domains: + - "carvel.dev" + - 8080 +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt-struct.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-struct.md new file mode 100644 index 000000000..a85bc898e --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-struct.md @@ -0,0 +1,203 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-struct] +title: Struct module +--- + +## Overview + +The `@ytt:struct` module provides functions for constructing and deconstructing [`struct`](lang-ref-structs.md) values. + +To use these functions, include the `@ytt:struct` module: + +```python +load("@ytt:struct", "struct") +``` + +--- +## struct.decode() + +Deconstructs a given value into plain/Starlark values, recursively. + +```python +struct.decode(struct_val) +``` + +- `struct_val` ([`struct`](lang-ref-structs.md)) — the value to decompose. + - `struct` values are converted into `dict` values where each attribute in the `struct` becomes a + key on the `dict`. + - if the value of an attribute is a `struct`, it is likewise converted to a `dict`. + - all other values are copied, as is. + +### Example + +```python +load("@ytt:struct", "struct") + +foo = struct.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +bar = struct.decode(foo) + +bar["a"] # <== [1, 2, 3, {"c": 456}] +``` + +--- +## struct.encode() + +Makes a `struct` of a given value, recursively. + +```python +struct.encode(value) +``` + +- `value` ([`dict`](lang-ref-dict.md) | [`list`](lang-ref-list.md) | scalar) — the value to encode. + - `dict` values are converted into `struct`s where each key in the `dict` becomes an attribute on the `struct`. + Keys of the items in `dict` values must be strings. + - if a `dict` or `list` contains a value that is a `dict`, it is likewise converted to a `struct`. + +Notes: + +- `encode()` cannot encode functions nor [YAML Fragments](lang-ref-yaml-fragment.md). If you wish to make a struct that + contains attributes that hold these types, consider [`make()`](#structmake). + +### Example: Data structure from a dictionary + +```python +load("@ytt:struct", "struct") + +d = struct.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +d.a # <== [1, 2, 3, c: (struct...)] +d.a[3].c # <== 456 +bar["b"] # <== "str" +``` + +## struct.make() + +Instantiates a `struct` based on the key/value pairs provided. + +```python +struct.make(key1=value1, key2=value2, ...) +``` + +- `keyN` (keyword argument name) — becomes the name of the Nth attribute in the constructed + `struct`. +- `=valueN`(`any`) — becomes the value of the Nth attribute in the constructed `struct`. + +Notes: + +- `make()` does not modify `values` in any way (e.g. if `valueN` is a dictionary, it is + _not_ converted into a `struct`). To recursively build a hierarchy of `struct`s from `dict`, + `list`, and scalars, see [`struct.encode()`](#structencode). + +### Example 1: Scalar values + +For visually pleasing collections of fields +```python +load("@ytt:struct", "struct") + +consts = struct.make(version="0.39.0", service_name="prometheus") + +consts.version # <== "0.39.0" +consts.service_name # <== "prometheus" +``` + +### Example 2: Data structures + +Dictionaries values remain instances of `dict`. +```python +load("@ytt:struct", "struct") + +consts = struct.make(service={"version": "0.39.0", "name": "prometheus"}) + +consts.service["version"] # <== "0.39.0" +consts.service["name"] # <== "prometheus" +# const.service.version # Error! "dict has no .version field or method" +``` + +### Example 3: Nested structs + +Nested invocations of `make()` to retain dot expression access. +```python +load("@ytt:struct", "struct") + +consts = struct.make(service=struct.make(version="0.39.0", name="prometheus")) + +consts.service.version # <== "0.39.0" +consts.service.name # <== "prometheus" +``` +See also: [`struct.encode()`](#structencode) to convert all `dict` values to `struct`s, recursively. + +### Example 4: Collection of functions + +"Export" a set of functions from a library file. + +`urls.star` +```python +load("@ytt:struct", "struct") + +def _valid_port(port): +... +end + +def _url_encode(url): +... +end + +urls = struct.make(valid_port= _valid_port, encode= _url_encode, ...) +``` +```yaml +#@ load("urls.star", "urls") + +#@ if/end urls.valid_port(...): +encoded: #@ urls.encode("encode_url") +``` +--- +## struct.make_and_bind() + +Binds one or more function(s) to a `struct`, making them method(s) on that struct. +This allows `struct`s to carry both data and behavior related to that data. + +```python +struct.make_and_bind(receiver, method_name1=function1, ...) +``` + +- `receiver` ([`struct`](lang-ref-structs.md)) — "object" to attach the function(s) to. +- `method_nameN` (keyword argument name) — the name that callers will specify to invoke the method. +- `functionN` ([`function`](lang-ref-def.md)) — the function value (either the name of a `function` or a `lambda` expression) + that will be bound to `receiver` by the name `method_nameN`. + - the first parameter of `functionN` is `receiver`, implicitly. + - the remaining parameters of `functionN` become the parameters of `receiver.method_nameN()` + +Notes: + +- Binding is useful for cases where a commonly desired value is a calculation of two or more + values on `receiver`. + +### Example 1: Binding a function value + +```python +load("@ytt:struct", "struct") + +conn = struct.make(hostname="svc.example.com", default_port=1022, protocol="https") + +def _url(self, port=None): + port = port or self.default_port + return "{}://{}:{}".format(self.protocol, self.hostname, port) +end + +conn = struct.make_and_bind(conn, url=_url) + +conn.url() # ==> https://svc.example.com:1022 +conn.url(8080) # ==> https://svc.example.com:8080 +``` + +### Example 2: Binding a lambda expression + +```python +load("@ytt:struct", "struct") + +_conn_data = struct.make(hostname="svc.example.com", default_port=1022, protocol="https") + +conn = struct.make_and_bind(_conn_data, url=lambda self, port=None: "{}://{}:{}".format(self.protocol, self.hostname, port or self.default_port)) + +conn.url() # ==> https://svc.example.com:1022 +conn.url(8080) # ==> https://svc.example.com:8080 +``` diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt-template.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-template.md new file mode 100644 index 000000000..b813ee923 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-template.md @@ -0,0 +1,71 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-template] +title: Template Module +--- + +## `@template` Functions + +The `@ytt:template` module provides a function that can be used to update templates. + +To use these functions, include the `@ytt:template` module: + +```python +#@ load("@ytt:template", "template") +``` + +The functions exported by this module are: +- [template.replace()](#templatereplace) + +### template.replace() +Replaces the existing yaml node with the yaml node(s) provided or returned from a function call, of the same type. +Underscore (`_`) is the conventional replacement key, though any key can be used. + +``` +template.replace(node) +``` + +* `node` ([`yamlfragment`](lang-ref-yaml-fragment.md)) — yaml fragment that will replace the existing node + +**Examples:** + +##### Add a new item to the `labels` mapping +```yaml +#@ load("@ytt:template", "template") + +labels: + another-label: true + _: #@ template.replace({"key2": "value2"}) +``` +results in: +```yaml +labels: + another-label: true + key2: value2 +``` + +Notice that the argument to the function entirely replaces the `_` map item. + +##### Use a function instead of providing the item(s) inline +```yaml +#@ load("@ytt:template", "template") + +#@ def my_labels(): +#@ return {"key2": "value2"} +#@ end + +labels: + another-label: true + key-will-disappear: #@ template.replace(my_labels()) +``` +results in: +```yaml +labels: + another-label: true + key2: value2 +``` + +Notice that the argument to the `replace` function entirely replaces the `key-will-disappear` map item. + +So, regardless of the node's key or value, `template.replace` will overwrite it with the argument provided. + +See also: [Replace example](/ytt/#example:example-replace) in the ytt Playground. diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt-version.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-version.md new file mode 100644 index 000000000..0b70659f2 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt-version.md @@ -0,0 +1,25 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-version] +title: Version module +--- + +Available in v0.26.0+ + +Version module provides a way to assert on minimum ytt binary version in your configuration. It could be placed into a conditional or just at the top level within `*.star`, `*.yml` or any other template file. + +Example configuration directory may look like this: + +- `config/` + - `0-min-version.star`: contents below + - `deployment.yml` + - `other.yml` + +```python +# filename starts with '0-' to make sure this file gets +# processed first, consequently forcing version check run first +load("@ytt:version", "version") + +version.require_at_least("0.26.0") +``` + +Note that ytt sorts files alphanumerically and executes templates in order. Most of the time it's best to check version first before processing other templates, hence, in the above example we've named file `0-min-version.star` so that it's first alphanumerically. diff --git a/site/content/ytt/docs/v0.38.0/lang-ref-ytt.md b/site/content/ytt/docs/v0.38.0/lang-ref-ytt.md new file mode 100644 index 000000000..a3442d988 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang-ref-ytt.md @@ -0,0 +1,227 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt] +title: Built-in ytt Library +--- + +## General modules + +### struct + +See [@ytt:struct module docs](lang-ref-ytt-struct.md). + +### assert + +```python +load("@ytt:assert", "assert") + +# stop execution and report a failure +assert.fail("expected value foo, but was {}".format(value)) # stops execution +x = data.values.env.mysql_password or assert.fail("missing env.mysql_password") + +# invoke a function value, catching failure if it occurs +x, err = assert.try_to(lambda : json.decode('{"key": "value"}')) +x # { "key" = "value" } (i.e. dict with one entry) +err # None + +x, err = assert.try_to(lambda : json.decode("(not JSON)")) +x # None +err # "json.decode: invalid character '(' looking for beginning of value" +``` + +### data + +See [Data Values](ytt-data-values.md) reference for more details + +```python +load("@ytt:data", "data") + +data.values # struct that has input values + +# relative to current package +data.list() # ["template.yml", "data/data.txt"] +data.read("data/data.txt") # "data-txt contents" + +# relative to library root (available in v0.27.1+) +data.list("/") # list files +data.read("/data/data.txt") # read file +``` + +### ip + +Parse and inspect Internet Protocol values. + +(available in v0.37.0+) + +```python +load("@ytt:ip", "ip") + +# Parse IP addresses... +addr = ip.parse_addr("192.0.2.1") +addr.is_ipv4() # True +addr.is_ipv6() # False +addr.string() # "192.0.2.1" + +addr = ip.parse_addr("2001:db8::1") +addr.is_ipv4() # False +addr.is_ipv6() # True +addr.string() # "2001:db8::1" + +# Parse CIDR notation into an IP Address and IP Network... +addr, net = ip.parse_cidr("192.0.2.1/24") +addr.string() # "192.0.2.1" +addr.is_ipv4() # True +addr.is_ipv6() # False +net.string() # "192.0.2.0/24" +net.addr().string() # "192.0.2.1" +net.addr().is_ipv4() # True +net.addr().is_ipv6() # False + +addr, net = ip.parse_cidr("2001:db8::1/96") +addr.string() # "2001:db8::1" +addr.is_ipv4() # False +addr.is_ipv6() # True +net.string() # "2001:db8::/96" +net.addr().string() # "2001:db8::" +net.addr().is_ipv4() # False +net.addr().is_ipv6() # True +``` + +### regexp + +```python +load("@ytt:regexp", "regexp") + +regexp.match("[a-z]+[0-9]+", "__hello123__") # True + +regexp.replace("[a-z]+[0-9]+", "__hello123__", "foo") # __foo__ +regexp.replace("(?i)[a-z]+[0-9]+", "__hello123__HI456__", "bye") # __bye__bye__ +regexp.replace("([a-z]+)[0-9]+", "__hello123__bye123__", "$1") # __hello__bye__ +regexp.replace("[a-z]+[0-9]+", "__hello123__", lambda s: str(len(s))) # __8__ + +# example of passing the "dot matches newline" flag and using replace to extract a single match from a multiline input string +input_str = "\\ multline string\n\nconst (\n\t// Value is what we want to scrape\n\tValue = 12\n)\n\nfunc main() {..." +regexp.replace("(?s).*Value = ([0-9]+).*", input_str, "$1") # 12 +``` + +See the [RE2 docs](https://github.com/google/re2/wiki/Syntax) for more on regex syntax. Note that flags such as multiline mode are passed in the pattern string as in the [golang regexp library](https://pkg.go.dev/regexp/syntax). + +When calling `replace` you can pass either a string or a lambda function as the third parameter. When given a string, `$` symbols are expanded, so that `$1` expands to the first submatch. When given a lambda function, the match is directly replaced by the result of the function. + +While `match` and `replace` are currently the only regexp verbs supported, it is possible to mimic `find` by using `replace` to replace all its input with a capture group (see example above). + +### url + +```python +load("@ytt:url", "url") + +url.path_segment_encode("part part") # "part%20part" +url.path_segment_decode("part%20part") # "part part" + +url.query_param_value_encode("part part") # "part+part" +url.query_param_value_decode("part+part") # "part part" + +url.query_params_encode({"x":["1"],"y":["2","3"],"z":[""]}) # "x=1&y=2&y=3&z=" +url.query_params_decode("x=1&y=2&y=3;z") # (DEPRECATED) # {"x":["1"],"y":["2","3"],"z":[""]} +url.query_params_decode("x=1&y=2&y=3&z") # {"x":["1"],"y":["2","3"],"z":[""]} + +u = url.parse("http://alice:secret@example.com") +u.string() # "http://alice:secret@example.com" +u.user.name # "alice" +u.user.password # "secret" +u.user.string() # "alice:secret" +u.without_user().string() # "http://example.com" +``` + +As of v0.38.0, including semicolons in query strings is deprecated behavior. +Allowing semicolons in query strings can [lead to cache poisoning attacks](https://snyk.io/blog/cache-poisoning-in-popular-open-source-packages/). Authors should use ampersands (i.e. `&`) exclusively to separate parameters. + +### version + +`load("@ytt:version", "version")` (see [version module doc](lang-ref-ytt-version.md)) + +--- +## Serialization modules + +### base64 + +```python +load("@ytt:base64", "base64") + +base64.encode("regular") # "cmVndWxhcg==" +base64.decode("cmVndWxhcg==") # "regular" +``` + +### json + +```python +load("@ytt:json", "json") + +json.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +json.encode({"a": [1,2,3,{"c":456}], "b": "str"}, indent=3) + +json.decode('{"a":[1,2,3,{"c":456}],"b":"str"}') +``` +As of v0.35.0, `json.encode()` with `indent` argument encodes result in multi-line string. + +### toml + +As of v0.38.0. + +```python +load("@ytt:toml", "toml") + +toml.encode({"a": [1,2,3,456], "b": "str"}) # 'a = [1, 2, 3, 456]\nb = "str"' +toml.encode({"metrics": {"address":"", "grpc_histogram": False}}, indent=4) + # '[metrics]\n address = ""\n grpc_histogram = false\n' + +toml.decode("[plugins]\n [plugins.cgroups]\n no_prometheus = false") + # {"plugins": {"cgroups": {"no_prometheus": False}}} +``` + + +### yaml + +```python +load("@ytt:yaml", "yaml") + +yaml.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +yaml.decode('{"a":[1,2,3,{"c":456}],"b":"str"}') +``` + +--- +## Hashing modules + +### md5 + +```python +load("@ytt:md5", "md5") + +md5.sum("data") # "8d777f385d3dfec8815d20f7496026dc" +``` + +### sha256 + +```python +load("@ytt:sha256", "sha256") + +sha256.sum("data") # "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7" +``` +--- +## Schema Module + +See [Schema specific docs](lang-ref-ytt-schema.md). + +--- +## Overlay module + +See [Overlay specific docs](lang-ref-ytt-overlay.md). + +--- +## Library module + +See [Library specific docs](lang-ref-ytt-library.md). + +--- +## Template module + +See [Template specific docs](lang-ref-ytt-template.md). diff --git a/site/content/ytt/docs/v0.38.0/lang.md b/site/content/ytt/docs/v0.38.0/lang.md new file mode 100644 index 000000000..d46f0a34c --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/lang.md @@ -0,0 +1,34 @@ +--- +aliases: [/ytt/docs/latest/lang] +title: Language +--- + +## Overview + +Templating language used in `ytt` is a slightly modified version of [Starlark](https://github.com/google/starlark-go/blob/master/doc/spec.md). Following modifications were made: + +- requires `end` keyword for block closing + - hence no longer whitespace sensitive (except new line breaks) +- does not allow use of `pass` keyword + +See [full Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#contents) for detailed reference. + +## Types + +- NoneType: `None` (equivalent to null in other languages) +- Bool: `True` or `False` +- Integer: `1` +- Float: `1.1` +- [String](lang-ref-string.md): `"string"` +- [List](lang-ref-list.md): `[1, 2, {"a":3}]` +- Tuple: `(1, 2, "a")` +- [Dictionary](lang-ref-dict.md): `{"a": 1, "b": "b"}` +- [Struct](lang-ref-structs.md): `struct.make(field1=123, field2="val2")` +- [YAML fragment](lang-ref-yaml-fragment.md) +- [Annotation](lang-ref-annotation.md): `@name arg1,arg2,keyword_arg3=123` + +## Control flow + +- [If conditional](lang-ref-if.md) +- [For loop](lang-ref-for.md) +- [Function](lang-ref-def.md) diff --git a/site/content/ytt/docs/v0.38.0/outputs.md b/site/content/ytt/docs/v0.38.0/outputs.md new file mode 100644 index 000000000..6c9548312 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/outputs.md @@ -0,0 +1,17 @@ +--- +aliases: [/ytt/docs/latest/outputs] +title: Outputs +--- + +ytt supports three different output destinations: + +- stdout, which is default + - All YAML documents are combined into one document set. Non-YAML files are not printed anywhere. +- output files, controlled via `--output-files` flag (v0.28.0+) + - Output files will be added to given directory, preserving file names. + - Example: `ytt -f config.yml --output-files tmp/`. +- output directory, controlled via `--dangerous-emptied-output-directory` flag + - Given directory will be _emptied out_ beforehand and output files will be added preserving file names. + - Example: `ytt -f config.yml --dangerous-emptied-output-directory tmp/ytt/`. + +If you want to control which files are included in the output use `--file-mark 'something.yml:exclusive-for-output=true'` flag to mark one or more files. diff --git a/site/content/ytt/docs/v0.38.0/security.md b/site/content/ytt/docs/v0.38.0/security.md new file mode 100644 index 000000000..0341b2ddd --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/security.md @@ -0,0 +1,34 @@ +--- +aliases: [/ytt/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `ytt`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). + +## Attack Vectors + +This section is a work-in-progress... + +- malicious template input + - input tries to exhaust cpu/mem/disk resources + - A: how does it affect go-yaml? ... https://en.wikipedia.org/wiki/Billion_laughs_attack + - input tries to use YAML tagging to initialize custom objects (std yaml concern) + - A: TBD + +- malicious template code + - code tries to load file contents from sensitive locations + - A: templating is constrained to seeing only files explicitly specified by the user via -f flag. unless user is tricked to provide sensitive file as input, template code is not able to access it. in other words, template runtime does not have facilities to access arbitrary filesystem locations. + - code tries to exfiltrate data over network + - A: template runtime does not have facilities to access network. + - code tries to exhaust cpu/mem/disk resources + - A: there are currently no resource constraints set by ytt itself for cpu/mem/disk. currently cpu can be pegged at 100% via an infinite loop. function recursion is also possible; however, it will be contstrained by Go stack space (and will exit the program). + - code tries to produce YAML that exhausts resources + - A: TBD + - meltdown/spectre style attacks + - A: TBD + +- CLI output directory + - user is tricked to set --output-files flag to a sensitive filesystem location + - A: template output is constrained to stdout or specified output directory via --output-files flag. if user is tricked to point --output-files flag to a sensitive filesystem location such as ~/.ssh/, attacker may be able to write templates (for example ~/.ssh/authorized_keys) that can be intepreted by the system as configuration/executable files. diff --git a/site/content/ytt/docs/v0.38.0/strict.md b/site/content/ytt/docs/v0.38.0/strict.md new file mode 100644 index 000000000..43a8b7e39 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/strict.md @@ -0,0 +1,48 @@ +--- +aliases: [/ytt/docs/latest/strict] +title: Strict YAML +--- + +## Overview + +ytt includes strict YAML subset mode that tries to remove any kind of ambiguity in user's intent when parsing YAML. + +Unlike full YAML, strict subset: + +- only supports specifying nulls as "" or `null` +- only supports specifying bools as `false` or `true` +- only support basic int and float declarations + - prefix, suffix, octal notation, etc are not supported +- requires strings with whitespace to be explicitly quoted +- requires strings with colon to be explicitly quoted +- requires strings with triple-dash (document start) to be explicitly quoted + +## Example + +Non-strict: + +```bash +$ echo 'key: yes' | ytt -f- +key: true +``` + +Strict: + +```bash +$ echo 'key: yes' | ytt -f- -s +Error: Unmarshaling YAML template 'stdin.yml': yaml: Strict parsing: Found 'yes' ambigious (could be !!str or !!bool) +``` + +To fix error, explicitly make it a string: + +```bash +$ echo 'key: "yes"' | ytt -f- -s +key: "yes" +``` + +or via YAML tag `!!str`: + +```bash +$ echo 'key: !!str yes' | ytt -f- -s +key: "yes" +``` diff --git a/site/content/ytt/docs/v0.38.0/yaml-primer.md b/site/content/ytt/docs/v0.38.0/yaml-primer.md new file mode 100644 index 000000000..cf95a8a6d --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/yaml-primer.md @@ -0,0 +1,228 @@ +--- +aliases: [/ytt/docs/latest/yaml-primer] +title: "YAML and Annotations" +--- + +## Overview + +Templates in `ytt` — rather than being YAML-like text files with templating injected _in_ — are well-formed YAML files with templating annotated _on_. + +It's reasonable to have expectations of how to write templates, especially if we have prior experience. But in the shift from a free-form text file to a structured YAML document set, some of those expectations are foiled. And when they are, it can be rather frustrating. + +If we take a few minutes to cover: + +- [what YAML is](#what-is-yaml) — its format and constituent parts; and +- [how to annotate YAML in `ytt`](#annotating-yaml), + +it can mitigate much of that frustration. + +## What is YAML? + +YAML is a tree of nodes. Each node is of one of these six types: + +- a **Document Set** (which is list of **Documents**) +- a **Map** that has **Map Items** +- an **Array** that has **Array Items** + +We'll start from the top of the tree... + +### Documents and Document Sets + +A file containing text in YAML format is parsed into a **Document Set**: a collection of **Document**s. + +![a file containing two YAML documents](/images/ytt/yaml-primer/two-docs.jpg) + +_Figure 1: A file, parsed into a **document set** (dotted blue) containing two **documents** (solid blue)._ + +Note: `---` marks the start of a new document. + +The corresponding tree of nodes looks something like this: + +![](/images/ytt/yaml-primer/two-docs-ast.jpg) + +_Figure 2: Corresponding tree: this document set has two documents._ + +A given document has exactly one (1) value. It is usually either: +- a **Map**, +- an **Array**, or +- a **Scalar** + +We'll look at each of these, in turn. Maps are most common... + +### Maps and Map Items + +A "map" is collection of key-value pairs. Each pair is referred to as a "map item". + +![](/images/ytt/yaml-primer/two-maps-and-items.jpg) + +_Figure 3: Each document has a **map** (dotted green) each containing two **map items** (solid green)._ + +The complete tree of nodes looks like this: + +![](/images/ytt/yaml-primer/two-maps-and-items-ast.jpg) + +_Figure 4: Corresponding tree: each document has a map for its value; each map has two items._ + +Let's zoom in on the first document, and explicitly reveal the contents of a map item: + +![](/images/ytt/yaml-primer/map-items-ast.jpg) + +_Figure 5: Each map item has a **key** and a **value**; here, both map items' key is a string and their value is an integer._ + +Like documents, a map item has exactly one (1) value. And just like documents, it's either: +- a **Map**, +- an **Array**, or +- a **Scalar** + +Let's see what it looks like for a **map item** to have a **map**, itself. + +![](/images/ytt/yaml-primer/map-item-has-map-ast.jpg) + +_Figure 6: The document's value is a map with two map items; the second map item has a key of "metadata" and a value that is another map._ + + +### Arrays and Array Items + +An "array" is a list of "array items" (zero-indexed). + +Like documents (and map items), an array item has exactly one (1) value. Once more, it's either: +- a **Map** +- an **Array**, or +- a **Scalar** + +![](/images/ytt/yaml-primer/map-item-has-array-ast.jpg) + +_Figure 7: The **map item** "foo" has an **array** (dotted gold) for a value; that array has three **array items** (solid gold)._ + +Let's see what it looks like when an **array item**'s value is a **map**... + +![](/images/ytt/yaml-primer/array-item-has-map-ast.jpg) + +_Figure 8: "foo" has an **array** with one **array item** whose value is a **map** containing two **map item**s: `name` and `factor`._ + +We've seen scalars in use above. Let's cover them explicitly. + +### Scalars + +A "scalar" is a fundamental value. YAML supports: +- integers (e.g. 13, 42, 137, 32767) +- floating point numbers (e.g. 1.0, 3.14, 137.03599913) +- strings (e.g. "", "fine structure", "$ecr3t") +- boolean values: `true` or `false` +- `null` (when a value is omitted, that implies `null`; `~` == `null`) + +### Summary + +In short: + +- a YAML file is parsed into a **Document Set** which is merely a list of **Document**s. +- a given **Document** has one _value_. +- Both **Map**s and **Array**s are nothing more than collections of items. + - a **Map Item** has a _key_ and a _value_ + - an **Array Item** has a _value_ +- a _value_ (whether held by a document, map item, or array item) is either a **Map**, an **Array**, or a **Scalar**. + +With this structure in mind, we can now look into how `ytt` annotates it. + +## Annotating YAML + +A `ytt` "annotation" is a YAML comment of a specific format; it attaches behavior to a node. + +For example: + +![](/images/ytt/yaml-primer/map-item-with-value.jpg) + +_Figure 9: A YAML file with a `ytt` annotation._ + +Can be understood to mean that the value of `foo` is the result of evaluating the expression `13 + 23 + 6`. + +The diagram reveals how the annotation is attached to the YAML tree: + +![](/images/ytt/yaml-primer/map-item-ann-with-value-ast.jpg) + +_Figure 10: The annotation (dotted black) is attached to the **map item** (solid green)._ + +This is a document, whose value is a map, that contains a single map item. To the right of the map item is an annotation that contains an expression that, when evaluated, becomes a _value_. The annotation is attached to the map item. + +### How an Annotation Finds its Node + +When attaching annotations, `ytt` follows these two rules (in order): +1. the annotation is attached to the value-holding node **on its left**, if there is one; otherwise, +2. the annotation is attached to the value-holding node **directly below** it. + +In the example above, the value-holding node to the left of the annotation is the map item. + +As noted in the summary [above](#summary), "value-holding" nodes are: +- **Document**s, +- **Map Item**s, and +- **Array Item**s. + +#### Example: Annotating Map Items + +Let's see these rules at play: + +![](/images/ytt/yaml-primer/overlay-ann-on-doc-and-map.jpg) + +_Figure 11: A templated document with three annotations._ + +We'll ignore _what_ these annotations mean, for now, and focus on _where_ they attach. There are three annotations in total. + +Visualizing the YAML tree can help: + +![](/images/ytt/yaml-primer/overlay-ann-on-doc-and-map-ast.jpg) + +_Figure 12: Each annotation attaches to the value-holding node to its left or bottom._ + +Taking each annotation in turn: +- `@overlay/match by=overlay.all`: + - is in the document set, but that is not a value-holding node + - has a **document** node just below it and so is attached to _that_ node. +- `@overlay/replace`: + - has no node to its left; + - has a **map** just below it but maps are _not_ value-holding; + - the next node is a **map item**, and so attaches to _that_ node. +- `@ 13 + 23 + 6`: + - has a **map item** to its left, and so attaches to _that_ node. + - there _is_ another **map item** below it (i.e. `bar: true`), but a home has already been found for this annotation. + +#### Example: Annotating an Array + +When an **array item** contains a **map**, it can be tricky to know which item a particular annotation attaches to. + +In this example, we reinforce the value of knowing the two rules that determine [how an annotation finds its node](#how-an-annotation-finds-its-node). + +![](/images/ytt/yaml-primer/overlay-ann-on-array-and-map.jpg) + +_Figure 13: An overlay ensuring each book has been marked as reviewed. The blank line after the array item is intentional._ + +There are three annotations in this example. The first one clearly annotates the document. But what about the next two? + +Visualizing the layer, it can start to become more clear... + +![](/images/ytt/yaml-primer/overlay-ann-on-array-and-map-ast.jpg) + +_Figure 14: The newline after the **array item** ensures the last annotation attaches to the **map item**._ + +Taking each annotation in turn: +- the first `@overlay/match by=overlay.all`: + - has a **document** node just below it and so is attached to _that_ node. +- the second `@overlay/match by=overlay.all`: + - has a **map item** (`books:`), but that is _above_ this annotation; + - `books:` contains an **array**, but arrays are _not_ value-holding nodes; + - that array's **array item** (denoted by `-`) is just below the annotation and so is attached to _that_ node. +- finally, `@overlay/match missing_ok=True`: + - if this _had_ been the same line as the **array item**, it _would_ have also attached to the same node as the previous annotation (but it is not, and is it won't); + - the next node is a **map**, but again, maps are _not_ value-holding; + - the next node is a **map item** (`reviewed:`), so the annotation attaches to _that_ node. + + +## Further Exploration + +While we've thoroughly covered the fundamentals here, these concepts only become real with use: + +- put your understanding to the test by picking an example from [the Playground](https://carvel.dev/ytt/#example:example-demo) and tweaking it in various ways to see what _actually_ happens. +- take a simple example and run it through `ytt --debug` and study the `### ast` section of the output. Note: + - in this output, annotations are referred to as `meta`. + - "AST" is short for "Abstract Syntax Tree" which is the technical name for the trees we've been visualizing in this guide. + - we recommend making small changes and study how it modifies the "AST" +- if you'd like to obsess over YAML itself, look no further than the [YAML spec](https://yaml.org/spec/1.2/spec.html). diff --git a/site/content/ytt/docs/v0.38.0/ytt-data-values.md b/site/content/ytt/docs/v0.38.0/ytt-data-values.md new file mode 100644 index 000000000..1017df87e --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/ytt-data-values.md @@ -0,0 +1,281 @@ +--- +aliases: [/ytt/docs/latest/ytt-data-values] +title: Data Values +--- + +## Overview + +A `ytt` run can be configured by supplying custom Data Values. + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +## Declaring Data Values + +Typically, Data Values are declared in a schema file. See the [Using Data Values](how-to-use-data-values.md) guide for more details. + +Note: `ytt` continues to support declaring Data Values without schema for backwards-compatibility. However, due to the significantly improved support for catching configuration errors that schema brings, it is the recommended method for doing so. + +## Configuring Data Values + +Data Values can be configured in one of two ways: + +- on the command-line via the family of [command-line `--data-value...` flags](#configuring-data-values-via-command-line-flags), +- in a ["Data Values Overlay" document](#configuring-data-values-via-data-values-overlays) and included via the `--file` flag, + + +### Configuring Data Values via command line flags + +The `--data-value...` family of command-line flags provides a means of configuring Data Values from: +- the command-line, itself; +- OS environment variables; +- "data values file"s — plain YAML files containing values for multiple Data Values + +Those flags are: + +`--data-value [@lib:]key=value` — sets a Data Value to a _string_ value +- `key` — name of Data Value. Use dot notation for nested values (e.g. `key2.nested=val`) +- `value` — value to set (always interpreted as a string) +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- examples: `instance.count=123`, `key=string`, `input=true`, all set to strings + +`--data-value-yaml [@lib:]key=value`) — sets a Data Value to a YAML-parsed value +- `key` — name of Data Value. +- `value` — value to set (decoded as a YAML value) +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- examples: `instance.count=123` sets as integer, `key=string` as string, `input=true` as bool + +`--data-value-file [@lib:]key=file-path` — sets a single Data Value to the _contents_ of a given file. +- `key` — name of Data Value. +- `file-path` — file-system path to a file whose contents will become the value of `key`. +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- particularly useful for loading multi-line string values from files such as private and public key files, certificates, etc. +- not to be confused with `--data-values-file` (described below). + +`--data-values-env [@lib:]PREFIX` — sets one or more Data Values to _string_ values from OS environment variables that start with the given prefix. +- `PREFIX` — the literal prefix used to select the set of environment variables from which to configure Data Values. +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- for nested values, use double-underscore (i.e. `__`) in environment variable names to denote a "dot". +- example: \ + with environment variables... + ```shell + DVAL_key1=blue + DVAL_key2__nested=1337 + ``` + ... and the parameter ... + ```console + $ ytt ... --data-values-env DVAL ... + ``` + ... would set two Data Values: \ + `key1=blue` and \ + `key2.nested="1337"`. (both as _strings_) + +`--data-values-env-yaml [@lib:]PREFIX` — sets one or more Data Values to the YAML-parsed values from OS environment variables that start with the given prefix. +- `PREFIX` — the literal prefix used to select the set of environment variables from which to configure Data Values. +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- for nested values, use double-underscore (i.e. `__`) in environment variable names to denote a "dot". +- example: \ + with environment variables... + ```shell + DVAL_key1=blue + DVAL_key2__nested=1337 + ``` + ... and the parameter ... + ```console + $ ytt ... --data-values-env DVAL ... + ``` + ... would set two Data Values: \ + `key1=blue` (a string value) and \ + `key2.nested=1337` (an integer value). + + +`--data-values-file [@lib:]file-path` — sets one or more Data Values from a plain YAML file. +- `file-path` — file-system path to a file which will be parsed as YAML structured identically to expected Data Values. + - file must be plain YAML (i.e. not a `ytt` template or Data Values Overlay); it cannot contain YAML comments starting with `#@`. + - array values _replace_ (rather than append to) any existing value. + - if there are more than one YAML documents in such a file, they are merged from top to bottom (last wins) +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- example: \ + with the file `prod-values.yml` + ```yaml + domain: example.com + client_opts: + timeout: 10 + retry: 5 + ``` + ... and the parameter ... + ```console + $ ytt ... --data-values-file prod-values.yml ... + ``` + ... would set all three Data Values: \ + `domain=example.com` (a string value) \ + `client_opts.timeout=10` (an integer value) \ + `client_opts.retry=5` (an integer value). + +Notes: +- As of v0.34.0+ Data Values passed via `--data-value...` flags do not _necessarily_ need to be declared beforehand. In prior versions of `ytt`, a Data Value _must_ be declared (back then, specified in a `@data/values` overlay, typically), before it could be configured through a flag. +- the `--data-value...` flags can be repeated multiple times and used in any combination. See [Data Values merge order](#data-values-merge-order) for details on how they combine. + ```bash + export STR_VALS_key6=true # will be string 'true' + export YAML_VALS_key6=true # will be boolean true + + ytt -f . \ + --data-value key1=val1-arg \ + --data-value-yaml key2.nested=123 \ # will be int 123 + --data-value-yaml 'key3.other={"nested": true}' \ + --data-value-file key4=/path \ + --data-values-env STR_VALS \ + --data-values-env-yaml YAML_VALS + ``` + + +### Configuring Data Values via Data Values Overlays + +Data Values can also be configured via a specific kind of `ytt` Overlay. + +A Data Values Overlay is a YAML document annotated with `@data/values`. + +```yaml +#@data/values +--- +key1: val1 +key2: + nested: val2 +key3: val3 +key4: +``` + +Note: +- `data.values` is a [`struct`](lang-ref-structs.md). + +#### Splitting Data Values Overlays into multiple files + +Available in v0.13.0. + +It's possible to split data values into multiple files (or specify multiple data values in the same file). `@ytt:data` library provides access to the _merged_ result. Merging is controlled via [overlay annotations](lang-ref-ytt-overlay.md) and follows same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order). Example: + +`values-default.yml`: + +```yaml +#@data/values +--- +key1: val1 +key2: + nested: val2 +key3: +key4: +``` + +`values-production.yml`: + +```yaml +#@data/values +--- +key3: new-val3 +#@overlay/remove +key4: +#@overlay/match missing_ok=True +key5: new-val5 +``` + +Note that `key4` is being removed, and `key5` is marked as `missing_ok=True` because it doesn't exist in `values-default.yml` (this is a safety feature to prevent accidental typos in keys). + +`config.yml`: + +```yaml +#@ load("@ytt:data", "data") + +first: #@ data.values.key1 +third: #@ data.values.key3 +fifth: #@ data.values.key5 +``` + +Running `ytt -f .` (or `ytt -f config.yml -f values-default.yml -f values-production.yml`) results in: + +```yaml +first: val1 +third: new-val3 +fifth: new-val5 +``` + +See [Multiple data values example](/ytt/#example:example-multiple-data-values) in the online playground. + + +### Data Values merge order + +Data values are merged in following order (latter one wins): + +1. default values from `@data/values-schema` files +2. `@data/values` overlays (same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order)) +3. `--data-values-file` specified files (left to right) +4. `--data-values-env` specified values (left to right) +5. `--data-values-env-yaml` specified values (left to right) +6. `--data-value` specified value (left to right) +7. `--data-value-yaml` specified value (left to right) +8. `--data-value-file` specified value (left to right) + +_(When configuring libraries, the [data values merge order](#library-data-values-merge-order), is the same, even if through different mechanisms.)_ + +--- +## Library data values + +Available in v0.28.0+ + +Each library may specify data values which will be evaluated separately from the root level library. + +### Setting library values via files + +To override library data values, add [`@library/ref`](lang-ref-ytt-library.md#libraryref) annotation to data values YAML document, like so: + +```yaml +#@library/ref "@lib1" +#@data/values +--- +key1: val1 +key2: val2 + +#@library/ref "@lib1" +#@data/values after_library_module=True +--- +key3: val3 +``` + +The `@data/values` annotation also supports a keyword argument `after_library_module`. If this keyword argument is specified, given data values will take precedence over data values passed to the `.with_data_values(...)` function when evaluating via the [library module](./lang-ref-ytt-library.md). + +### Setting library values via command line flags + +Data value flags support attaching values to libraries for use during [library module](./lang-ref-ytt-library.md) evaluation: + +```bash +export STR_VALS_key6=true # will be string 'true' +export YAML_VALS_key6=true # will be boolean true + +ytt -f . \ + --data-value @lib1:key1=val1-arg \ + --data-value-yaml @lib2:key2.nested=123 \ # will be int 123 + --data-value-yaml '@lib3:key3.other={"nested": true}' \ + --data-value-file @lib4:key4=/path \ + --data-values-env @lib5:STR_VALS \ + --data-values-env-yaml @lib6:YAML_VALS + --data-values-file @lib6:/path +``` + +### Library Data Values merge order + +For a given library instance, data values are merged in following order (latter one wins): + +1. default values from schema: + 1. `@data/values-schema` files within the library + 2. `@data/values-schema` files [externally referenced in](#setting-library-values-via-files). + 3. given through [`instance.with_data_values_schema()`](lang-ref-ytt-library.md#instancewith_data_values_schema) + 4. `@data/values-schema` files [externally referenced in `after_library_module=True`](#setting-library-values-via-files). +2. values from data value sources: + 1. `@data/values` overlays within the library (same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order)) + 2. `@data/values` overlays [externally referenced in](#setting-library-values-via-files) + 3. specified using [`instance.with_data_values()`](lang-ref-ytt-library.md#instancewith_data_values) + 4. `@data/values` overlays [externally referenced in `after_library_module=True`](#setting-library-values-via-files) + 5. `--data-values-file` specified files [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 6. `--data-values-env` specified values [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 7. `--data-values-env-yaml` specified values [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 8. `--data-value` specified value [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 9. `--data-value-yaml` specified value [referenced in](#setting-library-values-via-command-line-flags) (left to right) + diff --git a/site/content/ytt/docs/v0.38.0/ytt-overlays.md b/site/content/ytt/docs/v0.38.0/ytt-overlays.md new file mode 100644 index 000000000..1e1a28278 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/ytt-overlays.md @@ -0,0 +1,79 @@ +--- +aliases: [/ytt/docs/latest/ytt-overlays] +title: Overlays +--- + +## Overview + +The way to describe patch-like edits in `ytt` is via "Overlays". + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +## What are they? + +Sometimes it makes more sense to patch some YAML rather than template it. + +For example, when: +- the file should not be edited directly (e.g. from a third party); +- the edit will apply to most or all documents; or +- the specific variable is less commonly configured. + +Given a sample target YAML file: + +`config.yml` +```yaml +--- +id: 1 +contents: +- apple +--- +id: 2 +contents: +- pineapple +``` +... this overlay ... + +`add-content.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +contents: +- pen +``` + +Prior to v0.32.0 append array items with [`#@overlay/append`](lang-ref-ytt-overlay.md#overlayappend) + +_read as..._ +1. _"match all YAML documents, expecting to match _at least_ one;"_ +2. _"within _each_ document, merge the key named `contents`;"_ +3. _"append an array item with the content `"pen"`"_ + + +... when processed by `ytt` ... + +```console +$ ytt -f config.yml -f add-content.yml +``` + +... produces ... + +`config.yml` _(edited)_ +```yaml +id: 1 +contents: +- apple +- pen +--- +id: 2 +contents: +- pineapple +- pen +``` + +## Next Steps + +- [Overlay example](/ytt/#example:example-overlay-files) in the ytt Playground to try it out, yourself. +- [ytt Library: Overlay module](lang-ref-ytt-overlay.md) for reference of all overlay annotations and functions. +- [Data Values vs Overlays](data-values-vs-overlays.md) for when to use one mechanism over the other. diff --git a/site/content/ytt/docs/v0.38.0/ytt-playground-screenshot.png b/site/content/ytt/docs/v0.38.0/ytt-playground-screenshot.png new file mode 100644 index 000000000..92882a6a9 Binary files /dev/null and b/site/content/ytt/docs/v0.38.0/ytt-playground-screenshot.png differ diff --git a/site/content/ytt/docs/v0.38.0/ytt-text-templating.md b/site/content/ytt/docs/v0.38.0/ytt-text-templating.md new file mode 100644 index 000000000..f2e588508 --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/ytt-text-templating.md @@ -0,0 +1,54 @@ +--- +aliases: [/ytt/docs/latest/ytt-text-templating] +title: Text Templating +--- + +## Overview + +ytt supports text templating within YAML strings and `.txt` files. + +Text templating is controlled via `(@` and `@)` directives. These directives can be combined with following markers: + +- `=` to output result; result must be of type string +- `-` to trim space either to the left (if next to opening directive) or right (if next to closing directive) + +Examples: + +- `before (@ 123 @) middle (@= "tpl" @) after` produces `before middle tpl after` +- `before (@- 123 -@) middle (@-= "tpl" -@) after` produces `beforemiddletplafter` + +## Inside YAML strings + +`+` operand or [`string·format(...)`](lang-ref-string.md) method provide a good way to build strings: + +- `name: #@ name_prefix + "-secret"` +- `name: #@ name_prefix + "-" + str(1234)` +- `name: #@ "{}-secret-{}".format(name_prefix, name_suffix)` + +However, occasionally it might be useful to use text templating directly in YAML strings. To do so YAML node must be annotated with `@yaml/text-templated-strings` (v0.17.0+). Annotation will apply to node and its child nodes. + +Examples: + +- basic use +```yaml +#@ val1 = "val1" +#@ val2 = "val2" + +#@yaml/text-templated-strings +templated: "before (@= val1 @) middle (@= val2 @) after" + +non_templated: "(@ something" +``` + +- nested nodes +```yaml +#@ val1 = "val1" +#@ val2 = "val2" + +#@yaml/text-templated-strings +--- +key: + nested_key_(@= val1 @): "middle (@= val2 @) after" +``` + +See [Text template example](/ytt/#example:example-text-template) in online playground. diff --git a/site/content/ytt/docs/v0.38.0/ytt-vs-x.md b/site/content/ytt/docs/v0.38.0/ytt-vs-x.md new file mode 100644 index 000000000..fbafe2fcd --- /dev/null +++ b/site/content/ytt/docs/v0.38.0/ytt-vs-x.md @@ -0,0 +1,61 @@ +--- +aliases: [/ytt/docs/latest/ytt-vs-x] +title: ytt vs x +--- + + +## ytt vs Go text/template (and other text templating tools) + +- [Go's text/template](https://golang.org/pkg/text/template/) +- [Jinja](http://jinja.pocoo.org/) + +Most generic templating tools do not understand content that they are templating and consider it just plain text. ytt operates on YAML structures, hence typical escaping and formatting problems common to text templating tools are eliminated. Additionally, ytt provides a very easy way to make structures reusable in a much more readable way that's possible with some text templating tools. + +## ytt vs jsonnet + +- [Jsonnet](https://jsonnet.org/) + +ytt conceptually is very close to [jsonnet](https://jsonnet.org/). Both operate on data structures instead of text, hence are able to provide a better way to construct, compose and reuse structures. jsonnet introduces a custom language to help perform structure operations. ytt on the other hand, builds upon a Python-like language, which we think will be more familiar to the larger community. + +We also believe that transitioning from plain YAML to templated YAML with `ytt` is very easy and natural. + +## ytt vs Dhall + +- [Dhall](https://dhall-lang.org/) + +Dhall language is a configuration language that can output YAML, and JSON. One of its strong points is ability to provide scripting environment that is "hermetically sealed" and safe, even against malicious templates. `ytt` also embraces same goal (and builds upon the great work of Starlark community) by exposing small API in the template context. For example, there is no way to make network calls, read from file system, _or currently, even get time_. + +## ytt vs Kustomize (and CF BOSH ops files) + +- [Kustomize](https://kubernetes.io/blog/2018/05/29/introducing-kustomize-template-free-configuration-customization-for-kubernetes/) +- [CF BOSH's ops files](https://bosh.io/docs/cli-ops-files) + +Configuration customization tools are unique in a sense that they don't allow templating but rather build upon "base" configuration. `ytt` offers its own take on configuration customization via the ['overlay' feature](lang-ref-ytt-overlay.md). Unlike other tools, overlay operations (remove, replace, merge) in `ytt` mimic structure of the base configuration. For example in Kustomize to remove a particular map key, one has to use JSON patch syntax which is quite different from the normal document structure. On the other hand, `ytt` uses its ability to annotate YAML structures, hence it can mark map key that should be deleted. All in all, we think that `ytt`'s approach is superior. + +Here are a few more detailed differences: + +- `ytt` overlays + + - are not Kubernetes-specific so various types of configurations are covered that kustomize cannot deal with. + - cover all CRUD operations in one consistent style whereas kustomize needs varied syntaxes for varied types of modification (SMP vs jsonPatch, etc.). + - do not care about native kinds, CRDs vs something else since they are generic. + +- `ytt` is not just an overlay tool; it supports overlaying _and_ templating. We see configuration writing split into two categories: configuration authors, and configuration consumers. Configuration _authors_ are best supported by templating; however, configuration _consumers_ typically need more than just templating inputs. Overlaying provides the rest. Having one tool with consistent functionality across templating and overlays is powerful. +- `ytt` is more explicit about missing map keys, etc (avoids a lot of unnecessary typos early on). +- `ytt` allows you to define variables. +- `ytt` has facilties to inject data into overlays from a variety of inputs including command line arguments, environment variables, and files. + +## ytt vs Orchestration Tools (Pulumi / HELM) + +- [Pulumi](https://www.pulumi.com/) +- [HELM](https://helm.sh/) + +Orchestration tools like Pulumi, and HELM, have combined configuration management and workflow management into the same tool. There are advantages and disadvantages to that. `ytt` is designed specifically to only focus on configuration management. Though, YAML output can be used with HELM, Pulumi, or other tools. + +## ytt vs plain Ruby/Python/etc + +Key advantages for `ytt`: + +- provides an easy way to operate on structures (maps, lists, etc.). One can definitely use a regular language to do data manipulation. However, this is not what the language is optimized for, especially if data is heavily nested, typically leading to very verbose and less readable code. +- provides an _easy and safe_ way to execute templates without worrying that template code may be malicious. One can Dockerize execution of regular language templates but of course that brings in pretty heavy dependency. +- provides an _easy_ way to customize any part of configuration via [overlays](lang-ref-ytt-overlay.md). This is not possible to do with a regular language without parameterizing everything (a general anti-pattern) or bringing in an additional tool (e.g. BOSH ops files). diff --git a/site/data/imgpkg/docs/imgpkg-latest-toc.yml b/site/data/imgpkg/docs/imgpkg-develop-toc.yml similarity index 100% rename from site/data/imgpkg/docs/imgpkg-latest-toc.yml rename to site/data/imgpkg/docs/imgpkg-develop-toc.yml diff --git a/site/data/imgpkg/docs/imgpkg-v0-24-0-toc.yml b/site/data/imgpkg/docs/imgpkg-v0-24-0-toc.yml new file mode 100644 index 000000000..741386d1d --- /dev/null +++ b/site/data/imgpkg/docs/imgpkg-v0-24-0-toc.yml @@ -0,0 +1,35 @@ +toc: + - title: Introduction + subfolderitems: + - page: About imgpkg + url: / + - page: Install + url: /install + - title: Workflows + subfolderitems: + - page: Basic workflow + url: /basic-workflow + - page: Air-gapped workflow + url: /air-gapped-workflow + - page: Automation workflow + url: /automation-workflow + - title: Reference + subfolderitems: + - page: Authentication + url: /auth + - page: Resources + url: /resources + - page: Commands + url: /commands + - page: Working directly with images + url: /working-directly-with-images + - page: Proxy + url: /proxy + - title: FAQ + subfolderitems: + - page: Security + url: /security + - page: CA Certs on Windows + url: /ca-certs-windows + - page: Debugging + url: /debugging diff --git a/site/data/imgpkg/docs/toc-mapping.yml b/site/data/imgpkg/docs/toc-mapping.yml index d7ee5cf81..0218af62e 100644 --- a/site/data/imgpkg/docs/toc-mapping.yml +++ b/site/data/imgpkg/docs/toc-mapping.yml @@ -2,5 +2,5 @@ # (TOC). You'll want to use this after any revamps to information architecture, to ensure # that the navigation for older versions still work. -latest: imgpkg-latest-toc -# v0.3.0: imgpkg-v0-3-0-toc +develop: imgpkg-develop-toc +v0.24.0: imgpkg-v0-24-0-toc diff --git a/site/data/kapp-controller/docs/kapp-controller-latest-toc.yml b/site/data/kapp-controller/docs/kapp-controller-develop-toc.yml similarity index 100% rename from site/data/kapp-controller/docs/kapp-controller-latest-toc.yml rename to site/data/kapp-controller/docs/kapp-controller-develop-toc.yml diff --git a/site/data/kapp-controller/docs/kapp-controller-v0-31-0-toc.yml b/site/data/kapp-controller/docs/kapp-controller-v0-31-0-toc.yml new file mode 100644 index 000000000..9b071ae6f --- /dev/null +++ b/site/data/kapp-controller/docs/kapp-controller-v0-31-0-toc.yml @@ -0,0 +1,57 @@ +toc: + - title: Introduction + subfolderitems: + - page: About kapp-controller + url: / + - page: Install + url: /install + - title: Continuous Delivery + subfolderitems: + - page: App CR Overview + url: /app-overview + - page: App CR Detailed Spec + url: /app-spec + - page: Install an application + url: /walkthrough + - page: Example Usage + url: /app-examples + - page: Intergration with Mozilla's sops + url: /sops + - page: Security Model + url: /security-model + - title: Package Management + subfolderitems: + - page: Concepts & CRDs + url: /packaging + - page: "Tutorial: Create & Install a Package" + url: /packaging-tutorial + - page: OSS Carvel Packages + url: /oss-packages + - page: Artifact Formats + url: /packaging-artifact-formats + - page: Consumer Concepts + url: /package-consumer-concepts + - page: Overlays with PackageInstall + url: /package-install-extensions + - page: Air-gapped installations + url: /air-gapped-workflow + - page: Private registry authentication + url: /private-registry-auth + - page: Package Management with GitOps + url: /packaging-gitops + - title: Advanced Topics + subfolderitems: + - page: Debugging CRs + url: /debugging-crs + - page: Controller Configuration + url: /controller-config + - page: Controller Startup + url: /startup + - page: Debugging kapp-controller + url: /debugging-kc + - title: FAQ + subfolderitems: + - page: kapp-controller FAQ + url: /faq + - page: Security + url: /security diff --git a/site/data/kapp-controller/docs/kapp-controller-v0-32-0-toc.yml b/site/data/kapp-controller/docs/kapp-controller-v0-32-0-toc.yml new file mode 100644 index 000000000..9b071ae6f --- /dev/null +++ b/site/data/kapp-controller/docs/kapp-controller-v0-32-0-toc.yml @@ -0,0 +1,57 @@ +toc: + - title: Introduction + subfolderitems: + - page: About kapp-controller + url: / + - page: Install + url: /install + - title: Continuous Delivery + subfolderitems: + - page: App CR Overview + url: /app-overview + - page: App CR Detailed Spec + url: /app-spec + - page: Install an application + url: /walkthrough + - page: Example Usage + url: /app-examples + - page: Intergration with Mozilla's sops + url: /sops + - page: Security Model + url: /security-model + - title: Package Management + subfolderitems: + - page: Concepts & CRDs + url: /packaging + - page: "Tutorial: Create & Install a Package" + url: /packaging-tutorial + - page: OSS Carvel Packages + url: /oss-packages + - page: Artifact Formats + url: /packaging-artifact-formats + - page: Consumer Concepts + url: /package-consumer-concepts + - page: Overlays with PackageInstall + url: /package-install-extensions + - page: Air-gapped installations + url: /air-gapped-workflow + - page: Private registry authentication + url: /private-registry-auth + - page: Package Management with GitOps + url: /packaging-gitops + - title: Advanced Topics + subfolderitems: + - page: Debugging CRs + url: /debugging-crs + - page: Controller Configuration + url: /controller-config + - page: Controller Startup + url: /startup + - page: Debugging kapp-controller + url: /debugging-kc + - title: FAQ + subfolderitems: + - page: kapp-controller FAQ + url: /faq + - page: Security + url: /security diff --git a/site/data/kapp-controller/docs/toc-mapping.yml b/site/data/kapp-controller/docs/toc-mapping.yml index 9e3dff6e3..b0fdb0216 100644 --- a/site/data/kapp-controller/docs/toc-mapping.yml +++ b/site/data/kapp-controller/docs/toc-mapping.yml @@ -1,5 +1,3 @@ -# This file can be used to explicitly map a release to a specific table-of-contents -# (TOC). You'll want to use this after any revamps to information architecture, to ensure -# that the navigation for older versions still work. - -latest: kapp-controller-latest-toc +develop: kapp-controller-develop-toc +v0.31.0: kapp-controller-v0-31-0-toc +v0.32.0: kapp-controller-v0-32-0-toc diff --git a/site/data/kapp/docs/kapp-latest-toc.yml b/site/data/kapp/docs/kapp-develop-toc.yml similarity index 100% rename from site/data/kapp/docs/kapp-latest-toc.yml rename to site/data/kapp/docs/kapp-develop-toc.yml diff --git a/site/data/kapp/docs/kapp-v0-44-0-toc.yml b/site/data/kapp/docs/kapp-v0-44-0-toc.yml new file mode 100644 index 000000000..3aa836543 --- /dev/null +++ b/site/data/kapp/docs/kapp-v0-44-0-toc.yml @@ -0,0 +1,45 @@ +toc: + - title: Introduction + subfolderitems: + - page: About kapp + url: / + - page: Install + url: /install + - page: Applications + url: /apps + - title: Deploy command + subfolderitems: + - page: Diff stage + url: /diff + - page: Apply stage + url: /apply + - page: Apply Ordering + url: /apply-ordering + - page: Apply Waiting + url: /apply-waiting + - title: Reference + subfolderitems: + - page: Configuration + url: /config + - page: Permissions + url: /rbac + - page: Namespace for State Storage + url: /state-namespace + - page: Cheatsheet + url: /cheatsheet + - page: Dangerous Flags + url: /dangerous-flags + - title: Integrations + subfolderitems: + - page: Integrating With Other Tools + url: /integrating-with-other-tools + - page: GitOps + url: /gitops + - title: FAQ + subfolderitems: + - page: FAQ + url: /faq + - page: Resource Merge Method + url: /merge-method + - page: Security + url: /security \ No newline at end of file diff --git a/site/data/kapp/docs/kapp-v0-45-0-toc.yml b/site/data/kapp/docs/kapp-v0-45-0-toc.yml new file mode 100644 index 000000000..3aa836543 --- /dev/null +++ b/site/data/kapp/docs/kapp-v0-45-0-toc.yml @@ -0,0 +1,45 @@ +toc: + - title: Introduction + subfolderitems: + - page: About kapp + url: / + - page: Install + url: /install + - page: Applications + url: /apps + - title: Deploy command + subfolderitems: + - page: Diff stage + url: /diff + - page: Apply stage + url: /apply + - page: Apply Ordering + url: /apply-ordering + - page: Apply Waiting + url: /apply-waiting + - title: Reference + subfolderitems: + - page: Configuration + url: /config + - page: Permissions + url: /rbac + - page: Namespace for State Storage + url: /state-namespace + - page: Cheatsheet + url: /cheatsheet + - page: Dangerous Flags + url: /dangerous-flags + - title: Integrations + subfolderitems: + - page: Integrating With Other Tools + url: /integrating-with-other-tools + - page: GitOps + url: /gitops + - title: FAQ + subfolderitems: + - page: FAQ + url: /faq + - page: Resource Merge Method + url: /merge-method + - page: Security + url: /security \ No newline at end of file diff --git a/site/data/kapp/docs/toc-mapping.yml b/site/data/kapp/docs/toc-mapping.yml index 34a76d29b..ddc145298 100644 --- a/site/data/kapp/docs/toc-mapping.yml +++ b/site/data/kapp/docs/toc-mapping.yml @@ -1,5 +1,3 @@ -# This file can be used to explicitly map a release to a specific table-of-contents -# (TOC). You'll want to use this after any revamps to information architecture, to ensure -# that the navigation for older versions still work. - -latest: kapp-latest-toc +develop: kapp-develop-toc +v0.44.0: kapp-v0-44-0-toc +v0.45.0: kapp-v0-45-0-toc diff --git a/site/data/kbld/docs/kbld-latest-toc.yml b/site/data/kbld/docs/kbld-develop-toc.yml similarity index 100% rename from site/data/kbld/docs/kbld-latest-toc.yml rename to site/data/kbld/docs/kbld-develop-toc.yml diff --git a/site/data/kbld/docs/kbld-v0-32-0-toc.yml b/site/data/kbld/docs/kbld-v0-32-0-toc.yml new file mode 100644 index 000000000..82ad1cc39 --- /dev/null +++ b/site/data/kbld/docs/kbld-v0-32-0-toc.yml @@ -0,0 +1,27 @@ +toc: + - title: Introduction + subfolderitems: + - page: About kbld + url: / + - page: Install + url: /install + - title: Usage + subfolderitems: + - page: Resolving images + url: /resolving + - page: Building images + url: /building + - page: Packaging & Relocation + url: /packaging + - page: CNAB Image Mappings + url: /cnab-image-relocation + - title: Reference + subfolderitems: + - page: Configuration + url: /config + - page: Authentication + url: /auth + - title: FAQ + subfolderitems: + - page: Security + url: /security \ No newline at end of file diff --git a/site/data/kbld/docs/toc-mapping.yml b/site/data/kbld/docs/toc-mapping.yml index f2bdecc47..42ef25374 100644 --- a/site/data/kbld/docs/toc-mapping.yml +++ b/site/data/kbld/docs/toc-mapping.yml @@ -2,4 +2,5 @@ # (TOC). You'll want to use this after any revamps to information architecture, to ensure # that the navigation for older versions still work. -latest: kbld-latest-toc +develop: kbld-develop-toc +v0.32.0: kbld-v0-32-0-toc diff --git a/site/data/vendir/docs/toc-mapping.yml b/site/data/vendir/docs/toc-mapping.yml index 6438ad4f2..ced3f1ba7 100644 --- a/site/data/vendir/docs/toc-mapping.yml +++ b/site/data/vendir/docs/toc-mapping.yml @@ -2,4 +2,5 @@ # (TOC). You'll want to use this after any revamps to information architecture, to ensure # that the navigation for older versions still work. -latest: vendir-latest-toc +develop: vendir-develop-toc +v0.24.0: vendir-v0-24-0-toc diff --git a/site/data/vendir/docs/vendir-latest-toc.yml b/site/data/vendir/docs/vendir-develop-toc.yml similarity index 100% rename from site/data/vendir/docs/vendir-latest-toc.yml rename to site/data/vendir/docs/vendir-develop-toc.yml diff --git a/site/data/vendir/docs/vendir-v0-24-0-toc.yml b/site/data/vendir/docs/vendir-v0-24-0-toc.yml new file mode 100644 index 000000000..8d154b1fe --- /dev/null +++ b/site/data/vendir/docs/vendir-v0-24-0-toc.yml @@ -0,0 +1,23 @@ +toc: + - title: Introduction + subfolderitems: + - page: About vendir + url: / + - page: Install + url: /install + - title: Reference + subfolderitems: + - page: Sync command + url: /sync + - page: vendir.yml Spec + url: /vendir-spec + - page: vendir.lock.yml Spec + url: /vendir-lock-spec + - page: Versions + url: /versions + - page: Github Release + url: /github-release + - title: FAQ + subfolderitems: + - page: Security + url: /security \ No newline at end of file diff --git a/site/data/ytt/docs/toc-mapping.yml b/site/data/ytt/docs/toc-mapping.yml index 39d01529a..23650ab72 100644 --- a/site/data/ytt/docs/toc-mapping.yml +++ b/site/data/ytt/docs/toc-mapping.yml @@ -2,4 +2,5 @@ # (TOC). You'll want to use this after any revamps to information architecture, to ensure # that the navigation for older versions still work. -latest: ytt-latest-toc +develop: ytt-develop-toc +v0.38.0: ytt-v0-38-0-toc diff --git a/site/data/ytt/docs/ytt-latest-toc.yml b/site/data/ytt/docs/ytt-develop-toc.yml similarity index 100% rename from site/data/ytt/docs/ytt-latest-toc.yml rename to site/data/ytt/docs/ytt-develop-toc.yml diff --git a/site/data/ytt/docs/ytt-v0-38-0-toc.yml b/site/data/ytt/docs/ytt-v0-38-0-toc.yml new file mode 100644 index 000000000..2c578b2ca --- /dev/null +++ b/site/data/ytt/docs/ytt-v0-38-0-toc.yml @@ -0,0 +1,95 @@ +toc: + - title: Introduction + subfolderitems: + - page: About ytt + url: / + - page: Install + url: /install + - page: How it Works + url: /how-it-works + - title: Concepts + subfolderitems: + - page: YAML and Annotations + url: /yaml-primer + - title: How To + subfolderitems: + - page: Use Data Values + url: /how-to-use-data-values + - page: Write Schema + url: /how-to-write-schema + - page: Export Schema in OpenAPI format + url: /how-to-export-schema + - page: Use Overlays + url: /ytt-overlays + - page: Data Values vs. Overlays + url: /data-values-vs-overlays + - page: Inject secrets + url: /injecting-secrets + - title: Reference + subfolderitems: + - page: Language + url: /lang + - page: Annotations + url: /lang-ref-annotation + - page: Data Values + url: /ytt-data-values + - page: Data Values Schema + url: /lang-ref-ytt-schema + - page: Text Templating + url: /ytt-text-templating + - title: Built-in @ytt library + subfolderitems: + - page: All modules + url: /lang-ref-ytt + - page: Struct module + url: /lang-ref-ytt-struct + - page: Overlay module + url: /lang-ref-ytt-overlay + - page: Library module + url: /lang-ref-ytt-library + - page: Version module + url: /lang-ref-ytt-version + - page: Template module + url: /lang-ref-ytt-template + - title: Language + subfolderitems: + - page: General + url: /lang + - page: String + url: /lang-ref-string + - page: List + url: /lang-ref-list + - page: Dictionary + url: /lang-ref-dict + - page: Struct + url: /lang-ref-structs + - page: YAML Fragment + url: /lang-ref-yaml-fragment + - page: If conditional + url: /lang-ref-if + - page: For loop + url: /lang-ref-for + - page: Function + url: /lang-ref-def + - page: Load + url: /lang-ref-load + - title: CLI configuration + subfolderitems: + - page: Outputs + url: /outputs + - page: File Marks + url: /file-marks + - page: Strict yaml + url: /strict + - title: FAQ + subfolderitems: + - page: FAQ + url: /faq + - page: Why ytt vs. x? + url: /ytt-vs-x + - page: Schema Migration Guide + url: /data-values-schema-migration-guide + - page: Security + url: /security + - page: Known Limitations + url: /known-limitations diff --git a/site/hack/build.sh b/site/hack/build.sh new file mode 100755 index 000000000..164e32bee --- /dev/null +++ b/site/hack/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -o nounset +set -o pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +pushd "${SCRIPT_DIR}"/../../tools +export CGO_ENABLED=0 +go build -o "../validatedocs" -trimpath ./cmd/validate-docs/... +popd diff --git a/site/hack/release-docs.sh b/site/hack/release-docs.sh new file mode 100755 index 000000000..941a748e8 --- /dev/null +++ b/site/hack/release-docs.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +# How to run the command to generate new documentation of impkg from develop with the version v0.5.0 +# ./hack/release-docs.sh imgpkg v0.5.0 + +TOOL=$1 +NEW_VERSION=$2 +FROM_VERSION=${3:-develop} + +CONTENT_DIRECTORY=content/${TOOL}/docs +TOC_DIRECTORY=data/${TOOL}/docs + +# don't run if there's already a directory for the target docs version +if [[ -d $CONTENT_DIRECTORY/$NEW_VERSION ]]; then + echo "ERROR: $CONTENT_DIRECTORY/$NEW_VERSION already exists" + exit 1 +fi + +LATEST_VERSION=$(ytt -f version.yml=<(echo -e "#@ load('@ytt:data', 'data')\n--- #@ data.values.params[\"${TOOL}\"].version_latest") --data-values-file config.yaml) +# don't run if cannot find the origin version +if [[ ! -d $CONTENT_DIRECTORY/$FROM_VERSION ]]; then + echo "ERROR: $CONTENT_DIRECTORY/$FROM_VERSION from version folder does not exist" + exit 1 +fi + +PREVIOUS_VERSION="${LATEST_VERSION}" +COPY_FROM_FOLDER="${CONTENT_DIRECTORY}/${LATEST_VERSION}/" +if [ "${FROM_VERSION}" != "develop" ] && [ "${FROM_VERSION}" != "${LATEST_VERSION}" ]; then + PREVIOUS_VERSION=$(ytt -f version.yml=<(echo -e "#@ load('@ytt:data', 'data')\n--- #@ data.values.cascade.version") --data-values-file <(head -n8 content/ytt/docs/v0.38.0/_index.md)) + COPY_FROM_FOLDER="${CONTENT_DIRECTORY}/${FROM_VERSION}/" +fi + + +if [ "${FROM_VERSION}" == "develop" ]; then # When creating a new release from develop + echo "Copying the content of the develop documentation to latest documentation" + cp -rf "${CONTENT_DIRECTORY}"/develop "${CONTENT_DIRECTORY}"/${NEW_VERSION} + + echo "Update version on ${NEW_VERSION} docs" + sed -i.bak "s/version: develop/version: ${NEW_VERSION}/" "${CONTENT_DIRECTORY}"/${NEW_VERSION}/_index.md + rm "${CONTENT_DIRECTORY}"/${NEW_VERSION}/_index.md.bak + + dashedVersion=${NEW_VERSION//\./-} + echo "Copy table of content" + cp "${TOC_DIRECTORY}"/"${TOOL}"-develop-toc.yml "${TOC_DIRECTORY}"/"${TOOL}"-"${dashedVersion}"-toc.yml + git add "${TOC_DIRECTORY}"/"${TOOL}"-"${dashedVersion}"-toc.yml +elif [ "${FROM_VERSION}" == "${LATEST_VERSION}" ]; then # When creating a patch from the latest release + echo "Copying the content of the latest documentation to named version ${LATEST_VERSION}" + cp -rf "${CONTENT_DIRECTORY}"/${LATEST_VERSION} "${CONTENT_DIRECTORY}"/"${NEW_VERSION}" + + echo "Update version on ${NEW_VERSION} docs" + sed -i.bak "s/version: ${LATEST_VERSION}/version: ${NEW_VERSION}/" "${CONTENT_DIRECTORY}"/${NEW_VERSION}/_index.md + rm "${CONTENT_DIRECTORY}"/${NEW_VERSION}/_index.md.bak + + dashedVersion=${NEW_VERSION//\./-} + latestVersionDashedVersion=${LATEST_VERSION//\./-} + echo "Copy table of content" + cp "${TOC_DIRECTORY}/${TOOL}-${latestVersionDashedVersion}-toc.yml" "${TOC_DIRECTORY}/${TOOL}-${dashedVersion}-toc.yml" + git add "${TOC_DIRECTORY}"/"${TOOL}"-"${dashedVersion}"-toc.yml +else # When creating a patch for any other release + echo "Copying the content of the ${FROM_VERSION} documentation to named version ${NEW_VERSION}" + cp -rf "${CONTENT_DIRECTORY}/${FROM_VERSION}/" "${CONTENT_DIRECTORY}"/"${NEW_VERSION}" + sed -i.bak "s/version: ${FROM_VERSION}/version: ${NEW_VERSION}/" "${CONTENT_DIRECTORY}"/${NEW_VERSION}/_index.md + rm "${CONTENT_DIRECTORY}"/${NEW_VERSION}/_index.md.bak + + dashedVersion=${NEW_VERSION//\./-} + fromDashedVersion=${FROM_VERSION//\./-} + echo "Copy table of content" + cp "${TOC_DIRECTORY}/${TOOL}-${fromDashedVersion}-toc.yml" "${TOC_DIRECTORY}/${TOOL}-${dashedVersion}-toc.yml" + git add "${TOC_DIRECTORY}"/"${TOOL}"-"${dashedVersion}"-toc.yml +fi + +git add "${CONTENT_DIRECTORY}"/${NEW_VERSION} + +currentTOC="${TOC_DIRECTORY}"/toc-mapping.yml + +echo "Updating the TOC file" +ytt --ignore-unknown-comments -f"$currentTOC" -ftocOverlay.yml=<(cat < /tmp/newToc.yml +mv /tmp/newToc.yml $currentTOC +git add $currentTOC + +echo "Updating the configuration file" +if [ "${FROM_VERSION}" == "develop" ] || [ "${FROM_VERSION}" == "${LATEST_VERSION}" ]; then + echo "For latest version" + ytt -f config.yaml -fconfigOverlay.yml=<(cat < /tmp/newConfig.yml + mv /tmp/newConfig.yml config.yaml + git add config.yaml + + echo "Add redirection from latest to version ${NEW_VERSION}" + for file in $(find content/${TOOL}/docs/${NEW_VERSION} -name "*.md"); + do + filename=$(basename $file) + if [[ "$filename" == "_index.md" ]]; then + filename= + fi + + cat "$file" | awk 'BEGIN {t=0}; { print }; /---/ { t++; if ( t==1) { printf "aliases: [/%s/docs/latest/%s]\n", tool, filename } }' tool=${TOOL} filename="${filename%.md}" > "$file.bak" + mv "$file".bak "$file" + git add $file + done + + echo "Remove redirection from ${LATEST_VERSION} to latest" + for file in $(find content/${TOOL}/docs/${LATEST_VERSION} -name "*.md"); + do + sed -i.bak "s/aliases: \[[a-z\/\-]*\]//" $file + rm "$file.bak" + git add $file + done +else + echo "For patch version version" + ytt -f config.yaml -fconfigOverlay.yml=<(cat < /tmp/newConfig.yml + mv /tmp/newConfig.yml config.yaml + git add config.yaml +fi diff --git a/site/hack/validate-doc-change.sh b/site/hack/validate-doc-change.sh new file mode 100755 index 000000000..cd81ef0b6 --- /dev/null +++ b/site/hack/validate-doc-change.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -o nounset +set -o pipefail + +echo "=============== list modified files ===============" +git diff --name-only develop | head + +echo "========== check paths of modified files ==========" +git diff --name-only develop | head > filesChanged.txt + +./validatedocs -diff filesChanged.txt -cfg site/config.yaml + +echo "No frozen documentation changes" +exit 0 diff --git a/site/themes/carvel/assets/scss/_alerts.scss b/site/themes/carvel/assets/scss/_alerts.scss new file mode 100644 index 000000000..6da04818d --- /dev/null +++ b/site/themes/carvel/assets/scss/_alerts.scss @@ -0,0 +1,50 @@ +// +// Base styles +// +.alert { + position: relative; + padding: $alert-padding-y $alert-padding-x; + margin-bottom: $alert-margin-bottom; + border: $alert-border-width solid transparent; + @include border-radius($alert-border-radius); +} + +// Headings for larger alerts +.alert-heading { + // Specified to prevent conflicts of changing $headings-color + color: inherit; +} + +// Provide class for links that match alerts +.alert-link { + font-weight: $alert-link-font-weight; +} + + +// Dismissible alerts +// +// Expand the right padding and account for the close button's positioning. + +.alert-dismissible { + padding-right: $close-font-size + $alert-padding-x * 2; + + // Adjust close link position + .close { + position: absolute; + top: 0; + right: 0; + z-index: 2; + padding: $alert-padding-y $alert-padding-x; + color: inherit; + } +} + +// Alternate styles +// +// Generate contextual modifier classes for colorizing the alert. + +@each $color, $value in $theme-colors { + .alert-#{$color} { + @include alert-variant(theme-color-level($color, $alert-bg-level), theme-color-level($color, $alert-border-level), theme-color-level($color, $alert-color-level)); + } +} diff --git a/site/themes/carvel/assets/scss/_components.scss b/site/themes/carvel/assets/scss/_components.scss index 9667a06ac..df8d9743f 100644 --- a/site/themes/carvel/assets/scss/_components.scss +++ b/site/themes/carvel/assets/scss/_components.scss @@ -387,6 +387,7 @@ } } .dropdown-menu { + z-index: 1; position: absolute; border: 1px solid $grey; border-radius: 5px; @@ -403,6 +404,10 @@ &.dropdown-menu-visible { display: block; } + + .dropdown-item.deprecated { + color: $grey; + } } .form-control { display: block; diff --git a/site/themes/carvel/assets/scss/_functions.scss b/site/themes/carvel/assets/scss/_functions.scss new file mode 100644 index 000000000..13d9de0c1 --- /dev/null +++ b/site/themes/carvel/assets/scss/_functions.scss @@ -0,0 +1,190 @@ +// Bootstrap functions +// +// Utility mixins and functions for evaluating source code across our variables, maps, and mixins. + +// Ascending +// Used to evaluate Sass maps like our grid breakpoints. +@mixin _assert-ascending($map, $map-name) { + $prev-key: null; + $prev-num: null; + @each $key, $num in $map { + @if $prev-num == null or unit($num) == "%" or unit($prev-num) == "%" { + // Do nothing + } @else if not comparable($prev-num, $num) { + @warn "Potentially invalid value for #{$map-name}: This map must be in ascending order, but key '#{$key}' has value #{$num} whose unit makes it incomparable to #{$prev-num}, the value of the previous key '#{$prev-key}' !"; + } @else if $prev-num >= $num { + @warn "Invalid value for #{$map-name}: This map must be in ascending order, but key '#{$key}' has value #{$num} which isn't greater than #{$prev-num}, the value of the previous key '#{$prev-key}' !"; + } + $prev-key: $key; + $prev-num: $num; + } +} + +// Starts at zero +// Used to ensure the min-width of the lowest breakpoint starts at 0. +@mixin _assert-starts-at-zero($map, $map-name: "$grid-breakpoints") { + @if length($map) > 0 { + $values: map-values($map); + $first-value: nth($values, 1); + @if $first-value != 0 { + @warn "First breakpoint in #{$map-name} must start at 0, but starts at #{$first-value}."; + } + } +} + +// Replace `$search` with `$replace` in `$string` +// Used on our SVG icon backgrounds for custom forms. +// +// @author Hugo Giraudel +// @param {String} $string - Initial string +// @param {String} $search - Substring to replace +// @param {String} $replace ('') - New value +// @return {String} - Updated string +@function str-replace($string, $search, $replace: "") { + $index: str-index($string, $search); + + @if $index { + @return str-slice($string, 1, $index - 1) + $replace + str-replace(str-slice($string, $index + str-length($search)), $search, $replace); + } + + @return $string; +} + +// See https://codepen.io/kevinweber/pen/dXWoRw +// +// Requires the use of quotes around data URIs. + +@function escape-svg($string) { + @if str-index($string, "data:image/svg+xml") { + @each $char, $encoded in $escaped-characters { + // Do not escape the url brackets + @if str-index($string, "url(") == 1 { + $string: url("#{str-replace(str-slice($string, 6, -3), $char, $encoded)}"); + } @else { + $string: str-replace($string, $char, $encoded); + } + } + } + + @return $string; +} + +// Color contrast +@function color-yiq($color, $dark: $yiq-text-dark, $light: $yiq-text-light) { + $r: red($color); + $g: green($color); + $b: blue($color); + + $yiq: (($r * 299) + ($g * 587) + ($b * 114)) * .001; + + @if ($yiq >= $yiq-contrasted-threshold) { + @return $dark; + } @else { + @return $light; + } +} + +// Retrieve color Sass maps +@function color($key: "blue") { + @return map-get($colors, $key); +} + +@function theme-color($key: "primary") { + @return map-get($theme-colors, $key); +} + +@function gray($key: "100") { + @return map-get($grays, $key); +} + +// Request a theme color level +@function theme-color-level($color-name: "primary", $level: 0) { + $color: theme-color($color-name); + $color-base: if($level > 0, $black, $white); + $level: abs($level); + + @return mix($color-base, $color, $level * $theme-color-interval); +} + +// Return valid calc +@function add($value1, $value2, $return-calc: true) { + @if $value1 == null { + @return $value2; + } + + @if $value2 == null { + @return $value1; + } + + @if type-of($value1) == number and type-of($value2) == number and comparable($value1, $value2) { + @return $value1 + $value2; + } + + @return if($return-calc == true, calc(#{$value1} + #{$value2}), $value1 + unquote(" + ") + $value2); +} + +@function subtract($value1, $value2, $return-calc: true) { + @if $value1 == null and $value2 == null { + @return null; + } + + @if $value1 == null { + @return -$value2; + } + + @if $value2 == null { + @return $value1; + } + + @if type-of($value1) == number and type-of($value2) == number and comparable($value1, $value2) { + @return $value1 - $value2; + } + + @if type-of($value2) != number { + $value2: unquote("(") + $value2 + unquote(")"); + } + + @return if($return-calc == true, calc(#{$value1} - #{$value2}), $value1 + unquote(" - ") + $value2); +} + +@function divide($dividend, $divisor, $precision: 10) { + $sign: if($dividend > 0 and $divisor > 0 or $dividend < 0 and $divisor < 0, 1, -1); + $dividend: abs($dividend); + $divisor: abs($divisor); + @if $dividend == 0 { + @return 0; + } + @if $divisor == 0 { + @error "Cannot divide by 0"; + } + $remainder: $dividend; + $result: 0; + $factor: 10; + @while ($remainder > 0 and $precision >= 0) { + $quotient: 0; + @while ($remainder >= $divisor) { + $remainder: $remainder - $divisor; + $quotient: $quotient + 1; + } + $result: $result * 10 + $quotient; + $factor: $factor * .1; + $remainder: $remainder * 10; + $precision: $precision - 1; + @if ($precision < 0 and $remainder >= $divisor * 5) { + $result: $result + 1; + } + } + $result: $result * $factor * $sign; + $dividend-unit: unit($dividend); + $divisor-unit: unit($divisor); + $unit-map: ( + "px": 1px, + "rem": 1rem, + "em": 1em, + "%": 1% + ); + @if ($dividend-unit != $divisor-unit and map-has-key($unit-map, $dividend-unit)) { + $result: $result * map-get($unit-map, $dividend-unit); + } + @return $result; +} diff --git a/site/themes/carvel/assets/scss/_mixins.scss b/site/themes/carvel/assets/scss/_mixins.scss index 49380a36e..dd84ff441 100644 --- a/site/themes/carvel/assets/scss/_mixins.scss +++ b/site/themes/carvel/assets/scss/_mixins.scss @@ -33,4 +33,17 @@ &:after { clear: both; } -} \ No newline at end of file +} + +// Toggles +// +// Used in conjunction with global variables to enable certain theme features. + +// Components +@import "mixins/alert"; + +// Skins +@import "mixins/gradients"; +@import "mixins/border-radius"; + +// Layout diff --git a/site/themes/carvel/assets/scss/_variables.scss b/site/themes/carvel/assets/scss/_variables.scss index 380173106..e258b5473 100644 --- a/site/themes/carvel/assets/scss/_variables.scss +++ b/site/themes/carvel/assets/scss/_variables.scss @@ -7,3 +7,89 @@ $grey: #777777; $lightgrey: #F2F2F2; $green: #78BE20; $teal: #00C0D5; +$gray-100: #f8f9fa !default; +$gray-200: #e9ecef !default; +$gray-300: #dee2e6 !default; +$gray-400: #ced4da !default; +$gray-500: #adb5bd !default; +$gray-600: #6c757d !default; +$gray-700: #495057 !default; +$gray-800: #343a40 !default; +$gray-900: #212529 !default; +$cyan: #0dcaf0 !default; +$yellow: #ffc107 !default; + +$primary: $blue !default; +$secondary: $gray-600 !default; +$success: $green !default; +$info: $cyan !default; +$warning: $yellow !default; +$light: $gray-100 !default; +$dark: $gray-900 !default; + +$theme-colors: ( + "primary": $primary, + "secondary": $secondary, + "success": $success, + "info": $info, + "warning": $warning, + "light": $light, + "dark": $dark +) !default; + +// Set a specific jump point for requesting color jumps +$theme-color-interval: 8% !default; + +$variable-prefix: bs- !default; +$spacer: 1rem !default; +// scss-docs-start border-radius-variables +$border-radius: .25rem !default; +$border-radius-sm: .2rem !default; +$border-radius-lg: .3rem !default; +$border-radius-pill: 50rem !default; +// scss-docs-end border-radius-variables + +$font-weight-lighter: lighter !default; +$font-weight-light: 300 !default; +$font-weight-normal: 400 !default; +$font-weight-bold: 700 !default; +$font-weight-bolder: bolder !default; + +$font-weight-base: $font-weight-normal !default; +$border-width: 1px !default; +$border-widths: ( + 1: 1px, + 2: 2px, + 3: 3px, + 4: 4px, + 5: 5px +) !default; +$stretched-link-z-index: 1 !default; +// The contrast ratio to reach against white, to determine if color changes from "light" to "dark". Acceptable values for WCAG 2.0 are 3, 4.5 and 7. +// See https://www.w3.org/TR/WCAG20/#visual-audio-contrast-contrast +$min-contrast-ratio: 4.5 !default; +$font-size-base: 1rem !default; // Assumes the browser default, typically `16px` +$alert-bg-level: -10 !default; +$alert-border-level: -9 !default; +$alert-color-level: 6 !default; +$close-font-size: $font-size-base * 1.5 !default; + + +$enable-rounded: true !default; +$enable-gradients: false !default; + +// Alerts +// +// Define alert colors, border radius, and padding. + +// scss-docs-start alert-variables +$alert-padding-y: $spacer !default; +$alert-padding-x: $spacer !default; +$alert-margin-bottom: 1rem !default; +$alert-border-radius: $border-radius !default; +$alert-link-font-weight: $font-weight-bold !default; +$alert-border-width: $border-width !default; +$alert-bg-scale: -80% !default; +$alert-border-scale: -70% !default; +$alert-color-scale: 40% !default; +$alert-dismissible-padding-r: $alert-padding-x * 3 !default; // 3x covers width of x plus default padding on either side diff --git a/site/themes/carvel/assets/scss/mixins/_alert.scss b/site/themes/carvel/assets/scss/mixins/_alert.scss new file mode 100644 index 000000000..c74696430 --- /dev/null +++ b/site/themes/carvel/assets/scss/mixins/_alert.scss @@ -0,0 +1,13 @@ +@mixin alert-variant($background, $border, $color) { + color: $color; +@include gradient-bg($background); + border-color: $border; + + hr { + border-top-color: darken($border, 5%); + } + + .alert-link { + color: darken($color, 10%); + } +} diff --git a/site/themes/carvel/assets/scss/mixins/_border-radius.scss b/site/themes/carvel/assets/scss/mixins/_border-radius.scss new file mode 100644 index 000000000..4fad91d67 --- /dev/null +++ b/site/themes/carvel/assets/scss/mixins/_border-radius.scss @@ -0,0 +1,76 @@ +// stylelint-disable property-disallowed-list +// Single side border-radius + +// Helper function to replace negative values with 0 +@function valid-radius($radius) { + $return: (); + @each $value in $radius { + @if type-of($value) == number { + $return: append($return, max($value, 0)); + } @else { + $return: append($return, $value); + } + } + @return $return; +} + +@mixin border-radius($radius: $border-radius, $fallback-border-radius: false) { + @if $enable-rounded { + border-radius: valid-radius($radius); + } + @else if $fallback-border-radius != false { + border-radius: $fallback-border-radius; + } +} + +@mixin border-top-radius($radius) { + @if $enable-rounded { + border-top-left-radius: valid-radius($radius); + border-top-right-radius: valid-radius($radius); + } +} + +@mixin border-right-radius($radius) { + @if $enable-rounded { + border-top-right-radius: valid-radius($radius); + border-bottom-right-radius: valid-radius($radius); + } +} + +@mixin border-bottom-radius($radius) { + @if $enable-rounded { + border-bottom-right-radius: valid-radius($radius); + border-bottom-left-radius: valid-radius($radius); + } +} + +@mixin border-left-radius($radius) { + @if $enable-rounded { + border-top-left-radius: valid-radius($radius); + border-bottom-left-radius: valid-radius($radius); + } +} + +@mixin border-top-left-radius($radius) { + @if $enable-rounded { + border-top-left-radius: valid-radius($radius); + } +} + +@mixin border-top-right-radius($radius) { + @if $enable-rounded { + border-top-right-radius: valid-radius($radius); + } +} + +@mixin border-bottom-right-radius($radius) { + @if $enable-rounded { + border-bottom-right-radius: valid-radius($radius); + } +} + +@mixin border-bottom-left-radius($radius) { + @if $enable-rounded { + border-bottom-left-radius: valid-radius($radius); + } +} diff --git a/site/themes/carvel/assets/scss/mixins/_gradients.scss b/site/themes/carvel/assets/scss/mixins/_gradients.scss new file mode 100644 index 000000000..39480bb20 --- /dev/null +++ b/site/themes/carvel/assets/scss/mixins/_gradients.scss @@ -0,0 +1,45 @@ +// Gradients + +@mixin gradient-bg($color) { +@if $enable-gradients { + background: $color linear-gradient(180deg, mix($body-bg, $color, 15%), $color) repeat-x; +} @else { + background-color: $color; +} +} + +// Horizontal gradient, from left to right +// +// Creates two color stops, start and end, by specifying a color and position for each color stop. +@mixin gradient-x($start-color: $gray-700, $end-color: $gray-800, $start-percent: 0%, $end-percent: 100%) { + background-image: linear-gradient(to right, $start-color $start-percent, $end-color $end-percent); + background-repeat: repeat-x; +} + +// Vertical gradient, from top to bottom +// +// Creates two color stops, start and end, by specifying a color and position for each color stop. +@mixin gradient-y($start-color: $gray-700, $end-color: $gray-800, $start-percent: 0%, $end-percent: 100%) { + background-image: linear-gradient(to bottom, $start-color $start-percent, $end-color $end-percent); + background-repeat: repeat-x; +} + +@mixin gradient-directional($start-color: $gray-700, $end-color: $gray-800, $deg: 45deg) { + background-image: linear-gradient($deg, $start-color, $end-color); + background-repeat: repeat-x; +} +@mixin gradient-x-three-colors($start-color: $blue, $mid-color: $purple, $color-stop: 50%, $end-color: $red) { + background-image: linear-gradient(to right, $start-color, $mid-color $color-stop, $end-color); + background-repeat: no-repeat; +} +@mixin gradient-y-three-colors($start-color: $blue, $mid-color: $purple, $color-stop: 50%, $end-color: $red) { + background-image: linear-gradient($start-color, $mid-color $color-stop, $end-color); + background-repeat: no-repeat; +} +@mixin gradient-radial($inner-color: $gray-700, $outer-color: $gray-800) { + background-image: radial-gradient(circle, $inner-color, $outer-color); + background-repeat: no-repeat; +} +@mixin gradient-striped($color: rgba($white, .15), $angle: 45deg) { + background-image: linear-gradient($angle, $color 25%, transparent 25%, transparent 50%, $color 50%, $color 75%, transparent 75%, transparent); +} diff --git a/site/themes/carvel/assets/scss/site.scss b/site/themes/carvel/assets/scss/site.scss index 4ce7ebdc6..8da506c12 100644 --- a/site/themes/carvel/assets/scss/site.scss +++ b/site/themes/carvel/assets/scss/site.scss @@ -1,6 +1,8 @@ +@import 'variables'; +@import 'functions'; +@import 'mixins'; +@import 'alerts'; @import 'header'; @import 'footer'; @import 'base'; -@import 'variables'; @import 'components'; -@import 'mixins'; \ No newline at end of file diff --git a/site/themes/carvel/layouts/docs/_version-warning.html b/site/themes/carvel/layouts/docs/_version-warning.html index afe82aec7..6b932ceee 100644 --- a/site/themes/carvel/layouts/docs/_version-warning.html +++ b/site/themes/carvel/layouts/docs/_version-warning.html @@ -1,11 +1,20 @@ {{ $subprojectInfo := (index .Site.Params .Section) }} {{ if $subprojectInfo }} {{ if ne .CurrentSection.Params.version $subprojectInfo.version_latest }} + {{ if ne .CurrentSection.Params.version "develop" }} + {{ else }} + + {{ end }} {{ end }} {{ end }} diff --git a/site/themes/carvel/layouts/docs/_versions.html b/site/themes/carvel/layouts/docs/_versions.html index 7e1e3e8fe..a3dad90dd 100644 --- a/site/themes/carvel/layouts/docs/_versions.html +++ b/site/themes/carvel/layouts/docs/_versions.html @@ -1,19 +1,28 @@ {{ $subprojectInfo := (index .Site.Params .Section) }}
{{ if $subprojectInfo.versioning }} + {{ $is_latest := eq $subprojectInfo.version_latest .CurrentSection.Params.version }} {{ else }} diff --git a/site/themes/carvel/layouts/docs/docs.html b/site/themes/carvel/layouts/docs/docs.html index 9a3735b0a..f9bd0e156 100644 --- a/site/themes/carvel/layouts/docs/docs.html +++ b/site/themes/carvel/layouts/docs/docs.html @@ -16,6 +16,10 @@ {{ end }} {{ .Site.Title }} - {{ .Section }} - {{ .Title }} + + + + @@ -49,6 +53,7 @@