From d8dd1c7179390cf0d6a21bcdac1ddd692f8000aa Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 1 Oct 2024 10:36:26 +0100 Subject: [PATCH 01/51] added SSD guide for AWS --- .../install/helm/install-scalable/_index.md | 187 +++---- .../install/helm/install-scalable/aws.md | 502 ++++++++++++++++++ 2 files changed, 569 insertions(+), 120 deletions(-) create mode 100644 docs/sources/setup/install/helm/install-scalable/aws.md diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index a39b6580a90b2..bd15526168244 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -9,9 +9,9 @@ weight: 300 keywords: --- -# Install the simple scalable Helm chart +# Loki simple scalable Helm chart -This Helm Chart deploys Grafana Loki on Kubernetes. +This Helm Chart deploys Grafana Loki on Kubernetes in simple scalable mode. This chart configures Loki to run `read`, `write`, and `backend` targets in a [scalable mode]({{< relref "../../../../get-started/deployment-modes#simple-scalable" >}}). Loki’s simple scalable deployment mode separates execution paths into read, write, and backend targets. @@ -22,17 +22,21 @@ The default Helm chart deploys the following components: - Loki Canary (1 DaemonSet) - Gateway (1 NGINX replica) - Minio (optional, if `minio.enabled=true`) +- Index and Chunk cache (1 replica) +{{< admonition type="note" >}} It is not recommended to run scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +{{< /admonition >}} -**Prerequisites** +## Prerequisites - Helm 3 or above. See [Installing Helm](https://helm.sh/docs/intro/install/). - A running Kubernetes cluster. -- (Optional) A Memcached deployment for better query performance. For information on configuring Memcached, refer to [caching section]({{< relref "../../../../operations/caching" >}}). -**To deploy Loki in simple scalable mode:** +## Deploying the Helm chart for development and testing + +The following steps show how to deploy the Loki Helm chart in simple scalable mode using the included MinIO as the storage backend. Our recommendation is to start here for development and testing purposes. Then configure Loki with an object storage provider when moving to production. 1. Add [Grafana's chart repository](https://github.com/grafana/helm-charts) to Helm: @@ -49,72 +53,51 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu 3. Create the configuration file `values.yaml`. The example below illustrates how to deploy Loki in test mode using MinIO as storage: - ```yaml - loki: - schemaConfig: - configs: - - from: 2024-04-01 - store: tsdb - object_store: s3 - schema: v13 - index: - prefix: loki_index_ - period: 24h - ingester: - chunk_encoding: snappy - tracing: - enabled: true - querier: - # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing - max_concurrent: 4 - - #gateway: - # ingress: - # enabled: true - # hosts: - # - host: FIXME - # paths: - # - path: / - # pathType: Prefix - - deploymentMode: SimpleScalable - - backend: - replicas: 3 - read: - replicas: 3 - write: - replicas: 3 - - # Enable minio for storage - minio: - enabled: true - - # Zero out replica counts of other deployment modes - singleBinary: - replicas: 0 - - ingester: - replicas: 0 - querier: - replicas: 0 - queryFrontend: - replicas: 0 - queryScheduler: - replicas: 0 - distributor: - replicas: 0 - compactor: - replicas: 0 - indexGateway: - replicas: 0 - bloomCompactor: - replicas: 0 - bloomGateway: - replicas: 0 - ``` - -4. Install or upgrade the Loki deployment. + ```yaml + loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + querier: + # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing + max_concurrent: 4 + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h + compactor: + retention_enabled: true + delete_request_store: s3 + + deploymentMode: SimpleScalable + + backend: + replicas: 2 + read: + replicas: 2 + write: + replicas: 3 + + # Enable minio for storage + minio: + enabled: true + + gateway: + service: + type: LoadBalancer + ``` + +1. Install or upgrade the Loki deployment. - To install: ```bash helm install --values values.yaml loki grafana/loki @@ -155,11 +138,11 @@ loki: storage: type: s3 bucketNames: - chunks: "" + chunks: "" # ruler: "" admin: "" s3: - # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name + # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name # AWS endpoint URL endpoint: @@ -190,29 +173,6 @@ write: # Disable minio storage minio: enabled: false - -# Zero out replica counts of other deployment modes -singleBinary: - replicas: 0 - -ingester: - replicas: 0 -querier: - replicas: 0 -queryFrontend: - replicas: 0 -queryScheduler: - replicas: 0 -distributor: - replicas: 0 -compactor: - replicas: 0 -indexGateway: - replicas: 0 -bloomCompactor: - replicas: 0 -bloomGateway: - replicas: 0 ``` ```azure loki: @@ -269,34 +229,21 @@ write: minio: enabled: false -# Zero out replica counts of other deployment modes -singleBinary: - replicas: 0 - -ingester: - replicas: 0 -querier: - replicas: 0 -queryFrontend: - replicas: 0 -queryScheduler: - replicas: 0 -distributor: - replicas: 0 -compactor: - replicas: 0 -indexGateway: - replicas: 0 -bloomCompactor: - replicas: 0 -bloomGateway: - replicas: 0 -``` - {{< /code >}} To configure other storage providers, refer to the [Helm Chart Reference]({{< relref "../reference" >}}). +## Deploying the Loki Helm chart to a Production Environment + +{{< admonition type="note" >}} +We are activley working on providing more guides for deploying Loki in production. +{{< /admonition >}} + +It is recommended to run Loki at scale within in a cloud enviroment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. +- [Deploy Loki on AWS]({{< relref "./aws" >}}) + + + ## Next Steps * Configure an agent to [send log data to Loki](/docs/loki//send-data/). * Monitor the Loki deployment using the [Meta Monitoring Helm chart](/docs/loki//setup/install/helm/monitor-and-alert/) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md new file mode 100644 index 0000000000000..a6430bc05b987 --- /dev/null +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -0,0 +1,502 @@ +--- +title: Deploy the Simple Scalable Helm chart on AWS +menuTitle: Deploy on AWS +description: Installing Loki in Simple Scalable mode using the Helm chart on AWS. +aliases: + - ../../../installation/helm/scalable/ + - ../../../installation/helm/install-scalable/ +weight: 300 +keywords: +--- + +# Deploy the Simple Scalable Helm chart on AWS + +This guide shows how to deploy a minimally viable Loki in simple scalable mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: + +- Full access to EKS (Amazon Elastic Kubernetes Service) +- Full access to S3 (Amazon Simple Storage Service) +- Sufficient permissions to create IAM roles and policies + +There are two methods for authenticating and connecting Loki to AWS S3. We will guide you through the recommended method of granting access via an IAM role. + +## Prerequisites + +- Helm 3 or above. Refer to [Installing Helm](https://helm.sh/docs/intro/install/). This should be installed on your local machine. +- A running Kubernetes cluster on AWS. A simple way to get started is by using EKSctl. See [Getting started with EKSctl](https://eksctl.io/). +- Kubectl installed on your local machine. Refer to [Install and Set Up kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). +- (Optional) AWS CLI installed on your local machine. Refer to [Installing the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). This is required if you plan to use EKSctl to create the EKS cluster and modify the IAM roles and policies locally. + +### EKS Minimum Requirements + +{{< admonition type="warning" >}} +These EKS requirements are the minimum specification needed to deploy Loki using this guide. You may wish to adjust plugins and instance types based on your AWS environment and workload. **If you choose to do so, this may invalidate the guide.** + +In this guide, we deploy Loki using `m5.xlarge` instances. This is a middle-of-the-road instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. +{{< /admonition >}} + +Since EKS is manged and maintained by AWS, we can't provide a definitive method to deploying a cluster on EKS. However, we can provide you with the minimum requirements for deploying Loki on EKS: + +- Kubernetes version `1.30` or above. +- `3` nodes for the EKS cluster. +- Instance type depends on your workload. A good starting point is `m5.xlarge`. + +Here is the EKSctl cluster configuration file used in this guide: + +```yaml +# A simple example of ClusterConfig object: +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: loki + region: eu-west-2 + version: "1.31" + +iam: + withOIDC: true + +addons: + - name: aws-ebs-csi-driver + - name: eks-pod-identity-agent + +managedNodeGroups: + - name: loki-workers + instanceType: m5.xlarge + desiredCapacity: 3 + minSize: 2 + maxSize: 3 + amiFamily: AmazonLinux2 + iam: + withAddonPolicies: + ebs: true + volumeSize: 80 + volumeType: gp2 + ebsOptimized: true + +``` + + +The following plugins are also required to be installed within the EKS cluster: +- Amazon EBS CSI driver +- Amazon EKS Pod Identity Agent +- CoreDNS +- kube-proxy + +You must also install an OIDC provider on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using a command like this: + +```bash +eksctl utils associate-iam-oidc-provider --cluster loki --approve +``` +**This may be installed by default when creating the EKS cluster using EKSctl.** + +## Create an S3 bucket + +Before deploying Loki, you need to create an S3 bucket to store the logs. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket name `loki-aws-bucket`. + +```bash +aws s3api create-bucket --bucket loki-aws-bucket --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 +``` +Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. + + + +## Defining IAM roles and policies + +The recommended method for connecting Loki to AWS S3 is to use an IAM role. This method is more secure than using access keys and secret keys which are directly stored in the Loki configuration. The role and policy can be created using the AWS CLI or the AWS Management Console. The below steps show how to create the role and policy using the AWS CLI. + +1. Create a `loki-s3-policy.json` file with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "LokiStorage", + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::< Name of the bucket >", + "arn:aws:s3:::< Name of the bucket >/*" + ] + } + ] + } + ``` + + Make sure to replace the placeholder with the name of the bucket you created earlier. + +1. Create the IAM policy using the AWS CLI: + + ```bash + aws iam create-policy --policy-name LokiS3AccessPolicy --policy-document file://loki-s3-policy.json + ``` + +1. Create a trust policy document named `trust-policy.json` with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::< ACCOUNT ID >:oidc-provider/oidc.eks..amazonaws.com/id/< ID >" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks..amazonaws.com/id/< ID >:sub": "system:serviceaccount:loki:loki" + } + } + } + ] + } + ``` + **Note: that you need to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** + +1. Create the IAM role using the AWS CLI: + + ```bash + aws iam create-role --role-name LokiServiceAccountRole --assume-role-policy-document file://trust-policy.json + ``` + +1. Attach the policy to the role: + + ```bash + aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam:::policy/LokiS3AccessPolicy + ``` + Make sure to replace the placeholder with your AWS account ID. + +### Adding the policy to our S3 bucket + +To allow the IAM role to access the S3 bucket, you need to add the policy to the bucket. You can do this using the AWS Management Console or the AWS CLI. The below steps show how to add the policy using the AWS CLI. + +1. Create a bucket policy file named `bucket-policy.json` with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Statement1", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" + }, + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + } + ] + } + ``` + **Note: that you need to replace the placeholders with your AWS account ID and the bucket name.** + +1. Add the policy to the bucket: + + ```bash + aws s3api put-bucket-policy --bucket loki-aws-bucket --policy file://bucket-policy.json + ``` + +## Deploying the Helm chart + +1. Add the Grafana chart repository to Helm: + + ```bash + helm repo add grafana https://grafana.github.io/helm-charts + ``` +1. Update the chart repository: + + ```bash + helm repo update + ``` + +1. Create the `loki` namespace: + + ```bash + kubectl create namespace loki + ``` + **This is important as our trust policy is set to allow the role to be used by the `loki` service account in the `loki` namespace.** + +### Loki Helm chart configuration + +Create a `values.yaml` file with the following content: + +```yaml + loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage_config: + aws: + region: # eu-west-2 + bucketnames: # Your actual S3 bucket name (loki-aws-bucket) + s3forcepathstyle: false + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + compactor: + retention_enabled: true + delete_request_store: s3 + + + querier: + max_concurrent: 4 + + storage: + type: s3 + bucketNames: + chunks: "" # Your actual S3 bucket name (loki-aws-bucket) + ruler: "" # Your actual S3 bucket name (loki-aws-bucket) + admin: "" # Your actual S3 bucket name (loki-aws-bucket) + s3: + region: # eu-west-2 + #insecure: false + # s3forcepathstyle: false + + + deploymentMode: SimpleScalable + + serviceAccount: + create: true + annotations: + "eks.amazonaws.com/role-arn": "arn:aws:iam:::role/LokiServiceAccountRole" # The service role you created + + backend: + replicas: 2 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + read: + replicas: 2 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + write: + replicas: 3 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + + # This exposes the Loki gateway so it can be written to and queried externaly + gateway: + service: + type: LoadBalancer + + + # Enable minio for storage + minio: + enabled: false + +``` + +It is critical to define a valid `values.yaml` file for the Loki deployment. To remove the risk of misconfiguration, let's break down the configuration options to keep in mind when deploying to AWS: + +- **Loki Config vs. Values Config:** + - The `values.yaml` file contains a section called `loki`, which is a direct representation of the Loki configuration file. + - This section defines the Loki configuration, including the schema, storage, and querier configuration. + - The key configuration to focus on is the `storage_config` section, where you define the S3 bucket region and name. + +- **Storage:** + - Defines where the Helm chart stores data. + - Set the type to `s3` since we are using Amazon S3. + - Configure the bucket names for the chunks, ruler, and admin to match the bucket name created earlier. + - The `s3` section specifies the region of the bucket. + +- **Service Account:** + - The `serviceAccount` section is used to define the IAM role for the Loki service account. + - This is where the IAM role created earlier is linked. + +- **Gateway:** + - Defines how the Loki gateway will be exposed. + - We are using a `LoadBalancer` service type in this configuration. + +**Important:** + - Make sure to replace the placeholders with your actual values. + +### Deploy Loki + +Now that you have created the `values.yaml` file, you can deploy Loki using the Helm chart. + +1. Deploy using the newly created `values.yaml` file: + + ```bash + helm install --values values.yaml loki grafana/loki + ``` + **Note: this may take a few minutes to complete.** + +1. Verify the deployment: + + ```bash + kubectl get pods -n loki + ``` + You should see the Loki pods running. + ```console + NAME READY STATUS RESTARTS AGE + loki-backend-0 2/2 Running 0 43m + loki-backend-1 2/2 Running 0 43m + loki-canary-c7hbw 1/1 Running 0 43m + loki-canary-wtxb8 1/1 Running 0 43m + loki-chunks-cache-0 2/2 Running 0 43m + loki-gateway-844ff6d84-9zg5k 1/1 Running 0 43m + loki-read-848c686c9d-kd9xt 1/1 Running 0 43m + loki-read-848c686c9d-vx697 1/1 Running 0 43m + loki-results-cache-0 2/2 Running 0 43m + loki-write-0 1/1 Running 0 43m + loki-write-1 1/1 Running 0 43m + loki-write-2 1/1 Running 0 43m + ``` + +### Find the Loki Gateway Service + +To find the Loki Gateway service, run the following command: + +```bash +kubectl get svc -n loki +``` +You should see the Loki Gateway service with an external IP address. This is the address you will use to write to and query Loki. + +```console + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +loki-backend ClusterIP 10.100.97.223 3100/TCP,9095/TCP 46m +loki-backend-headless ClusterIP None 3100/TCP,9095/TCP 46m +loki-canary ClusterIP 10.100.121.159 3500/TCP 46m +loki-chunks-cache ClusterIP None 11211/TCP,9150/TCP 46m +loki-gateway LoadBalancer 10.100.201.74 12345678975675456-1433434453245433545656563.eu-west-2.elb.amazonaws.com 80:30707/TCP 46m +loki-memberlist ClusterIP None 7946/TCP 46m +loki-query-scheduler-discovery ClusterIP None 3100/TCP,9095/TCP 46m +loki-read ClusterIP 10.100.187.114 3100/TCP,9095/TCP 46m +loki-read-headless ClusterIP None 3100/TCP,9095/TCP 46m +loki-results-cache ClusterIP None 11211/TCP,9150/TCP 46m +loki-write ClusterIP 10.100.217.163 3100/TCP,9095/TCP 46m +loki-write-headless ClusterIP None 3100/TCP,9095/TCP 46m +``` + +## Testing You're Loki Deployment + +k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: + +1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension]({{< relref "../../../../send-data/k6/" >}}). + +2. Create a `aws-test.js` file with the following content: + + ```javascript + import {sleep, check} from 'k6'; + import loki from 'k6/x/loki'; + + /** + * URL used for push and query requests + * Path is automatically appended by the client + * @constant {string} + */ + const BASE_URL = `http:///loki/api/v1/push`; + + /** + * Helper constant for byte values + * @constant {number} + */ + const KB = 1024; + + /** + * Helper constant for byte values + * @constant {number} + */ + const MB = KB * KB; + + /** + * Instantiate config and Loki client + */ + const conf = new loki.Config(BASE_URL); + const client = new loki.Client(conf); + + /** + * Define test scenario + */ + export const options = { + vus: 10, + iterations: 10, + }; + + /** + * "main" function for each VU iteration + */ + export default () => { + // Push request with 10 streams and uncompressed logs between 800KB and 2MB + var res = client.pushParameterized(10, 800 * KB, 2 * MB); + // Check for successful write + check(res, { 'successful write': (res) => res.status == 204 }); + + // Pick a random log format from label pool + let format = randomChoice(conf.labels["format"]); + + // Execute instant query with limit 1 + res = client.instantQuery(`count_over_time({format="${format}"}[1m])`, 1) + // Check for successful read + check(res, { 'successful instant query': (res) => res.status == 200 }); + + // Execute range query over last 5m and limit 1000 + res = client.rangeQuery(`{format="${format}"}`, "5m", 1000) + // Check for successful read + check(res, { 'successful range query': (res) => res.status == 200 }); + + // Wait before next iteration + sleep(1); + } + + /** + * Helper function to get random item from array + */ + function randomChoice(items) { + return items[Math.floor(Math.random() * items.length)]; + } + ``` + + Replace `` with the external IP address of the Loki Gateway service. + +## Considerations + +- **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. + +- **Authentication:** Grafana Loki doesn't come with any included authentication layer. Within the simple scalable mode, the Loki gateway (NGINX) is exposed to the internet. It is advised to add a username and password via the `basicAuth` section in the `values.yaml` file. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication/" >}}) for more information. + +- **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. + +- **Costs:** Running Loki on AWS will incur costs. Make sure to monitor your usage and costs to avoid any unexpected bills. In this guide we have used a simple EKS cluster with 3 nodes and m5.xlarge instances. You may wish to adjust the instance types and number of nodes based on your workload. + +- **Guide:** Note that this guide was created on the first `1st of October 2024`. Cloud providers are constantly updating their services and offerings. This may mean that some of the steps require updating. + +## Next steps + +If you have successfully deployed Loki in simple scalable mode on AWS, you may wish to explore the following: + +- [Sanding data to Loki]({{< relref "../../../../send-data/" >}}) +- [Querying Loki]({{< relref "../../../../query/" >}}) +- [Operations]({{< relref "../../../../operations/" >}}) \ No newline at end of file From 3305c96a00314ef172fda778b74f0930c1d44354 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 1 Oct 2024 10:46:47 +0100 Subject: [PATCH 02/51] fixed title --- docs/sources/setup/install/helm/install-scalable/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index bd15526168244..f49222559d77b 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -1,5 +1,5 @@ --- -title: Install the simple scalable Helm chart +title: Loki Simple Scalable Helm chart menuTitle: Install scalable Loki description: Installing Loki in simple scalable mode using the Helm chart. aliases: @@ -9,7 +9,7 @@ weight: 300 keywords: --- -# Loki simple scalable Helm chart +# Loki Simple Scalable Helm chart This Helm Chart deploys Grafana Loki on Kubernetes in simple scalable mode. From ff2b1db0b3b5b5892314a60f7a469c221a4ef3f6 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 1 Oct 2024 10:47:53 +0100 Subject: [PATCH 03/51] fixed trailing slashes --- .../sources/setup/install/helm/install-scalable/aws.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index a6430bc05b987..9d6d8dfbf3eb0 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -404,7 +404,7 @@ loki-write-headless ClusterIP None k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: -1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension]({{< relref "../../../../send-data/k6/" >}}). +1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension]({{< relref "../../../../send-data/k6" >}}). 2. Create a `aws-test.js` file with the following content: @@ -485,7 +485,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t - **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. -- **Authentication:** Grafana Loki doesn't come with any included authentication layer. Within the simple scalable mode, the Loki gateway (NGINX) is exposed to the internet. It is advised to add a username and password via the `basicAuth` section in the `values.yaml` file. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication/" >}}) for more information. +- **Authentication:** Grafana Loki doesn't come with any included authentication layer. Within the simple scalable mode, the Loki gateway (NGINX) is exposed to the internet. It is advised to add a username and password via the `basicAuth` section in the `values.yaml` file. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. - **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. @@ -497,6 +497,6 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t If you have successfully deployed Loki in simple scalable mode on AWS, you may wish to explore the following: -- [Sanding data to Loki]({{< relref "../../../../send-data/" >}}) -- [Querying Loki]({{< relref "../../../../query/" >}}) -- [Operations]({{< relref "../../../../operations/" >}}) \ No newline at end of file +- [Sanding data to Loki]({{< relref "../../../../send-data" >}}) +- [Querying Loki]({{< relref "../../../../query" >}}) +- [Operations]({{< relref "../../../../operations" >}}) \ No newline at end of file From 933ee34d7d3b8dcca10a6b6b3b3f67bc93545082 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 1 Oct 2024 11:49:29 +0100 Subject: [PATCH 04/51] updated microservice ref --- .../helm/install-microservices/_index.md | 156 +++++++++--------- .../install/helm/install-scalable/_index.md | 15 +- .../install/helm/install-scalable/aws.md | 2 +- 3 files changed, 91 insertions(+), 82 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 4afca42d10b3e..20a920887c691 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -1,12 +1,12 @@ --- -title: Install the microservice Helm chart +title: Loki Microservice Helm chart menuTitle: Install microservice Loki description: Installing Loki in microservice (distributed) mode using the Helm chart. weight: 300 keywords: --- -# Install the microservice Helm chart +# Loki Microservice Helm chart This Helm Chart deploys Grafana Loki on Kubernetes. @@ -21,16 +21,17 @@ The default Helm chart deploys the following components: - **QueryFrontend component** (2 replicas, maxUnavailable: 1): Manages frontend queries. Up to 1 replica can be unavailable during updates. - **QueryScheduler component** (2 replicas): Schedules queries. +{{< admonition type="note" >}} It is not recommended to run scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +{{< /admonition >}} -**Prerequisites** +## Prerequisites - Helm 3 or above. See [Installing Helm](https://helm.sh/docs/intro/install/). - A running Kubernetes cluster. -- (Optional) A Memcached deployment for better query performance. For information on configuring Memcached, refer to the [caching section](https://grafana.com/docs/loki//operations/caching/). -**To deploy Loki in microservice mode (with MinIO):** +## Deploying the Helm chart for development and testing 1. Add [Grafana's chart repository](https://github.com/grafana/helm-charts) to Helm: @@ -49,31 +50,30 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu ```yaml loki: - schemaConfig: - configs: - - from: 2024-04-01 - store: tsdb - object_store: s3 - schema: v13 - index: - prefix: loki_index_ - period: 24h - ingester: - chunk_encoding: snappy - tracing: - enabled: true - querier: - # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing - max_concurrent: 4 - - #gateway: - # ingress: - # enabled: true - # hosts: - # - host: FIXME - # paths: - # - path: / - # pathType: Prefix + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + querier: + # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing + max_concurrent: 4 + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h + compactor: + retention_enabled: true + delete_request_store: s3 + deploymentMode: Distributed @@ -101,20 +101,15 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu bloomGateway: replicas: 0 + # This exposes the Loki gateway so it can be written to and queried externaly + gateway: + service: + type: LoadBalancer + + # Enable minio for storage minio: enabled: true - - # Zero out replica counts of other deployment modes - backend: - replicas: 0 - read: - replicas: 0 - write: - replicas: 0 - - singleBinary: - replicas: 0 ``` 4. Install or upgrade the Loki deployment. @@ -138,7 +133,7 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu loki-canary-8thrx 1/1 Running 0 167m loki-canary-h965l 1/1 Running 0 167m loki-canary-th8kb 1/1 Running 0 167m - loki-chunks-cache-0 0/2 Pending 0 167m + loki-chunks-cache-0 2/2 Running 0 167m loki-compactor-0 1/1 Running 0 167m loki-compactor-1 1/1 Running 0 167m loki-distributor-7c9bb8f4dd-bcwc5 1/1 Running 0 167m @@ -165,7 +160,7 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu ## Object Storage Configuration -After testing Loki with MinIO, it is recommended to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with MinIO, we recommended to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. @@ -176,48 +171,55 @@ When deploying Loki using S3 Storage **DO NOT** use the default bucket names; ` ```s3 # Example configuration for Loki with S3 storage - loki: - schemaConfig: - configs: - - from: 2024-04-01 - store: tsdb - object_store: s3 - schema: v13 - index: - prefix: loki_index_ - period: 24h - ingester: - chunk_encoding: snappy - tracing: +loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage_config: + aws: + region: eu-west-2 + bucketnames: loki-aws-bucket # Define your AWS bucket here + s3forcepathstyle: false + pattern_ingester: enabled: true - querier: + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + querier: max_concurrent: 4 - storage: + storage: type: s3 bucketNames: chunks: "" ruler: "" admin: "" - s3: - # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name - s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name - # AWS endpoint URL - endpoint: - # AWS region where the S3 bucket is located - region: - # AWS secret access key - secretAccessKey: - # AWS access key ID - accessKeyId: - # AWS signature version (e.g., v2 or v4) - signatureVersion: - # Forces the path style for S3 (true/false) - s3ForcePathStyle: false - # Allows insecure (HTTP) connections (true/false) - insecure: false - # HTTP configuration settings - http_config: {} + s3: + # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). + s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name + # AWS endpoint URL + endpoint: + # AWS region where the S3 bucket is located + region: + # AWS secret access key + secretAccessKey: + # AWS access key ID + accessKeyId: + # AWS signature version (e.g., v2 or v4) + signatureVersion: + # Forces the path style for S3 (true/false) + s3ForcePathStyle: false + # Allows insecure (HTTP) connections (true/false) + insecure: false + # HTTP configuration settings + http_config: {} deploymentMode: Distributed diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index f49222559d77b..87e0389da8ce4 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -128,10 +128,17 @@ loki: index: prefix: loki_index_ period: 24h - ingester: - chunk_encoding: snappy - tracing: - enabled: true + storage_config: + aws: + region: eu-west-2 + bucketnames: loki-aws-bucket # Define your AWS bucket here + s3forcepathstyle: false + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention querier: max_concurrent: 4 diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 9d6d8dfbf3eb0..a4febb92d05a1 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -321,7 +321,7 @@ Create a `values.yaml` file with the following content: It is critical to define a valid `values.yaml` file for the Loki deployment. To remove the risk of misconfiguration, let's break down the configuration options to keep in mind when deploying to AWS: - **Loki Config vs. Values Config:** - - The `values.yaml` file contains a section called `loki`, which is a direct representation of the Loki configuration file. + - The `values.yaml` file contains a section called `loki`, which contains a direct representation of the Loki configuration file. - This section defines the Loki configuration, including the schema, storage, and querier configuration. - The key configuration to focus on is the `storage_config` section, where you define the S3 bucket region and name. From edf4605337394dd4cbed16ecdd048994d51cc0d5 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 1 Oct 2024 12:02:08 +0100 Subject: [PATCH 05/51] updated index --- docs/sources/setup/install/helm/_index.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/sources/setup/install/helm/_index.md b/docs/sources/setup/install/helm/_index.md index ab06ae644a801..d6c0c184aa7f6 100644 --- a/docs/sources/setup/install/helm/_index.md +++ b/docs/sources/setup/install/helm/_index.md @@ -22,6 +22,17 @@ This guide references the Loki Helm chart version 3.0 or greater and contains th If you are installing Grafana Enterprise Logs, follow the [GEL Helm installation](https://grafana.com/docs/enterprise-logs//setup/helm/). +## Recommended Installation + +The recommended installation method for initial deployments is to use the [Loki Simple Scalable Helm chart]({{< relref "./install-scalable" >}}). This chart provides a simple scalable deployment mode for Loki, separating execution paths into read, write, and backend targets. For small to medium-sized deployments, this chart is a good starting point. + +### Cloud Deployment Guides + +The following guides provide step-by-step instructions for deploying Loki on cloud providers: + +- [Deploy Loki Simple Scalable Helm chart on AWS]({{< relref "./install-scalable/aws" >}}) + + ## Reference [Values reference]({{< relref "./reference" >}}) From 7886f0d481eefd87e70234dbe632213294f86cac Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:26:45 +0100 Subject: [PATCH 06/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: Tom Glenn <289945+tomglenn@users.noreply.github.com> --- docs/sources/setup/install/helm/install-microservices/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 20a920887c691..d10343d683051 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -160,7 +160,7 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu ## Object Storage Configuration -After testing Loki with MinIO, we recommended to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with MinIO, we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. From c7cd341284837c7b4095abd6568d12b7b689b414 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:27:28 +0100 Subject: [PATCH 07/51] Update docs/sources/setup/install/helm/install-scalable/aws.md Co-authored-by: Tom Glenn <289945+tomglenn@users.noreply.github.com> --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index a4febb92d05a1..f9c6afdb4331a 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -83,7 +83,7 @@ The following plugins are also required to be installed within the EKS cluster: - CoreDNS - kube-proxy -You must also install an OIDC provider on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using a command like this: +You must also install an OIDC provider on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: ```bash eksctl utils associate-iam-oidc-provider --cluster loki --approve From bcbc2af2d5178b845d630c3d7dde7c9002a0d7cc Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:27:42 +0100 Subject: [PATCH 08/51] Update docs/sources/setup/install/helm/install-scalable/aws.md Co-authored-by: Tom Glenn <289945+tomglenn@users.noreply.github.com> --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index f9c6afdb4331a..6ee5157b37f26 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -92,7 +92,7 @@ eksctl utils associate-iam-oidc-provider --cluster loki --approve ## Create an S3 bucket -Before deploying Loki, you need to create an S3 bucket to store the logs. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket name `loki-aws-bucket`. +Before deploying Loki, you need to create an S3 bucket to store the logs. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket name `loki-aws-bucket` but you should choose your own unique name when creating your own bucket. ```bash aws s3api create-bucket --bucket loki-aws-bucket --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 From a94257cca1c2e5d5d57ef9ad00a10272f780cffb Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:28:13 +0100 Subject: [PATCH 09/51] Update docs/sources/setup/install/helm/install-scalable/aws.md Co-authored-by: Tom Glenn <289945+tomglenn@users.noreply.github.com> --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 6ee5157b37f26..c8743a913ff64 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -491,7 +491,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t - **Costs:** Running Loki on AWS will incur costs. Make sure to monitor your usage and costs to avoid any unexpected bills. In this guide we have used a simple EKS cluster with 3 nodes and m5.xlarge instances. You may wish to adjust the instance types and number of nodes based on your workload. -- **Guide:** Note that this guide was created on the first `1st of October 2024`. Cloud providers are constantly updating their services and offerings. This may mean that some of the steps require updating. +- **Guide:** Note that this guide was **last updated on 1st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. ## Next steps From 119676b1ab07245c1953d20052bd294d09471dc0 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:28:21 +0100 Subject: [PATCH 10/51] Update docs/sources/setup/install/helm/install-scalable/aws.md Co-authored-by: Tom Glenn <289945+tomglenn@users.noreply.github.com> --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index c8743a913ff64..24836afdc8f29 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -495,7 +495,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t ## Next steps -If you have successfully deployed Loki in simple scalable mode on AWS, you may wish to explore the following: +Now that you have successfully deployed Loki in simple scalable mode on AWS, you may wish to explore the following: - [Sanding data to Loki]({{< relref "../../../../send-data" >}}) - [Querying Loki]({{< relref "../../../../query" >}}) From 108a3f03d9f3f3e9bcefd9335a53de5a316ef350 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:13:06 +0100 Subject: [PATCH 11/51] Update docs/sources/setup/install/helm/install-scalable/aws.md Co-authored-by: Tom Glenn <289945+tomglenn@users.noreply.github.com> --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 24836afdc8f29..3c0639c975f36 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -34,7 +34,7 @@ These EKS requirements are the minimum specification needed to deploy Loki using In this guide, we deploy Loki using `m5.xlarge` instances. This is a middle-of-the-road instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. {{< /admonition >}} -Since EKS is manged and maintained by AWS, we can't provide a definitive method to deploying a cluster on EKS. However, we can provide you with the minimum requirements for deploying Loki on EKS: +The minimum requirements for deploying Loki on EKS are: - Kubernetes version `1.30` or above. - `3` nodes for the EKS cluster. From f77d2d73872262809d8b85570ea8cb2e1008bf09 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 3 Oct 2024 13:34:17 +0100 Subject: [PATCH 12/51] added recommended changes --- .../install/helm/install-scalable/aws.md | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 24836afdc8f29..46afaf3d79327 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -78,10 +78,10 @@ managedNodeGroups: The following plugins are also required to be installed within the EKS cluster: -- Amazon EBS CSI driver -- Amazon EKS Pod Identity Agent -- CoreDNS -- kube-proxy +- Amazon EBS CSI Driver: Enables Kubernetes to dynamically provision and manage EBS volumes as persistent storage for applications. We use this to provision the node volumes for Loki. +- Amazon EKS Pod Identity Agent: Manages AWS IAM roles for pods, allowing fine-grained access control to AWS resources without needing to store credentials in containers. This is how Loki will access the S3 bucket. +- CoreDNS: Provides internal DNS service for Kubernetes clusters, ensuring that services and pods can communicate with each other using DNS names. +- kube-proxy: Maintains network rules on nodes, enabling communication between pods and services within the cluster. You must also install an OIDC provider on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: @@ -105,6 +105,10 @@ Make sure to replace the region and bucket name with your desired values. We wil The recommended method for connecting Loki to AWS S3 is to use an IAM role. This method is more secure than using access keys and secret keys which are directly stored in the Loki configuration. The role and policy can be created using the AWS CLI or the AWS Management Console. The below steps show how to create the role and policy using the AWS CLI. +{{< admonition type="tip" >}} +Create a new directory and navigate to it. Make sure to create the files in this directory. All commands in this guide assume you are in this directory. +{{< /admonition >}} + 1. Create a `loki-s3-policy.json` file with the following content: ```json @@ -158,7 +162,7 @@ The recommended method for connecting Loki to AWS S3 is to use an IAM role. This ] } ``` - **Note: that you need to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** + **Make sure to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** 1. Create the IAM role using the AWS CLI: @@ -171,7 +175,7 @@ The recommended method for connecting Loki to AWS S3 is to use an IAM role. This ```bash aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam:::policy/LokiS3AccessPolicy ``` - Make sure to replace the placeholder with your AWS account ID. + **Make sure to replace the placeholder with your AWS account ID.** ### Adding the policy to our S3 bucket @@ -203,7 +207,7 @@ To allow the IAM role to access the S3 bucket, you need to add the policy to the ] } ``` - **Note: that you need to replace the placeholders with your AWS account ID and the bucket name.** + **Note: Make sure to replace the placeholders with your AWS account ID and the bucket name.** 1. Add the policy to the bucket: @@ -324,6 +328,7 @@ It is critical to define a valid `values.yaml` file for the Loki deployment. To - The `values.yaml` file contains a section called `loki`, which contains a direct representation of the Loki configuration file. - This section defines the Loki configuration, including the schema, storage, and querier configuration. - The key configuration to focus on is the `storage_config` section, where you define the S3 bucket region and name. + - For the full Loki configuration, refer to the [Loki Configuration]({{< relref "../../../../configure" >}}) documentation. - **Storage:** - Defines where the Helm chart stores data. @@ -339,8 +344,9 @@ It is critical to define a valid `values.yaml` file for the Loki deployment. To - Defines how the Loki gateway will be exposed. - We are using a `LoadBalancer` service type in this configuration. -**Important:** - - Make sure to replace the placeholders with your actual values. +{{< admonition type="warning" >}} +Make sure to replace the placeholders with your actual values. +{{< /admonition >}} ### Deploy Loki @@ -377,6 +383,12 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the ### Find the Loki Gateway Service +The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. + +{{< admonition type="note" >}} +The Loki Gateway service is exposed to the internet. It is recommended to secure the gateway with authentication. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. +{{< /admonition >}} + To find the Loki Gateway service, run the following command: ```bash @@ -400,6 +412,8 @@ loki-write ClusterIP 10.100.217.163 loki-write-headless ClusterIP None 3100/TCP,9095/TCP 46m ``` +Congratulations! You have successfully deployed Loki in simple scalable mode on AWS using the Helm chart. Before we finish, let's test the deployment. + ## Testing You're Loki Deployment k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: @@ -479,7 +493,19 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t } ``` - Replace `` with the external IP address of the Loki Gateway service. + **Replace `` with the external IP address of the Loki Gateway service.** + + This script will write logs to Loki and query logs from Loki. It will write logs in a random format between 800KB and 2MB and query logs in a random format over the last 5 minutes. + +3. Run the test: + + ```bash + ./k6 run aws-test.js + ``` + + This will run the test and output the results. You should see the test writing logs to Loki and querying logs from Loki. + + ## Considerations From 7103f1c33de0d48b1e9b235be72be3f1a1ece68e Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 3 Oct 2024 13:35:41 +0100 Subject: [PATCH 13/51] fixed formatting --- docs/sources/setup/install/helm/install-scalable/aws.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index ff250c03e99ad..71c0259d64533 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -133,7 +133,7 @@ Create a new directory and navigate to it. Make sure to create the files in this } ``` - Make sure to replace the placeholder with the name of the bucket you created earlier. + **Make sure to replace the placeholder with the name of the bucket you created earlier.** 1. Create the IAM policy using the AWS CLI: @@ -207,9 +207,9 @@ To allow the IAM role to access the S3 bucket, you need to add the policy to the ] } ``` - **Note: Make sure to replace the placeholders with your AWS account ID and the bucket name.** + **Make sure to replace the placeholders with your AWS account ID and the bucket name.** -1. Add the policy to the bucket: +2. Add the policy to the bucket: ```bash aws s3api put-bucket-policy --bucket loki-aws-bucket --policy file://bucket-policy.json From 8225c4cb05e3047e59b3f85802dcf1946be289a8 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 3 Oct 2024 15:26:34 +0100 Subject: [PATCH 14/51] fixed namespace --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 71c0259d64533..8127a429663cd 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -355,7 +355,7 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the 1. Deploy using the newly created `values.yaml` file: ```bash - helm install --values values.yaml loki grafana/loki + helm install --values values.yaml loki grafana/loki -n loki ``` **Note: this may take a few minutes to complete.** From 271ef6dc2fca8af145ee0964f346d730938210d1 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Mon, 7 Oct 2024 16:40:19 +0100 Subject: [PATCH 15/51] added suggestion around seperating chunk and ruler bucket --- .../install/helm/install-scalable/aws.md | 90 +++++++++++++++---- 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 8127a429663cd..ac35483dc04d9 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -90,12 +90,17 @@ eksctl utils associate-iam-oidc-provider --cluster loki --approve ``` **This may be installed by default when creating the EKS cluster using EKSctl.** -## Create an S3 bucket +## Create three S3 buckets -Before deploying Loki, you need to create an S3 bucket to store the logs. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket name `loki-aws-bucket` but you should choose your own unique name when creating your own bucket. +Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket names `loki-aws-dev-chunks` and `loki-aws-dev-ruler` **but you should choose your own unique names when creating your own buckets**. + +{{}} +GEL customers will require a third bucket to store the admin data. This bucket is not required for OSS users. +{{}} ```bash -aws s3api create-bucket --bucket loki-aws-bucket --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 +aws s3api create-bucket --bucket loki-aws-dev-chunks --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 \ +aws s3api create-bucket --bucket loki-aws-dev-ruler --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 \ ``` Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. @@ -125,15 +130,17 @@ Create a new directory and navigate to it. Make sure to create the files in this "s3:DeleteObject" ], "Resource": [ - "arn:aws:s3:::< Name of the bucket >", - "arn:aws:s3:::< Name of the bucket >/*" + "arn:aws:s3:::< Name of the chunk bucket >", + "arn:aws:s3:::< Name of the chunk bucket >/*", + "arn:aws:s3:::< Name of the ruler bucket >", + "arn:aws:s3:::< Name of the ruler bucket >/*" ] } ] } ``` - **Make sure to replace the placeholder with the name of the bucket you created earlier.** + **Make sure to replace the placeholder with the name of the buckets you created earlier.** 1. Create the IAM policy using the AWS CLI: @@ -177,11 +184,11 @@ Create a new directory and navigate to it. Make sure to create the files in this ``` **Make sure to replace the placeholder with your AWS account ID.** -### Adding the policy to our S3 bucket +### Adding the policy to the S3 buckets -To allow the IAM role to access the S3 bucket, you need to add the policy to the bucket. You can do this using the AWS Management Console or the AWS CLI. The below steps show how to add the policy using the AWS CLI. +To allow the IAM role to access the S3 buckets, you need to add the policy to the bucket. You can do this using the AWS Management Console or the AWS CLI. The below steps show how to add the policy using the AWS CLI. -1. Create a bucket policy file named `bucket-policy.json` with the following content: +1. Create a bucket policy file named `bucket-policy-chunk.json` with the following content: ```json { @@ -200,8 +207,8 @@ To allow the IAM role to access the S3 bucket, you need to add the policy to the "s3:ListBucket" ], "Resource": [ - "arn:aws:s3:::", - "arn:aws:s3:::/*" + "arn:aws:s3:::", + "arn:aws:s3:::/*" ] } ] @@ -209,10 +216,43 @@ To allow the IAM role to access the S3 bucket, you need to add the policy to the ``` **Make sure to replace the placeholders with your AWS account ID and the bucket name.** -2. Add the policy to the bucket: +1. Add the policy to the bucket: ```bash - aws s3api put-bucket-policy --bucket loki-aws-bucket --policy file://bucket-policy.json + aws s3api put-bucket-policy --bucket loki-aws-dev-chunk --policy file://bucket-policy-chunk.json + ``` +1. Create a bucket policy file named `bucket-policy-ruler.json` with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Statement1", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" + }, + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + } + ] + } + ``` + **Make sure to replace the placeholders with your AWS account ID and the bucket name.** + +1. Add the policy to the bucket: + + ```bash + aws s3api put-bucket-policy --bucket loki-aws-dev-ruler --policy file://bucket-policy-ruler.json ``` ## Deploying the Helm chart @@ -253,7 +293,7 @@ Create a `values.yaml` file with the following content: storage_config: aws: region: # eu-west-2 - bucketnames: # Your actual S3 bucket name (loki-aws-bucket) + bucketnames: # Your actual S3 bucket name (loki-aws-dev-chunks) s3forcepathstyle: false pattern_ingester: enabled: true @@ -264,6 +304,17 @@ Create a `values.yaml` file with the following content: compactor: retention_enabled: true delete_request_store: s3 + ruler: + enable_api: true + storage: + type: s3 + s3: + region: eu-west-2 + bucketnames: # Your actual S3 bucket name (loki-aws-dev-ruler) + s3forcepathstyle: false + alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) + + querier: @@ -272,9 +323,9 @@ Create a `values.yaml` file with the following content: storage: type: s3 bucketNames: - chunks: "" # Your actual S3 bucket name (loki-aws-bucket) - ruler: "" # Your actual S3 bucket name (loki-aws-bucket) - admin: "" # Your actual S3 bucket name (loki-aws-bucket) + chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) + ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) + # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only s3: region: # eu-west-2 #insecure: false @@ -327,13 +378,14 @@ It is critical to define a valid `values.yaml` file for the Loki deployment. To - **Loki Config vs. Values Config:** - The `values.yaml` file contains a section called `loki`, which contains a direct representation of the Loki configuration file. - This section defines the Loki configuration, including the schema, storage, and querier configuration. - - The key configuration to focus on is the `storage_config` section, where you define the S3 bucket region and name. + - The key configuration to focus on for chunks is the `storage_config` section, where you define the S3 bucket region and name. This tells Loki where to store the chunks. + - The `ruler` section defines the configuration for the ruler, including the S3 bucket region and name. This tells Loki where to store the alert and recording rules. - For the full Loki configuration, refer to the [Loki Configuration]({{< relref "../../../../configure" >}}) documentation. - **Storage:** - Defines where the Helm chart stores data. - Set the type to `s3` since we are using Amazon S3. - - Configure the bucket names for the chunks, ruler, and admin to match the bucket name created earlier. + - Configure the bucket names for the chunks and ruler to match the buckets created earlier. - The `s3` section specifies the region of the bucket. - **Service Account:** From d9767d2288d912ad44a42078ed96e86a5721fa8b Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 8 Oct 2024 10:49:01 +0100 Subject: [PATCH 16/51] Added suggestions by trevor --- .../helm/install-microservices/_index.md | 18 ++++++-- .../install/helm/install-scalable/_index.md | 10 ++--- .../install/helm/install-scalable/aws.md | 42 +++++++++---------- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index d10343d683051..ac5af1de66c38 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -22,7 +22,7 @@ The default Helm chart deploys the following components: - **QueryScheduler component** (2 replicas): Schedules queries. {{< admonition type="note" >}} -It is not recommended to run scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +We do not recommended to run Microservice mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. {{< /admonition >}} ## Prerequisites @@ -78,9 +78,9 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu deploymentMode: Distributed ingester: - replicas: 3 + replicas: 3 # To ensure data durability with replication querier: - replicas: 3 + replicas: 3 # Improve query performance via parallelism maxUnavailable: 2 queryFrontend: replicas: 2 @@ -88,7 +88,7 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu queryScheduler: replicas: 2 distributor: - replicas: 3 + replicas: 3 # To ensure data durability with replication maxUnavailable: 2 compactor: replicas: 1 @@ -100,6 +100,16 @@ It is not recommended to run scalable mode with `filesystem` storage. For the pu replicas: 0 bloomGateway: replicas: 0 + + backend: + replicas: 0 + read: + replicas: 0 + write: + replicas: 0 + + singleBinary: + replicas: 0 # This exposes the Loki gateway so it can be written to and queried externaly gateway: diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index 87e0389da8ce4..7391999c17c9f 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -25,7 +25,7 @@ The default Helm chart deploys the following components: - Index and Chunk cache (1 replica) {{< admonition type="note" >}} -It is not recommended to run scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +We do not recommended to run scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. {{< /admonition >}} ## Prerequisites @@ -86,7 +86,7 @@ The following steps show how to deploy the Loki Helm chart in simple scalable mo read: replicas: 2 write: - replicas: 3 + replicas: 3 # To ensure data durability with replication # Enable minio for storage minio: @@ -131,7 +131,7 @@ loki: storage_config: aws: region: eu-west-2 - bucketnames: loki-aws-bucket # Define your AWS bucket here + bucketnames: loki-aws-dev-chunk # Define your AWS bucket here (Must be unique) s3forcepathstyle: false pattern_ingester: enabled: true @@ -145,8 +145,8 @@ loki: storage: type: s3 bucketNames: - chunks: "" # - ruler: "" + chunks: "" # The S3 bucket name for storing chunks (Must be unique) + ruler: "" # The S3 bucket name for storing ruler data (Must be unique) admin: "" s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index ac35483dc04d9..37b3451f5c433 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -28,7 +28,7 @@ There are two methods for authenticating and connecting Loki to AWS S3. We will ### EKS Minimum Requirements -{{< admonition type="warning" >}} +{{< admonition type="caution" >}} These EKS requirements are the minimum specification needed to deploy Loki using this guide. You may wish to adjust plugins and instance types based on your AWS environment and workload. **If you choose to do so, this may invalidate the guide.** In this guide, we deploy Loki using `m5.xlarge` instances. This is a middle-of-the-road instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. @@ -78,20 +78,24 @@ managedNodeGroups: The following plugins are also required to be installed within the EKS cluster: -- Amazon EBS CSI Driver: Enables Kubernetes to dynamically provision and manage EBS volumes as persistent storage for applications. We use this to provision the node volumes for Loki. -- Amazon EKS Pod Identity Agent: Manages AWS IAM roles for pods, allowing fine-grained access control to AWS resources without needing to store credentials in containers. This is how Loki will access the S3 bucket. -- CoreDNS: Provides internal DNS service for Kubernetes clusters, ensuring that services and pods can communicate with each other using DNS names. -- kube-proxy: Maintains network rules on nodes, enabling communication between pods and services within the cluster. +- **Amazon EBS CSI Driver**: Enables Kubernetes to dynamically provision and manage EBS volumes as persistent storage for applications. We use this to provision the node volumes for Loki. +- **Amazon EKS Pod Identity Agent**: Manages AWS IAM roles for pods, allowing fine-grained access control to AWS resources without needing to store credentials in containers. This is how Loki will access the S3 bucket. +- **CoreDNS**: Provides internal DNS service for Kubernetes clusters, ensuring that services and pods can communicate with each other using DNS names. +- **kube-proxy**: Maintains network rules on nodes, enabling communication between pods and services within the cluster. -You must also install an OIDC provider on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: +You must also install an **OIDC (OpenID Connect) provider** on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: ```bash eksctl utils associate-iam-oidc-provider --cluster loki --approve ``` -**This may be installed by default when creating the EKS cluster using EKSctl.** +**This may be installed by default when creating the EKS cluster using EKSctl and the above config.** ## Create three S3 buckets +{{< admonition type="WARNING" >}} + **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a **unique** name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). +{{< /admonition >}} + Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket names `loki-aws-dev-chunks` and `loki-aws-dev-ruler` **but you should choose your own unique names when creating your own buckets**. {{}} @@ -100,7 +104,7 @@ GEL customers will require a third bucket to store the admin data. This bucket i ```bash aws s3api create-bucket --bucket loki-aws-dev-chunks --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 \ -aws s3api create-bucket --bucket loki-aws-dev-ruler --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 \ +aws s3api create-bucket --bucket loki-aws-dev-ruler --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 ``` Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. @@ -267,13 +271,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th ```bash helm repo update ``` - -1. Create the `loki` namespace: - - ```bash - kubectl create namespace loki - ``` - **This is important as our trust policy is set to allow the role to be used by the `loki` service account in the `loki` namespace.** + ### Loki Helm chart configuration @@ -314,9 +312,6 @@ Create a `values.yaml` file with the following content: s3forcepathstyle: false alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) - - - querier: max_concurrent: 4 @@ -373,6 +368,10 @@ Create a `values.yaml` file with the following content: ``` +{{< admonition type="caution" >}} +Make sure to replace the placeholders with your actual values. +{{< /admonition >}} + It is critical to define a valid `values.yaml` file for the Loki deployment. To remove the risk of misconfiguration, let's break down the configuration options to keep in mind when deploying to AWS: - **Loki Config vs. Values Config:** @@ -396,9 +395,6 @@ It is critical to define a valid `values.yaml` file for the Loki deployment. To - Defines how the Loki gateway will be exposed. - We are using a `LoadBalancer` service type in this configuration. -{{< admonition type="warning" >}} -Make sure to replace the placeholders with your actual values. -{{< /admonition >}} ### Deploy Loki @@ -407,9 +403,9 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the 1. Deploy using the newly created `values.yaml` file: ```bash - helm install --values values.yaml loki grafana/loki -n loki + helm install --values values.yaml loki grafana/loki -n loki --create-namespace ``` - **Note: this may take a few minutes to complete.** + **It is important to create a namespace called `loki` as our trust policy is set to allow the IAM role to be used by the `loki` service account in the `loki` namespace. This is configurable but make sure to update your service account** 1. Verify the deployment: From f5538948c0d1bf052b2cd857efdea29ae1ea12af Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:54:28 +0100 Subject: [PATCH 17/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: Poyzan <31743851+poyzannur@users.noreply.github.com> --- docs/sources/setup/install/helm/install-microservices/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index ac5af1de66c38..c83e011ca432b 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -193,7 +193,7 @@ loki: period: 24h storage_config: aws: - region: eu-west-2 + region: bucketnames: loki-aws-bucket # Define your AWS bucket here s3forcepathstyle: false pattern_ingester: From dd57de1e949deddd1d927d607f2422e93f252ce8 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 8 Oct 2024 13:44:37 +0100 Subject: [PATCH 18/51] added Poyzans recommendations --- .../install/helm/install-microservices/_index.md | 4 +++- .../setup/install/helm/install-scalable/aws.md | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index c83e011ca432b..e9c59fba4c45d 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -88,7 +88,7 @@ We do not recommended to run Microservice mode with `filesystem` storage. For th queryScheduler: replicas: 2 distributor: - replicas: 3 # To ensure data durability with replication + replicas: 3 maxUnavailable: 2 compactor: replicas: 1 @@ -196,6 +196,8 @@ loki: region: bucketnames: loki-aws-bucket # Define your AWS bucket here s3forcepathstyle: false + ingester: + chunk_encoding: snappy pattern_ingester: enabled: true limits_config: diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 37b3451f5c433..396b764222ef2 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -92,19 +92,19 @@ eksctl utils associate-iam-oidc-provider --cluster loki --approve ## Create three S3 buckets -{{< admonition type="WARNING" >}} +{{< admonition type="warning" >}} **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a **unique** name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). {{< /admonition >}} -Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. For this guide, we will use the bucket names `loki-aws-dev-chunks` and `loki-aws-dev-ruler` **but you should choose your own unique names when creating your own buckets**. +Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. {{}} GEL customers will require a third bucket to store the admin data. This bucket is not required for OSS users. {{}} ```bash -aws s3api create-bucket --bucket loki-aws-dev-chunks --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 \ -aws s3api create-bucket --bucket loki-aws-dev-ruler --region eu-west-2 --create-bucket-configuration LocationConstraint=eu-west-2 +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= ``` Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. @@ -223,7 +223,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th 1. Add the policy to the bucket: ```bash - aws s3api put-bucket-policy --bucket loki-aws-dev-chunk --policy file://bucket-policy-chunk.json + aws s3api put-bucket-policy --bucket --policy file://bucket-policy-chunk.json ``` 1. Create a bucket policy file named `bucket-policy-ruler.json` with the following content: @@ -256,7 +256,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th 1. Add the policy to the bucket: ```bash - aws s3api put-bucket-policy --bucket loki-aws-dev-ruler --policy file://bucket-policy-ruler.json + aws s3api put-bucket-policy --bucket --policy file://bucket-policy-ruler.json ``` ## Deploying the Helm chart @@ -293,6 +293,8 @@ Create a `values.yaml` file with the following content: region: # eu-west-2 bucketnames: # Your actual S3 bucket name (loki-aws-dev-chunks) s3forcepathstyle: false + ingester: + chunk_encoding: snappy pattern_ingester: enabled: true limits_config: @@ -433,7 +435,7 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. -{{< admonition type="note" >}} +{{< admonition type="caution" >}} The Loki Gateway service is exposed to the internet. It is recommended to secure the gateway with authentication. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. {{< /admonition >}} From 972878fe71cc4ad86e51d580fe6551581f1c14ec Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 10 Oct 2024 09:38:58 +0100 Subject: [PATCH 19/51] added Davids suggestions --- .../install/helm/install-microservices/_index.md | 14 +++++++------- .../setup/install/helm/install-scalable/_index.md | 12 ++++++------ .../setup/install/helm/install-scalable/aws.md | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index e9c59fba4c45d..6cca3ee0fe191 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -170,7 +170,7 @@ We do not recommended to run Microservice mode with `filesystem` storage. For th ## Object Storage Configuration -After testing Loki with MinIO, we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with [MinIO](https://min.io/docs/minio/kubernetes/upstream/index.html), we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. @@ -194,7 +194,7 @@ loki: storage_config: aws: region: - bucketnames: loki-aws-bucket # Define your AWS bucket here + bucketnames: s3forcepathstyle: false ingester: chunk_encoding: snappy @@ -208,11 +208,11 @@ loki: max_concurrent: 4 storage: - type: s3 - bucketNames: - chunks: "" - ruler: "" - admin: "" + type: s3 + bucketNames: + chunks: + ruler: + admin: s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index 7391999c17c9f..9e81359891df5 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -109,7 +109,7 @@ The following steps show how to deploy the Loki Helm chart in simple scalable mo ## Object Storage Configuration -After testing Loki with MinIO, it is recommended to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with MinIO, we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. @@ -130,8 +130,8 @@ loki: period: 24h storage_config: aws: - region: eu-west-2 - bucketnames: loki-aws-dev-chunk # Define your AWS bucket here (Must be unique) + region: + bucketnames: s3forcepathstyle: false pattern_ingester: enabled: true @@ -145,9 +145,9 @@ loki: storage: type: s3 bucketNames: - chunks: "" # The S3 bucket name for storing chunks (Must be unique) - ruler: "" # The S3 bucket name for storing ruler data (Must be unique) - admin: "" + chunks: + ruler: + admin: s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 396b764222ef2..8142827b54bb4 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -436,7 +436,7 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. {{< admonition type="caution" >}} -The Loki Gateway service is exposed to the internet. It is recommended to secure the gateway with authentication. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. +The Loki Gateway service is exposed to the internet. We recommend to secure the gateway with authentication. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. {{< /admonition >}} To find the Loki Gateway service, run the following command: From b5144ec829d79a642cdb0237f6416fe7b56306ee Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Tue, 15 Oct 2024 18:19:09 +0100 Subject: [PATCH 20/51] fixed typo --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 8142827b54bb4..696913fa3f7d2 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -464,7 +464,7 @@ loki-write-headless ClusterIP None Congratulations! You have successfully deployed Loki in simple scalable mode on AWS using the Helm chart. Before we finish, let's test the deployment. -## Testing You're Loki Deployment +## Testing Your Loki Deployment k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: From 109c349df3ed6060614aac326b83f18b91386f45 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:11:55 +0100 Subject: [PATCH 21/51] Update docs/sources/setup/install/helm/install-scalable/aws.md Co-authored-by: Jens Horstmann --- docs/sources/setup/install/helm/install-scalable/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 696913fa3f7d2..806092a00951b 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -573,6 +573,6 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t Now that you have successfully deployed Loki in simple scalable mode on AWS, you may wish to explore the following: -- [Sanding data to Loki]({{< relref "../../../../send-data" >}}) +- [Sending data to Loki]({{< relref "../../../../send-data" >}}) - [Querying Loki]({{< relref "../../../../query" >}}) - [Operations]({{< relref "../../../../operations" >}}) \ No newline at end of file From 1a9970ba777363397319ca3fbe10c75eed16a02a Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 17 Oct 2024 15:55:12 +0100 Subject: [PATCH 22/51] added authentication --- .../install/helm/install-scalable/aws.md | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md index 806092a00951b..7fa4675fb6ebf 100644 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ b/docs/sources/setup/install/helm/install-scalable/aws.md @@ -261,6 +261,8 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th ## Deploying the Helm chart +Before we can deploy the Loki Helm chart, we need to add the Grafana chart repository to Helm. This repository contains the Loki Helm chart. + 1. Add the Grafana chart repository to Helm: ```bash @@ -271,6 +273,34 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th ```bash helm repo update ``` +1. Create a new namespace for Loki: + + ```bash + kubectl create namespace loki + ``` +### Loki Basic Authentication + +Loki by default does not come with any authentication. Since we will be deploying Loki to AWS and exposing the gateway to the internet, we recommend adding atleast basic authentication. In this guide we will give Loki a username and password: + +1. To start we will need create a `.htpasswd` file with the username and password. You can use the `htpasswd` command to create the file: + + {{< admonition type="tip" >}} + If you don't have the `htpasswd` command installed, you can install it using `brew` or `apt-get` or `yum` depending on your OS. + {{< /admonition >}} + + ```bash + htpasswd -c .htpasswd + ``` + This will create a file called `auth` with the username `loki`. You will be prompted to enter a password. + + 1. Create a Kubernetes secret with the `.htpasswd` file: + + ```bash + kubectl create secret generic loki-basic-auth --from-file=.htpasswd -n loki + ``` + + This will create a secret called `loki-basic-auth` in the `loki` namespace. We will reference this secret in the Loki Helm chart configuration. + ### Loki Helm chart configuration @@ -360,8 +390,11 @@ Create a `values.yaml` file with the following content: # This exposes the Loki gateway so it can be written to and queried externaly gateway: - service: - type: LoadBalancer + service: + type: LoadBalancer + basicAuth: + enabled: true + existingSecret: loki-basic-auth # Change this is you used a different secret name # Enable minio for storage @@ -436,7 +469,7 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. {{< admonition type="caution" >}} -The Loki Gateway service is exposed to the internet. We recommend to secure the gateway with authentication. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. +The Loki Gateway service is exposed to the internet. We provide basic authentication using a username and password in this tutorial. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. {{< /admonition >}} To find the Loki Gateway service, run the following command: @@ -481,7 +514,14 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t * Path is automatically appended by the client * @constant {string} */ - const BASE_URL = `http:///loki/api/v1/push`; + + const username = ''; + const password = ''; + const external_ip = ''; + + const credentials = `${username}:${password}`; + + const BASE_URL = `http://${credentials}@${external_ip}`; /** * Helper constant for byte values @@ -498,6 +538,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t /** * Instantiate config and Loki client */ + const conf = new loki.Config(BASE_URL); const client = new loki.Client(conf); @@ -541,13 +582,13 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t function randomChoice(items) { return items[Math.floor(Math.random() * items.length)]; } - ``` + ``` **Replace `` with the external IP address of the Loki Gateway service.** This script will write logs to Loki and query logs from Loki. It will write logs in a random format between 800KB and 2MB and query logs in a random format over the last 5 minutes. -3. Run the test: +1. Run the test: ```bash ./k6 run aws-test.js @@ -561,7 +602,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t - **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. -- **Authentication:** Grafana Loki doesn't come with any included authentication layer. Within the simple scalable mode, the Loki gateway (NGINX) is exposed to the internet. It is advised to add a username and password via the `basicAuth` section in the `values.yaml` file. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. +- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. - **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. From 4397b1dae57b6544fb30f28375d2620f24d9629d Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 17 Oct 2024 17:02:07 +0100 Subject: [PATCH 23/51] added distributed aws guide --- docs/sources/setup/install/helm/_index.md | 7 +- .../install/helm/install-microservices/aws.md | 653 ++++++++++++++++++ 2 files changed, 655 insertions(+), 5 deletions(-) create mode 100644 docs/sources/setup/install/helm/install-microservices/aws.md diff --git a/docs/sources/setup/install/helm/_index.md b/docs/sources/setup/install/helm/_index.md index d6c0c184aa7f6..0f582d6260ebc 100644 --- a/docs/sources/setup/install/helm/_index.md +++ b/docs/sources/setup/install/helm/_index.md @@ -22,16 +22,13 @@ This guide references the Loki Helm chart version 3.0 or greater and contains th If you are installing Grafana Enterprise Logs, follow the [GEL Helm installation](https://grafana.com/docs/enterprise-logs//setup/helm/). -## Recommended Installation -The recommended installation method for initial deployments is to use the [Loki Simple Scalable Helm chart]({{< relref "./install-scalable" >}}). This chart provides a simple scalable deployment mode for Loki, separating execution paths into read, write, and backend targets. For small to medium-sized deployments, this chart is a good starting point. - -### Cloud Deployment Guides +## Cloud Deployment Guides The following guides provide step-by-step instructions for deploying Loki on cloud providers: - [Deploy Loki Simple Scalable Helm chart on AWS]({{< relref "./install-scalable/aws" >}}) - +- [Deploy Loki Distributed Helm chart on AWS]({{< relref "./install-microservices/aws" >}}) ## Reference diff --git a/docs/sources/setup/install/helm/install-microservices/aws.md b/docs/sources/setup/install/helm/install-microservices/aws.md new file mode 100644 index 0000000000000..8cbb6d244c9ae --- /dev/null +++ b/docs/sources/setup/install/helm/install-microservices/aws.md @@ -0,0 +1,653 @@ +--- +title: Deploy the distributed Helm chart on AWS +menuTitle: Deploy on AWS +description: Installing Loki in distributed mode using the Helm chart on AWS. +weight: 300 +keywords: +--- + +# Deploy the distributed Helm chart on AWS + +This guide shows how to deploy a minimally viable Loki in distributed mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: + +- Full access to EKS (Amazon Elastic Kubernetes Service) +- Full access to S3 (Amazon Simple Storage Service) +- Sufficient permissions to create IAM roles and policies + +There are two methods for authenticating and connecting Loki to AWS S3. We will guide you through the recommended method of granting access via an IAM role. + +## Prerequisites + +- Helm 3 or above. Refer to [Installing Helm](https://helm.sh/docs/intro/install/). This should be installed on your local machine. +- A running Kubernetes cluster on AWS. A simple way to get started is by using EKSctl. See [Getting started with EKSctl](https://eksctl.io/). +- Kubectl installed on your local machine. Refer to [Install and Set Up kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). +- (Optional) AWS CLI installed on your local machine. Refer to [Installing the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). This is required if you plan to use EKSctl to create the EKS cluster and modify the IAM roles and policies locally. + +### EKS Minimum Requirements + +{{< admonition type="caution" >}} +These EKS requirements are the minimum specification needed to deploy Loki using this guide. You may wish to adjust plugins and instance types based on your AWS environment and workload. **If you choose to do so, this may invalidate the guide.** + +In this guide, we deploy Loki using `m5.xlarge` instances. This is a middle-of-the-road instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. +{{< /admonition >}} + +The minimum requirements for deploying Loki on EKS are: + +- Kubernetes version `1.30` or above. +- `3` nodes for the EKS cluster. +- Instance type depends on your workload. A good starting point is `m5.xlarge`. + +Here is the EKSctl cluster configuration file used in this guide: + +```yaml +# A simple example of ClusterConfig object: +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: loki + region: eu-west-2 + version: "1.31" + +iam: + withOIDC: true + +addons: + - name: aws-ebs-csi-driver + - name: eks-pod-identity-agent + +managedNodeGroups: + - name: loki-workers + instanceType: m5.xlarge + desiredCapacity: 3 + minSize: 2 + maxSize: 3 + amiFamily: AmazonLinux2 + iam: + withAddonPolicies: + ebs: true + volumeSize: 80 + volumeType: gp2 + ebsOptimized: true + +``` + + +The following plugins are also required to be installed within the EKS cluster: +- **Amazon EBS CSI Driver**: Enables Kubernetes to dynamically provision and manage EBS volumes as persistent storage for applications. We use this to provision the node volumes for Loki. +- **Amazon EKS Pod Identity Agent**: Manages AWS IAM roles for pods, allowing fine-grained access control to AWS resources without needing to store credentials in containers. This is how Loki will access the S3 bucket. +- **CoreDNS**: Provides internal DNS service for Kubernetes clusters, ensuring that services and pods can communicate with each other using DNS names. +- **kube-proxy**: Maintains network rules on nodes, enabling communication between pods and services within the cluster. + +You must also install an **OIDC (OpenID Connect) provider** on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: + +```bash +eksctl utils associate-iam-oidc-provider --cluster loki --approve +``` +**This may be installed by default when creating the EKS cluster using EKSctl and the above config.** + +## Create three S3 buckets + +{{< admonition type="warning" >}} + **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a **unique** name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). +{{< /admonition >}} + +Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. + +{{}} +GEL customers will require a third bucket to store the admin data. This bucket is not required for OSS users. +{{}} + +```bash +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= +``` +Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. + + + +## Defining IAM roles and policies + +The recommended method for connecting Loki to AWS S3 is to use an IAM role. This method is more secure than using access keys and secret keys which are directly stored in the Loki configuration. The role and policy can be created using the AWS CLI or the AWS Management Console. The below steps show how to create the role and policy using the AWS CLI. + +{{< admonition type="tip" >}} +Create a new directory and navigate to it. Make sure to create the files in this directory. All commands in this guide assume you are in this directory. +{{< /admonition >}} + +1. Create a `loki-s3-policy.json` file with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "LokiStorage", + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::< Name of the chunk bucket >", + "arn:aws:s3:::< Name of the chunk bucket >/*", + "arn:aws:s3:::< Name of the ruler bucket >", + "arn:aws:s3:::< Name of the ruler bucket >/*" + ] + } + ] + } + ``` + + **Make sure to replace the placeholder with the name of the buckets you created earlier.** + +1. Create the IAM policy using the AWS CLI: + + ```bash + aws iam create-policy --policy-name LokiS3AccessPolicy --policy-document file://loki-s3-policy.json + ``` + +1. Create a trust policy document named `trust-policy.json` with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::< ACCOUNT ID >:oidc-provider/oidc.eks..amazonaws.com/id/< ID >" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks..amazonaws.com/id/< ID >:sub": "system:serviceaccount:loki:loki" + } + } + } + ] + } + ``` + **Make sure to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** + +1. Create the IAM role using the AWS CLI: + + ```bash + aws iam create-role --role-name LokiServiceAccountRole --assume-role-policy-document file://trust-policy.json + ``` + +1. Attach the policy to the role: + + ```bash + aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam:::policy/LokiS3AccessPolicy + ``` + **Make sure to replace the placeholder with your AWS account ID.** + +### Adding the policy to the S3 buckets + +To allow the IAM role to access the S3 buckets, you need to add the policy to the bucket. You can do this using the AWS Management Console or the AWS CLI. The below steps show how to add the policy using the AWS CLI. + +1. Create a bucket policy file named `bucket-policy-chunk.json` with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Statement1", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" + }, + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + } + ] + } + ``` + **Make sure to replace the placeholders with your AWS account ID and the bucket name.** + +1. Add the policy to the bucket: + + ```bash + aws s3api put-bucket-policy --bucket --policy file://bucket-policy-chunk.json + ``` +1. Create a bucket policy file named `bucket-policy-ruler.json` with the following content: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Statement1", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" + }, + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + } + ] + } + ``` + **Make sure to replace the placeholders with your AWS account ID and the bucket name.** + +1. Add the policy to the bucket: + + ```bash + aws s3api put-bucket-policy --bucket --policy file://bucket-policy-ruler.json + ``` + +## Deploying the Helm chart + +Before we can deploy the Loki Helm chart, we need to add the Grafana chart repository to Helm. This repository contains the Loki Helm chart. + +1. Add the Grafana chart repository to Helm: + + ```bash + helm repo add grafana https://grafana.github.io/helm-charts + ``` +1. Update the chart repository: + + ```bash + helm repo update + ``` +1. Create a new namespace for Loki: + + ```bash + kubectl create namespace loki + ``` +### Loki Basic Authentication + +Loki by default does not come with any authentication. Since we will be deploying Loki to AWS and exposing the gateway to the internet, we recommend adding atleast basic authentication. In this guide we will give Loki a username and password: + +1. To start we will need create a `.htpasswd` file with the username and password. You can use the `htpasswd` command to create the file: + + {{< admonition type="tip" >}} + If you don't have the `htpasswd` command installed, you can install it using `brew` or `apt-get` or `yum` depending on your OS. + {{< /admonition >}} + + ```bash + htpasswd -c .htpasswd + ``` + This will create a file called `auth` with the username `loki`. You will be prompted to enter a password. + + 1. Create a Kubernetes secret with the `.htpasswd` file: + + ```bash + kubectl create secret generic loki-basic-auth --from-file=.htpasswd -n loki + ``` + + This will create a secret called `loki-basic-auth` in the `loki` namespace. We will reference this secret in the Loki Helm chart configuration. + + + +### Loki Helm chart configuration + +Create a `values.yaml` file with the following content: + +```yaml + loki: + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage_config: + aws: + region: # eu-west-2 + bucketnames: # Your actual S3 bucket name (loki-aws-dev-chunks) + s3forcepathstyle: false + ingester: + chunk_encoding: snappy + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + enable_api: true + storage: + type: s3 + s3: + region: eu-west-2 + bucketnames: # Your actual S3 bucket name (loki-aws-dev-ruler) + s3forcepathstyle: false + alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) + + querier: + max_concurrent: 4 + + storage: + type: s3 + bucketNames: + chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) + ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) + # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only + s3: + region: # eu-west-2 + #insecure: false + # s3forcepathstyle: false + + serviceAccount: + create: true + annotations: + "eks.amazonaws.com/role-arn": "arn:aws:iam:::role/LokiServiceAccountRole" # The service role you created + + deploymentMode: Distributed + + ingester: + replicas: 3 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + querier: + replicas: 3 + maxUnavailable: 2 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + queryFrontend: + replicas: 2 + maxUnavailable: 1 + queryScheduler: + replicas: 2 + distributor: + replicas: 3 + maxUnavailable: 2 + compactor: + replicas: 1 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + indexGateway: + replicas: 2 + maxUnavailable: 1 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + ruler: + replicas: 1 + maxUnavailable: 1 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + + + # This exposes the Loki gateway so it can be written to and queried externaly + gateway: + service: + type: LoadBalancer + basicAuth: + enabled: true + existingSecret: loki-basic-auth + + + + # Enable minio for storage + minio: + enabled: false + + backend: + replicas: 0 + read: + replicas: 0 + write: + replicas: 0 + + singleBinary: + replicas: 0 + +``` + +{{< admonition type="caution" >}} +Make sure to replace the placeholders with your actual values. +{{< /admonition >}} + +It is critical to define a valid `values.yaml` file for the Loki deployment. To remove the risk of misconfiguration, let's break down the configuration options to keep in mind when deploying to AWS: + +- **Loki Config vs. Values Config:** + - The `values.yaml` file contains a section called `loki`, which contains a direct representation of the Loki configuration file. + - This section defines the Loki configuration, including the schema, storage, and querier configuration. + - The key configuration to focus on for chunks is the `storage_config` section, where you define the S3 bucket region and name. This tells Loki where to store the chunks. + - The `ruler` section defines the configuration for the ruler, including the S3 bucket region and name. This tells Loki where to store the alert and recording rules. + - For the full Loki configuration, refer to the [Loki Configuration]({{< relref "../../../../configure" >}}) documentation. + +- **Storage:** + - Defines where the Helm chart stores data. + - Set the type to `s3` since we are using Amazon S3. + - Configure the bucket names for the chunks and ruler to match the buckets created earlier. + - The `s3` section specifies the region of the bucket. + +- **Service Account:** + - The `serviceAccount` section is used to define the IAM role for the Loki service account. + - This is where the IAM role created earlier is linked. + +- **Gateway:** + - Defines how the Loki gateway will be exposed. + - We are using a `LoadBalancer` service type in this configuration. + + +### Deploy Loki + +Now that you have created the `values.yaml` file, you can deploy Loki using the Helm chart. + +1. Deploy using the newly created `values.yaml` file: + + ```bash + helm install --values values.yaml loki grafana/loki -n loki --create-namespace + ``` + **It is important to create a namespace called `loki` as our trust policy is set to allow the IAM role to be used by the `loki` service account in the `loki` namespace. This is configurable but make sure to update your service account** + +1. Verify the deployment: + + ```bash + kubectl get pods -n loki + ``` + You should see the Loki pods running. + + ```console + NAME READY STATUS RESTARTS AGE + loki-canary-crqpg 1/1 Running 0 10m + loki-canary-hm26p 1/1 Running 0 10m + loki-canary-v9wv9 1/1 Running 0 10m + loki-chunks-cache-0 2/2 Running 0 10m + loki-compactor-0 1/1 Running 0 10m + loki-distributor-78ccdcc9b4-9wlhl 1/1 Running 0 10m + loki-distributor-78ccdcc9b4-km6j2 1/1 Running 0 10m + loki-distributor-78ccdcc9b4-ptwrb 1/1 Running 0 10m + loki-gateway-5f97f78755-hm6mx 1/1 Running 0 10m + loki-index-gateway-0 1/1 Running 0 10m + loki-index-gateway-1 1/1 Running 0 10m + loki-ingester-zone-a-0 1/1 Running 0 10m + loki-ingester-zone-b-0 1/1 Running 0 10m + loki-ingester-zone-c-0 1/1 Running 0 10m + loki-querier-89d4ff448-4vr9b 1/1 Running 0 10m + loki-querier-89d4ff448-7nvrf 1/1 Running 0 10m + loki-querier-89d4ff448-q89kh 1/1 Running 0 10m + loki-query-frontend-678899db5-n5wc4 1/1 Running 0 10m + loki-query-frontend-678899db5-tf69b 1/1 Running 0 10m + loki-query-scheduler-7d666bf759-9xqb5 1/1 Running 0 10m + loki-query-scheduler-7d666bf759-kpb5q 1/1 Running 0 10m + loki-results-cache-0 2/2 Running 0 10m + loki-ruler-0 1/1 Running 0 10m + ``` + +### Find the Loki Gateway Service + +The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. + +{{< admonition type="caution" >}} +The Loki Gateway service is exposed to the internet. We provide basic authentication using a username and password in this tutorial. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. +{{< /admonition >}} + +To find the Loki Gateway service, run the following command: + +```bash +kubectl get svc -n loki +``` +You should see the Loki Gateway service with an external IP address. This is the address you will use to write to and query Loki. + +```console + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +loki-gateway LoadBalancer 10.100.201.74 12345678975675456-1433434453245433545656563.eu-west-2.elb.amazonaws.com 80:30707/TCP 46m +``` + +Congratulations! You have successfully deployed Loki in distributed mode on AWS using the Helm chart. Before we finish, let's test the deployment. + +## Testing Your Loki Deployment + +k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: + +1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension]({{< relref "../../../../send-data/k6" >}}). + +2. Create a `aws-test.js` file with the following content: + + ```javascript + import {sleep, check} from 'k6'; + import loki from 'k6/x/loki'; + + /** + * URL used for push and query requests + * Path is automatically appended by the client + * @constant {string} + */ + + const username = ''; + const password = ''; + const external_ip = ''; + + const credentials = `${username}:${password}`; + + const BASE_URL = `http://${credentials}@${external_ip}`; + + /** + * Helper constant for byte values + * @constant {number} + */ + const KB = 1024; + + /** + * Helper constant for byte values + * @constant {number} + */ + const MB = KB * KB; + + /** + * Instantiate config and Loki client + */ + + const conf = new loki.Config(BASE_URL); + const client = new loki.Client(conf); + + /** + * Define test scenario + */ + export const options = { + vus: 10, + iterations: 10, + }; + + /** + * "main" function for each VU iteration + */ + export default () => { + // Push request with 10 streams and uncompressed logs between 800KB and 2MB + var res = client.pushParameterized(10, 800 * KB, 2 * MB); + // Check for successful write + check(res, { 'successful write': (res) => res.status == 204 }); + + // Pick a random log format from label pool + let format = randomChoice(conf.labels["format"]); + + // Execute instant query with limit 1 + res = client.instantQuery(`count_over_time({format="${format}"}[1m])`, 1) + // Check for successful read + check(res, { 'successful instant query': (res) => res.status == 200 }); + + // Execute range query over last 5m and limit 1000 + res = client.rangeQuery(`{format="${format}"}`, "5m", 1000) + // Check for successful read + check(res, { 'successful range query': (res) => res.status == 200 }); + + // Wait before next iteration + sleep(1); + } + + /** + * Helper function to get random item from array + */ + function randomChoice(items) { + return items[Math.floor(Math.random() * items.length)]; + } + ``` + + **Replace `` with the external IP address of the Loki Gateway service.** + + This script will write logs to Loki and query logs from Loki. It will write logs in a random format between 800KB and 2MB and query logs in a random format over the last 5 minutes. + +1. Run the test: + + ```bash + ./k6 run aws-test.js + ``` + + This will run the test and output the results. You should see the test writing logs to Loki and querying logs from Loki. + + + +## Considerations + +- **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. + +- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. + +- **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. + +- **Costs:** Running Loki on AWS will incur costs. Make sure to monitor your usage and costs to avoid any unexpected bills. In this guide we have used a simple EKS cluster with 3 nodes and m5.xlarge instances. You may wish to adjust the instance types and number of nodes based on your workload. + +- **Guide:** Note that this guide was **last updated on 1st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. + +## Next steps + +Now that you have successfully deployed Loki in distributed mode on AWS, you may wish to explore the following: + +- [Sending data to Loki]({{< relref "../../../../send-data" >}}) +- [Querying Loki]({{< relref "../../../../query" >}}) +- [Operations]({{< relref "../../../../operations" >}}) \ No newline at end of file From a9bc24af20a7af557689c78969dc863fdcf0b0a6 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Fri, 18 Oct 2024 17:25:05 +0100 Subject: [PATCH 24/51] updated single node helm, moved deployment guide, removed ssd guide --- docs/sources/setup/install/helm/_index.md | 3 +- .../install/helm/deployment-guides/_index.md | 13 + .../aws.md | 125 +++- .../helm/install-microservices/_index.md | 27 +- .../install/helm/install-monolithic/_index.md | 470 ++++++++++--- .../install/helm/install-scalable/_index.md | 29 +- .../install/helm/install-scalable/aws.md | 619 ------------------ 7 files changed, 550 insertions(+), 736 deletions(-) create mode 100644 docs/sources/setup/install/helm/deployment-guides/_index.md rename docs/sources/setup/install/helm/{install-microservices => deployment-guides}/aws.md (87%) delete mode 100644 docs/sources/setup/install/helm/install-scalable/aws.md diff --git a/docs/sources/setup/install/helm/_index.md b/docs/sources/setup/install/helm/_index.md index 0f582d6260ebc..3896d866b6cb6 100644 --- a/docs/sources/setup/install/helm/_index.md +++ b/docs/sources/setup/install/helm/_index.md @@ -27,8 +27,7 @@ If you are installing Grafana Enterprise Logs, follow the [GEL Helm installation The following guides provide step-by-step instructions for deploying Loki on cloud providers: -- [Deploy Loki Simple Scalable Helm chart on AWS]({{< relref "./install-scalable/aws" >}}) -- [Deploy Loki Distributed Helm chart on AWS]({{< relref "./install-microservices/aws" >}}) +- [Amazon EKS]({{< relref "./deployment-guides/aws.md" >}}) ## Reference diff --git a/docs/sources/setup/install/helm/deployment-guides/_index.md b/docs/sources/setup/install/helm/deployment-guides/_index.md new file mode 100644 index 0000000000000..119fa67d2a77e --- /dev/null +++ b/docs/sources/setup/install/helm/deployment-guides/_index.md @@ -0,0 +1,13 @@ +--- +title: Cloud Deployment Guides +menuTitle: Cloud Deployment Guides +description: Step-by-step instructions for deploying Loki on cloud providers. +weight: 500 +keywords: +--- + +# Cloud Deployment Guides + +The following guides provide step-by-step instructions for deploying Loki on cloud providers: + +- [Deploy Loki on AWS]({{< relref "./aws" >}}) \ No newline at end of file diff --git a/docs/sources/setup/install/helm/install-microservices/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md similarity index 87% rename from docs/sources/setup/install/helm/install-microservices/aws.md rename to docs/sources/setup/install/helm/deployment-guides/aws.md index 8cbb6d244c9ae..a05fc5e02f6b9 100644 --- a/docs/sources/setup/install/helm/install-microservices/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -1,14 +1,13 @@ --- -title: Deploy the distributed Helm chart on AWS +title: Deploy the Loki Helm chart on AWS menuTitle: Deploy on AWS -description: Installing Loki in distributed mode using the Helm chart on AWS. -weight: 300 +description: Installing the Loki Helm chart on AWS. keywords: --- -# Deploy the distributed Helm chart on AWS +# Deploy the Loki Helm chart on AWS -This guide shows how to deploy a minimally viable Loki in distributed mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: +This guide shows how to deploy a minimally viable Loki in either **distributed** or **monolithic** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: - Full access to EKS (Amazon Elastic Kubernetes Service) - Full access to S3 (Amazon Simple Storage Service) @@ -302,7 +301,9 @@ Loki by default does not come with any authentication. Since we will be deployin ### Loki Helm chart configuration -Create a `values.yaml` file with the following content: +Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in distributed mode the second is Loki in monolithic mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in distributed mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. + +{{< collapse title="Distributed" >}} ```yaml loki: @@ -336,7 +337,7 @@ Create a `values.yaml` file with the following content: storage: type: s3 s3: - region: eu-west-2 + region: # eu-west-2 bucketnames: # Your actual S3 bucket name (loki-aws-dev-ruler) s3forcepathstyle: false alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) @@ -435,6 +436,114 @@ Create a `values.yaml` file with the following content: replicas: 0 ``` +{{< /collapse >}} + +{{< collapse title="Monolithic" >}} + +```yaml +loki: + commonConfig: + replication_factor: 3 + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage_config: + aws: + region: # eu-west-2 + bucketnames: # Your actual S3 bucket name for chunks + s3forcepathstyle: false + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + enable_api: true + storage: + type: s3 + s3: + region: # eu-west-2 + bucketnames: # Your actual S3 bucket name for ruler + s3forcepathstyle: false + alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) + + storage: + type: s3 + bucketNames: + chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) + ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) + # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only + s3: + region: # eu-west-2 + #insecure: false + # s3forcepathstyle: false + + +serviceAccount: + create: true + annotations: + "eks.amazonaws.com/role-arn": "arn:aws:iam::182399686258:role/LokiServiceAccountRole" # The service role you created + +deploymentMode: SingleBinary + + +# This exposes the Loki gateway so it can be written to and queried externaly +gateway: + service: + type: LoadBalancer + basicAuth: + enabled: true + existingSecret: loki-basic-auth + + +singleBinary: + replicas: 3 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 30Gi + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 + +``` + +{{< /collapse >}} {{< admonition type="caution" >}} Make sure to replace the placeholders with your actual values. @@ -529,7 +638,7 @@ You should see the Loki Gateway service with an external IP address. This is the loki-gateway LoadBalancer 10.100.201.74 12345678975675456-1433434453245433545656563.eu-west-2.elb.amazonaws.com 80:30707/TCP 46m ``` -Congratulations! You have successfully deployed Loki in distributed mode on AWS using the Helm chart. Before we finish, let's test the deployment. +Congratulations! You have successfully deployed Loki on AWS using the Helm chart. Before we finish, let's test the deployment. ## Testing Your Loki Deployment diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 146c01813a27c..1b7e8ac30f426 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -41,13 +41,13 @@ We do not recommended to run Microservice mode with `filesystem` storage. For th helm repo add grafana https://grafana.github.io/helm-charts ``` -2. Update the chart repository: +1. Update the chart repository: ```bash helm repo update ``` -3. Create the configuration file `values.yaml`. The example below illustrates how to deploy Loki in test mode using MinIO as storage: +1. Create the configuration file `values.yaml`. The example below illustrates how to deploy Loki in test mode using MinIO as storage: ```yaml loki: @@ -122,7 +122,7 @@ We do not recommended to run Microservice mode with `filesystem` storage. For th enabled: true ``` -4. Install or upgrade the Loki deployment. +1. Install or upgrade the Loki deployment. - To install: ```bash helm install --values values.yaml loki grafana/loki @@ -176,9 +176,10 @@ After testing Loki with [MinIO](https://min.io/docs/minio/kubernetes/upstream/in When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. {{< /admonition >}} -{{< code >}} -```s3 +{{< collapse title="S3" >}} + +```yaml # Example configuration for Loki with S3 storage loki: @@ -274,8 +275,11 @@ loki: replicas: 0 ``` +{{< /collapse >}} + +{{< collapse title="Azure" >}} -```azure +```yaml # Example configuration for Loki with Azure Blob Storage loki: @@ -355,10 +359,19 @@ singleBinary: replicas: 0 ``` -{{< /code >}} +{{< /collapse >}} To configure other storage providers, refer to the [Helm Chart Reference]({{< relref "../reference" >}}). +## Deploying the Loki Helm chart to a Production Environment + +{{< admonition type="note" >}} +We are actively working on providing more guides for deploying Loki in production. +{{< /admonition >}} + +It is recommended to run Loki at scale within in a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. +- [Deploy Loki on AWS]({{< relref "../deployment-guides/aws" >}}) + ## Next Steps * Configure an agent to [send log data to Loki](/docs/loki//send-data/). * Monitor the Loki deployment using the [Meta Monitoring Helm chart](/docs/loki//setup/install/helm/monitor-and-alert/) diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 4373907dcfbb1..3d4132792ceab 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -10,107 +10,411 @@ weight: 100 # Install the monolithic Helm chart -This Helm Chart installation runs the Grafana Loki *single binary* within a Kubernetes cluster. +This Helm Chart installation deploys Grafana Loki in [monolithic mode](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) within a Kubernetes cluster. -If you set the `singleBinary.replicas` value to 1 and set the deployment mode to `SingleBinary`, this chart configures Loki to run the `all` target in a [monolithic mode](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode), designed to work with the filesystem storage configuration. It will also configure meta-monitoring of metrics and logs. +## Prerequisites + +- Helm 3 or above. See [Installing Helm](https://helm.sh/docs/intro/install/). +- A running Kubernetes cluster. + +## Single Replica or Multiple Replicas + +There are two ways to deploy Loki in monolithic mode: +1. **Single Replica**: Run Loki with a single replica. This mode is useful for testing and development or if you are planning to run Loki as a meta-monitoring system. +2. **Multiple Replicas**: Run Loki with multiple replicas. This mode is useful for high availability and scalability. This mode is less economical than distributed mode, but it is simpler to operate. + +Once you have selected choose the appropriate `values.yaml` configuration file below and then continue with the deployment steps. + +### Single Replica + +The default Helm chart deploys the following components: +- Loki (1 replica) +- Loki Canary (1 DaemonSet) +- Loki Gateway (1 NGINX replica) +- Loki Chunk and Result Cache (1 DaemonSet) +- Minio (optional, if `minio.enabled=true`) + +Create the configuration file `values.yaml`: {{< admonition type="note" >}} You must specify `commonConfig.replication_factor: 1` if you are only using 1 replica, otherwise requests will fail. {{< /admonition >}} -If you set the `singleBinary.replicas` value to 2 or more, this chart configures Loki to run a *single binary* in a replicated, highly available mode. When running replicas of a single binary, you must configure object storage. +```yaml +loki: + commonConfig: + replication_factor: 1 + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + enable_api: true -**Before you begin: Software Requirements** +minio: + enabled: true + +deploymentMode: SingleBinary -- Helm 3 or above. See [Installing Helm](https://helm.sh/docs/intro/install/). -- A running Kubernetes cluster +singleBinary: + replicas: 1 + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 +``` + +In this configuration, we are deploying Loki with MinIO as the object storage. We recommend configuring object storage via cloud provider or pointing Loki at a MinIO cluster for production deployments. + +### Multi Replica + +The default Helm chart deploys the following components: +- Loki (3 replicas) +- Loki Canary (1 DaemonSet) +- Loki Gateway (1 NGINX replica) +- Loki Chunk and Result Cache (1 DaemonSet) +- Minio (optional, if `minio.enabled=true`) + +Create the configuration file `values.yaml`: -**To deploy Loki in monolithic mode:** +{{< admonition type="note" >}} +If you set the `singleBinary.replicas` value to 2 or more, this chart configures Loki to run a *single binary* in a replicated, highly available mode. When running replicas of a single binary, you must configure object storage. +{{< /admonition >}} + +```yaml +loki: + commonConfig: + replication_factor: 3 + schemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + compactor: + retention_enabled: true + delete_request_store: s3 + ruler: + enable_api: true + +minio: + enabled: true + +deploymentMode: SingleBinary + +singleBinary: + replicas: 3 + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 +``` +In this configuration, we need to make sure to update the `commonConfig.replication_factor` and `singleBinary.replicas` to the desired number of replicas. We are deploying Loki with MinIO as the object storage. We recommend configuring object storage via cloud provider or pointing Loki at a MinIO cluster for production deployments. + +## Deploying the Helm chart for development and testing 1. Add [Grafana's chart repository](https://github.com/grafana/helm-charts) to Helm: - ```bash - helm repo add grafana https://grafana.github.io/helm-charts - ``` + ```bash + helm repo add grafana https://grafana.github.io/helm-charts + ``` 1. Update the chart repository: + ```bash + helm repo update + ``` + +1. Deploy Loki using the configuration file `values.yaml`: + + ```bash + helm install loki grafana/loki-stack -f values.yaml + ``` +1. Install or upgrade the Loki deployment. + - To install: + ```bash + helm install --values values.yaml loki grafana/loki + ``` + - To upgrade: + ```bash + helm upgrade --values values.yaml loki grafana/loki + ``` + +1. Verify that Loki is running: ```bash - helm repo update + kubectl get pods -n loki ``` -1. Create the configuration file `values.yaml`: - - - If running a single replica of Loki, configure the `filesystem` storage: - - ```yaml - deploymentMode: SingleBinary - loki: - commonConfig: - replication_factor: 1 - storage: - type: 'filesystem' - schemaConfig: - configs: - - from: "2024-01-01" - store: tsdb - index: - prefix: loki_index_ - period: 24h - object_store: filesystem # we're storing on filesystem so there's no real persistence here. - schema: v13 - singleBinary: - replicas: 1 - read: - replicas: 0 - backend: - replicas: 0 - write: - replicas: 0 - ``` - - - If running Loki with a replication factor greater than 1, set the desired number replicas and provide object storage credentials: - - ```yaml - loki: - commonConfig: - replication_factor: 3 - schemaConfig: - configs: - - from: "2024-01-01" - store: tsdb - index: - prefix: loki_index_ - period: 24h - object_store: s3 - schema: v13 - storage: - type: 's3' - bucketNames: - chunks: loki-chunks - ruler: loki-ruler - admin: loki-admin - s3: - endpoint: foo.aws.com - region: - secretAccessKey: supersecret - accessKeyId: secret - s3ForcePathStyle: false - insecure: false - singleBinary: - replicas: 3 - ``` - -1. Deploy the Loki cluster using one of these commands. - - - Deploy with the defined configuration: +## Object Storage Configuration - ```bash - helm install --values values.yaml loki grafana/loki - ``` +After testing Loki with MinIO, we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: + +{{< admonition type="caution" >}} +When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. +{{< /admonition >}} - - Deploy with the defined configuration in a custom Kubernetes cluster namespace: +{{< collapse title="S3" >}} + +```yaml +loki: + commonConfig: + replication_factor: 3 + schemaConfig: + configs: + - from: "2024-04-01" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + storage_config: + aws: + region: + bucketnames: + s3forcepathstyle: false + pattern_ingester: + enabled: true + limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 672h # 28 days retention + + storage: + type: s3 + bucketNames: + chunks: + ruler: + admin: + s3: + # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). + s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name + # AWS endpoint URL + endpoint: + # AWS region where the S3 bucket is located + region: + # AWS secret access key + secretAccessKey: + # AWS access key ID + accessKeyId: + # AWS signature version (e.g., v2 or v4) + signatureVersion: + # Forces the path style for S3 (true/false) + s3ForcePathStyle: false + # Allows insecure (HTTP) connections (true/false) + insecure: false + # HTTP configuration settings + http_config: {} + +# Disable minio storage +minio: + enabled: false + +singleBinary: + replicas: 3 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 30Gi + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 +``` + +{{< /collapse >}} + +{{< collapse title="Azure" >}} + +```yaml +loki: + schemaConfig: + configs: + - from: "2024-04-01" + store: tsdb + object_store: azure + schema: v13 + index: + prefix: loki_index_ + period: 24h + ingester: + chunk_encoding: snappy + + storage: + type: azure + azure: + # Name of the Azure Blob Storage account + accountName: + # Key associated with the Azure Blob Storage account + accountKey: + # Comprehensive connection string for Azure Blob Storage account (Can be used to replace endpoint, accountName, and accountKey) + connectionString: + # Flag indicating whether to use Azure Managed Identity for authentication + useManagedIdentity: false + # Flag indicating whether to use a federated token for authentication + useFederatedToken: false + # Client ID of the user-assigned managed identity (if applicable) + userAssignedId: + # Timeout duration for requests made to the Azure Blob Storage account (in seconds) + requestTimeout: + # Domain suffix of the Azure Blob Storage service endpoint (e.g., core.windows.net) + endpointSuffix: + bucketNames: + chunks: "chunks" + ruler: "ruler" + admin: "admin" + +# Disable minio storage +minio: + enabled: false + +singleBinary: + replicas: 3 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 30Gi + +# Zero out replica counts of other deployment modes +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +ingester: + replicas: 0 +querier: + replicas: 0 +queryFrontend: + replicas: 0 +queryScheduler: + replicas: 0 +distributor: + replicas: 0 +compactor: + replicas: 0 +indexGateway: + replicas: 0 +bloomCompactor: + replicas: 0 +bloomGateway: + replicas: 0 + +``` + +{{< /collapse >}} + + + +To configure other storage providers, refer to the [Helm Chart Reference]({{< relref "../reference" >}}). + +## Deploying the Loki Helm chart to a Production Environment + +{{< admonition type="note" >}} +We are actively working on providing more guides for deploying Loki in production. +{{< /admonition >}} + +It is recommended to run Loki at scale within in a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. +- [Deploy Loki on AWS]({{< relref "../deployment-guides/aws" >}}) + + +## Next Steps +* Configure an agent to [send log data to Loki](/docs/loki//send-data/). +* Monitor the Loki deployment using the [Meta Monitoring Helm chart](/docs/loki//setup/install/helm/monitor-and-alert/) - ```bash - helm install --values values.yaml loki --namespace=loki grafana/loki - ``` diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index 588ee8b7be848..636c6295c0783 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -33,8 +33,6 @@ We do not recommended to run scalable mode with `filesystem` storage. For the pu - Helm 3 or above. See [Installing Helm](https://helm.sh/docs/intro/install/). - A running Kubernetes cluster (must have at least 3 nodes). - - ## Deploying the Helm chart for development and testing The following steps show how to deploy the Loki Helm chart in simple scalable mode using the included MinIO as the storage backend. Our recommendation is to start here for development and testing purposes. Then configure Loki with an object storage provider when moving to production. @@ -116,9 +114,9 @@ After testing Loki with MinIO, we recommend to configure Loki with an object sto When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. {{< /admonition >}} -{{< code >}} +{{< collapse title="S3" >}} -```s3 +```yaml loki: schemaConfig: configs: @@ -182,7 +180,13 @@ write: minio: enabled: false ``` -```azure + +{{< /collapse >}} + +{{< collapse title="Azure" >}} + +```yaml + loki: schemaConfig: configs: @@ -237,20 +241,11 @@ write: minio: enabled: false -{{< /code >}} - -To configure other storage providers, refer to the [Helm Chart Reference]({{< relref "../reference" >}}). - -## Deploying the Loki Helm chart to a Production Environment - -{{< admonition type="note" >}} -We are activley working on providing more guides for deploying Loki in production. -{{< /admonition >}} - -It is recommended to run Loki at scale within in a cloud enviroment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. -- [Deploy Loki on AWS]({{< relref "./aws" >}}) +``` +{{< /collapse >}} +To configure other storage providers, refer to the [Helm Chart Reference]({{< relref "../reference" >}}). ## Next Steps * Configure an agent to [send log data to Loki](/docs/loki//send-data/). diff --git a/docs/sources/setup/install/helm/install-scalable/aws.md b/docs/sources/setup/install/helm/install-scalable/aws.md deleted file mode 100644 index 7fa4675fb6ebf..0000000000000 --- a/docs/sources/setup/install/helm/install-scalable/aws.md +++ /dev/null @@ -1,619 +0,0 @@ ---- -title: Deploy the Simple Scalable Helm chart on AWS -menuTitle: Deploy on AWS -description: Installing Loki in Simple Scalable mode using the Helm chart on AWS. -aliases: - - ../../../installation/helm/scalable/ - - ../../../installation/helm/install-scalable/ -weight: 300 -keywords: ---- - -# Deploy the Simple Scalable Helm chart on AWS - -This guide shows how to deploy a minimally viable Loki in simple scalable mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: - -- Full access to EKS (Amazon Elastic Kubernetes Service) -- Full access to S3 (Amazon Simple Storage Service) -- Sufficient permissions to create IAM roles and policies - -There are two methods for authenticating and connecting Loki to AWS S3. We will guide you through the recommended method of granting access via an IAM role. - -## Prerequisites - -- Helm 3 or above. Refer to [Installing Helm](https://helm.sh/docs/intro/install/). This should be installed on your local machine. -- A running Kubernetes cluster on AWS. A simple way to get started is by using EKSctl. See [Getting started with EKSctl](https://eksctl.io/). -- Kubectl installed on your local machine. Refer to [Install and Set Up kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). -- (Optional) AWS CLI installed on your local machine. Refer to [Installing the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). This is required if you plan to use EKSctl to create the EKS cluster and modify the IAM roles and policies locally. - -### EKS Minimum Requirements - -{{< admonition type="caution" >}} -These EKS requirements are the minimum specification needed to deploy Loki using this guide. You may wish to adjust plugins and instance types based on your AWS environment and workload. **If you choose to do so, this may invalidate the guide.** - -In this guide, we deploy Loki using `m5.xlarge` instances. This is a middle-of-the-road instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. -{{< /admonition >}} - -The minimum requirements for deploying Loki on EKS are: - -- Kubernetes version `1.30` or above. -- `3` nodes for the EKS cluster. -- Instance type depends on your workload. A good starting point is `m5.xlarge`. - -Here is the EKSctl cluster configuration file used in this guide: - -```yaml -# A simple example of ClusterConfig object: ---- -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: loki - region: eu-west-2 - version: "1.31" - -iam: - withOIDC: true - -addons: - - name: aws-ebs-csi-driver - - name: eks-pod-identity-agent - -managedNodeGroups: - - name: loki-workers - instanceType: m5.xlarge - desiredCapacity: 3 - minSize: 2 - maxSize: 3 - amiFamily: AmazonLinux2 - iam: - withAddonPolicies: - ebs: true - volumeSize: 80 - volumeType: gp2 - ebsOptimized: true - -``` - - -The following plugins are also required to be installed within the EKS cluster: -- **Amazon EBS CSI Driver**: Enables Kubernetes to dynamically provision and manage EBS volumes as persistent storage for applications. We use this to provision the node volumes for Loki. -- **Amazon EKS Pod Identity Agent**: Manages AWS IAM roles for pods, allowing fine-grained access control to AWS resources without needing to store credentials in containers. This is how Loki will access the S3 bucket. -- **CoreDNS**: Provides internal DNS service for Kubernetes clusters, ensuring that services and pods can communicate with each other using DNS names. -- **kube-proxy**: Maintains network rules on nodes, enabling communication between pods and services within the cluster. - -You must also install an **OIDC (OpenID Connect) provider** on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: - -```bash -eksctl utils associate-iam-oidc-provider --cluster loki --approve -``` -**This may be installed by default when creating the EKS cluster using EKSctl and the above config.** - -## Create three S3 buckets - -{{< admonition type="warning" >}} - **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a **unique** name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). -{{< /admonition >}} - -Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. - -{{}} -GEL customers will require a third bucket to store the admin data. This bucket is not required for OSS users. -{{}} - -```bash -aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ -aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= -``` -Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. - - - -## Defining IAM roles and policies - -The recommended method for connecting Loki to AWS S3 is to use an IAM role. This method is more secure than using access keys and secret keys which are directly stored in the Loki configuration. The role and policy can be created using the AWS CLI or the AWS Management Console. The below steps show how to create the role and policy using the AWS CLI. - -{{< admonition type="tip" >}} -Create a new directory and navigate to it. Make sure to create the files in this directory. All commands in this guide assume you are in this directory. -{{< /admonition >}} - -1. Create a `loki-s3-policy.json` file with the following content: - - ```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "LokiStorage", - "Effect": "Allow", - "Action": [ - "s3:ListBucket", - "s3:PutObject", - "s3:GetObject", - "s3:DeleteObject" - ], - "Resource": [ - "arn:aws:s3:::< Name of the chunk bucket >", - "arn:aws:s3:::< Name of the chunk bucket >/*", - "arn:aws:s3:::< Name of the ruler bucket >", - "arn:aws:s3:::< Name of the ruler bucket >/*" - ] - } - ] - } - ``` - - **Make sure to replace the placeholder with the name of the buckets you created earlier.** - -1. Create the IAM policy using the AWS CLI: - - ```bash - aws iam create-policy --policy-name LokiS3AccessPolicy --policy-document file://loki-s3-policy.json - ``` - -1. Create a trust policy document named `trust-policy.json` with the following content: - - ```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::< ACCOUNT ID >:oidc-provider/oidc.eks..amazonaws.com/id/< ID >" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks..amazonaws.com/id/< ID >:sub": "system:serviceaccount:loki:loki" - } - } - } - ] - } - ``` - **Make sure to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** - -1. Create the IAM role using the AWS CLI: - - ```bash - aws iam create-role --role-name LokiServiceAccountRole --assume-role-policy-document file://trust-policy.json - ``` - -1. Attach the policy to the role: - - ```bash - aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam:::policy/LokiS3AccessPolicy - ``` - **Make sure to replace the placeholder with your AWS account ID.** - -### Adding the policy to the S3 buckets - -To allow the IAM role to access the S3 buckets, you need to add the policy to the bucket. You can do this using the AWS Management Console or the AWS CLI. The below steps show how to add the policy using the AWS CLI. - -1. Create a bucket policy file named `bucket-policy-chunk.json` with the following content: - - ```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Statement1", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" - }, - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:DeleteObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::", - "arn:aws:s3:::/*" - ] - } - ] - } - ``` - **Make sure to replace the placeholders with your AWS account ID and the bucket name.** - -1. Add the policy to the bucket: - - ```bash - aws s3api put-bucket-policy --bucket --policy file://bucket-policy-chunk.json - ``` -1. Create a bucket policy file named `bucket-policy-ruler.json` with the following content: - - ```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Statement1", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" - }, - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:DeleteObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::", - "arn:aws:s3:::/*" - ] - } - ] - } - ``` - **Make sure to replace the placeholders with your AWS account ID and the bucket name.** - -1. Add the policy to the bucket: - - ```bash - aws s3api put-bucket-policy --bucket --policy file://bucket-policy-ruler.json - ``` - -## Deploying the Helm chart - -Before we can deploy the Loki Helm chart, we need to add the Grafana chart repository to Helm. This repository contains the Loki Helm chart. - -1. Add the Grafana chart repository to Helm: - - ```bash - helm repo add grafana https://grafana.github.io/helm-charts - ``` -1. Update the chart repository: - - ```bash - helm repo update - ``` -1. Create a new namespace for Loki: - - ```bash - kubectl create namespace loki - ``` -### Loki Basic Authentication - -Loki by default does not come with any authentication. Since we will be deploying Loki to AWS and exposing the gateway to the internet, we recommend adding atleast basic authentication. In this guide we will give Loki a username and password: - -1. To start we will need create a `.htpasswd` file with the username and password. You can use the `htpasswd` command to create the file: - - {{< admonition type="tip" >}} - If you don't have the `htpasswd` command installed, you can install it using `brew` or `apt-get` or `yum` depending on your OS. - {{< /admonition >}} - - ```bash - htpasswd -c .htpasswd - ``` - This will create a file called `auth` with the username `loki`. You will be prompted to enter a password. - - 1. Create a Kubernetes secret with the `.htpasswd` file: - - ```bash - kubectl create secret generic loki-basic-auth --from-file=.htpasswd -n loki - ``` - - This will create a secret called `loki-basic-auth` in the `loki` namespace. We will reference this secret in the Loki Helm chart configuration. - - - -### Loki Helm chart configuration - -Create a `values.yaml` file with the following content: - -```yaml - loki: - schemaConfig: - configs: - - from: 2024-04-01 - store: tsdb - object_store: s3 - schema: v13 - index: - prefix: loki_index_ - period: 24h - storage_config: - aws: - region: # eu-west-2 - bucketnames: # Your actual S3 bucket name (loki-aws-dev-chunks) - s3forcepathstyle: false - ingester: - chunk_encoding: snappy - pattern_ingester: - enabled: true - limits_config: - allow_structured_metadata: true - volume_enabled: true - retention_period: 672h # 28 days retention - compactor: - retention_enabled: true - delete_request_store: s3 - ruler: - enable_api: true - storage: - type: s3 - s3: - region: eu-west-2 - bucketnames: # Your actual S3 bucket name (loki-aws-dev-ruler) - s3forcepathstyle: false - alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) - - querier: - max_concurrent: 4 - - storage: - type: s3 - bucketNames: - chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) - ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) - # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only - s3: - region: # eu-west-2 - #insecure: false - # s3forcepathstyle: false - - - deploymentMode: SimpleScalable - - serviceAccount: - create: true - annotations: - "eks.amazonaws.com/role-arn": "arn:aws:iam:::role/LokiServiceAccountRole" # The service role you created - - backend: - replicas: 2 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - read: - replicas: 2 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - write: - replicas: 3 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - - # This exposes the Loki gateway so it can be written to and queried externaly - gateway: - service: - type: LoadBalancer - basicAuth: - enabled: true - existingSecret: loki-basic-auth # Change this is you used a different secret name - - - # Enable minio for storage - minio: - enabled: false - -``` - -{{< admonition type="caution" >}} -Make sure to replace the placeholders with your actual values. -{{< /admonition >}} - -It is critical to define a valid `values.yaml` file for the Loki deployment. To remove the risk of misconfiguration, let's break down the configuration options to keep in mind when deploying to AWS: - -- **Loki Config vs. Values Config:** - - The `values.yaml` file contains a section called `loki`, which contains a direct representation of the Loki configuration file. - - This section defines the Loki configuration, including the schema, storage, and querier configuration. - - The key configuration to focus on for chunks is the `storage_config` section, where you define the S3 bucket region and name. This tells Loki where to store the chunks. - - The `ruler` section defines the configuration for the ruler, including the S3 bucket region and name. This tells Loki where to store the alert and recording rules. - - For the full Loki configuration, refer to the [Loki Configuration]({{< relref "../../../../configure" >}}) documentation. - -- **Storage:** - - Defines where the Helm chart stores data. - - Set the type to `s3` since we are using Amazon S3. - - Configure the bucket names for the chunks and ruler to match the buckets created earlier. - - The `s3` section specifies the region of the bucket. - -- **Service Account:** - - The `serviceAccount` section is used to define the IAM role for the Loki service account. - - This is where the IAM role created earlier is linked. - -- **Gateway:** - - Defines how the Loki gateway will be exposed. - - We are using a `LoadBalancer` service type in this configuration. - - -### Deploy Loki - -Now that you have created the `values.yaml` file, you can deploy Loki using the Helm chart. - -1. Deploy using the newly created `values.yaml` file: - - ```bash - helm install --values values.yaml loki grafana/loki -n loki --create-namespace - ``` - **It is important to create a namespace called `loki` as our trust policy is set to allow the IAM role to be used by the `loki` service account in the `loki` namespace. This is configurable but make sure to update your service account** - -1. Verify the deployment: - - ```bash - kubectl get pods -n loki - ``` - You should see the Loki pods running. - ```console - NAME READY STATUS RESTARTS AGE - loki-backend-0 2/2 Running 0 43m - loki-backend-1 2/2 Running 0 43m - loki-canary-c7hbw 1/1 Running 0 43m - loki-canary-wtxb8 1/1 Running 0 43m - loki-chunks-cache-0 2/2 Running 0 43m - loki-gateway-844ff6d84-9zg5k 1/1 Running 0 43m - loki-read-848c686c9d-kd9xt 1/1 Running 0 43m - loki-read-848c686c9d-vx697 1/1 Running 0 43m - loki-results-cache-0 2/2 Running 0 43m - loki-write-0 1/1 Running 0 43m - loki-write-1 1/1 Running 0 43m - loki-write-2 1/1 Running 0 43m - ``` - -### Find the Loki Gateway Service - -The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. - -{{< admonition type="caution" >}} -The Loki Gateway service is exposed to the internet. We provide basic authentication using a username and password in this tutorial. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. -{{< /admonition >}} - -To find the Loki Gateway service, run the following command: - -```bash -kubectl get svc -n loki -``` -You should see the Loki Gateway service with an external IP address. This is the address you will use to write to and query Loki. - -```console - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -loki-backend ClusterIP 10.100.97.223 3100/TCP,9095/TCP 46m -loki-backend-headless ClusterIP None 3100/TCP,9095/TCP 46m -loki-canary ClusterIP 10.100.121.159 3500/TCP 46m -loki-chunks-cache ClusterIP None 11211/TCP,9150/TCP 46m -loki-gateway LoadBalancer 10.100.201.74 12345678975675456-1433434453245433545656563.eu-west-2.elb.amazonaws.com 80:30707/TCP 46m -loki-memberlist ClusterIP None 7946/TCP 46m -loki-query-scheduler-discovery ClusterIP None 3100/TCP,9095/TCP 46m -loki-read ClusterIP 10.100.187.114 3100/TCP,9095/TCP 46m -loki-read-headless ClusterIP None 3100/TCP,9095/TCP 46m -loki-results-cache ClusterIP None 11211/TCP,9150/TCP 46m -loki-write ClusterIP 10.100.217.163 3100/TCP,9095/TCP 46m -loki-write-headless ClusterIP None 3100/TCP,9095/TCP 46m -``` - -Congratulations! You have successfully deployed Loki in simple scalable mode on AWS using the Helm chart. Before we finish, let's test the deployment. - -## Testing Your Loki Deployment - -k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: - -1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension]({{< relref "../../../../send-data/k6" >}}). - -2. Create a `aws-test.js` file with the following content: - - ```javascript - import {sleep, check} from 'k6'; - import loki from 'k6/x/loki'; - - /** - * URL used for push and query requests - * Path is automatically appended by the client - * @constant {string} - */ - - const username = ''; - const password = ''; - const external_ip = ''; - - const credentials = `${username}:${password}`; - - const BASE_URL = `http://${credentials}@${external_ip}`; - - /** - * Helper constant for byte values - * @constant {number} - */ - const KB = 1024; - - /** - * Helper constant for byte values - * @constant {number} - */ - const MB = KB * KB; - - /** - * Instantiate config and Loki client - */ - - const conf = new loki.Config(BASE_URL); - const client = new loki.Client(conf); - - /** - * Define test scenario - */ - export const options = { - vus: 10, - iterations: 10, - }; - - /** - * "main" function for each VU iteration - */ - export default () => { - // Push request with 10 streams and uncompressed logs between 800KB and 2MB - var res = client.pushParameterized(10, 800 * KB, 2 * MB); - // Check for successful write - check(res, { 'successful write': (res) => res.status == 204 }); - - // Pick a random log format from label pool - let format = randomChoice(conf.labels["format"]); - - // Execute instant query with limit 1 - res = client.instantQuery(`count_over_time({format="${format}"}[1m])`, 1) - // Check for successful read - check(res, { 'successful instant query': (res) => res.status == 200 }); - - // Execute range query over last 5m and limit 1000 - res = client.rangeQuery(`{format="${format}"}`, "5m", 1000) - // Check for successful read - check(res, { 'successful range query': (res) => res.status == 200 }); - - // Wait before next iteration - sleep(1); - } - - /** - * Helper function to get random item from array - */ - function randomChoice(items) { - return items[Math.floor(Math.random() * items.length)]; - } - ``` - - **Replace `` with the external IP address of the Loki Gateway service.** - - This script will write logs to Loki and query logs from Loki. It will write logs in a random format between 800KB and 2MB and query logs in a random format over the last 5 minutes. - -1. Run the test: - - ```bash - ./k6 run aws-test.js - ``` - - This will run the test and output the results. You should see the test writing logs to Loki and querying logs from Loki. - - - -## Considerations - -- **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. - -- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. - -- **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. - -- **Costs:** Running Loki on AWS will incur costs. Make sure to monitor your usage and costs to avoid any unexpected bills. In this guide we have used a simple EKS cluster with 3 nodes and m5.xlarge instances. You may wish to adjust the instance types and number of nodes based on your workload. - -- **Guide:** Note that this guide was **last updated on 1st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. - -## Next steps - -Now that you have successfully deployed Loki in simple scalable mode on AWS, you may wish to explore the following: - -- [Sending data to Loki]({{< relref "../../../../send-data" >}}) -- [Querying Loki]({{< relref "../../../../query" >}}) -- [Operations]({{< relref "../../../../operations" >}}) \ No newline at end of file From 6b08dd185bae1389befeef01c8e0af46ce7200f8 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Mon, 21 Oct 2024 13:42:36 +0100 Subject: [PATCH 25/51] fixed typos --- .../setup/install/helm/deployment-guides/aws.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index a05fc5e02f6b9..126d4cf88038b 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -157,12 +157,13 @@ Create a new directory and navigate to it. Make sure to create the files in this { "Effect": "Allow", "Principal": { - "Federated": "arn:aws:iam::< ACCOUNT ID >:oidc-provider/oidc.eks..amazonaws.com/id/< ID >" + "Federated": "arn:aws:iam::< ACCOUNT ID >:oidc-provider/oidc.eks..amazonaws.com/id/< OIDC ID >" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { - "oidc.eks..amazonaws.com/id/< ID >:sub": "system:serviceaccount:loki:loki" + "oidc.eks..amazonaws.com/id/< OIDC ID >:sub": "system:serviceaccount:loki:loki", + "oidc.eks..amazonaws.com/id/< OIDC ID >:aud": "sts.amazonaws.com" } } } @@ -301,7 +302,7 @@ Loki by default does not come with any authentication. Since we will be deployin ### Loki Helm chart configuration -Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in distributed mode the second is Loki in monolithic mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in distributed mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. +Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in [distributed](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode the second is Loki in [monolithic](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in distributed mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. {{< collapse title="Distributed" >}} @@ -745,13 +746,13 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t - **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. -- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. +- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. - **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. - **Costs:** Running Loki on AWS will incur costs. Make sure to monitor your usage and costs to avoid any unexpected bills. In this guide we have used a simple EKS cluster with 3 nodes and m5.xlarge instances. You may wish to adjust the instance types and number of nodes based on your workload. -- **Guide:** Note that this guide was **last updated on 1st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. +- **Guide:** Note that this guide was **last updated on 21st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. ## Next steps From 28b3fa50ddc29ccc7160ab151ad7ed6fa789b103 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Mon, 21 Oct 2024 15:12:26 +0100 Subject: [PATCH 26/51] fixed naming --- docs/sources/setup/install/helm/deployment-guides/aws.md | 6 +++--- .../setup/install/helm/install-microservices/_index.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index 126d4cf88038b..f9fb66db8140e 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -7,7 +7,7 @@ keywords: # Deploy the Loki Helm chart on AWS -This guide shows how to deploy a minimally viable Loki in either **distributed** or **monolithic** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: +This guide shows how to deploy a minimally viable Loki in either **microservice** or **monolithic** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: - Full access to EKS (Amazon Elastic Kubernetes Service) - Full access to S3 (Amazon Simple Storage Service) @@ -302,7 +302,7 @@ Loki by default does not come with any authentication. Since we will be deployin ### Loki Helm chart configuration -Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in [distributed](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode the second is Loki in [monolithic](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in distributed mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. +Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in [microservices](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode the second is Loki in [monolithic](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in microservices mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. {{< collapse title="Distributed" >}} @@ -756,7 +756,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t ## Next steps -Now that you have successfully deployed Loki in distributed mode on AWS, you may wish to explore the following: +Now that you have successfully deployed Loki in microservices mode on AWS, you may wish to explore the following: - [Sending data to Loki]({{< relref "../../../../send-data" >}}) - [Querying Loki]({{< relref "../../../../query" >}}) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 1b7e8ac30f426..96be45d380ae9 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -1,7 +1,7 @@ --- title: Loki Microservice Helm chart menuTitle: Install microservice Loki -description: Installing Loki in microservice (distributed) mode using the Helm chart. +description: Installing Loki in microservice mode using the Helm chart. weight: 300 keywords: --- @@ -10,7 +10,7 @@ keywords: This Helm Chart deploys Grafana Loki on Kubernetes. -This chart configures Loki to run Loki in [microservice / distributed mode]({{< relref "../../../../get-started/deployment-modes#microservices-mode" >}}). The microservices deployment mode runs components of Loki as distinct processes. +This chart configures Loki to run Loki in [microservice mode]({{< relref "../../../../get-started/deployment-modes#microservices-mode" >}}). The microservices deployment mode runs components of Loki as distinct processes. The default Helm chart deploys the following components: - **Compactor component** (1 replica): Compacts and processes stored data. From 2a72d396318c3866c1e1021d6ad1795a54e16709 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:17:01 +0100 Subject: [PATCH 27/51] Update docs/sources/setup/install/helm/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/_index.md b/docs/sources/setup/install/helm/_index.md index 3896d866b6cb6..dd58d03235f5f 100644 --- a/docs/sources/setup/install/helm/_index.md +++ b/docs/sources/setup/install/helm/_index.md @@ -27,7 +27,7 @@ If you are installing Grafana Enterprise Logs, follow the [GEL Helm installation The following guides provide step-by-step instructions for deploying Loki on cloud providers: -- [Amazon EKS]({{< relref "./deployment-guides/aws.md" >}}) +- [Amazon EKS](https://grafana.com/docs/loki//setup/install/helm/deployment-guides/aws/) ## Reference From 8beeec6221c07de07911afba7b2687a861da7ef1 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:17:07 +0100 Subject: [PATCH 28/51] Update docs/sources/setup/install/helm/deployment-guides/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/deployment-guides/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/_index.md b/docs/sources/setup/install/helm/deployment-guides/_index.md index 119fa67d2a77e..4ff7d5dcaa983 100644 --- a/docs/sources/setup/install/helm/deployment-guides/_index.md +++ b/docs/sources/setup/install/helm/deployment-guides/_index.md @@ -10,4 +10,4 @@ keywords: The following guides provide step-by-step instructions for deploying Loki on cloud providers: -- [Deploy Loki on AWS]({{< relref "./aws" >}}) \ No newline at end of file +- [Deploy Loki on AWS](https://grafana.com/docs/loki//setup/install/helm/deployment-guides/aws/) \ No newline at end of file From 65a7a73ccbe83016fb5cf9852db2117bdcf0e98e Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:17:17 +0100 Subject: [PATCH 29/51] Update docs/sources/setup/install/helm/deployment-guides/aws.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/deployment-guides/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index f9fb66db8140e..a2258087db2f0 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -18,7 +18,7 @@ There are two methods for authenticating and connecting Loki to AWS S3. We will ## Prerequisites - Helm 3 or above. Refer to [Installing Helm](https://helm.sh/docs/intro/install/). This should be installed on your local machine. -- A running Kubernetes cluster on AWS. A simple way to get started is by using EKSctl. See [Getting started with EKSctl](https://eksctl.io/). +- A running Kubernetes cluster on AWS. A simple way to get started is by using EKSctl. Refer to [Getting started with EKSctl](https://eksctl.io/). - Kubectl installed on your local machine. Refer to [Install and Set Up kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). - (Optional) AWS CLI installed on your local machine. Refer to [Installing the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). This is required if you plan to use EKSctl to create the EKS cluster and modify the IAM roles and policies locally. From cfd2a5affd1b3bb72b6fe5b5afaa4bda1c3ca73a Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:17:30 +0100 Subject: [PATCH 30/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-microservices/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 96be45d380ae9..8250198db1413 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -369,7 +369,7 @@ To configure other storage providers, refer to the [Helm Chart Reference]({{< re We are actively working on providing more guides for deploying Loki in production. {{< /admonition >}} -It is recommended to run Loki at scale within in a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. +It is recommended to run Loki at scale within a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. - [Deploy Loki on AWS]({{< relref "../deployment-guides/aws" >}}) ## Next Steps From 8a3a082e9a8b9f2c667c5910d9163e5903b7902f Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:17:39 +0100 Subject: [PATCH 31/51] Update docs/sources/setup/install/helm/install-monolithic/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-monolithic/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 3d4132792ceab..f458b334193b4 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -103,7 +103,7 @@ bloomGateway: In this configuration, we are deploying Loki with MinIO as the object storage. We recommend configuring object storage via cloud provider or pointing Loki at a MinIO cluster for production deployments. -### Multi Replica +### Multiple Replicas The default Helm chart deploys the following components: - Loki (3 replicas) From 9f53e020bba3fd088c0f77c4a480386104f74891 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:17:54 +0100 Subject: [PATCH 32/51] Update docs/sources/setup/install/helm/install-monolithic/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-monolithic/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index f458b334193b4..a0ee2cca26028 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -216,7 +216,7 @@ In this configuration, we need to make sure to update the `commonConfig.replicat ## Object Storage Configuration -After testing Loki with MinIO, we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with MinIO, we recommend configuring Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. From 913e4a541fe0510356079904d9e604f40a859aac Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Wed, 23 Oct 2024 09:36:08 +0100 Subject: [PATCH 33/51] Made cloud deployment microservice only --- .../install/helm/deployment-guides/aws.md | 157 +++--------------- .../install/helm/install-monolithic/_index.md | 4 +- 2 files changed, 25 insertions(+), 136 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index a2258087db2f0..278bc39f216c9 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -7,11 +7,11 @@ keywords: # Deploy the Loki Helm chart on AWS -This guide shows how to deploy a minimally viable Loki in either **microservice** or **monolithic** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: +This guide shows how to deploy a minimally viable Loki in either **microservice** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: - Full access to EKS (Amazon Elastic Kubernetes Service) - Full access to S3 (Amazon Simple Storage Service) -- Sufficient permissions to create IAM roles and policies +- Sufficient permissions to create IAM (Identity Access Management) roles and policies There are two methods for authenticating and connecting Loki to AWS S3. We will guide you through the recommended method of granting access via an IAM role. @@ -81,10 +81,13 @@ The following plugins are also required to be installed within the EKS cluster: You must also install an **OIDC (OpenID Connect) provider** on the EKS cluster. This is required for the IAM roles and policies to work correctly. If you are using EKSctl, you can install the OIDC provider using the following command: +{{< admonition type="tip" >}} +If you used the above EKSctl configuration file to create the cluster, you do not need to run this command. The OIDC provider is automatically installed. +{{< /admonition >}} + ```bash eksctl utils associate-iam-oidc-provider --cluster loki --approve ``` -**This may be installed by default when creating the EKS cluster using EKSctl and the above config.** ## Create three S3 buckets @@ -94,25 +97,21 @@ eksctl utils associate-iam-oidc-provider --cluster loki --approve Before deploying Loki, you need to create two S3 buckets; one to store logs (chunks), the second to store alert rules. You can create the bucket using the AWS Management Console or the AWS CLI. The bucket name must be globally unique. -{{}} +{{< admonition type="note" >}} GEL customers will require a third bucket to store the admin data. This bucket is not required for OSS users. -{{}} +{{< /admonition >}} ```bash aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= ``` -Make sure to replace the region and bucket name with your desired values. We will revisit the bucket policy later in this guide. - - +Make sure to replace the `region` and `bucket` name with your desired values. We will revisit the bucket policy later in this guide. ## Defining IAM roles and policies The recommended method for connecting Loki to AWS S3 is to use an IAM role. This method is more secure than using access keys and secret keys which are directly stored in the Loki configuration. The role and policy can be created using the AWS CLI or the AWS Management Console. The below steps show how to create the role and policy using the AWS CLI. -{{< admonition type="tip" >}} -Create a new directory and navigate to it. Make sure to create the files in this directory. All commands in this guide assume you are in this directory. -{{< /admonition >}} +1. Create a new directory and navigate to it. Make sure to create the files in this directory. All commands in this guide assume you are in this directory. 1. Create a `loki-s3-policy.json` file with the following content: @@ -142,13 +141,13 @@ Create a new directory and navigate to it. Make sure to create the files in this **Make sure to replace the placeholder with the name of the buckets you created earlier.** -1. Create the IAM policy using the AWS CLI: - +2. Create the IAM policy using the AWS CLI: + **Make sure to replace the placeholders with the names of the buckets you created earlier.** ```bash aws iam create-policy --policy-name LokiS3AccessPolicy --policy-document file://loki-s3-policy.json ``` -1. Create a trust policy document named `trust-policy.json` with the following content: +3. Create a trust policy document named `trust-policy.json` with the following content: ```json { @@ -172,13 +171,13 @@ Create a new directory and navigate to it. Make sure to create the files in this ``` **Make sure to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** -1. Create the IAM role using the AWS CLI: +4. Create the IAM role using the AWS CLI: ```bash aws iam create-role --role-name LokiServiceAccountRole --assume-role-policy-document file://trust-policy.json ``` -1. Attach the policy to the role: +5. Attach the policy to the role: ```bash aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam:::policy/LokiS3AccessPolicy @@ -277,10 +276,10 @@ Before we can deploy the Loki Helm chart, we need to add the Grafana chart repos ``` ### Loki Basic Authentication -Loki by default does not come with any authentication. Since we will be deploying Loki to AWS and exposing the gateway to the internet, we recommend adding atleast basic authentication. In this guide we will give Loki a username and password: +Loki by default does not come with any authentication. Since we will be deploying Loki to AWS and exposing the gateway to the internet, we recommend adding at least basic authentication. In this guide we will give Loki a username and password: 1. To start we will need create a `.htpasswd` file with the username and password. You can use the `htpasswd` command to create the file: - + {{< admonition type="tip" >}} If you don't have the `htpasswd` command installed, you can install it using `brew` or `apt-get` or `yum` depending on your OS. {{< /admonition >}} @@ -304,8 +303,6 @@ Loki by default does not come with any authentication. Since we will be deployin Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in [microservices](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode the second is Loki in [monolithic](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in microservices mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. -{{< collapse title="Distributed" >}} - ```yaml loki: schemaConfig: @@ -319,8 +316,8 @@ Create a `values.yaml` file choosing the configuration options that best suit yo period: 24h storage_config: aws: - region: # eu-west-2 - bucketnames: # Your actual S3 bucket name (loki-aws-dev-chunks) + region: # for example, eu-west-2 + bucketnames: # Your actual S3 bucket name, for example, loki-aws-dev-chunks s3forcepathstyle: false ingester: chunk_encoding: snappy @@ -338,8 +335,8 @@ Create a `values.yaml` file choosing the configuration options that best suit yo storage: type: s3 s3: - region: # eu-west-2 - bucketnames: # Your actual S3 bucket name (loki-aws-dev-ruler) + region: # for example, eu-west-2 + bucketnames: # Your actual S3 bucket name, for example, loki-aws-dev-ruler s3forcepathstyle: false alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) @@ -437,114 +434,6 @@ Create a `values.yaml` file choosing the configuration options that best suit yo replicas: 0 ``` -{{< /collapse >}} - -{{< collapse title="Monolithic" >}} - -```yaml -loki: - commonConfig: - replication_factor: 3 - schemaConfig: - configs: - - from: 2024-04-01 - store: tsdb - object_store: s3 - schema: v13 - index: - prefix: loki_index_ - period: 24h - storage_config: - aws: - region: # eu-west-2 - bucketnames: # Your actual S3 bucket name for chunks - s3forcepathstyle: false - pattern_ingester: - enabled: true - limits_config: - allow_structured_metadata: true - volume_enabled: true - retention_period: 672h # 28 days retention - compactor: - retention_enabled: true - delete_request_store: s3 - ruler: - enable_api: true - storage: - type: s3 - s3: - region: # eu-west-2 - bucketnames: # Your actual S3 bucket name for ruler - s3forcepathstyle: false - alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) - - storage: - type: s3 - bucketNames: - chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) - ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) - # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only - s3: - region: # eu-west-2 - #insecure: false - # s3forcepathstyle: false - - -serviceAccount: - create: true - annotations: - "eks.amazonaws.com/role-arn": "arn:aws:iam::182399686258:role/LokiServiceAccountRole" # The service role you created - -deploymentMode: SingleBinary - - -# This exposes the Loki gateway so it can be written to and queried externaly -gateway: - service: - type: LoadBalancer - basicAuth: - enabled: true - existingSecret: loki-basic-auth - - -singleBinary: - replicas: 3 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 30Gi - -# Zero out replica counts of other deployment modes -backend: - replicas: 0 -read: - replicas: 0 -write: - replicas: 0 - -ingester: - replicas: 0 -querier: - replicas: 0 -queryFrontend: - replicas: 0 -queryScheduler: - replicas: 0 -distributor: - replicas: 0 -compactor: - replicas: 0 -indexGateway: - replicas: 0 -bloomCompactor: - replicas: 0 -bloomGateway: - replicas: 0 - -``` - -{{< /collapse >}} {{< admonition type="caution" >}} Make sure to replace the placeholders with your actual values. @@ -694,7 +583,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t iterations: 10, }; - /** + **It is important to create a namespace called `loki` as our trust policy is set to allow the IAM role to be used by the `loki` service account in the `loki` namespace. This is configurable but make sure to update your service account.** * "main" function for each VU iteration */ export default () => { @@ -754,7 +643,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t - **Guide:** Note that this guide was **last updated on 21st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. -## Next steps +k6 is one of the fastest ways to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: Now that you have successfully deployed Loki in microservices mode on AWS, you may wish to explore the following: diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index a0ee2cca26028..9fe13db4034b0 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -27,7 +27,7 @@ Once you have selected choose the appropriate `values.yaml` configuration file b ### Single Replica -The default Helm chart deploys the following components: +Deploying the Helm chart with a single replica deploys the following components: - Loki (1 replica) - Loki Canary (1 DaemonSet) - Loki Gateway (1 NGINX replica) @@ -105,7 +105,7 @@ In this configuration, we are deploying Loki with MinIO as the object storage. W ### Multiple Replicas -The default Helm chart deploys the following components: +Deploying the Helm chart with multiple replicas deploys the following components: - Loki (3 replicas) - Loki Canary (1 DaemonSet) - Loki Gateway (1 NGINX replica) From cec22d15c31c24d2d30e2cff1929527692d195c9 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Wed, 23 Oct 2024 09:42:18 +0100 Subject: [PATCH 34/51] fixed typos in deployments and changed url --- .../setup/install/helm/install-microservices/_index.md | 8 ++++---- .../setup/install/helm/install-monolithic/_index.md | 6 +++--- .../sources/setup/install/helm/install-scalable/_index.md | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 8250198db1413..3fff8e21ee6c6 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -1,12 +1,12 @@ --- -title: Loki Microservice Helm chart +title: Install the microservice Helm chart menuTitle: Install microservice Loki description: Installing Loki in microservice mode using the Helm chart. weight: 300 keywords: --- -# Loki Microservice Helm chart +# Install the microservice Helm chart This Helm Chart deploys Grafana Loki on Kubernetes. @@ -22,7 +22,7 @@ The default Helm chart deploys the following components: - **QueryScheduler component** (2 replicas): Schedules queries. {{< admonition type="note" >}} -We do not recommended to run Microservice mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +We do not recommend running Microservice mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. {{< /admonition >}} ## Prerequisites @@ -53,7 +53,7 @@ We do not recommended to run Microservice mode with `filesystem` storage. For th loki: schemaConfig: configs: - - from: 2024-04-01 + - from: "2024-04-01" store: tsdb object_store: s3 schema: v13 diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 9fe13db4034b0..a6f62d039fbf8 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -21,9 +21,9 @@ This Helm Chart installation deploys Grafana Loki in [monolithic mode](https://g There are two ways to deploy Loki in monolithic mode: 1. **Single Replica**: Run Loki with a single replica. This mode is useful for testing and development or if you are planning to run Loki as a meta-monitoring system. -2. **Multiple Replicas**: Run Loki with multiple replicas. This mode is useful for high availability and scalability. This mode is less economical than distributed mode, but it is simpler to operate. +2. **Multiple Replicas**: Run Loki with multiple replicas. This mode is useful for high availability and scalability. This mode is less economical than microservice mode, but it is simpler to operate. We recommend running at least three replicas for high availability. -Once you have selected choose the appropriate `values.yaml` configuration file below and then continue with the deployment steps. +Once you have selected how many replicas you would like to deploy choose the appropriate `values.yaml` configuration file below and then continue with the deployment steps. ### Single Replica @@ -103,7 +103,7 @@ bloomGateway: In this configuration, we are deploying Loki with MinIO as the object storage. We recommend configuring object storage via cloud provider or pointing Loki at a MinIO cluster for production deployments. -### Multiple Replicas +### Multiple Replicas Deploying the Helm chart with multiple replicas deploys the following components: - Loki (3 replicas) diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index 636c6295c0783..63a9f96ca3d8c 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -1,5 +1,5 @@ --- -title: Loki Simple Scalable Helm chart +title: Install the simple scalable Helm chart menuTitle: Install scalable Loki description: Installing Loki in simple scalable mode using the Helm chart. aliases: @@ -9,7 +9,7 @@ weight: 300 keywords: --- -# Loki Simple Scalable Helm chart +# Install the simple scalable Helm chart This Helm Chart deploys Grafana Loki on Kubernetes in simple scalable mode. From edcaa115cf0a0ec910471d7c7cfaa5b8ceb7262a Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Thu, 24 Oct 2024 16:15:43 +0100 Subject: [PATCH 35/51] fixed URL's and other recommendations --- .../install/helm/deployment-guides/aws.md | 106 +++++++++--------- .../install/helm/install-monolithic/_index.md | 2 +- 2 files changed, 53 insertions(+), 55 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index 278bc39f216c9..b4f6c0d4940ad 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -7,7 +7,7 @@ keywords: # Deploy the Loki Helm chart on AWS -This guide shows how to deploy a minimally viable Loki in either **microservice** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: +This guide shows how to deploy a minimally viable Loki in **microservice** mode on AWS using the Helm chart. To run through this guide, we expect you to have the necessary tools and permissions to deploy resources on AWS, such as: - Full access to EKS (Amazon Elastic Kubernetes Service) - Full access to S3 (Amazon Simple Storage Service) @@ -15,6 +15,20 @@ This guide shows how to deploy a minimally viable Loki in either **microservice* There are two methods for authenticating and connecting Loki to AWS S3. We will guide you through the recommended method of granting access via an IAM role. +## Considerations + +{{< admonition type="caution" >}} +This guide was accurate at the time it was last updated on **21st October, 2024**. As cloud providers frequently update their services and offerings, as a best practice, you should refer to the [AWS S3 documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html) before creating your buckets and assigning roles. +{{< /admonition >}} + +- **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. + +- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication](https://grafana.com/docs/loki/}} -These EKS requirements are the minimum specification needed to deploy Loki using this guide. You may wish to adjust plugins and instance types based on your AWS environment and workload. **If you choose to do so, this may invalidate the guide.** +These EKS requirements are the minimum specification needed to deploy Loki using this guide. You may wish to adjust plugins and instance types based on your AWS environment and workload. **If you choose to do so, we cannot guarantee that this sample configuration will still meet your needs.** -In this guide, we deploy Loki using `m5.xlarge` instances. This is a middle-of-the-road instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. +In this guide, we deploy Loki using `m5.xlarge` instances. This is a instance type that should work for most scenarios. However, you can modify the instance types and count based on your specific needs. {{< /admonition >}} The minimum requirements for deploying Loki on EKS are: @@ -45,8 +59,8 @@ apiVersion: eksctl.io/v1alpha5 kind: ClusterConfig metadata: - name: loki - region: eu-west-2 + name: + region: version: "1.31" iam: @@ -73,7 +87,7 @@ managedNodeGroups: ``` -The following plugins are also required to be installed within the EKS cluster: +The following plugins must also be installed within the EKS cluster: - **Amazon EBS CSI Driver**: Enables Kubernetes to dynamically provision and manage EBS volumes as persistent storage for applications. We use this to provision the node volumes for Loki. - **Amazon EKS Pod Identity Agent**: Manages AWS IAM roles for pods, allowing fine-grained access control to AWS resources without needing to store credentials in containers. This is how Loki will access the S3 bucket. - **CoreDNS**: Provides internal DNS service for Kubernetes clusters, ensuring that services and pods can communicate with each other using DNS names. @@ -89,7 +103,7 @@ If you used the above EKSctl configuration file to create the cluster, you do no eksctl utils associate-iam-oidc-provider --cluster loki --approve ``` -## Create three S3 buckets +## Create S3 buckets {{< admonition type="warning" >}} **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a **unique** name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). @@ -102,8 +116,8 @@ GEL customers will require a third bucket to store the admin data. This bucket i {{< /admonition >}} ```bash -aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ -aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= +aws s3api create-bucket --bucket < YOUR CHUNK BUCKET NAME eg. `loki-aws-dev-chunks`> --region --create-bucket-configuration LocationConstraint= \ +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= ``` Make sure to replace the `region` and `bucket` name with your desired values. We will revisit the bucket policy later in this guide. @@ -129,10 +143,10 @@ The recommended method for connecting Loki to AWS S3 is to use an IAM role. This "s3:DeleteObject" ], "Resource": [ - "arn:aws:s3:::< Name of the chunk bucket >", - "arn:aws:s3:::< Name of the chunk bucket >/*", - "arn:aws:s3:::< Name of the ruler bucket >", - "arn:aws:s3:::< Name of the ruler bucket >/*" + "arn:aws:s3:::< CHUNK BUCKET NAME >", + "arn:aws:s3:::< CHUNK BUCKET NAME >/*", + "arn:aws:s3:::< RULER BUCKET NAME >", + "arn:aws:s3:::< RULER BUCKET NAME >/*" ] } ] @@ -141,13 +155,13 @@ The recommended method for connecting Loki to AWS S3 is to use an IAM role. This **Make sure to replace the placeholder with the name of the buckets you created earlier.** -2. Create the IAM policy using the AWS CLI: - **Make sure to replace the placeholders with the names of the buckets you created earlier.** +1. Create the IAM policy using the AWS CLI: + ```bash aws iam create-policy --policy-name LokiS3AccessPolicy --policy-document file://loki-s3-policy.json ``` -3. Create a trust policy document named `trust-policy.json` with the following content: +2. Create a trust policy document named `trust-policy.json` with the following content: ```json { @@ -171,13 +185,13 @@ The recommended method for connecting Loki to AWS S3 is to use an IAM role. This ``` **Make sure to replace the placeholders with your AWS account ID, region, and the OIDC ID (you can find this in the EKS cluster configuration).** -4. Create the IAM role using the AWS CLI: +3. Create the IAM role using the AWS CLI: ```bash aws iam create-role --role-name LokiServiceAccountRole --assume-role-policy-document file://trust-policy.json ``` -5. Attach the policy to the role: +4. Attach the policy to the role: ```bash aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam:::policy/LokiS3AccessPolicy @@ -198,7 +212,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th "Sid": "Statement1", "Effect": "Allow", "Principal": { - "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" + "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" }, "Action": [ "s3:PutObject", @@ -207,8 +221,8 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th "s3:ListBucket" ], "Resource": [ - "arn:aws:s3:::", - "arn:aws:s3:::/*" + "arn:aws:s3:::< CHUNK BUCKET NAME >", + "arn:aws:s3:::< CHUNK BUCKET NAME >/*" ] } ] @@ -219,7 +233,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th 1. Add the policy to the bucket: ```bash - aws s3api put-bucket-policy --bucket --policy file://bucket-policy-chunk.json + aws s3api put-bucket-policy --bucket --policy file://bucket-policy-chunk.json ``` 1. Create a bucket policy file named `bucket-policy-ruler.json` with the following content: @@ -231,7 +245,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th "Sid": "Statement1", "Effect": "Allow", "Principal": { - "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" + "AWS": "arn:aws:iam:::role/LokiServiceAccountRole" }, "Action": [ "s3:PutObject", @@ -240,8 +254,8 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th "s3:ListBucket" ], "Resource": [ - "arn:aws:s3:::", - "arn:aws:s3:::/*" + "arn:aws:s3:::< RULER BUCKET NAME >", + "arn:aws:s3:::< RULER BUCKET NAME >/*" ] } ] @@ -252,7 +266,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th 1. Add the policy to the bucket: ```bash - aws s3api put-bucket-policy --bucket --policy file://bucket-policy-ruler.json + aws s3api put-bucket-policy --bucket --policy file://bucket-policy-ruler.json ``` ## Deploying the Helm chart @@ -316,8 +330,8 @@ Create a `values.yaml` file choosing the configuration options that best suit yo period: 24h storage_config: aws: - region: # for example, eu-west-2 - bucketnames: # Your actual S3 bucket name, for example, loki-aws-dev-chunks + region: # for example, eu-west-2 + bucketnames: # Your actual S3 bucket name, for example, loki-aws-dev-chunks s3forcepathstyle: false ingester: chunk_encoding: snappy @@ -346,11 +360,11 @@ Create a `values.yaml` file choosing the configuration options that best suit yo storage: type: s3 bucketNames: - chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) - ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) + chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) + ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only s3: - region: # eu-west-2 + region: # eu-west-2 #insecure: false # s3forcepathstyle: false @@ -446,7 +460,7 @@ It is critical to define a valid `values.yaml` file for the Loki deployment. To - This section defines the Loki configuration, including the schema, storage, and querier configuration. - The key configuration to focus on for chunks is the `storage_config` section, where you define the S3 bucket region and name. This tells Loki where to store the chunks. - The `ruler` section defines the configuration for the ruler, including the S3 bucket region and name. This tells Loki where to store the alert and recording rules. - - For the full Loki configuration, refer to the [Loki Configuration]({{< relref "../../../../configure" >}}) documentation. + - For the full Loki configuration, refer to the [Loki Configuration](https://grafana.com/docs/loki//configure/) documentation. - **Storage:** - Defines where the Helm chart stores data. @@ -513,7 +527,7 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the The Loki Gateway service is a LoadBalancer service that exposes the Loki gateway to the internet. This is where you will write logs to and query logs from. By default NGINX is used as the gateway. {{< admonition type="caution" >}} -The Loki Gateway service is exposed to the internet. We provide basic authentication using a username and password in this tutorial. Refer to the [Authentication]({{< relref "../../../../operations/authentication" >}}) documentation for more information. +The Loki Gateway service is exposed to the internet. We provide basic authentication using a username and password in this tutorial. Refer to the [Authentication](https://grafana.com/docs/loki//operations/authentication/) documentation for more information. {{< /admonition >}} To find the Loki Gateway service, run the following command: @@ -534,7 +548,7 @@ Congratulations! You have successfully deployed Loki on AWS using the Helm chart k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: -1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension]({{< relref "../../../../send-data/k6" >}}). +1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension](https://grafana.com/docs/loki//send-data/k6/). 2. Create a `aws-test.js` file with the following content: @@ -621,7 +635,7 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t This script will write logs to Loki and query logs from Loki. It will write logs in a random format between 800KB and 2MB and query logs in a random format over the last 5 minutes. -1. Run the test: +3. Run the test: ```bash ./k6 run aws-test.js @@ -629,24 +643,8 @@ k6 is one of the fastest way to test your Loki deployment. This will allow you t This will run the test and output the results. You should see the test writing logs to Loki and querying logs from Loki. - - -## Considerations - -- **IAM Role:** The IAM role created in this guide is a basic role that allows Loki to read and write to the S3 bucket. You may wish to add more granular permissions based on your requirements. - -- **Authentication:** Grafana Loki comes with a basic authentication layer. The Loki gateway (NGINX) is exposed to the internet using basic authentication in this example. NGINX can also be replaced with other open-source reverse proxies. Refer to [Authentication]({{< relref "../../../../operations/authentication" >}}) for more information. - -- **Retention:** The retention period is set to 28 days in the `values.yaml` file. You may wish to adjust this based on your requirements. - -- **Costs:** Running Loki on AWS will incur costs. Make sure to monitor your usage and costs to avoid any unexpected bills. In this guide we have used a simple EKS cluster with 3 nodes and m5.xlarge instances. You may wish to adjust the instance types and number of nodes based on your workload. - -- **Guide:** Note that this guide was **last updated on 21st October 2024**. As cloud providers frequently update their services and offerings, some steps in this guide may need adjustments over time. - -k6 is one of the fastest ways to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: - Now that you have successfully deployed Loki in microservices mode on AWS, you may wish to explore the following: -- [Sending data to Loki]({{< relref "../../../../send-data" >}}) -- [Querying Loki]({{< relref "../../../../query" >}}) -- [Operations]({{< relref "../../../../operations" >}}) \ No newline at end of file +- [Sending data to Loki](https://grafana.com/docs/loki//query/) +- [Manage](https://grafana.com/docs/loki/ Date: Thu, 24 Oct 2024 19:00:31 +0100 Subject: [PATCH 36/51] fixed indentaiton --- .../install/helm/deployment-guides/aws.md | 182 +++++++++--------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index b4f6c0d4940ad..54295bab949e8 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -318,7 +318,7 @@ Loki by default does not come with any authentication. Since we will be deployin Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in [microservices](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode the second is Loki in [monolithic](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in microservices mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. ```yaml - loki: +loki: schemaConfig: configs: - from: 2024-04-01 @@ -355,98 +355,98 @@ Create a `values.yaml` file choosing the configuration options that best suit yo alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) querier: - max_concurrent: 4 + max_concurrent: 4 storage: - type: s3 - bucketNames: - chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) - ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) - # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only - s3: - region: # eu-west-2 - #insecure: false - # s3forcepathstyle: false - - serviceAccount: - create: true - annotations: - "eks.amazonaws.com/role-arn": "arn:aws:iam:::role/LokiServiceAccountRole" # The service role you created - - deploymentMode: Distributed - - ingester: - replicas: 3 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - querier: - replicas: 3 - maxUnavailable: 2 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - queryFrontend: - replicas: 2 - maxUnavailable: 1 - queryScheduler: - replicas: 2 - distributor: - replicas: 3 - maxUnavailable: 2 - compactor: - replicas: 1 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - indexGateway: - replicas: 2 - maxUnavailable: 1 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - ruler: - replicas: 1 - maxUnavailable: 1 - persistence: - storageClass: gp2 - accessModes: - - ReadWriteOnce - size: 10Gi - - - # This exposes the Loki gateway so it can be written to and queried externaly - gateway: - service: - type: LoadBalancer - basicAuth: - enabled: true - existingSecret: loki-basic-auth - - - - # Enable minio for storage - minio: - enabled: false - - backend: - replicas: 0 - read: - replicas: 0 - write: - replicas: 0 - - singleBinary: - replicas: 0 - + type: s3 + bucketNames: + chunks: "" # Your actual S3 bucket name (loki-aws-dev-chunks) + ruler: "" # Your actual S3 bucket name (loki-aws-dev-ruler) + # admin: "" # Your actual S3 bucket name (loki-aws-dev-admin) - GEL customers only + s3: + region: # eu-west-2 + #insecure: false + # s3forcepathstyle: false + +serviceAccount: + create: true + annotations: + "eks.amazonaws.com/role-arn": "arn:aws:iam:::role/LokiServiceAccountRole" # The service role you created + +deploymentMode: Distributed + +ingester: + replicas: 3 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + +querier: + replicas: 3 + maxUnavailable: 2 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi +queryFrontend: + replicas: 2 + maxUnavailable: 1 +queryScheduler: + replicas: 2 +distributor: + replicas: 3 + maxUnavailable: 2 +compactor: + replicas: 1 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi +indexGateway: + replicas: 2 + maxUnavailable: 1 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi +ruler: + replicas: 1 + maxUnavailable: 1 + persistence: + storageClass: gp2 + accessModes: + - ReadWriteOnce + size: 10Gi + + +# This exposes the Loki gateway so it can be written to and queried externaly +gateway: + service: + type: LoadBalancer + basicAuth: + enabled: true + existingSecret: loki-basic-auth + + + +# Enable minio for storage +minio: + enabled: false + +backend: + replicas: 0 +read: + replicas: 0 +write: + replicas: 0 + +singleBinary: + replicas: 0 ``` {{< admonition type="caution" >}} From c643b4879f6c60a177da04d31565da011cae60ab Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:10:53 +0000 Subject: [PATCH 37/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-microservices/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index b53d415ba4819..2ff3ee4c72b42 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -173,7 +173,7 @@ We do not recommend running Microservice mode with `filesystem` storage. For the ## Object Storage Configuration -After testing Loki with [MinIO](https://min.io/docs/minio/kubernetes/upstream/index.html), we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with [MinIO](https://min.io/docs/minio/kubernetes/upstream/index.html), we recommend configuring Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. From 73fd3f3e1991fbaf1fb5553f687dab85e340b927 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:00 +0000 Subject: [PATCH 38/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: J Stickler --- .../setup/install/helm/install-microservices/_index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 2ff3ee4c72b42..1b3e6822dd587 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -214,9 +214,9 @@ loki: storage: type: s3 bucketNames: - chunks: - ruler: - admin: + chunks: + ruler: + admin: s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name From 56f8895c03447e6be83e4cd703ab32cae475a238 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:07 +0000 Subject: [PATCH 39/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-microservices/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index 1b3e6822dd587..e264a2a3b06cb 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -218,7 +218,7 @@ loki: ruler: admin: s3: - # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). + # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storage or if you are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name # AWS endpoint URL endpoint: From 8dcca5751b73230ae9cf2f1eeb2c6d171eae3dfd Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:13 +0000 Subject: [PATCH 40/51] Update docs/sources/setup/install/helm/install-microservices/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-microservices/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index e264a2a3b06cb..edb1abd427163 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -376,7 +376,7 @@ To configure other storage providers, refer to the [Helm Chart Reference]({{< re We are actively working on providing more guides for deploying Loki in production. {{< /admonition >}} -It is recommended to run Loki at scale within a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. +We recommend running Loki at scale within a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. - [Deploy Loki on AWS]({{< relref "../deployment-guides/aws" >}}) ## Next Steps From 6d32293acf3e3daaddf3cd8c93e9332724b69f96 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:19 +0000 Subject: [PATCH 41/51] Update docs/sources/setup/install/helm/install-monolithic/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-monolithic/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 748f89fac62cb..23835a8a2216e 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -23,7 +23,7 @@ There are two ways to deploy Loki in monolithic mode: 1. **Single Replica**: Run Loki with a single replica. This mode is useful for testing and development or if you are planning to run Loki as a meta-monitoring system. 2. **Multiple Replicas**: Run Loki with multiple replicas. This mode is useful for high availability. This mode is less economical than microservice mode, but it is simpler to operate. We recommend running at least three replicas for high availability. -Once you have selected how many replicas you would like to deploy choose the appropriate `values.yaml` configuration file below and then continue with the deployment steps. +Once you have selected how many replicas you would like to deploy, choose the appropriate `values.yaml` configuration file below and then continue with the deployment steps. ### Single Replica From c99ffe5c2e127d2f6bfc06e21aa6a7f7c38e5b4a Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:25 +0000 Subject: [PATCH 42/51] Update docs/sources/setup/install/helm/install-scalable/_index.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/install-scalable/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index 03649d181dbd9..fefe55a1538bc 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -149,7 +149,7 @@ loki: ruler: admin: s3: - # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). + # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or if you are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name # AWS endpoint URL endpoint: From 23acdef14e099a3fceccf3b12d9304c07a01c566 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:31 +0000 Subject: [PATCH 43/51] Update docs/sources/setup/install/helm/deployment-guides/aws.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/deployment-guides/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index 54295bab949e8..05d9d62381856 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -116,7 +116,7 @@ GEL customers will require a third bucket to store the admin data. This bucket i {{< /admonition >}} ```bash -aws s3api create-bucket --bucket < YOUR CHUNK BUCKET NAME eg. `loki-aws-dev-chunks`> --region --create-bucket-configuration LocationConstraint= \ +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= ``` Make sure to replace the `region` and `bucket` name with your desired values. We will revisit the bucket policy later in this guide. From 19ab1471c8beecd0b1d33d11f1d987e41f1942f6 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:37 +0000 Subject: [PATCH 44/51] Update docs/sources/setup/install/helm/deployment-guides/aws.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/deployment-guides/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index 05d9d62381856..af53550aef917 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -117,7 +117,7 @@ GEL customers will require a third bucket to store the admin data. This bucket i ```bash aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= \ -aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= +aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= ``` Make sure to replace the `region` and `bucket` name with your desired values. We will revisit the bucket policy later in this guide. From aba3b50ee9c84333e44aa1a23eb8ad502990aedb Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:44 +0000 Subject: [PATCH 45/51] Update docs/sources/setup/install/helm/deployment-guides/aws.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/deployment-guides/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index af53550aef917..21868fb3daab2 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -321,7 +321,7 @@ Create a `values.yaml` file choosing the configuration options that best suit yo loki: schemaConfig: configs: - - from: 2024-04-01 + - from: "2024-04-01" store: tsdb object_store: s3 schema: v13 From 7cc5504ed71a39c8e392bafce4cae144804f83c7 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:11:51 +0000 Subject: [PATCH 46/51] Update docs/sources/setup/install/helm/deployment-guides/aws.md Co-authored-by: J Stickler --- docs/sources/setup/install/helm/deployment-guides/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index 21868fb3daab2..ab1e3dbb5371e 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -315,7 +315,7 @@ Loki by default does not come with any authentication. Since we will be deployin ### Loki Helm chart configuration -Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there are two examples of `values.yaml` files for the Loki Helm chart. The first template is Loki in [microservices](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode the second is Loki in [monolithic](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode with a replication factor of 3. The rest of this guide will focus on deploying Loki in microservices mode. However, there is no difference in the deployment process between the two modes apart from the components you are deploying. +Create a `values.yaml` file choosing the configuration options that best suit your requirements. Below there is an example of `values.yaml` files for the Loki Helm chart in [microservices](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode. ```yaml loki: From 905551bc23d83a1127c38cabde1ff22e69df8e54 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:10:58 +0000 Subject: [PATCH 47/51] Apply suggestions from code review Co-authored-by: J Stickler --- .../install/helm/install-monolithic/_index.md | 14 +++++++------- .../setup/install/helm/install-scalable/_index.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 23835a8a2216e..7f0628ef1b853 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -239,8 +239,8 @@ loki: period: 24h storage_config: aws: - region: - bucketnames: + region: + bucketnames: s3forcepathstyle: false pattern_ingester: enabled: true @@ -252,9 +252,9 @@ loki: storage: type: s3 bucketNames: - chunks: - ruler: - admin: + chunks: + ruler: + admin: s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name @@ -402,7 +402,7 @@ bloomGateway: -To configure other storage providers, refer to the [Helm Chart Reference]({{< relref "../reference" >}}). +To configure other storage providers, refer to the [Helm Chart Reference](https://grafana.com/docs/loki//setup/install/helm/reference/). ## Deploying the Loki Helm chart to a Production Environment @@ -411,7 +411,7 @@ We are actively working on providing more guides for deploying Loki in productio {{< /admonition >}} It is recommended to run Loki at scale within in a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. -- [Deploy Loki on AWS]({{< relref "../deployment-guides/aws" >}}) +- [Deploy Loki on AWS](https://grafana.com/docs/loki//setup/install/helm/deployment-guides/aws) ## Next Steps diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index fefe55a1538bc..7d993b2ff9ac8 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -25,7 +25,7 @@ The default Helm chart deploys the following components: - Index and Chunk cache (1 replica) {{< admonition type="note" >}} -We do not recommended to run scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +We do not recommended running scalable mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. {{< /admonition >}} ## Prerequisites @@ -56,7 +56,7 @@ The following steps show how to deploy the Loki Helm chart in simple scalable mo loki: schemaConfig: configs: - - from: 2024-04-01 + - from: "2024-04-01" store: tsdb object_store: s3 schema: v13 @@ -130,8 +130,8 @@ loki: period: 24h storage_config: aws: - region: - bucketnames: + region: + bucketnames: s3forcepathstyle: false pattern_ingester: enabled: true @@ -145,9 +145,9 @@ loki: storage: type: s3 bucketNames: - chunks: - ruler: - admin: + chunks: + ruler: + admin: s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storages or if you are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name From 8f0dc753b9b8df8f81d52f1d804264a6df8547f6 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:14:01 +0000 Subject: [PATCH 48/51] Apply suggestions from code review Co-authored-by: J Stickler --- .../setup/install/helm/deployment-guides/aws.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index ab1e3dbb5371e..e7e570a7c8067 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -153,7 +153,7 @@ The recommended method for connecting Loki to AWS S3 is to use an IAM role. This } ``` - **Make sure to replace the placeholder with the name of the buckets you created earlier.** + **Make sure to replace the placeholders with the names of the buckets you created earlier.** 1. Create the IAM policy using the AWS CLI: @@ -228,7 +228,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th ] } ``` - **Make sure to replace the placeholders with your AWS account ID and the bucket name.** + **Make sure to replace the placeholders with your AWS account ID and the bucket names.** 1. Add the policy to the bucket: @@ -261,7 +261,7 @@ To allow the IAM role to access the S3 buckets, you need to add the policy to th ] } ``` - **Make sure to replace the placeholders with your AWS account ID and the bucket name.** + **Make sure to replace the placeholders with your AWS account ID and the bucket names.** 1. Add the policy to the bucket: @@ -486,7 +486,7 @@ Now that you have created the `values.yaml` file, you can deploy Loki using the ```bash helm install --values values.yaml loki grafana/loki -n loki --create-namespace ``` - **It is important to create a namespace called `loki` as our trust policy is set to allow the IAM role to be used by the `loki` service account in the `loki` namespace. This is configurable but make sure to update your service account** + **It is important to create a namespace called `loki` as our trust policy is set to allow the IAM role to be used by the `loki` service account in the `loki` namespace. This is configurable but make sure to update your service account.** 1. Verify the deployment: @@ -546,7 +546,7 @@ Congratulations! You have successfully deployed Loki on AWS using the Helm chart ## Testing Your Loki Deployment -k6 is one of the fastest way to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: +k6 is one of the fastest ways to test your Loki deployment. This will allow you to both write and query logs to Loki. To get started with k6, follow the steps below: 1. Install k6 with the Loki extension on your local machine. Refer to [Installing k6 and the xk6-loki extension](https://grafana.com/docs/loki//send-data/k6/). From d209efb1ba2ad70a677a772fcdc4c7216602f005 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 29 Oct 2024 18:15:19 +0000 Subject: [PATCH 49/51] Update docs/sources/setup/install/helm/_index.md --- docs/sources/setup/install/helm/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/_index.md b/docs/sources/setup/install/helm/_index.md index dd58d03235f5f..5392838e2ca74 100644 --- a/docs/sources/setup/install/helm/_index.md +++ b/docs/sources/setup/install/helm/_index.md @@ -31,4 +31,4 @@ The following guides provide step-by-step instructions for deploying Loki on clo ## Reference -[Values reference]({{< relref "./reference" >}}) +[Values reference](https://grafana.com/docs/loki//setup/install/helm/reference/) From a097bf8ad888cf836ba9148b0be2c62d838df1f4 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Wed, 30 Oct 2024 09:11:47 +0000 Subject: [PATCH 50/51] Apply suggestions from code review Co-authored-by: J Stickler --- .../install/helm/install-microservices/_index.md | 12 ++++++------ .../setup/install/helm/install-monolithic/_index.md | 6 +++--- .../setup/install/helm/install-scalable/_index.md | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/sources/setup/install/helm/install-microservices/_index.md b/docs/sources/setup/install/helm/install-microservices/_index.md index edb1abd427163..37ee90d25b291 100644 --- a/docs/sources/setup/install/helm/install-microservices/_index.md +++ b/docs/sources/setup/install/helm/install-microservices/_index.md @@ -10,7 +10,7 @@ keywords: This Helm Chart deploys Grafana Loki on Kubernetes. -This chart configures Loki to run Loki in [microservice mode]({{< relref "../../../../get-started/deployment-modes#microservices-mode" >}}). The microservices deployment mode runs components of Loki as distinct processes. +This Helm chart deploys Loki to run Loki in [microservice mode](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) within a Kubernetes cluster. The microservices deployment mode runs components of Loki as distinct processes. The default Helm chart deploys the following components: - **Compactor component** (1 replica): Compacts and processes stored data. @@ -22,7 +22,7 @@ The default Helm chart deploys the following components: - **QueryScheduler component** (2 replicas): Schedules queries. {{< admonition type="note" >}} -We do not recommend running Microservice mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. +We do not recommend running in Microservice mode with `filesystem` storage. For the purpose of this guide, we will use MinIO as the object storage to provide a complete example. {{< /admonition >}} @@ -197,8 +197,8 @@ loki: period: 24h storage_config: aws: - region: - bucketnames: + region: + bucketnames: s3forcepathstyle: false ingester: chunk_encoding: snappy @@ -215,8 +215,8 @@ loki: type: s3 bucketNames: chunks: - ruler: - admin: + ruler: + admin: s3: # s3 URL can be used to specify the endpoint, access key, secret key, and bucket name this works well for S3 compatible storage or if you are hosting Loki on-premises and want to use S3 as the storage backend. Either use the s3 URL or the individual fields below (AWS endpoint, region, secret). s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 7f0628ef1b853..fe790b0b6f890 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -46,7 +46,7 @@ loki: replication_factor: 1 schemaConfig: configs: - - from: 2024-04-01 + - from: "2024-04-01" store: tsdb object_store: s3 schema: v13 @@ -124,7 +124,7 @@ loki: replication_factor: 3 schemaConfig: configs: - - from: 2024-04-01 + - from: "2024-04-01" store: tsdb object_store: s3 schema: v13 @@ -410,7 +410,7 @@ To configure other storage providers, refer to the [Helm Chart Reference](https: We are actively working on providing more guides for deploying Loki in production. {{< /admonition >}} -It is recommended to run Loki at scale within in a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. +We recommend running Loki at scale within a cloud environment like AWS, Azure, or GCP. The below guides will show you how to deploy a minimally viable production environment. - [Deploy Loki on AWS](https://grafana.com/docs/loki//setup/install/helm/deployment-guides/aws) diff --git a/docs/sources/setup/install/helm/install-scalable/_index.md b/docs/sources/setup/install/helm/install-scalable/_index.md index 7d993b2ff9ac8..31461b33e06f8 100644 --- a/docs/sources/setup/install/helm/install-scalable/_index.md +++ b/docs/sources/setup/install/helm/install-scalable/_index.md @@ -11,7 +11,7 @@ keywords: # Install the simple scalable Helm chart -This Helm Chart deploys Grafana Loki on Kubernetes in simple scalable mode. +This Helm Chart deploys Grafana Loki in [simple scalable mode](https://grafana.com/docs/loki//get-started/deployment-modes/#simple-scalable) within a Kubernetes cluster. This chart configures Loki to run `read`, `write`, and `backend` targets in a [scalable mode]({{< relref "../../../../get-started/deployment-modes#simple-scalable" >}}). Loki’s simple scalable deployment mode separates execution paths into read, write, and backend targets. @@ -109,7 +109,7 @@ The following steps show how to deploy the Loki Helm chart in simple scalable mo ## Object Storage Configuration -After testing Loki with MinIO, we recommend to configure Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: +After testing Loki with MinIO, we recommend configuring Loki with an object storage provider. The following examples shows how to configure Loki with different object storage providers: {{< admonition type="caution" >}} When deploying Loki using S3 Storage **DO NOT** use the default bucket names; `chunk`, `ruler` and `admin`. Choose a unique name for each bucket. For more information see the following [security update](https://grafana.com/blog/2024/06/27/grafana-security-update-grafana-loki-and-unintended-data-write-attempts-to-amazon-s3-buckets/). This caution does not apply when you are using MinIO. When using MinIO we recommend using the default bucket names. From 7bee5e35f2883912228a2c02e5b133017466d1d1 Mon Sep 17 00:00:00 2001 From: Jayclifford345 Date: Wed, 30 Oct 2024 09:53:18 +0000 Subject: [PATCH 51/51] added canary secret --- .../install/helm/deployment-guides/aws.md | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/sources/setup/install/helm/deployment-guides/aws.md b/docs/sources/setup/install/helm/deployment-guides/aws.md index e7e570a7c8067..bbe80da5e7761 100644 --- a/docs/sources/setup/install/helm/deployment-guides/aws.md +++ b/docs/sources/setup/install/helm/deployment-guides/aws.md @@ -310,6 +310,17 @@ Loki by default does not come with any authentication. Since we will be deployin ``` This will create a secret called `loki-basic-auth` in the `loki` namespace. We will reference this secret in the Loki Helm chart configuration. + +1. Create a `canary-basic-auth` secret for the canary: + + ```bash + kubectl create secret generic canary-basic-auth \ + --from-literal=username= \ + --from-literal=password= \ + -n loki + ``` + We create a literal secret with the username and password for Loki canary to authenticate with the Loki gateway. + **Make sure to replace the placeholders with your desired username and password.** @@ -349,8 +360,8 @@ loki: storage: type: s3 s3: - region: # for example, eu-west-2 - bucketnames: # Your actual S3 bucket name, for example, loki-aws-dev-ruler + region: # for example, eu-west-2 + bucketnames: # Your actual S3 bucket name, for example, loki-aws-dev-ruler s3forcepathstyle: false alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.) @@ -432,7 +443,22 @@ gateway: enabled: true existingSecret: loki-basic-auth - +# Since we are using basic auth, we need to pass the username and password to the canary +lokiCanary: + extraArgs: + - -pass=$(LOKI_PASS) + - -user=$(LOKI_USER) + extraEnv: + - name: LOKI_PASS + valueFrom: + secretKeyRef: + name: canary-basic-auth + key: password + - name: LOKI_USER + valueFrom: + secretKeyRef: + name: canary-basic-auth + key: username # Enable minio for storage minio: