From 03a05876975d5367f9b10cf10e9506b09f01e70b Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Mon, 9 Dec 2024 13:46:44 -0700 Subject: [PATCH] ci: fix nix build (#15308) --- .github/workflows/nix-ci.yaml | 3 + CODEOWNERS | 4 + flake.nix | 81 +++++++-------- nix/default.nix | 179 +++++++++++++++++----------------- nix/packages/faillint.nix | 16 --- nix/packages/loki.nix | 98 ++++++------------- pkg/loki/loki_test.go | 1 + pkg/loki/modules_test.go | 1 + 8 files changed, 162 insertions(+), 221 deletions(-) delete mode 100644 nix/packages/faillint.nix diff --git a/.github/workflows/nix-ci.yaml b/.github/workflows/nix-ci.yaml index ba08c719ee3dc..c76e45a199495 100644 --- a/.github/workflows/nix-ci.yaml +++ b/.github/workflows/nix-ci.yaml @@ -1,6 +1,9 @@ --- name: "Lint And Build Nix Flake" on: + push: + branches: + - main pull_request: paths: - "flake.nix" diff --git a/CODEOWNERS b/CODEOWNERS index a118c5a40c0b1..1d122e1f580c5 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -12,5 +12,9 @@ # The observability logs team is listed as co-codeowner for grammar file. This is to receive notifications about updates, so these can be implemented in https://github.com/grafana/lezer-logql /pkg/logql/syntax/expr.y @grafana/observability-logs @grafana/loki-team +# Nix +/nix/ @trevorwhitney +flake.nix @trevorwhitney + # No owners - allows sub-maintainers to merge changes. CHANGELOG.md diff --git a/flake.nix b/flake.nix index c9e52eb589af3..32642f11d110d 100644 --- a/flake.nix +++ b/flake.nix @@ -6,41 +6,21 @@ flake-utils.url = "github:numtide/flake-utils"; }; - # Nixpkgs / NixOS version to use. - outputs = { self, nixpkgs, flake-utils }: - let - nix = import ./nix { inherit self; }; - in - { - overlays = { - default = nix.overlay; - }; - } // flake-utils.lib.eachDefaultSystem (system: let - pkgs = import nixpkgs { - inherit system; - overlays = [ - nix.overlay - ]; - config = { allowUnfree = true; }; - }; + pkgs = import nixpkgs + { + inherit system; + config = { allowUnfree = true; }; + }; in - { - # The default package for 'nix build'. This makes sense if the - # flake provides only one package or there is a clear "main" - # package. + rec { defaultPackage = pkgs.loki; - packages = with pkgs; { - inherit - logcli - loki - loki-canary - loki-helm-test - loki-helm-test-docker - promtail; + packages = import ./nix { + inherit self pkgs; + inherit (pkgs) lib; }; apps = { @@ -56,18 +36,31 @@ test = { type = "app"; - program = with pkgs; "${ - (writeShellScriptBin "test.sh" '' - ${loki.overrideAttrs(old: { - buildInputs = - let - inherit (old) buildInputs; - in - if pkgs.stdenv.hostPlatform.isLinux then - buildInputs ++ (with pkgs; [ systemd ]) - else buildInputs; - doCheck = true; - })}/bin/loki --version + program = + let + loki = packages.loki.overrideAttrs (old: { + buildInputs = with pkgs; lib.optionals stdenv.hostPlatform.isLinux [ systemd.dev ]; + doCheck = true; + checkFlags = [ + "-covermode=atomic" + "-coverprofile=coverage.txt" + "-p=4" + ]; + subPackages = [ + "./..." # for tests + "cmd/loki" + "cmd/logcli" + "cmd/loki-canary" + "clients/cmd/promtail" + ]; + }); + in + "${ + (pkgs.writeShellScriptBin "test.sh" '' + ${loki}/bin/loki --version + ${loki}/bin/logcli --version + ${loki}/bin/loki-canary --version + ${loki}/bin/promtail --version '') }/bin/test.sh"; }; @@ -80,10 +73,6 @@ inherit (pkgs) buildGoModule fetchFromGitHub; }) - (pkgs.callPackage ./nix/packages/faillint.nix { - inherit (pkgs) lib buildGoModule fetchFromGitHub; - }) - chart-testing gcc go @@ -94,7 +83,7 @@ nixpkgs-fmt statix yamllint - ]; + ] // packages; }; }); } diff --git a/nix/default.nix b/nix/default.nix index d2c45ae2af383..1b00605962cb8 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,92 +1,91 @@ -{ self }: +{ self, pkgs, lib }: +let + # self.rev is only set on a clean git tree + gitRevision = if (self ? rev) then self.rev else "dirty"; + shortGitRevsion = with lib; + if (self ? rev) then + (strings.concatStrings + (lists.take 8 (strings.stringToCharacters gitRevision))) + else + "dirty"; + + # the image tag script is hard coded to take only 7 characters + imageTagVersion = with lib; + if (self ? rev) then + (strings.concatStrings + (lists.take 8 (strings.stringToCharacters gitRevision))) + else + "dirty"; + + imageTag = + if (self ? rev) then + "${imageTagVersion}" + else + "${imageTagVersion}-WIP"; + + meta = with lib; { + homepage = "https://grafana.com/oss/loki/"; + changelog = "https://github.com/grafana/loki/commit/${shortGitRevsion}"; + maintainers = with maintainers; [ trevorwhitney ]; + + }; + + loki-helm-test = pkgs.callPackage ../production/helm/loki/src/helm-test { + inherit pkgs; + inherit (pkgs) lib buildGoModule dockerTools; + rev = gitRevision; + }; +in { - overlay = final: prev: - let - # self.rev is only set on a clean git tree - gitRevision = if (self ? rev) then self.rev else "dirty"; - shortGitRevsion = with prev.lib; - if (self ? rev) then - (strings.concatStrings - (lists.take 8 (strings.stringToCharacters gitRevision))) - else - "dirty"; - - # the image tag script is hard coded to take only 7 characters - imageTagVersion = with prev.lib; - if (self ? rev) then - (strings.concatStrings - (lists.take 8 (strings.stringToCharacters gitRevision))) - else - "dirty"; - - imageTag = - if (self ? rev) then - "${imageTagVersion}" - else - "${imageTagVersion}-WIP"; - - loki-helm-test = prev.callPackage ../production/helm/loki/src/helm-test { - inherit (prev) pkgs lib buildGoModule dockerTools; - rev = gitRevision; - }; - in - { - inherit (loki-helm-test) loki-helm-test loki-helm-test-docker; - } // rec { - loki = prev.callPackage ./packages/loki.nix { - inherit imageTag; - version = shortGitRevsion; - pkgs = prev; - }; - - logcli = loki.overrideAttrs (oldAttrs: { - pname = "logcli"; - - buildPhase = '' - export GOCACHE=$TMPDIR/go-cache - make clean logcli - ''; - - installPhase = '' - mkdir -p $out/bin - install -m755 cmd/logcli/logcli $out/bin/logcli - ''; - }); - - loki-canary = loki.overrideAttrs (oldAttrs: { - pname = "loki-canary"; - - buildPhase = '' - export GOCACHE=$TMPDIR/go-cache - make clean loki-canary - ''; - - installPhase = '' - mkdir -p $out/bin - install -m755 cmd/loki-canary/loki-canary $out/bin/loki-canary - ''; - }); - - promtail = loki.overrideAttrs (oldAttrs: { - pname = "promtail"; - - buildInputs = - let - inherit (oldAttrs) buildInputs; - in - if prev.stdenv.hostPlatform.isLinux then - buildInputs ++ (with prev; [ systemd ]) - else buildInputs; - - buildPhase = '' - export GOCACHE=$TMPDIR/go-cache - make clean promtail - ''; - - installPhase = '' - mkdir -p $out/bin - install -m755 clients/cmd/promtail/promtail $out/bin/promtail - ''; - }); - }; + inherit (loki-helm-test) loki-helm-test loki-helm-test-docker; +} // rec { + loki = pkgs.callPackage ./packages/loki.nix { + inherit imageTag pkgs; + version = shortGitRevsion; + }; + + logcli = loki.overrideAttrs (oldAttrs: { + pname = "logcli"; + + subPackages = [ "cmd/logcli" ]; + + meta = with lib; { + description = "LogCLI is a command line tool for interacting with Loki."; + mainProgram = "logcli"; + license = with licenses; [ agpl3Only ]; + } // meta; + }); + + loki-canary = loki.overrideAttrs (oldAttrs: { + pname = "loki-canary"; + + subPackages = [ "cmd/loki-canary" ]; + + meta = with lib; { + description = "Loki Canary is a canary for the Loki project."; + mainProgram = "loki-canary"; + license = with licenses; [ agpl3Only ]; + } // meta; + }); + + promtail = loki.overrideAttrs (oldAttrs: { + pname = "promtail"; + + buildInputs = with pkgs; lib.optionals stdenv.hostPlatform.isLinux [ systemd.dev ]; + + tags = [ "promtail_journal_enabled" ]; + + subPackages = [ "clients/cmd/promtail" ]; + + preFixup = lib.optionalString pkgs.stdenv.hostPlatform.isLinux '' + wrapProgram $out/bin/promtail \ + --prefix LD_LIBRARY_PATH : "${lib.getLib pkgs.systemd}/lib" + ''; + + meta = with lib; { + description = "Client for sending logs to Loki"; + mainProgram = "promtail"; + license = with licenses; [ asl20 ]; + } // meta; + }); } diff --git a/nix/packages/faillint.nix b/nix/packages/faillint.nix deleted file mode 100644 index 0931fc08fccfa..0000000000000 --- a/nix/packages/faillint.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ lib, buildGoModule, fetchFromGitHub }: - -buildGoModule rec { - pname = "faillint"; - version = "v1.14.0"; - - src = fetchFromGitHub { - owner = "fatih"; - repo = "faillint"; - rev = "${version}"; - sha256 = "NV+wbu547mtTa6dTGv7poBwWXOmu5YjqbauzolCg5qs="; - }; - - vendorHash = "sha256-vWt4HneDA7YwXYnn8TbfWCKzSv7RcgXxn/HAh6a+htQ="; - doCheck = false; -} diff --git a/nix/packages/loki.nix b/nix/packages/loki.nix index 7da43f05d3ede..28f215155b36a 100644 --- a/nix/packages/loki.nix +++ b/nix/packages/loki.nix @@ -1,78 +1,38 @@ -{ pkgs, version, imageTag }: -let - lambda-promtail-gomod = pkgs.buildGoModule { - inherit version; - pname = "lambda-promtail"; - - src = ./../../tools/lambda-promtail; - vendorHash = "sha256-yQIRFUhod91HiPS5IKm7eNeIXJzBWVcvIXf9qMncTKw="; - - doCheck = false; - - installPhase = '' - runHook preInstall - cp -r --reflink=auto vendor $out - runHook postInstall - ''; - }; -in -pkgs.stdenv.mkDerivation { +{ pkgs, version, imageTag, lib }: +pkgs.buildGo123Module { inherit version; pname = "loki"; src = ./../..; - - buildInputs = with pkgs; [ - bash - gcc - git - go_1_23 - golangci-lint - gotools - nettools - yamllint - - (import ./faillint.nix { - inherit (pkgs) lib buildGoModule fetchFromGitHub; - }) - ]; - - configurePhase = with pkgs; '' - patchShebangs tools - - substituteInPlace Makefile \ - --replace "SHELL = /usr/bin/env bash -o pipefail" "SHELL = ${bash}/bin/bash -o pipefail" \ - --replace "IMAGE_TAG ?= \$(shell ./tools/image-tag)" "IMAGE_TAG ?= ${imageTag}" \ - --replace "GIT_REVISION := \$(shell git rev-parse --short HEAD)" "GIT_REVISION := ${version}" \ - --replace "GIT_BRANCH := \$(shell git rev-parse --abbrev-ref HEAD)" "GIT_BRANCH := nix" - - substituteInPlace clients/cmd/fluentd/Makefile \ - --replace "SHELL = /usr/bin/env bash -o pipefail" "SHELL = ${bash}/bin/bash -o pipefail" - ''; - - buildPhase = '' - export GOCACHE=$TMPDIR/go-cache - export GOMODCACHE=$TMPDIR/gomodcache - export GOPROXY=off - - cp -r ${lambda-promtail-gomod} tools/lambda-promtail/vendor - make clean loki - ''; + vendorHash = null; + + ldflags = + let + prefix = "github.com/grafana/loki/v3/pkg/util/build"; + in + [ + "-s" + "-w" + "-X ${prefix}.Branch=nix" + "-X ${prefix}.Version=${imageTag}" + "-X ${prefix}.Revision=${version}" + "-X ${prefix}.BuildUser=nix@nixpkgs" + "-X ${prefix}.BuildDate=unknown" + ]; + + subPackages = [ "cmd/loki" ]; + + nativeBuildInputs = with pkgs; [ makeWrapper ]; doCheck = false; - checkPhase = '' - export GOCACHE=$TMPDIR/go-cache - export GOMODCACHE=$TMPDIR/gomodcache - export GOLANGCI_LINT_CACHE=$TMPDIR/go-cache - export GOPROXY=off - export BUILD_IN_CONTAINER=false - - make lint test - ''; - installPhase = '' - mkdir -p $out/bin - install -m755 cmd/loki/loki $out/bin/loki - ''; + meta = with lib; { + description = "Like Prometheus, but for logs"; + mainProgram = "loki"; + license = with licenses; [ agpl3Only ]; + homepage = "https://grafana.com/oss/loki/"; + changelog = "https://github.com/grafana/loki/commit/${version}"; + maintainers = with maintainers; [ trevorwhitney ]; + }; } diff --git a/pkg/loki/loki_test.go b/pkg/loki/loki_test.go index b29d2aad22065..3d5f774247754 100644 --- a/pkg/loki/loki_test.go +++ b/pkg/loki/loki_test.go @@ -188,6 +188,7 @@ server: common: compactor_address: http://localhost:%d path_prefix: /tmp/loki + instance_addr: localhost ring: kvstore: store: inmemory diff --git a/pkg/loki/modules_test.go b/pkg/loki/modules_test.go index 64241443d3439..f3b9fa9a98d51 100644 --- a/pkg/loki/modules_test.go +++ b/pkg/loki/modules_test.go @@ -406,6 +406,7 @@ func minimalWorkingConfig(t *testing.T, dir, target string, cfgTransformers ...f } cfg.Common.InstanceAddr = localhost + cfg.MemberlistKV.AdvertiseAddr = localhost cfg.Ingester.LifecyclerConfig.Addr = localhost cfg.Distributor.DistributorRing.InstanceAddr = localhost cfg.IndexGateway.Mode = indexgateway.SimpleMode