diff --git a/lib/tests/release.nix b/lib/tests/release.nix index 843180490bb2b..96d34be8c2d35 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -67,5 +67,17 @@ let in pkgs.symlinkJoin { name = "nixpkgs-lib-tests"; - paths = map testWithNix nixVersions; + paths = map testWithNix nixVersions ++ + + # + # TEMPORARY MIGRATION MECHANISM + # + # This comment and the expression which follows it should be + # removed as part of resolving this issue: + # + # https://github.com/NixOS/nixpkgs/issues/272591 + # + [(import ../../pkgs/test/release {})] + ; + } diff --git a/lib/types.nix b/lib/types.nix index 51e58eaa8ab51..4378568c141f6 100644 --- a/lib/types.nix +++ b/lib/types.nix @@ -67,6 +67,7 @@ let ; outer_types = rec { + __attrsFailEvaluation = true; isType = type: x: (x._type or "") == type; setType = typeName: value: value // { diff --git a/pkgs/applications/editors/eclipse/default.nix b/pkgs/applications/editors/eclipse/default.nix index 72ecd8e094f4d..dd33558e06f65 100644 --- a/pkgs/applications/editors/eclipse/default.nix +++ b/pkgs/applications/editors/eclipse/default.nix @@ -226,6 +226,6 @@ in rec { ### Plugins - plugins = callPackage ./plugins.nix { }; + plugins = callPackage ./plugins.nix { } // { __attrsFailEvaluation = true; }; } diff --git a/pkgs/applications/editors/emacs/elisp-packages/elpa-devel-packages.nix b/pkgs/applications/editors/emacs/elisp-packages/elpa-devel-packages.nix index 60c257e365ced..3d0073bf81432 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/elpa-devel-packages.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/elpa-devel-packages.nix @@ -89,4 +89,4 @@ self: let in elpaDevelPackages // { inherit elpaBuild; }); -in generateElpa { } +in (generateElpa { }) // { __attrsFailEvaluation = true; } diff --git a/pkgs/applications/editors/emacs/elisp-packages/elpa-packages.nix b/pkgs/applications/editors/emacs/elisp-packages/elpa-packages.nix index 2a6cb016cdc8a..112445453abb4 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/elpa-packages.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/elpa-packages.nix @@ -181,4 +181,4 @@ self: let in elpaPackages // { inherit elpaBuild; }); -in generateElpa { } +in (generateElpa { }) // { __attrsFailEvaluation = true; } diff --git a/pkgs/applications/editors/emacs/elisp-packages/manual-packages.nix b/pkgs/applications/editors/emacs/elisp-packages/manual-packages.nix index d53ee7056a890..bd0f1114b1cd1 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/manual-packages.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/manual-packages.nix @@ -113,4 +113,6 @@ in emacsSessionManagement = self.session-management-for-emacs; rectMark = self.rect-mark; sunriseCommander = self.sunrise-commander; + + __attrsFailEvaluation = true; } diff --git a/pkgs/applications/editors/emacs/elisp-packages/melpa-packages.nix b/pkgs/applications/editors/emacs/elisp-packages/melpa-packages.nix index aa1fb2f8050bc..bb45c383487cc 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/melpa-packages.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/melpa-packages.nix @@ -735,4 +735,5 @@ let in lib.mapAttrs (n: v: if lib.hasAttr n overrides then overrides.${n} else v) super); in -generateMelpa { } +(generateMelpa { }) +// { __attrsFailEvaluation = true; } diff --git a/pkgs/applications/editors/emacs/elisp-packages/nongnu-packages.nix b/pkgs/applications/editors/emacs/elisp-packages/nongnu-packages.nix index cd32a8bd39754..beca93ea4c35f 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/nongnu-packages.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/nongnu-packages.nix @@ -20,12 +20,12 @@ self: let generated ? ./nongnu-generated.nix }: let - imported = import generated { + imported = (import generated { callPackage = pkgs: args: self.callPackage pkgs (args // { # Use custom elpa url fetcher with fallback/uncompress fetchurl = buildPackages.callPackage ./fetchelpa.nix { }; }); - }; + }) // { __attrsFailEvaluation = true; }; super = imported; diff --git a/pkgs/applications/editors/jetbrains/default.nix b/pkgs/applications/editors/jetbrains/default.nix index 7ce12807985b0..e40fa4c68d721 100644 --- a/pkgs/applications/editors/jetbrains/default.nix +++ b/pkgs/applications/editors/jetbrains/default.nix @@ -249,6 +249,6 @@ rec { webstorm = mkJetBrainsProduct { pname = "webstorm"; extraBuildInputs = [ stdenv.cc.cc musl ]; }; - plugins = callPackage ./plugins { }; + plugins = callPackage ./plugins { } // { __attrsFailEvaluation = true; }; } diff --git a/pkgs/desktops/gnome/extensions/default.nix b/pkgs/desktops/gnome/extensions/default.nix index 686f8031ec865..bfb5f4ff33e63 100644 --- a/pkgs/desktops/gnome/extensions/default.nix +++ b/pkgs/desktops/gnome/extensions/default.nix @@ -35,6 +35,7 @@ let lib.trivial.pipe extensions [ (map (extension: lib.nameValuePair extension.extensionUuid extension)) builtins.listToAttrs + (attrs: attrs // { __attrsFailEvaluation = true; }) ]; # Map the list of extensions to an attrset based on the pname as key, which is more human readable than the UUID @@ -66,6 +67,7 @@ in rec { # Keep the last three versions in here gnomeExtensions = lib.trivial.pipe (gnome43Extensions // gnome44Extensions // gnome45Extensions) [ + (v: builtins.removeAttrs v [ "__attrsFailEvaluation" ]) # Apply some custom patches for automatically packaged extensions (callPackage ./extensionOverrides.nix {}) # Add all manually packaged extensions @@ -88,4 +90,5 @@ in rec { # Make the set "public" lib.recurseIntoAttrs ]; + } diff --git a/pkgs/development/compilers/chicken/4/default.nix b/pkgs/development/compilers/chicken/4/default.nix index de64d20d3e059..7502a7a9f1e72 100644 --- a/pkgs/development/compilers/chicken/4/default.nix +++ b/pkgs/development/compilers/chicken/4/default.nix @@ -3,7 +3,7 @@ let callPackage = newScope self; self = { - pkgs = self; + pkgs = self // { recurseForDerivations = false; }; fetchegg = callPackage ./fetchegg { }; diff --git a/pkgs/development/compilers/rust/default.nix b/pkgs/development/compilers/rust/default.nix index d56b6fb9e08e6..c08ffa848ef5e 100644 --- a/pkgs/development/compilers/rust/default.nix +++ b/pkgs/development/compilers/rust/default.nix @@ -62,7 +62,7 @@ in bootRustPlatform = makeRustPlatform bootstrapRustPackages; in { # Packages suitable for build-time, e.g. `build.rs`-type stuff. - buildRustPackages = (selectRustPackage buildPackages).packages.stable; + buildRustPackages = (selectRustPackage buildPackages).packages.stable // { __attrsFailEvaluation = true; }; # Analogous to stdenv rustPlatform = makeRustPlatform self.buildRustPackages; rustc-unwrapped = self.callPackage ./rustc.nix ({ diff --git a/pkgs/development/compilers/swift/default.nix b/pkgs/development/compilers/swift/default.nix index 0da2510f09a00..afb8ce39dcced 100644 --- a/pkgs/development/compilers/swift/default.nix +++ b/pkgs/development/compilers/swift/default.nix @@ -55,7 +55,8 @@ let darwin = pkgs.darwin.overrideScope (_: prev: { inherit apple_sdk; inherit (apple_sdk) Libsystem LibsystemCross libcharset libunwind objc4 configd IOKit Security; - CF = apple_sdk.CoreFoundation; + CF = apple_sdk.CoreFoundation // { __attrsFailEvaluation = true; }; + __attrsFailEvaluation = true; }); xcodebuild = pkgs.xcbuild.override { inherit (apple_sdk.frameworks) CoreServices CoreGraphics ImageIO; diff --git a/pkgs/development/haskell-modules/default.nix b/pkgs/development/haskell-modules/default.nix index 7bc1fa0f7e832..e766203bd1f0a 100644 --- a/pkgs/development/haskell-modules/default.nix +++ b/pkgs/development/haskell-modules/default.nix @@ -18,7 +18,8 @@ let haskellPackages = pkgs.callPackage makePackageSet { package-set = initialPackages; - inherit stdenv haskellLib ghc buildHaskellPackages extensible-self all-cabal-hashes; + inherit stdenv haskellLib ghc extensible-self all-cabal-hashes; + buildHaskellPackages = buildHaskellPackages // { __attrsFailEvaluation = true; }; }; platformConfigurations = lib.optionals stdenv.hostPlatform.isAarch [ diff --git a/pkgs/development/haskell-modules/make-package-set.nix b/pkgs/development/haskell-modules/make-package-set.nix index c39c934bed643..294ca295f22bf 100644 --- a/pkgs/development/haskell-modules/make-package-set.nix +++ b/pkgs/development/haskell-modules/make-package-set.nix @@ -614,7 +614,7 @@ in package-set { inherit pkgs lib callPackage; } self // { Type: [str] -> drv -> drv */ generateOptparseApplicativeCompletions = - self.callPackage ( + (self.callPackage ( { stdenv }: commands: @@ -623,7 +623,7 @@ in package-set { inherit pkgs lib callPackage; } self // { if stdenv.buildPlatform.canExecute stdenv.hostPlatform then lib.foldr haskellLib.__generateOptparseApplicativeCompletion pkg commands else pkg - ) { }; + ) { }) // { __attrsFailEvaluation = true; }; /* Modify given Haskell package to force GHC to employ the LLVM diff --git a/pkgs/development/interpreters/lua-5/default.nix b/pkgs/development/interpreters/lua-5/default.nix index 2fda54bef530f..ce91b024ce841 100644 --- a/pkgs/development/interpreters/lua-5/default.nix +++ b/pkgs/development/interpreters/lua-5/default.nix @@ -65,7 +65,8 @@ let inherit (luaPackages) requiredLuaModules; }; withPackages = import ./with-packages.nix { inherit buildEnv luaPackages;}; - pkgs = luaPackages; + pkgs = let lp = luaPackages; + in lp // { luaPackages = lp.luaPackages // { __attrsFailEvaluation = true; }; }; interpreter = "${self}/bin/${executable}"; inherit executable luaversion; luaOnBuild = luaOnBuildForHost.override { inherit packageOverrides; self = luaOnBuild; }; diff --git a/pkgs/development/libraries/qt-5/5.15/default.nix b/pkgs/development/libraries/qt-5/5.15/default.nix index 68a3e3531e19a..7a5ff60c9acbb 100644 --- a/pkgs/development/libraries/qt-5/5.15/default.nix +++ b/pkgs/development/libraries/qt-5/5.15/default.nix @@ -23,7 +23,7 @@ Check for any minor version changes. let - srcs = import ./srcs.nix { inherit lib fetchgit fetchFromGitHub; }; + srcs = import ./srcs.nix { inherit lib fetchgit fetchFromGitHub; } // { __attrsFailEvaluation = true; }; qtCompatVersion = srcs.qtbase.version; diff --git a/pkgs/test/cuda/default.nix b/pkgs/test/cuda/default.nix index be88bd3820a90..8431e4b9207db 100644 --- a/pkgs/test/cuda/default.nix +++ b/pkgs/test/cuda/default.nix @@ -27,4 +27,6 @@ rec { cuda-library-samples_cudatoolkit_11_3 cuda-library-samples_cudatoolkit_11_4 ; + + __attrsFailEvaluation = true; } diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 29dc4b5192ec9..cfae7a1edd8ce 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -119,11 +119,11 @@ with pkgs; macOSSierraShared = callPackage ./macos-sierra-shared {}; - cross = callPackage ./cross {}; + cross = callPackage ./cross {} // { __attrsFailEvaluation = true; }; php = recurseIntoAttrs (callPackages ./php {}); - pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }); + pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }) // { __recurseIntoDerivationForReleaseJobs = true; }; buildRustCrate = callPackage ../build-support/rust/build-rust-crate/test { }; importCargoLock = callPackage ../build-support/rust/test/import-cargo-lock { }; diff --git a/pkgs/test/release/default.nix b/pkgs/test/release/default.nix new file mode 100644 index 0000000000000..f112ee6b92124 --- /dev/null +++ b/pkgs/test/release/default.nix @@ -0,0 +1,46 @@ +# Adapted from lib/tests/release.nix +{ pkgs-path ? ../../.. +, pkgs ? import pkgs-path {} +, lib ? pkgs.lib +, nix ? pkgs.nix +}: + +# +# This verifies that release-attrpaths-superset.nix does not encounter +# infinite recursion or non-tryEval-able failures. +# +pkgs.runCommand "all-attrs-eval-under-tryEval" { + nativeBuildInputs = [ + nix + pkgs.gitMinimal + ] ++ lib.optional pkgs.stdenv.isLinux pkgs.inotify-tools; + strictDeps = true; +} +'' + datadir="${nix}/share" + export TEST_ROOT=$(pwd)/test-tmp + export HOME=$(mktemp -d) + export NIX_BUILD_HOOK= + export NIX_CONF_DIR=$TEST_ROOT/etc + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_STORE_DIR=$TEST_ROOT/store + export PAGER=cat + cacheDir=$TEST_ROOT/binary-cache + + nix-store --init + + cp -r ${pkgs-path + "/lib"} lib + cp -r ${pkgs-path + "/pkgs"} pkgs + cp -r ${pkgs-path + "/default.nix"} default.nix + cp -r ${pkgs-path + "/nixos"} nixos + cp -r ${pkgs-path + "/maintainers"} maintainers + cp -r ${pkgs-path + "/.version"} .version + cp -r ${pkgs-path + "/doc"} doc + echo "Running pkgs/top-level/release-attrpaths-superset.nix" + nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names > /dev/null + + mkdir $out + echo success > $out/${nix.version} +'' diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c0a22df227409..dfa420b45ad81 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -104,10 +104,10 @@ with pkgs; ### Evaluating the entire Nixpkgs naively will fail, make failure fast AAAAAASomeThingsFailToEvaluate = throw '' - Please be informed that this pseudo-package is not the only part of - Nixpkgs that fails to evaluate. You should not evaluate entire Nixpkgs - without some special measures to handle failing packages, like those taken - by Hydra. + Please be informed that this pseudo-package is not the only part + of Nixpkgs that fails to evaluate. You should not evaluate + entire Nixpkgs without some special measures to handle failing + packages, like using pkgs/top-level/release-attrpaths.nix. ''; tests = callPackages ../test { }; @@ -116,7 +116,7 @@ with pkgs; # We don't want nix-env -q to enter this, because all of these are aliases. dontRecurseIntoAttrs ( import ./pkg-config/defaultPkgConfigPackages.nix pkgs - ); + ) // { __attrsFailEvaluation = true; }; ### Nixpkgs maintainer tools @@ -159,6 +159,7 @@ with pkgs; system = stdenv.hostPlatform.system; callTest = config: config.test.driver; }; + __attrsFailEvaluation = true; }; ### BUILD SUPPORT @@ -902,7 +903,7 @@ with pkgs; dockerTools = callPackage ../build-support/docker { writePython3 = buildPackages.writers.writePython3; - }; + } // { __attrsFailEvaluation = true; }; fakeNss = callPackage ../build-support/fake-nss { }; @@ -10346,9 +10347,9 @@ with pkgs; inherit (callPackages ../build-support/node/fetch-npm-deps { }) fetchNpmDeps prefetch-npm-deps; - nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs; + nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs // { __attrsFailEvaluation = true; }; - nodePackages = dontRecurseIntoAttrs nodejs.pkgs; + nodePackages = dontRecurseIntoAttrs nodejs.pkgs // { __attrsFailEvaluation = true; }; node2nix = nodePackages.node2nix; @@ -16251,7 +16252,8 @@ with pkgs; # Prefer native-bignum to avoid linking issues with gmp else if stdenv.hostPlatform.isStatic then haskell.packages.native-bignum.ghc94 - else haskell.packages.ghc94); + else haskell.packages.ghc94) + // { __recurseIntoDerivationForReleaseJobs = true; }; # haskellPackages.ghc is build->host (it exposes the compiler used to build the # set, similarly to stdenv.cc), but pkgs.ghc should be host->target to be more @@ -16832,7 +16834,7 @@ with pkgs; ocamlPackages = ocaml-ng.ocamlPackages_4_14; }; - ocaml-ng = callPackage ./ocaml-packages.nix { }; + ocaml-ng = callPackage ./ocaml-packages.nix { } // { __attrsFailEvaluation = true; }; ocaml = ocamlPackages.ocaml; ocamlPackages = recurseIntoAttrs ocaml-ng.ocamlPackages; @@ -17616,8 +17618,9 @@ with pkgs; inherit (beam.packages.erlang) erlang-ls erlfmt elvis-erlang rebar rebar3 rebar3WithPlugins - fetchHex beamPackages + fetchHex lfe lfe_2_1; + beamPackages = beam.packages.erlang // { __attrsFailEvaluation = true; }; expr = callPackage ../development/interpreters/expr { }; @@ -17899,19 +17902,19 @@ with pkgs; # List of extensions with overrides to apply to all Python package sets. pythonPackagesExtensions = [ ]; # Python package sets. - python27Packages = python27.pkgs; - python38Packages = python38.pkgs; - python39Packages = python39.pkgs; - python310Packages = recurseIntoAttrs python310.pkgs; - python311Packages = recurseIntoAttrs python311.pkgs; - python312Packages = python312.pkgs; - python313Packages = python313.pkgs; - pypyPackages = pypy.pkgs; - pypy2Packages = pypy2.pkgs; - pypy27Packages = pypy27.pkgs; - pypy3Packages = pypy3.pkgs; - pypy39Packages = pypy39.pkgs; - pypy310Packages = pypy310.pkgs; + python27Packages = python27.pkgs // { __attrsFailEvaluation = true; }; + python38Packages = python38.pkgs // { __attrsFailEvaluation = true; }; + python39Packages = python39.pkgs // { __attrsFailEvaluation = true; }; + python310Packages = recurseIntoAttrs python310.pkgs // { pythonPackages = python310.pkgs // { __attrsFailEvaluation = true; }; }; + python311Packages = recurseIntoAttrs python311.pkgs // { pythonPackages = python311.pkgs // { __attrsFailEvaluation = true; }; }; + python312Packages = python312.pkgs // { __attrsFailEvaluation = true; }; + python313Packages = python313.pkgs // { __attrsFailEvaluation = true; }; + pypyPackages = pypy.pkgs // { __attrsFailEvaluation = true; }; + pypy2Packages = pypy2.pkgs // { __attrsFailEvaluation = true; }; + pypy27Packages = pypy27.pkgs // { __attrsFailEvaluation = true; }; + pypy3Packages = pypy3.pkgs // { __attrsFailEvaluation = true; }; + pypy39Packages = pypy39.pkgs // { __attrsFailEvaluation = true; }; + pypy310Packages = pypy310.pkgs // { __attrsFailEvaluation = true; }; py3c = callPackage ../development/libraries/py3c { }; @@ -24663,9 +24666,9 @@ with pkgs; stdenv = if stdenv.isDarwin then overrideSDK stdenv "11.0" else stdenv; }); - libsForQt5 = recurseIntoAttrs (import ./qt5-packages.nix { + libsForQt5 = (recurseIntoAttrs (import ./qt5-packages.nix { inherit lib __splicedPackages makeScopeWithSplicing' generateSplicesForMkScope pkgsHostTarget; - }); + })) // { __recurseIntoDerivationForReleaseJobs = true; }; # plasma5Packages maps to the Qt5 packages set that is used to build the plasma5 desktop plasma5Packages = libsForQt5; @@ -25980,11 +25983,11 @@ with pkgs; }; lispPackages = quicklispPackages // - (lispPackagesFor (wrapLisp_old sbcl)); + (lispPackagesFor (wrapLisp_old sbcl)) // { __attrsFailEvaluation = true; }; quicklispPackagesFor = clwrapper: callPackage ../development/lisp-modules-obsolete/quicklisp-to-nix.nix { inherit clwrapper; - }; + } // { __attrsFailEvaluation = true; }; quicklispPackagesClisp = dontRecurseIntoAttrs (quicklispPackagesFor (wrapLisp_old clisp)); quicklispPackagesSBCL = dontRecurseIntoAttrs (quicklispPackagesFor (wrapLisp_old sbcl)); quicklispPackagesECL = dontRecurseIntoAttrs (quicklispPackagesFor (wrapLisp_old ecl)); @@ -25994,7 +25997,8 @@ with pkgs; quicklispPackages = quicklispPackagesSBCL; # Alternative lisp-modules implementation - lispPackages_new = callPackage ../development/lisp-modules-new-obsolete/lisp-packages.nix {}; + lispPackages_new = callPackage ../development/lisp-modules-new-obsolete/lisp-packages.nix {} + // { __attrsFailEvaluation = true; }; ## End of DEPRECATED @@ -26140,9 +26144,9 @@ with pkgs; rstudioServerWrapper = rstudioWrapper.override { rstudio = rstudio-server; }; - rPackages = dontRecurseIntoAttrs (callPackage ../development/r-modules { + rPackages = (dontRecurseIntoAttrs (callPackage ../development/r-modules { overrides = (config.rPackageOverrides or (_: {})) pkgs; - }); + })) // { __attrsFailEvaluation = true; }; ### SERVERS @@ -31383,7 +31387,7 @@ with pkgs; # This alias should live in aliases.nix but that would cause Hydra not to evaluate/build the packages. # If you turn this into "real" alias again, please add it to pkgs/top-level/packages-config.nix again too - emacsPackages = emacs.pkgs; + emacsPackages = emacs.pkgs // { __recurseIntoDerivationForReleaseJobs = true; }; emptty = callPackage ../applications/display-managers/emptty { }; @@ -33816,9 +33820,9 @@ with pkgs; mop = callPackage ../applications/misc/mop { }; - mopidyPackages = callPackages ../applications/audio/mopidy { + mopidyPackages = (callPackages ../applications/audio/mopidy { python = python3; - }; + }) // { __attrsFailEvaluation = true; }; inherit (mopidyPackages) mopidy @@ -34725,7 +34729,10 @@ with pkgs; picosnitch = callPackage ../tools/networking/picosnitch { }; - pidginPackages = recurseIntoAttrs (callPackage ../applications/networking/instant-messengers/pidgin/pidgin-plugins { }); + pidginPackages = + let pidgin-plugins = + recurseIntoAttrs (callPackage ../applications/networking/instant-messengers/pidgin/pidgin-plugins { }); + in pidgin-plugins // { pidginPackages = pidgin-plugins.pidginPackages // { __attrsFailEvaluation = true; }; }; inherit (pidginPackages) pidgin; diff --git a/pkgs/top-level/beam-packages.nix b/pkgs/top-level/beam-packages.nix index ff9d1f62ab1f3..62ab2a63ae746 100644 --- a/pkgs/top-level/beam-packages.nix +++ b/pkgs/top-level/beam-packages.nix @@ -101,4 +101,6 @@ in erlang_25 = self.packagesWith self.interpreters.erlang_25; erlang_24 = self.packagesWith self.interpreters.erlang_24; } // packagesAliases; + + __attrsFailEvaluation = true; } diff --git a/pkgs/top-level/coq-packages.nix b/pkgs/top-level/coq-packages.nix index cf3817a3d7e46..64cbe925b5189 100644 --- a/pkgs/top-level/coq-packages.nix +++ b/pkgs/top-level/coq-packages.nix @@ -8,7 +8,7 @@ let mkCoqPackages' = self: coq: let callPackage = self.callPackage; in { inherit coq lib; - coqPackages = self; + coqPackages = self // { coqPackages = self.coqPackages // { recurseForDerivations = false; }; }; metaFetch = import ../build-support/coq/meta-fetch/default.nix {inherit lib stdenv fetchzip; }; @@ -190,21 +190,23 @@ in rec { coq_8_17 = mkCoq "8.17"; coq_8_18 = mkCoq "8.18"; - coqPackages_8_5 = mkCoqPackages coq_8_5; - coqPackages_8_6 = mkCoqPackages coq_8_6; - coqPackages_8_7 = mkCoqPackages coq_8_7; - coqPackages_8_8 = mkCoqPackages coq_8_8; - coqPackages_8_9 = mkCoqPackages coq_8_9; - coqPackages_8_10 = mkCoqPackages coq_8_10; - coqPackages_8_11 = mkCoqPackages coq_8_11; - coqPackages_8_12 = mkCoqPackages coq_8_12; - coqPackages_8_13 = mkCoqPackages coq_8_13; - coqPackages_8_14 = mkCoqPackages coq_8_14; - coqPackages_8_15 = mkCoqPackages coq_8_15; - coqPackages_8_16 = mkCoqPackages coq_8_16; - coqPackages_8_17 = mkCoqPackages coq_8_17; - coqPackages_8_18 = mkCoqPackages coq_8_18; - coqPackages = recurseIntoAttrs coqPackages_8_18; + coqPackages_8_5 = mkCoqPackages coq_8_5 // { __attrsFailEvaluation = true; }; + coqPackages_8_6 = mkCoqPackages coq_8_6 // { __attrsFailEvaluation = true; }; + coqPackages_8_7 = mkCoqPackages coq_8_7 // { __attrsFailEvaluation = true; }; + coqPackages_8_8 = mkCoqPackages coq_8_8 // { __attrsFailEvaluation = true; }; + coqPackages_8_9 = mkCoqPackages coq_8_9 // { __attrsFailEvaluation = true; }; + coqPackages_8_10 = mkCoqPackages coq_8_10 // { __attrsFailEvaluation = true; }; + coqPackages_8_11 = mkCoqPackages coq_8_11 // { __attrsFailEvaluation = true; }; + coqPackages_8_12 = mkCoqPackages coq_8_12 // { __attrsFailEvaluation = true; }; + coqPackages_8_13 = mkCoqPackages coq_8_13 // { __attrsFailEvaluation = true; }; + coqPackages_8_14 = mkCoqPackages coq_8_14 // { __attrsFailEvaluation = true; }; + coqPackages_8_15 = mkCoqPackages coq_8_15 // { __attrsFailEvaluation = true; }; + coqPackages_8_16 = mkCoqPackages coq_8_16 // { __attrsFailEvaluation = true; }; + coqPackages_8_17 = mkCoqPackages coq_8_17 // { __attrsFailEvaluation = true; }; + coqPackages_8_18 = mkCoqPackages coq_8_18 // { __attrsFailEvaluation = true; }; + coqPackages = + let cp = recurseIntoAttrs coqPackages_8_18; + in cp // { coqPackages = cp.coqPackages // { __attrsFailEvaluation = true; }; } // { __recurseIntoDerivationForReleaseJobs = true; }; coq = coqPackages.coq; } diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index 19ff558afbc7d..d474cf852e55f 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -26,6 +26,7 @@ lib, newScope, pkgs, + __attrsFailEvaluation ? true, }: let inherit (lib) @@ -118,4 +119,4 @@ let fixedPoints.extends composedExtension passthruFunction ); in -cudaPackages +cudaPackages // { inherit __attrsFailEvaluation; } diff --git a/pkgs/top-level/java-packages.nix b/pkgs/top-level/java-packages.nix index 88c95457afee6..f2c725bfb4c2f 100644 --- a/pkgs/top-level/java-packages.nix +++ b/pkgs/top-level/java-packages.nix @@ -35,6 +35,7 @@ in { else package-darwin; in { inherit package-linux package-darwin; + __attrsFailEvaluation = true; jdk-hotspot = callPackage package.jdk-hotspot {}; jre-hotspot = callPackage package.jre-hotspot {}; diff --git a/pkgs/top-level/linux-kernels.nix b/pkgs/top-level/linux-kernels.nix index 27496b9104440..64e0dee475147 100644 --- a/pkgs/top-level/linux-kernels.nix +++ b/pkgs/top-level/linux-kernels.nix @@ -398,7 +398,8 @@ in { nvidiabl = callPackage ../os-specific/linux/nvidiabl { }; - nvidiaPackages = dontRecurseIntoAttrs (lib.makeExtensible (_: callPackage ../os-specific/linux/nvidia-x11 { })); + nvidiaPackages = dontRecurseIntoAttrs (lib.makeExtensible (_: callPackage ../os-specific/linux/nvidia-x11 { })) + // { __attrsFailEvaluation = true; }; nvidia_x11 = nvidiaPackages.stable; nvidia_x11_beta = nvidiaPackages.beta; @@ -589,6 +590,7 @@ in { linux_6_1 = recurseIntoAttrs (packagesFor kernels.linux_6_1); linux_6_5 = recurseIntoAttrs (packagesFor kernels.linux_6_5); linux_6_6 = recurseIntoAttrs (packagesFor kernels.linux_6_6); + __attrsFailEvaluation = true; } // lib.optionalAttrs config.allowAliases { linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11"; # Added 2022-11-08 linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11"; # Added 2023-10-11 @@ -606,6 +608,7 @@ in { linux_rt_5_10 = packagesFor kernels.linux_rt_5_10; linux_rt_5_15 = packagesFor kernels.linux_rt_5_15; linux_rt_6_1 = packagesFor kernels.linux_rt_6_1; + __attrsFailEvaluation = true; }; rpiPackages = { @@ -613,6 +616,7 @@ in { linux_rpi2 = packagesFor kernels.linux_rpi2; linux_rpi3 = packagesFor kernels.linux_rpi3; linux_rpi4 = packagesFor kernels.linux_rpi4; + __attrsFailEvaluation = true; }; packages = recurseIntoAttrs (vanillaPackages // rtPackages // rpiPackages // { @@ -641,6 +645,7 @@ in { linux_libre = recurseIntoAttrs (packagesFor kernels.linux_libre); linux_latest_libre = recurseIntoAttrs (packagesFor kernels.linux_latest_libre); + __recurseIntoDerivationForReleaseJobs = true; } // lib.optionalAttrs config.allowAliases { linux_5_18_hardened = throw "linux 5.18 was removed because it has reached its end of life upstream"; linux_5_19_hardened = throw "linux 5.19 was removed because it has reached its end of life upstream"; @@ -655,7 +660,7 @@ in { linux_mptcp = throw "'linux_mptcp' has been moved to https://github.com/teto/mptcp-flake"; linux_rt_default = packages.linux_rt_5_4; linux_rt_latest = packages.linux_rt_6_1; - }; + } // { __attrsFailEvaluation = true; }; manualConfig = callPackage ../os-specific/linux/kernel/manual-config.nix {}; diff --git a/pkgs/top-level/perl-packages.nix b/pkgs/top-level/perl-packages.nix index 558eedeff4073..2cb7718f892c3 100644 --- a/pkgs/top-level/perl-packages.nix +++ b/pkgs/top-level/perl-packages.nix @@ -23,7 +23,7 @@ in with self; { inherit perl; - perlPackages = self; + perlPackages = self // { perlPackages = self.perlPackages // { __attrsFailEvaluation = true; }; }; # Check whether a derivation provides a perl module. hasPerlModule = drv: drv ? perlModule ; diff --git a/pkgs/top-level/pkg-config/tests.nix b/pkgs/top-level/pkg-config/tests.nix index ec1d445c5903b..786e2ecc534b8 100644 --- a/pkgs/top-level/pkg-config/tests.nix +++ b/pkgs/top-level/pkg-config/tests.nix @@ -17,5 +17,5 @@ let }; in lib.recurseIntoAttrs { - defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { }; + defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { } // { __recurseIntoDerivationForReleaseJobs = true; }; } diff --git a/pkgs/top-level/qt5-packages.nix b/pkgs/top-level/qt5-packages.nix index 9acaacea24a4d..f38a4c1c49078 100644 --- a/pkgs/top-level/qt5-packages.nix +++ b/pkgs/top-level/qt5-packages.nix @@ -69,7 +69,9 @@ makeScopeWithSplicing' { }; in (lib.makeOverridable mkMaui attrs); - noExtraAttrs = set: lib.attrsets.removeAttrs set [ "extend" "override" "overrideScope" "overrideScope'" "overrideDerivation" ]; + noExtraAttrs = set: + lib.attrsets.removeAttrs set [ "extend" "override" "overrideScope" "overrideScope'" "overrideDerivation" ] + // { __attrsFailEvaluation = true; }; in (noExtraAttrs (kdeFrameworks // plasmaMobileGear // plasma5 // plasma5.thirdParty // kdeGear // mauiPackages // qt5 // { diff --git a/pkgs/top-level/release-attrpaths-superset.nix b/pkgs/top-level/release-attrpaths-superset.nix new file mode 100644 index 0000000000000..673b63a5ac34f --- /dev/null +++ b/pkgs/top-level/release-attrpaths-superset.nix @@ -0,0 +1,192 @@ +# This expression will, as efficiently as possible, dump a +# *superset* of all attrpaths of derivations which might be +# part of a release on *any* platform. +# +# Both this expression and what ofborg uses (release-outpaths.nix) +# are essentially single-threaded (under the current cppnix +# implementation). +# +# This expression runs much, much, much faster and uses much, much +# less memory than the ofborg script by skipping the +# platform-relevance checks. The ofborg outpaths.nix script takes +# half an hour on a 3ghz core and peaks at 60gbytes of memory; this +# expression runs on the same machine in 44 seconds with peak memory +# usage of 5gbytes. +# +# Once you have the list of attrnames you can split it up into +# $NUM_CORES batches and run the platform checks separately for each +# batch, in parallel. +# +# To dump the attrnames: +# +# nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names +# +{ lib ? import (path + "/lib") +, trace ? false +, enableWarnings ? true +, checkMeta ? true +, path ? ./../.. +}: +let + + # No release package attrpath may have any of these attrnames as + # its initial component. + # + # If you can find a way to remove any of these entries without + # causing CI to fail, please do so. + # + excluded-toplevel-attrs = { + # spliced packagesets + __splicedPackages = true; + pkgsBuildBuild = true; + pkgsBuildHost = true; + pkgsBuildTarget = true; + pkgsHostHost = true; + pkgsHostTarget = true; + pkgsTargetTarget = true; + buildPackages = true; + targetPackages = true; + + # cross packagesets + pkgsLLVM = true; + pkgsMusl = true; + pkgsStatic = true; + pkgsCross = true; + pkgsi686Linux = true; + }; + + # No release package attrname may have any of these at a component + # anywhere in its attrpath. These are the names of gigantic + # top-level attrsets that have leaked into so many sub-packagesets + # that it's easier to simply exclude them entirely. + # + # If you can find a way to remove any of these entries without + # causing CI to fail, please do so. + # + excluded-attrnames-at-any-depth = { + lib = true; + override = true; + __functor = true; + __functionArgs = true; + newScope = true; + scope = true; + pkgs = true; + + buildHaskellPackages = true; + buildPackages = true; + generateOptparseApplicativeCompletions = true; + + callPackage = true; + mkDerivation = true; + overrideDerivation = true; + overrideScope = true; + overrideScope' = true; + + # Special case: lib/types.nix leaks into a lot of nixos-related + # derivations, and does not eval deeply. + type = true; + }; + + # __attrsFailEvaluation is a temporary workaround to get top-level + # eval to succeed (under builtins.tryEval) for the entire + # packageset, without deep invasve changes into individual + # packages. + # + # Now that CI has been added, ensuring that top-level eval will + # not be broken by any new commits, you should not add any new + # occurrences of __attrsFailEvaluation, and should remove them + # wherever you are able to (doing so will likely require deep + # adjustments within packages). Once all of the uses of + # __attrsFailEvaluation are removed, it will be deleted from the + # routine below. In the meantime, + # + # The intended semantics are that an attrpath rooted at pkgs is + # part of the (unfiltered) release jobset iff all of the following + # are true: + # + # 1. The first component of the attrpath is not in + # `excluded-toplevel-attrs` + # + # 2. No attrname in the attrpath belongs to the list of forbidden + # attrnames `excluded-attrnames-at-any-depth` + # + # 3. The attrpath leads to a value for which lib.isDerivation is true + # + # 4. No proper prefix of the attrpath has __attrsFailEvaluation=true + # + # 5. Any proper prefix of the attrpath at which lib.isDerivation + # is true also has __recurseIntoDerivationForReleaseJobs=true. + # + # The last condition is unfortunately necessary because there are + # Hydra release jobnames which have proper prefixes which are + # attrnames of derivations (!). We should probably restructure + # the job tree so that this is not the case. + # + justAttrNames = path: value: + let + attempt = + if lib.isDerivation value && + # in some places we have *derivations* with jobsets as subattributes, ugh + !(value.__recurseIntoDerivationForReleaseJobs or false) then + [ path ] + + # Even wackier case: we have meta.broken==true jobs with + # !meta.broken jobs as subattributes with license=unfree, and + # check-meta.nix won't throw an "unfree" failure because the + # enclosing derivation is marked broken. Yeah. Bonkers. + # We should just forbid jobsets enclosed by derivations. + else if lib.isDerivation value && + !value.meta.available then [] + + else if !(lib.isAttrs value) then [] + else if (value.__attrsFailEvaluation or false) then [] + else lib.pipe value [ + (builtins.mapAttrs + (name: value: + if excluded-attrnames-at-any-depth.${name} or false then [] else + (justAttrNames (path ++ [name]) value))) + builtins.attrValues + builtins.concatLists + ]; + + seq = builtins.deepSeq attempt attempt; + tried = builtins.tryEval seq; + + result = + if tried.success + then tried.value + else if enableWarnings && path != [ "AAAAAASomeThingsFailToEvaluate" ] + then lib.warn "tryEval failed at: ${lib.concatStringsSep "." path}" [] + else []; + in + if !trace + then result + else lib.trace "** ${lib.concatStringsSep "." path}" result; + + unfiltered = import ./release-outpaths.nix { + inherit checkMeta; + attrNamesOnly = true; + inherit path; + }; + + filtered = lib.pipe unfiltered [ + (pkgs: builtins.removeAttrs pkgs (builtins.attrNames excluded-toplevel-attrs)) + ]; + + paths = + [ + # I am not entirely sure why these three packages end up in + # the Hydra jobset. But they do, and they don't meet the + # criteria above, so at the moment they are special-cased. + [ "pkgsLLVM" "stdenv" ] + [ "pkgsStatic" "stdenv" ] + [ "pkgsMusl" "stdenv" ] + ] ++ justAttrNames [] filtered; + + names = + map (path: (lib.concatStringsSep "." path)) paths; + +in +{ + inherit paths names; +} diff --git a/pkgs/top-level/release-outpaths.nix b/pkgs/top-level/release-outpaths.nix new file mode 100644 index 0000000000000..5c433fa542e0c --- /dev/null +++ b/pkgs/top-level/release-outpaths.nix @@ -0,0 +1,87 @@ +#!/usr/bin/env nix-shell +# When using as a callable script, passing `--argstr path some/path` overrides $PWD. +#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path --arg checkMeta true -f pkgs/top-level/release-outpaths.nix" + +# Vendored from: +# https://raw.githubusercontent.com/NixOS/ofborg/74f38efa7ef6f0e8e71ec3bfc675ae4fb57d7491/ofborg/src/outpaths.nix +{ checkMeta +, includeBroken ? true # set this to false to exclude meta.broken packages from the output +, path ? ./../.. + +# used by pkgs/top-level/release-attrnames-superset.nix +, attrNamesOnly ? false + +# Set this to `null` to build for builtins.currentSystem only +, systems ? [ + "aarch64-linux" + "aarch64-darwin" + #"i686-linux" # !!! + "x86_64-linux" + "x86_64-darwin" + ] +}: +let + lib = import (path + "/lib"); + hydraJobs = import (path + "/pkgs/top-level/release.nix") + # Compromise: accuracy vs. resources needed for evaluation. + { + inherit attrNamesOnly; + supportedSystems = + if systems == null + then [ builtins.currentSystem ] + else systems; + nixpkgsArgs = { + config = { + allowAliases = false; + allowBroken = includeBroken; + allowUnfree = false; + allowInsecurePredicate = x: true; + checkMeta = checkMeta; + + handleEvalIssue = reason: errormsg: + let + fatalErrors = [ + "unknown-meta" + "broken-outputs" + ]; + in + if builtins.elem reason fatalErrors + then abort errormsg + # hydra does not build unfree packages, so tons of them are broken yet not marked meta.broken. + else if !includeBroken && builtins.elem reason [ "broken" "unfree" ] + then throw "broken" + else if builtins.elem reason [ "unsupported" ] + then throw "unsupported" + else true; + + inHydra = true; + }; + }; + }; + recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; + + # hydraJobs leaves recurseForDerivations as empty attrmaps; + # that would break nix-env and we also need to recurse everywhere. + tweak = lib.mapAttrs + (name: val: + if name == "recurseForDerivations" then true + else if lib.isAttrs val && val.type or null != "derivation" + then recurseIntoAttrs (tweak val) + else val + ); + + # Some of these contain explicit references to platform(s) we want to avoid; + # some even (transitively) depend on ~/.nixpkgs/config.nix (!) + blacklist = [ + "tarball" + "metrics" + "manual" + "darwin-tested" + "unstable" + "stdenvBootstrapTools" + "moduleSystem" + "lib-tests" # these just confuse the output + ]; + +in +tweak (builtins.removeAttrs hydraJobs blacklist) diff --git a/pkgs/top-level/release-python.nix b/pkgs/top-level/release-python.nix index ec5255362bccb..bc54860d45f6e 100644 --- a/pkgs/top-level/release-python.nix +++ b/pkgs/top-level/release-python.nix @@ -29,7 +29,7 @@ let let res = builtins.tryEval ( if isDerivation value then value.meta.isBuildPythonPackage or [] - else if value.recurseForDerivations or false || value.recurseForRelease or false then + else if value.recurseForDerivations or false || value.recurseForRelease or false || value.__recurseIntoDerivationForReleaseJobs or false then packagePython value else []); diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index 9b5c601415202..f2b38b4a89645 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -37,9 +37,23 @@ "openssl-1.1.1w" ]; }; } + + # This flag, if set to true, will inhibit the use of `mapTestOn` + # and `release-lib.packagePlatforms`. Generally, it causes the + # resulting tree of attributes to *not* have a ".${system}" + # suffixed upon every job name like Hydra expects. + # + # This flag exists mainly for use by + # pkgs/top-level/release-attrnames-superset.nix; see that file for + # full details. The exact behavior of this flag may change; it + # should be considered an internal implementation detail of + # pkgs/top-level/. + # +, attrNamesOnly ? false }: -with import ./release-lib.nix { inherit supportedSystems scrubJobs nixpkgsArgs; }; +let release-lib = import ./release-lib.nix { inherit supportedSystems scrubJobs nixpkgsArgs; }; in +with release-lib; let @@ -239,9 +253,9 @@ let # 'nonPackageAttrs' and jobs pulled in from 'pkgs'. # Conflicts usually cause silent job drops like in # https://github.com/NixOS/nixpkgs/pull/182058 - jobs = lib.attrsets.unionOfDisjoint - nonPackageJobs - (mapTestOn ((packagePlatforms pkgs) // { + jobs = let + packagePlatforms = if attrNamesOnly then lib.id else release-lib.packagePlatforms; + packageJobs = { haskell.compiler = packagePlatforms pkgs.haskell.compiler; haskellPackages = packagePlatforms pkgs.haskellPackages; # Build selected packages (HLS) for multiple Haskell compilers to rebuild @@ -275,6 +289,14 @@ let darwin = packagePlatforms pkgs.darwin // { xcode = {}; }; - } )); + }; + mapTestOn-packages = + if attrNamesOnly + then pkgs // packageJobs + else mapTestOn ((packagePlatforms pkgs) // packageJobs); + in + lib.attrsets.unionOfDisjoint + nonPackageJobs + mapTestOn-packages; in jobs