Move Nix build to be based on cabal (IntersectMBO#2659)

This moves to basing the haskell.nix build on cabal. This meant I had to fix a bunch of solver issues and then compilation issues due to missing bounds etc, which is fine; but it also means that the stack build will tend to go out of date. (There are a bunch of `allow-newer`s etc. in our `cabal.project`. It would be good not to need those, but that's future work.) Also, irritatingly the packge sets generated by haskell.nix from cabal are *not* platform-independent. E.g. the musl one doesn't have terminfo. So to keep using materialization we may need to have multiple sets of files for different systems. Fortunately: 1. Hamish showed me how to persuade Nix to generate the materialized files as if you were on a different system: this allows us to write an `updateMaterialized` that updates all platforms. 2. Linux/Darwin are actually the same, only musl differs slightly. If we could stop disabling `terminfo` on musl, even that would disappear. So it's not *too* bad.
pb99u069 · Jan 28, 2021 · baaa66f · baaa66f
1 parent a8a98c4
commit baaa66f
Show file tree

Hide file tree

Showing 168 changed files with 10,159 additions and 854 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -13,12 +13,12 @@ Pre-submit checklist:
     - [ ] Useful pull request description
     - [ ] Reviewer requested
 - If you updated any cabal files or added Haskell packages:
-    - [ ] `$(nix-build default.nix -A plutus.updateMaterialized)` to update the materialized Nix files
-   - [ ] Update `hie-*.yaml` files if needed
+    - [ ] `nix-shell shell.nix --run updateMaterialized` to update the materialized Nix files
+    - [ ] Update `hie-*.yaml` files if needed
 - If you changed any Haskell files:
-   - [ ] `$(nix-shell shell.nix --run fix-stylish-haskell)` to fix any formatting issues
+    - [ ] `nix-shell shell.nix --run fix-stylish-haskell` to fix any formatting issues
 - If you changed any Purescript files:
-   - [ ] `$(nix-shell shell.nix --run fix-purty)` to fix any formatting issues
+    - [ ] `nix-shell shell.nix --run fix-purty` to fix any formatting issues
 
 Pre-merge checklist:
 - [ ] Someone approved it

diff --git a/.ignore b/.ignore
@@ -1,4 +1,4 @@
-nix/stack.materialized/*.nix
+nix/pkgs/haskell/materialized*/*.nix
 nix/sources.nix
 **/spago-packages.nix
 **/packages.nix
diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc
@@ -70,18 +70,18 @@ You can find some documentation in the [hie-bios](https://github.com/mpickering/
 The Nix code that builds all the Haskell packages and their dependencies is generated automatically.
 However, to avoid doing too much work all the time, we have checked the generated output in.
 
-IMPORTANT: These files needs to be regenerated if you change any dependencies in cabal files or change the Stackage resolver.
+IMPORTANT: These files needs to be regenerated if you change any dependencies in cabal files.
 But the CI will tell you if you've failed to do so.
 
-You can regenerate the files by running `$(nix-build default.nix -A plutus.updateMaterialized)`.
+You can regenerate the files by running `updateMaterialized` (provided by `nix-shell`) from the repository root.
 
 === How to add a new Haskell package
 
 You need to do a few things when adding a new package, in the following order:
 
 . Add the cabal file for the new package.
-. Add the package to link:stack.yaml[`stack.yaml`].
 . Add the package to link:cabal.project[`cabal.project`].
+. Add the package to link:stack.yaml[`stack.yaml`].
 . Update the xref:update-generated[package set].
 . Update the `xref:update-hie[hie-*.yaml` files].
 . Check that you can run `nix build -f default.nix plutus.haskell.projectPackages.<package name>` successfully.

diff --git a/cabal.project b/cabal.project
@@ -58,7 +58,7 @@ source-repository-package
 -- newer version of persistent. See stack.yaml for the mirrored
 -- configuration.
 package eventful-sql-common
-  ghc-options: -XDerivingStrategies -XStandaloneDeriving -XUndecidableInstances
+  ghc-options: -XDerivingStrategies -XStandaloneDeriving -XUndecidableInstances -XDataKinds -XFlexibleInstances
 
 -- TODO: remove whenn inline-r version>0.10.3  is pushed to hackage
 package inline-r
@@ -79,16 +79,22 @@ allow-newer: Win32-network:base
            , prometheus:base
            -- restricts to ghc<8.10
            , monoidal-containers:base
+           , monoidal-containers:aeson
 
            -- we make use of an older servant. this makes the older servant build for ghc-8.10
            , servant:base
+           , servant:aeson
            , servant-server:base
+           , servant-server:aeson
            , servant-foreign:base
            , servant-foreign:lens
            , servant-client-core:base
+           , servant-client-core:aeson
            , servant-client-core:template-haskell
            , servant-client:base
+           , servant-client:aeson
            , servant-swagger:base
+           , servant-swagger:aeson
            , servant-swagger:swagger2
            , servant-swagger:lens
            , servant-swagger:Cabal
@@ -102,18 +108,26 @@ allow-newer: Win32-network:base
 
            -- size-based restricts TemplateHaskell to that of  ghc<8.10
            , size-based:template-haskell
+
 constraints:
   -- cardano-prelude currently redefines some instances that appear in 0.3.4
   nonempty-containers < 0.3.4
   -- cardano-binary, of the cardano-base repository, cannot build with 5.2 but Cabal picks that one
   , recursion-schemes < 5.2
-  -- A bunch of onstraints to make our forked servant-purescript build using an older servant
+  -- A bunch of constraints to make our forked servant-purescript build using an older servant
   , servant < 0.17
   , servant-server < 0.17
   , servant-foreign < 0.15.1
   , servant-client-core < 0.17
   , servant-client < 0.17
   , servant-swagger < 1.1.8
+  -- aws-lambda-haskell-runtime-wai doesn't compile with newer versions
+  , aws-lambda-haskell-runtime <= 3.0.3
+
+-- These are dependencies of the (disabled) tests of some of our source-repository-package dependencies.
+-- But shellFor still wants them around (https://github.com/input-output-hk/haskell.nix/issues/993), so
+-- we ensure they're included.
+extra-packages: data-ordlist, sequence, tasty-expected-failure, doctest-discover
 
 package ouroboros-network
   tests: False
@@ -217,6 +231,7 @@ source-repository-package
     typed-protocols
     typed-protocols-examples
     ouroboros-network
+    ouroboros-network-testing
     ouroboros-network-framework
     io-sim
     io-sim-classes

diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix
@@ -68,7 +68,17 @@ let
   #
   fixPurty = pkgs.callPackage ./fix-purty { inherit purty; };
   fixStylishHaskell = pkgs.callPackage ./fix-stylish-haskell { inherit stylish-haskell; };
-  updateMaterialized = haskell.project.stack-nix.passthru.updateMaterialized;
+  updateMaterialized = pkgs.writeShellScriptBin "updateMaterialized" ''
+    # This runs the 'updateMaterialize' script in all platform combinations we care about.
+    # See the comment in ./haskell/haskell.nix
+
+    # Update the linux files (will do for all unixes atm).
+    $(nix-build default.nix -A plutus.haskell.project.plan-nix.passthru.updateMaterialized --argstr system x86_64-linux)
+
+    # Update the musl files. This isn't quite as simple as just passing a system, since it's inherently a "cross" build,
+    # but we actually have a variant project exposed already, so we can just use that.
+    $(nix-build default.nix -A plutus.haskell.muslProject.plan-nix.passthru.updateMaterialized --argstr system x86_64-linux)
+  '';
   updateHie = pkgs.callPackage ./update-hie { inherit gen-hie; };
   updateMetadataSamples = pkgs.callPackage ./update-metadata-samples { };
   updateClientDeps = pkgs.callPackage ./update-client-deps {

diff --git a/nix/pkgs/haskell/haskell.nix b/nix/pkgs/haskell/haskell.nix
@@ -15,7 +15,7 @@
 }:
 let
   r-packages = with rPackages; [ R tidyverse dplyr stringr MASS plotly shiny shinyjs purrr ];
-  project = haskell-nix.stackProject' {
+  project = haskell-nix.cabalProject' {
     inherit compiler-nix-name;
     # This is incredibly difficult to get right, almost everything goes wrong, see https://github.com/input-output-hk/haskell.nix/issues/496
     src = let root = ../../../.; in
@@ -26,9 +26,15 @@ let
         # particularly bad on Hercules, see https://github.com/hercules-ci/support/issues/40
         name = "plutus";
       };
-    # These files need to be regenerated when you change the cabal files or stack resolver.
+    # These files need to be regenerated when you change the cabal files.
     # See ../CONTRIBUTING.doc for more information.
-    materialized = ../../stack.materialized;
+    # Unfortuntely, they are *not* constant across all possible systems, so in some circumstances we need different sets of files
+    # At the moment, we only need a different one for musl, but conceivably we might need one for darwin in future.
+    # See https://github.com/input-output-hk/nix-tools/issues/97
+    materialized =
+      if stdenv.hostPlatform.isMusl then ./materialized-musl
+      else if stdenv.hostPlatform.isUnix then ./materialized-unix
+      else builtins.error "Don't have materialized files for this platform";
     # If true, we check that the generated files are correct. Set in the CI so we don't make mistakes.
     inherit checkMaterialization;
     sha256map = {
@@ -41,62 +47,12 @@ let
       "https://github.com/raduom/cardano-ledger-specs"."2cac85306d8b3e07006e9081f36ce7ebf2d9d0a3" = "0w6z1va6a93f818m9byh49yxkkpd9q3xlxk5irpq3d42vmfpy447";
       "https://github.com/input-output-hk/iohk-monitoring-framework"."5c9627b6aee487f9b7ec44981aba57a6afc659b1" = "0ndnhff32h37xsc61b181m4vwaj4vm1z04p2rfwffnjjmgz23584";
       "https://github.com/input-output-hk/ouroboros-network"."75153affa23a0e68e035d7bb19880fe1ae35b1d4" = "0aj6rsqp93k2079bipv2ia7m56h2xwwlcjffr7mr99cz6l9xj96i";
+      "https://github.com/input-output-hk/goblins"."26d35ad52fe9ade3391532dbfeb2f416f07650bc" = "17p5x0hj6c67jkdqx0cysqlwq2zs2l87azihn1alzajy9ak6ii0b";
     };
     modules = [
       {
-        # Borrowed from https://github.com/input-output-hk/haskell.nix/pull/427
-        # This corresponds to the set of packages that comes with GHC. We are
-        # here saying that we must get them from GHC itself, rather than trying
-        # to "re-install" them into the package database.
-        nonReinstallablePkgs =
-          [
-            "rts"
-            "ghc-heap"
-            "ghc-prim"
-            "integer-gmp"
-            "integer-simple"
-            "base"
-            "deepseq"
-            "array"
-            "ghc-boot-th"
-            "pretty"
-            "template-haskell"
-            "ghc-boot"
-            "ghc"
-            "Cabal"
-            "Win32"
-            "array"
-            "binary"
-            "bytestring"
-            "containers"
-            "directory"
-            "filepath"
-            "ghc-boot"
-            "ghc-compact"
-            "ghc-prim"
-            "ghci"
-            "haskeline"
-            "hpc"
-            "mtl"
-            "parsec"
-            "process"
-            "text"
-            "time"
-            "transformers"
-            "unix"
-            "xhtml"
-            "stm"
-            "terminfo"
-          ];
-
+        reinstallableLibGhc = false;
         packages = {
-          # Using https connections ultimately requires x509. But on
-          # OSX, a pure build can't find the package. This is the
-          # solution used by the wallet build, and we reuse it here.
-          x509-system.components.library.preBuild = lib.optionalString (stdenv.isDarwin) ''
-            substituteInPlace System/X509/MacOS.hs --replace security /usr/bin/security
-          '';
-          inline-r.package.ghcOptions = "-XStandaloneKindSignatures";
           # See https://github.com/input-output-hk/plutus/issues/1213
           marlowe.doHaddock = false;
           plutus-use-cases.doHaddock = false;
@@ -171,6 +127,18 @@ let
           plutus-tx-plugin.package.ghcOptions = "-Werror";
           plutus-doc.package.ghcOptions = "-Werror";
           plutus-use-cases.package.ghcOptions = "-Werror";
+
+          # External package settings
+
+          # Using https connections ultimately requires x509. But on
+          # OSX, a pure build can't find the package. This is the
+          # solution used by the wallet build, and we reuse it here.
+          x509-system.components.library.preBuild = lib.optionalString (stdenv.isDarwin) ''
+            substituteInPlace System/X509/MacOS.hs --replace security /usr/bin/security
+          '';
+          inline-r.package.ghcOptions = "-XStandaloneKindSignatures";
+
+          eventful-sql-common.package.ghcOptions = "-XDerivingStrategies -XStandaloneDeriving -XUndecidableInstances -XDataKinds -XFlexibleInstances -XMultiParamTypeClasses";
         };
       }
     ];

diff --git a/...stack.materialized/.stack-to-nix.cache.11 → ...rialized-musl/.plan.nix/Win32-network.nix b/...stack.materialized/.stack-to-nix.cache.11 → ...rialized-musl/.plan.nix/Win32-network.nix
@@ -21,6 +21,14 @@
       synopsis = "Win32 network API";
       description = "";
       buildType = "Simple";
+      isLocal = true;
+      detailLevel = "FullDetails";
+      licenseFiles = [ "LICENSE" "NOTICE" ];
+      dataDir = "";
+      dataFiles = [];
+      extraSrcFiles = [ "include/Win32-network.h" ];
+      extraTmpFiles = [];
+      extraDocFiles = [];
       };
     components = {
       "library" = {
@@ -34,6 +42,26 @@
           else [ (hsPkgs."base" or (errorHandler.buildDepError "base")) ];
         libs = (pkgs.lib).optional (system.isWindows) (pkgs."ws2_32" or (errorHandler.sysDepError "ws2_32"));
         buildable = true;
+        modules = if system.isWindows
+          then [
+            "System/Win32/Async/IOData"
+            "System/Win32/Async/IOManager"
+            "System/Win32/Async/Overlapped"
+            "System/Win32/Async/Socket/Syscalls"
+            "System/Win32/Async/WSABuf"
+            "System/IOManager"
+            "System/Win32/NamedPipes"
+            "System/Win32/Async"
+            "System/Win32/Async/File"
+            "System/Win32/Async/ErrCode"
+            "System/Win32/Async/Socket"
+            "System/Win32/Async/Socket/ByteString"
+            "System/Win32/Async/Socket/ByteString/Lazy"
+            "System/Win32/Async/Internal"
+            ]
+          else [ "System/IOManager" ];
+        hsSourceDirs = [ "src" ];
+        includeDirs = (pkgs.lib).optional (system.isWindows) "include";
         };
       exes = {
         "named-pipe-demo" = {
@@ -47,6 +75,8 @@
               ]
             else [ (hsPkgs."base" or (errorHandler.buildDepError "base")) ];
           buildable = true;
+          hsSourceDirs = [ "demo" ];
+          mainPath = [ "named-pipe-demo.hs" ] ++ [ "" ];
           };
         };
       tests = {
@@ -67,9 +97,15 @@
             (hsPkgs."Win32-network" or (errorHandler.buildDepError "Win32-network"))
             ];
           buildable = if system.isWindows then true else false;
+          modules = (pkgs.lib).optionals (system.isWindows) [
+            "Test/Generators"
+            "Test/Async/PingPong"
+            "Test/Async/Handle"
+            "Test/Async/Socket"
+            ];
+          hsSourceDirs = [ "test" ];
+          mainPath = [ "Main.hs" ];
           };
         };
       };
-    } // rec {
-    src = (pkgs.lib).mkDefault /nix/store/fyxakg4qyqyg8c2crvcp97nnvfv38ifi-ouroboros-network-75153af/Win32-network;
-    }
+    } // rec { src = (pkgs.lib).mkDefault .././.source-repository-packages/17; }
diff --git a/...stack.materialized/.stack-to-nix.cache.18 → ...lized-musl/.plan.nix/byron-spec-chain.nix b/...stack.materialized/.stack-to-nix.cache.18 → ...lized-musl/.plan.nix/byron-spec-chain.nix
@@ -21,6 +21,14 @@
       synopsis = "Executable specification of the Cardano blockchain";
       description = "";
       buildType = "Simple";
+      isLocal = true;
+      detailLevel = "FullDetails";
+      licenseFiles = [ "LICENSE" ];
+      dataDir = "";
+      dataFiles = [];
+      extraSrcFiles = [ "ChangeLog.md" ];
+      extraTmpFiles = [];
+      extraDocFiles = [];
       };
     components = {
       "library" = {
@@ -37,6 +45,16 @@
           (hsPkgs."small-steps" or (errorHandler.buildDepError "small-steps"))
           ];
         buildable = true;
+        modules = [
+          "Byron/Spec/Chain/STS/Block"
+          "Byron/Spec/Chain/STS/Rule/BBody"
+          "Byron/Spec/Chain/STS/Rule/Bupi"
+          "Byron/Spec/Chain/STS/Rule/Chain"
+          "Byron/Spec/Chain/STS/Rule/Epoch"
+          "Byron/Spec/Chain/STS/Rule/Pbft"
+          "Byron/Spec/Chain/STS/Rule/SigCnt"
+          ];
+        hsSourceDirs = [ "src" ];
         };
       tests = {
         "chain-rules-test" = {
@@ -54,9 +72,13 @@
             (hsPkgs."small-steps" or (errorHandler.buildDepError "small-steps"))
             ];
           buildable = true;
+          modules = [
+            "Test/Byron/Spec/Chain/STS/Properties"
+            "Test/Byron/AbstractSize/Properties"
+            ];
+          hsSourceDirs = [ "test" ];
+          mainPath = [ "Main.hs" ];
           };
         };
       };
-    } // rec {
-    src = (pkgs.lib).mkDefault /nix/store/83bbr6a01zn2135qncr6v6znv7650msi-cardano-ledger-specs-2cac853/byron/chain/executable-spec;
-    }
+    } // rec { src = (pkgs.lib).mkDefault .././.source-repository-packages/22; }