Test improvements

.github/workflows/ci.yml

@@ -16,6 +16,16 @@ jobs:
       if: "!contains(github.event.head_commit.message, 'noci')"
       run: ./.aliases/verify-default-nix
 
+  nix-flake-check:
+    runs-on: self-hosted
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: nix flake check
+      if: "!contains(github.event.head_commit.message, 'noci')"
+      run: nix flake check
+
   build-ui-metadata:
     runs-on: self-hosted
 

default.nix

@@ -1,4 +1,4 @@
-{ fetchFromGitHub
+{ fetchFromGitHub ? (import <nixpkgs> {}).fetchFromGitHub
 , isCodeDown ? true # For introspection using builtins.functionArgs
 , ...
 }:

flake.nix

@@ -15,14 +15,16 @@
 
         codedown = import ./codedown.nix { inherit pkgsStable pkgsMaster; };
 
+        sampleOutputs = pkgsStable.callPackage ./nix/sample-outputs.nix { inherit codedown pkgsStable; };
+
       in
         rec {
-          packages = {
+          packages = rec {
             # For nix repl debugging
             # inherit codedown;
 
             # Tests use flake to do packageSearch builds
-            inherit (codedown) kernels codedownSearcher;
+            inherit (codedown) codedownSearcher;
 
             # For .envrc
             nixpkgsPath = pkgsStable.writeShellScriptBin "nixpkgsPath.sh" "echo -n ${pkgsStable.path}";
@@ -31,12 +33,21 @@
 
             notebook = with pkgsStable; python3.pkgs.toPythonModule (
               python3.pkgs.notebook.overridePythonAttrs (oldAttrs: {
-                makeWrapperArgs = ["--set JUPYTER_PATH ${sample_environments.mega}/lib/codedown"];
+                makeWrapperArgs = ["--set JUPYTER_PATH ${sampleOutputs.sample_environments.mega}/lib/codedown"];
               })
             );
           }
-          // (pkgsStable.callPackage ./nix/sample-outputs.nix { inherit codedown pkgsStable; }).inner
+          // sampleOutputs.inner
           ;
+
+          checks = {
+            customScript = pkgsStable.writeShellScript "test-script" ''
+              #!/bin/sh
+              echo "Running custom test..."
+              # Add your test logic here
+              exit 0
+            '';
+          };
         }
     );
 }

nix/sample-outputs.nix

@@ -19,6 +19,8 @@ let
 in
 
 {
+  inherit sample_environments;
+
   inner = {
     sample_environments_farm = linkFarm "sample_environments_farm" (
       lib.mapAttrsToList (name: path: { inherit name path; })
@@ -30,18 +32,32 @@ in
         sample_environments
     );
 
-    all_settings_schemas = writeTextFile {
-      name = "all_settings_schemas.json";
-      text = builtins.toJSON (
-        lib.mapAttrs (n: v: lib.listToAttrs (map (x: lib.nameValuePair x.name x.meta.settings_schema) v.ui_metadata.kernels))
-          sample_environments
-      );
-    };
+    # TODO: add test that all settings schemas are valid in certain ways:
+    # - they all have a title
+    all_settings_schemas = let
+      gatherSchemas = prefix: pkg:
+        (if !(lib.hasAttrByPath ["meta" "settings_schema"] pkg) then []
+         else [{ name = prefix + pkg.name; value = pkg.meta.settings_schema; }])
+        ++ builtins.concatLists (map (gatherSchemas (prefix + pkg.name + ".")) (pkg.packages or []))
+      ;
+
+      gatherSchemasFromEnvironment = prefix: env:
+        builtins.concatLists (lib.mapAttrsToList (n: v: gatherSchemas prefix v) env.ui_metadata.packages);
+
+    in
+      writeTextFile {
+        name = "all_settings_schemas.json";
+        text = builtins.toJSON (
+          lib.listToAttrs (
+            builtins.concatLists (lib.mapAttrsToList (n: v: gatherSchemasFromEnvironment (n + ".") v) sample_environments)
+          )
+        );
+      };
 
     printVersions = let
       versionsMap = with lib;
         mapAttrs (lang: value: if (hasAttr "versions" value) then value.versions else {})
-          (filterAttrs (k: _: !(hasPrefix "override") k) codedown.languages);
+          (filterAttrs (k: _: !(hasPrefix "override") k) codedown.kernels);
 
       file = writeTextFile {
         name = "versions.yaml";

tests/app/Spec/Tests.hs

@@ -27,6 +27,7 @@ tests :: forall context. (
 tests =
   introduceJupyterRunner $
   introduceJustBubblewrap $
+  introduceBootstrapNixpkgs $
     introduce' (defaultNodeOptions { nodeOptionsCreateFolder = False }) "Introduce parallel semaphore" parallelSemaphore getQSem (const $ return ()) $
       $(getSpecFromFolder $ defaultGetSpecFromFolderOptions {
            getSpecCombiner = 'describeParallel

tests/app/Spec/Tests/Searchers.hs

@@ -3,12 +3,16 @@
 module Spec.Tests.Searchers (tests) where
 
 import Test.Sandwich as Sandwich
+import TestLib.JupyterRunnerContext
 import TestLib.TestSearchers
+import TestLib.Types
 
 
-tests :: TopSpec
+tests :: SimpleSpec
 tests = describe "Searchers" $ do
   it "searcher has some results" $ testSearcherHasNonemptyResults ".#codedownSearcher"
 
 main :: IO ()
-main = runSandwichWithCommandLineArgs Sandwich.defaultOptions tests
+main = runSandwichWithCommandLineArgs Sandwich.defaultOptions $
+  introduceBootstrapNixpkgs $
+  tests

tests/package.yaml

@@ -73,7 +73,6 @@ executables:
     - -with-rtsopts=-N
     build-tools: sandwich:sandwich-discover
     dependencies:
-    - monad-logger
     - row-types
     - safe
     - tests

tests/src/TestLib/JupyterRunnerContext.hs

@@ -44,6 +44,15 @@ introduceJupyterRunner = introduceWith [i|Jupyter runner|] jupyterRunner $ \acti
   createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess)
   void $ action runnerPath
 
+introduceBootstrapNixpkgs :: (
+  HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  ) => SpecFree (LabelValue "bootstrapNixpkgs" FilePath :> context) m () -> SpecFree context m ()
+introduceBootstrapNixpkgs = introduceWith [i|Jupyter runner|] bootstrapNixpkgs $ \action -> do
+  rootDir <- findFirstParentMatching (\x -> doesPathExist (x </> ".git"))
+
+  out <- readCreateProcessWithLogging ((proc "nix" ["run", ".#nixpkgsPath"]) { cwd = Just rootDir }) ""
+  void $ action (T.unpack $ T.strip $ T.pack out)
+
 -- | TODO: pipe through a command-line argument to control whether bwrap is used?
 introduceJustBubblewrap :: (
   HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
@@ -319,5 +328,6 @@ notebookWithCode kernel code = A.object [
 jupyterMain :: LanguageSpec -> IO ()
 jupyterMain tests = runSandwichWithCommandLineArgs' Sandwich.defaultOptions specialOptions $
   introduceJupyterRunner $
-  introduceJustBubblewrap
+  introduceJustBubblewrap $
+  introduceBootstrapNixpkgs
   tests

tests/src/TestLib/TestBuilding.hs

@@ -4,31 +4,62 @@ module TestLib.TestBuilding where
 
 import Conduit as C
 import Control.Monad.Logger
+import Control.Monad.Reader
+import Data.Function
+import qualified Data.List as L
+import GHC.Stack
 import System.Exit
 import System.FilePath
 import Test.Sandwich
+import TestLib.Types
 import TestLib.Util
 import UnliftIO.Directory
+import UnliftIO.Environment
 import UnliftIO.Process
 
 
-testBuild :: (MonadUnliftIO m, MonadLogger m) => String -> m ()
+testBuild :: (
+  HasCallStack, MonadUnliftIO m, MonadLogger m, MonadReader context m
+  , HasBaseContext context, HasBootstrapNixpkgs context
+  ) => String -> m ()
 testBuild = testBuild' LevelDebug
 
-testBuild' :: (MonadUnliftIO m, MonadLogger m) => LogLevel -> String -> m ()
+testBuild' :: (
+  HasCallStack, MonadUnliftIO m, MonadLogger m, MonadReader context m
+  , HasBaseContext context, HasBootstrapNixpkgs context
+  ) => LogLevel -> String -> m ()
 testBuild' logLevel expr = do
   rootDir <- findFirstParentMatching (\x -> doesPathExist (x </> ".git"))
+  env <- getEnvWithNixPath
 
-  p <- createProcessWithLogging' logLevel $ (proc "nix" ["build", expr, "--json", "--no-link"]) {
+  p <- createProcessWithLogging' logLevel $ (proc "nix-build" [".", "-A", expr, "--no-out-link"]) {
     cwd = Just rootDir
+    , env = Just env
     }
   waitForProcess p >>= (`shouldBe` ExitSuccess)
 
-testEval :: (MonadUnliftIO m, MonadLogger m) => String -> m ()
+testEval :: (
+  HasCallStack, MonadUnliftIO m, MonadLogger m, MonadReader context m
+  , HasBaseContext context, HasBootstrapNixpkgs context
+  ) => String -> m ()
 testEval expr = do
   rootDir <- findFirstParentMatching (\x -> doesPathExist (x </> ".git"))
+  env <- getEnvWithNixPath
 
-  p <- createProcessWithLogging $ (proc "nix" ["eval", expr, "--json"]) {
+  p <- createProcessWithLogging $ (proc "nix-instantiate" ["--eval", "--strict", ".", "-A", expr, "--json"]) {
     cwd = Just rootDir
+    , env = Just env
     }
   waitForProcess p >>= (`shouldBe` ExitSuccess)
+
+
+getEnvWithNixPath :: (
+  MonadUnliftIO m, MonadReader context m
+  , HasBaseContext context, HasBootstrapNixpkgs context
+  ) => m [(String, String)]
+getEnvWithNixPath = do
+  nixpkgsPath <- getContext bootstrapNixpkgs
+  baseEnv <- getEnvironment
+  return $ baseEnv
+         & (("NIX_PATH", "nixpkgs=" <> nixpkgsPath) :)
+         & L.nubBy (\x y -> fst x == fst y)

tests/src/TestLib/TestSearchers.hs

@@ -5,18 +5,18 @@ module TestLib.TestSearchers where
 import Conduit as C
 import Control.Monad.Catch (MonadMask)
 import Control.Monad.Logger
+import Control.Monad.Reader
 import Control.Monad.Trans.Control (MonadBaseControl)
 import qualified Data.Aeson as A
-import qualified Data.ByteString.Lazy.Char8 as BL8
 import Data.Conduit.Aeson as C
 import qualified Data.List as L
 import Data.String.Interpolate
 import Data.Text as T
 import qualified Data.Text.IO as T
-import qualified Data.Vector as V
 import System.FilePath
 import Test.Sandwich
 import TestLib.TestBuilding
+import TestLib.Types
 import TestLib.Util
 import UnliftIO.Directory
 import UnliftIO.IO
@@ -26,53 +26,67 @@ import UnliftIO.Process
 -- Testing for successful build
 
 testKernelSearchersBuild :: (
-  HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  , HasBaseContext context, HasBootstrapNixpkgs context
   ) => Text -> SpecFree context m ()
 testKernelSearchersBuild kernel = it [i|#{kernel}: package searchers build|] $ do
-  testBuild [i|.\#kernels."#{kernel}".packageSearch|]
+  testBuild [i|kernels."#{kernel}".packageSearch|]
 
 testHasExpectedFields :: (
-  HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  , HasBaseContext context, HasBootstrapNixpkgs context
   ) => Text -> SpecFree context m ()
 testHasExpectedFields kernel = it [i|#{kernel}: has expected fields|] $ do
-  testEval [i|.\#kernels."#{kernel}".settingsSchema|]
-  testEval [i|.\#kernels."#{kernel}".modes|]
-  testEval [i|.\#kernels."#{kernel}".settings|]
-  testEval [i|.\#kernels."#{kernel}".args|]
-  testEval [i|.\#kernels."#{kernel}".meta|]
+  testEval [i|kernels."#{kernel}".settingsSchema|]
+  testEval [i|kernels."#{kernel}".modes|]
+  testEval [i|kernels."#{kernel}".settings|]
+  testEval [i|kernels."#{kernel}".args|]
+  testEval [i|kernels."#{kernel}".meta|]
 
   -- Used to view all versions in codedown-languages
-  testEval [i|.\#kernels."#{kernel}".versions|]
+  testEval [i|kernels."#{kernel}".versions|]
 
 -- Testing for nonempty results
 
 -- | A stronger version of testKernelSearchersBuild
 testKernelSearchersNonempty :: (
-  HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  , HasBaseContext context, HasBootstrapNixpkgs context
   ) => Text -> SpecFree context m ()
 testKernelSearchersNonempty kernel = describe [i|#{kernel}: package and LSP searchers build and have results|] $ do
   it "package searcher" $ testPackageSearchNonempty kernel
 
 testPackageSearchNonempty :: (
-  HasBaseContext context, MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  MonadIO m, MonadMask m, MonadUnliftIO m, MonadBaseControl IO m
+  , HasBaseContext context, HasBootstrapNixpkgs context
   ) => Text -> ExampleT context m ()
-testPackageSearchNonempty kernel = testSearcherHasNonemptyResults [i|.\#kernels."#{kernel}".packageSearch|]
+testPackageSearchNonempty kernel = testSearcherHasNonemptyResults [i|kernels."#{kernel}".packageSearch|]
 
-testSearcherHasNonemptyResults :: (MonadUnliftIO m, MonadThrow m, MonadLogger m, MonadFail m) => String -> m ()
+testSearcherHasNonemptyResults :: (
+  MonadUnliftIO m, MonadThrow m, MonadLogger m, MonadFail m, MonadReader context m
+  , HasBaseContext context, HasBootstrapNixpkgs context
+  ) => String -> m ()
 testSearcherHasNonemptyResults expr = searcherResults expr >>= \case
   xs | not (L.null xs) -> return ()
   x -> expectationFailure [i|Expected searcher to output an array with >=1 elem, but got #{x}|]
 
-searcherResults :: (MonadUnliftIO m, MonadThrow m, MonadLogger m, MonadFail m) => String -> m [A.Object]
+searcherResults :: (
+  MonadUnliftIO m, MonadThrow m, MonadLogger m, MonadFail m, MonadReader context m
+  , HasBaseContext context, HasBootstrapNixpkgs context
+  ) => String -> m [A.Object]
 searcherResults expr = do
   rootDir <- findFirstParentMatching (\x -> doesPathExist (x </> ".git"))
 
-  built <- ((A.eitherDecode . BL8.pack) <$> readCreateProcessWithLogging ((proc "nix" ["build", expr, "--no-link", "--json"]) { cwd = Just rootDir, std_err = CreatePipe }) "") >>= \case
-    Left err -> expectationFailure [i|Failed to decode JSON: #{err}|]
-    Right (A.Array ((V.! 0) -> (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String x))))))) -> pure x
-    Right x -> expectationFailure [i|Unexpected JSON: #{x}|]
+  env <- getEnvWithNixPath
+  let cp = (proc "nix-build" [".", "-A", expr, "--no-out-link"]) {
+        cwd = Just rootDir
+        , std_err = CreatePipe
+        , env = Just env
+        }
+  built <- (T.unpack . T.strip . T.pack) <$> (readCreateProcessWithLogging cp "")
+  info [i|Got built searcher: #{built}|]
 
-  let cp = (proc (T.unpack built </> "bin" </> "searcher") []) {
+  let cp' = (proc (built </> "bin" </> "searcher") []) {
         cwd = Just rootDir
         , std_in = CreatePipe
         , std_out = CreatePipe
@@ -81,7 +95,7 @@ searcherResults expr = do
         , close_fds = True
         }
 
-  withCreateProcess cp $ \(Just hin) (Just hout) (Just _herr) _p -> do
+  withCreateProcess cp' $ \(Just hin) (Just hout) (Just _herr) _p -> do
     liftIO $ T.hPutStrLn hin "50"
     liftIO $ T.hPutStrLn hin "0"
     liftIO $ T.hPutStrLn hin ""

tests/src/TestLib/Types.hs

@@ -65,12 +65,19 @@ maybeBubblewrap :: Label "maybeBubblewrap" (Maybe FilePath)
 maybeBubblewrap = Label
 type HasMaybeBubblewrap context = HasLabel context "maybeBubblewrap" (Maybe FilePath)
 
+bootstrapNixpkgs :: Label "bootstrapNixpkgs" FilePath
+bootstrapNixpkgs = Label
+type HasBootstrapNixpkgs context = HasLabel context "bootstrapNixpkgs" FilePath
+
 -- * Spec types
 
 type SomeLanguageSpec context = (
   HasBaseContext context
   , HasJupyterRunner context
   , HasMaybeBubblewrap context
+  , HasBootstrapNixpkgs context
   )
 
 type LanguageSpec = forall context. SomeLanguageSpec context => SpecFree context IO ()
+
+type SimpleSpec = forall context. (HasBaseContext context, HasBootstrapNixpkgs context) => SpecFree context IO ()

tests/tests.cabal

@@ -143,7 +143,6 @@ executable tests
     , lsp-test
     , lsp-types
     , monad-control
-    , monad-logger
     , mtl
     , optparse-applicative
     , row-types