Merge pull request #100 from Mic92/gha-eval-fixes

ci/parallel.nix: nixfmt

.github/workflows/eval.yml

@@ -40,14 +40,6 @@ jobs:
       - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
         if: env.mergedSha
 
-      - name: Enable swap
-        if: env.mergedSha
-        run: |
-          sudo fallocate -l 10G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-
       - name: Check eval
         if: env.mergedSha
         run: ./ci/eval-nixpkgs.sh --system "${{ matrix.system }}"

ci/eval-nixpkgs.sh

@@ -1,60 +1,88 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i bash -p moreutils -I nixpkgs=channel:nixpkgs-unstable
+#!nix-shell -i bash -p coreutils moreutils -I nixpkgs=channel:nixpkgs-unstable
 
 set -euxo pipefail
 
-system="x86_64-linux"
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 NIXPKGS_PATH="$(readlink -f "$SCRIPT_DIR"/..)"
 
+system="x86_64-linux"
+quick_test=0
+CORES=$(nproc)
+
 parseArgs() {
-  while [[ $# -gt 0 ]]; do
-    case $1 in
-      --system)
-        system=$2
-        shift 2
-        ;;
-      *)
-        echo "Unknown argument: $1"
-        exit 1
-        ;;
-    esac
-  done
+    while [[ $# -gt 0 ]]; do
+        arg=$1
+        shift
+        case "$arg" in
+        --system)
+            system=$1
+            shift 1
+            ;;
+        --cores)
+            CORES=$1
+            shift 1
+            ;;
+        --quick-test)
+            quick_test=1
+            ;;
+        *)
+            echo "Unknown argument: $arg"
+            exit 1
+            ;;
+        esac
+    done
 }
 
 main() {
     parseArgs "$@"
     tmpdir=$(mktemp -d)
     trap 'rm -rf "$tmpdir"' EXIT
 
-    nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths > "$tmpdir/paths.json"
+    nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths >"$tmpdir/paths.json"
 
-    CORES=$(nproc)
     # Originally @amjoseph: note that the number of processes spawned is four times
     # the number of cores -- this helps in two ways:
     # 1. Keeping cores busy while I/O operations are in flight
     # 2. Since the amount of time needed for the jobs is *not* balanced
     # this minimizes the "tail latency" for the very last job to finish
     # (on one core) by making the job size smaller.
-    NUM_CHUNKS=$(( 4 * CORES ))
-
+    local num_chunks=$((4 * CORES))
+    local seq_end=$((num_chunks - 1))
+    if [[ $quick_test -eq 1 ]]; then
+        seq_end=0
+    fi
 
     (
-      set +e
-      parallel -j "$CORES" \
-          nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \
-          --arg systems "[\"$system\"]" \
-          -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \
-          --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk \
-          -- $(seq 0 $(( NUM_CHUNKS - 1 ))) > "$tmpdir/paths"
-      echo $? > "$tmpdir/exit-code"
+        set +e
+        seq 0 $seq_end | xargs -P "$CORES" -I {} nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \
+            --arg systems "[\"$system\"]" \
+            -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \
+            --arg numChunks "$num_chunks" --show-trace --arg myChunk {} >"$tmpdir/paths"
+        echo $? >"$tmpdir/exit-code"
     ) &
     pid=$!
     while kill -0 "$pid"; do
         free -g >&2
         sleep 20
     done
-    jq --raw-input --slurp 'split("\n") | map(select(. != "") | split(" ") | map(select(. != "")) | { key: .[0], value: .[1] }) | from_entries' "$tmpdir/paths"
+
+    # Extract derivation outputs for each attribute into a JSON object
+    # i.e. { "attr1": { "out": "/nix/store/...", "dev": "/nix/store/..." }, ... }
+    jq --raw-input --slurp '
+    split("\n") |
+    map(select(. != "") | split(" ") | map(select(. != ""))) |
+    map(
+        {
+            key: .[0],
+            value: .[1] | split(";") | map(split("=") |
+               if length == 1 then
+                   { key: "out", value: .[0] }
+               else
+                   { key: .[0], value: .[1] }
+               end) | from_entries}
+    ) | from_entries
+    ' "$tmpdir/paths"
     exit "$(cat "$tmpdir/exit-code")"
 }
 

ci/parallel.nix

@@ -1,61 +1,63 @@
 /*
-Invocation:
+  Invocation:
 
+  Invocation; note that the number of processes spawned is four times
+  the number of cores -- this helps in two ways:
 
-Invocation; note that the number of processes spawned is four times
-the number of cores -- this helps in two ways:
-
-1. Keeping cores busy while I/O operations are in flight
-
-2. Since the amount of time needed for the jobs is *not* balanced
-   this minimizes the "tail latency" for the very last job to finish
-   (on one core) by making the job size smaller.
+  1. Keeping cores busy while I/O operations are in flight
 
+  2. Since the amount of time needed for the jobs is *not* balanced
+     this minimizes the "tail latency" for the very last job to finish
+     (on one core) by making the job size smaller.
 */
 # see pkgs/top-level/nohydra
-{ lib ? import ../lib
-, checkMeta
-, includeBroken ? true
-, path ? ./..
-, systems
-, myChunk
-, numChunks
-, attrPathFile
+{
+  lib ? import ../lib,
+  checkMeta,
+  includeBroken ? true,
+  path ? ./..,
+  systems,
+  myChunk,
+  numChunks,
+  attrPathFile,
 }:
 
 let
   attrPaths = builtins.fromJSON (builtins.readFile attrPathFile);
   chunkSize = (lib.length attrPaths) / numChunks;
   myPaths =
     let
-      dropped = lib.drop (chunkSize*myChunk) attrPaths;
+      dropped = lib.drop (chunkSize * myChunk) attrPaths;
     in
-      if myChunk == numChunks - 1
-      then dropped
-      else lib.take chunkSize dropped;
+    if myChunk == numChunks - 1 then dropped else lib.take chunkSize dropped;
 
   unfiltered = import ../pkgs/top-level/release-outpaths.nix {
-    inherit checkMeta path includeBroken systems;
+    inherit
+      checkMeta
+      path
+      includeBroken
+      systems
+      ;
   };
 
-  f = i: m: a:
-    lib.mapAttrs (name: values:
+  f =
+    i: m: a:
+    lib.mapAttrs (
+      name: values:
       if a ? ${name} then
-        if lib.any (value: lib.length value <= i + 1) values then
-          a.${name}
-        else
-          f (i + 1) values a.${name}
+        if lib.any (value: lib.length value <= i + 1) values then a.${name} else f (i + 1) values a.${name}
       else
         null
     ) (lib.groupBy (a: lib.elemAt a i) m);
 
   filtered = f 0 myPaths unfiltered;
 
-  recurseEverywhere = val:
-    if lib.isDerivation val || !(lib.isAttrs val)
-    then val
-    else (builtins.mapAttrs (_: v: recurseEverywhere v) val)
-         // { recurseForDerivations = true; };
+  recurseEverywhere =
+    val:
+    if lib.isDerivation val || !(lib.isAttrs val) then
+      val
+    else
+      (builtins.mapAttrs (_: v: recurseEverywhere v) val) // { recurseForDerivations = true; };
 
 in
-  recurseEverywhere filtered
+recurseEverywhere filtered