Skip to content

Commit

Permalink
Pick up latest julia-modules-indexing branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@thomasjm
thomasjm committed Dec 4, 2024
1 parent 8d45fad commit 61e88e7
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 165 deletions.
41 changes: 20 additions & 21 deletions modules/kernels/julia/julia-modules/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,7 @@ packageNames:
let
util = callPackage ./util.nix {};

in

let
# Some Julia packages require access to Python. Provide a Nixpkgs version so it
# doesn't try to install its own.
pythonToUse = let
Expand All @@ -54,7 +52,7 @@ let
juliaWrapped = let
cpuTargetFlag = lib.optionalString (juliaCpuTarget != null) "--set JULIA_CPU_TARGET ${juliaCpuTarget}";
in
runCommand "julia-${julia.version}-wrapped" { buildInputs = [makeWrapper]; inherit makeWrapperArgs; } ''
runCommand "julia-${julia.version}-wrapped" { nativeBuildInputs = [makeWrapper]; inherit makeWrapperArgs; } ''
mkdir -p $out/bin
makeWrapper ${julia}/bin/julia $out/bin/julia ${cpuTargetFlag} \
--suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \
Expand Down Expand Up @@ -177,24 +175,25 @@ let
in

runCommand "julia-${julia.version}-env" {
buildInputs = [makeWrapper];

inherit julia;
inherit juliaWrapped;
version = julia.version;
meta = julia.meta;

# Expose the steps we used along the way in case the user wants to use them, for example to build
# expressions and build them separately to avoid IFD.
inherit dependencies;
inherit closureYaml;
inherit dependencyUuidToInfoYaml;
inherit dependencyUuidToRepoYaml;
inherit minimalRegistry;
inherit artifactsNix;
inherit overridesJson;
inherit overridesToml;
inherit projectAndDepot;
nativeBuildInputs = [makeWrapper];

passthru = {
inherit julia;
inherit juliaWrapped;
inherit (julia) pname version meta;

# Expose the steps we used along the way in case the user wants to use them, for example to build
# expressions and build them separately to avoid IFD.
inherit dependencies;
inherit closureYaml;
inherit dependencyUuidToInfoYaml;
inherit dependencyUuidToRepoYaml;
inherit minimalRegistry;
inherit artifactsNix;
inherit overridesJson;
inherit overridesToml;
inherit projectAndDepot;
};
} (''
mkdir -p $out/bin
makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \
Expand Down
9 changes: 3 additions & 6 deletions modules/kernels/julia/julia-modules/extra-python-packages.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
}:

# This file contains an extra mapping from Julia packages to the Python packages they depend on.

with lib;

rec {
packageMapping = {
ExcelFiles = ["xlrd"];
Expand All @@ -14,9 +11,9 @@ rec {
SymPy = ["sympy"];
};

getExtraPythonPackages = names: concatMap (name: let
allCandidates = if hasAttr name packageMapping then getAttr name packageMapping else [];
getExtraPythonPackages = names: lib.concatMap (name: let
allCandidates = if lib.hasAttr name packageMapping then lib.getAttr name packageMapping else [];
in
filter (x: hasAttr x python3.pkgs) allCandidates
lib.filter (x: lib.hasAttr x python3.pkgs) allCandidates
) names;
}
150 changes: 13 additions & 137 deletions modules/kernels/julia/julia-modules/package-closure.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,131 +10,6 @@
}:

let
# The specific package resolution code depends on the Julia version
# These are pretty similar and could be combined to reduce duplication
resolveCode = if lib.versionOlder julia.version "1.7" then resolveCode1_6 else resolveCode1_8;

resolveCode1_6 = ''
import Pkg.API: check_package_name
import Pkg.Types: Context!, PRESERVE_NONE, manifest_info, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
import Pkg.Operations: _resolve, assert_can_add, is_dep, update_package_add
foreach(pkg -> check_package_name(pkg.name, :add), pkgs)
pkgs = deepcopy(pkgs) # deepcopy for avoid mutating PackageSpec members
Context!(ctx)
project_deps_resolve!(ctx, pkgs)
registry_resolve!(ctx, pkgs)
stdlib_resolve!(pkgs)
ensure_resolved(ctx, pkgs, registry=true)
assert_can_add(ctx, pkgs)
for (i, pkg) in pairs(pkgs)
entry = manifest_info(ctx, pkg.uuid)
pkgs[i] = update_package_add(ctx, pkg, entry, is_dep(ctx, pkg))
end
foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs)
pkgs, deps_map = _resolve(ctx, pkgs, PRESERVE_NONE)
'';

resolveCode1_8 = ''
import Pkg.API: handle_package_input!
import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
import Pkg.Operations: _resolve, assert_can_add, update_package_add
import TOML
foreach(handle_package_input!, pkgs)
# The handle_package_input! call above clears pkg.path, so we have to apply package overrides after
overrides = Dict{String, String}(${builtins.concatStringsSep ", " (lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides)})
println("Package overrides: ")
println(overrides)
for pkg in pkgs
if pkg.name in keys(overrides)
pkg.path = overrides[pkg.name]
# Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps.
# This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even
# if it does not appear in the registry.
# See https://github.com/NixOS/nixpkgs/issues/279853
project_toml = joinpath(pkg.path, "Project.toml")
if isfile(project_toml)
toml_data = TOML.parsefile(project_toml)
if haskey(toml_data, "uuid")
ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"])
end
end
end
end
project_deps_resolve!(ctx.env, pkgs)
registry_resolve!(ctx.registries, pkgs)
stdlib_resolve!(pkgs)
ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true)
assert_can_add(ctx, pkgs)
for (i, pkg) in pairs(pkgs)
entry = Pkg.Types.manifest_info(ctx.env.manifest, pkg.uuid)
is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps])
pkgs[i] = update_package_add(ctx, pkg, entry, is_dep)
end
foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs)
# Save the original pkgs for later. We might need to augment it with the weak dependencies
orig_pkgs = pkgs
pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_NONE, ctx.julia_version)
if VERSION >= VersionNumber("1.9")
while true
# Check for weak dependencies, which appear on the RHS of the deps_map but not in pkgs.
# Build up weak_name_to_uuid
uuid_to_name = Dict()
for pkg in pkgs
uuid_to_name[pkg.uuid] = pkg.name
end
weak_name_to_uuid = Dict()
for (uuid, deps) in pairs(deps_map)
for (dep_name, dep_uuid) in pairs(deps)
if !haskey(uuid_to_name, dep_uuid)
weak_name_to_uuid[dep_name] = dep_uuid
end
end
end
if isempty(weak_name_to_uuid)
break
end
# We have nontrivial weak dependencies, so add each one to the initial pkgs and then re-run _resolve
println("Found weak dependencies: $(keys(weak_name_to_uuid))")
orig_uuids = Set([pkg.uuid for pkg in orig_pkgs])
for (name, uuid) in pairs(weak_name_to_uuid)
if uuid in orig_uuids
continue
end
pkg = PackageSpec(name, uuid)
push!(orig_uuids, uuid)
push!(orig_pkgs, pkg)
ctx.env.project.deps[name] = uuid
entry = Pkg.Types.manifest_info(ctx.env.manifest, uuid)
orig_pkgs[length(orig_pkgs)] = update_package_add(ctx, pkg, entry, false)
end
global pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, orig_pkgs, PRESERVE_NONE, ctx.julia_version)
end
end
'';

juliaExpression = packageNames: ''
import Pkg
Pkg.Registry.add(Pkg.RegistrySpec(path="${augmentedRegistry}"))
Expand All @@ -144,7 +19,7 @@ let
input = ${lib.generators.toJSON {} packageNames}
if isfile("extra_package_names.txt")
append!(input, readlines("extra_package_names.txt"))
append!(input, readlines("extra_package_names.txt"))
end
input = unique(input)
Expand All @@ -155,20 +30,21 @@ let
ctx = Context()
${resolveCode}
overrides = Dict{String, String}(${builtins.concatStringsSep ", " (lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides)})
${builtins.readFile ./resolve_packages.jl}
open(ENV["out"], "w") do io
for spec in pkgs
println(io, "- name: " * spec.name)
println(io, " uuid: " * string(spec.uuid))
println(io, " version: " * string(spec.version))
if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid)
println(io, " depends_on: ")
for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])
println(io, " \"$(dep_name)\": \"$(dep_uuid)\"")
end
for spec in pkgs
println(io, "- name: " * spec.name)
println(io, " uuid: " * string(spec.uuid))
println(io, " version: " * string(spec.version))
if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid)
println(io, " depends_on: ")
for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])
println(io, " \"$(dep_name)\": \"$(dep_uuid)\"")
end
end
end
end
end
'';
in
Expand Down
1 change: 0 additions & 1 deletion modules/kernels/julia/julia-modules/tests/top-julia-packages.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ let
buildCommand = ''
wget https://julialang-logs.s3.amazonaws.com/public_outputs/current/package_requests.csv.gz
gunzip package_requests.csv.gz
ls -lh
cp package_requests.csv $out
'';
};
Expand Down

0 comments on commit 61e88e7

Please sign in to comment.