diff --git a/default.nix b/default.nix index ac38de969..4c17420f3 100644 --- a/default.nix +++ b/default.nix @@ -5,10 +5,10 @@ let inherit (poetryLib) isCompatible readTOML; - pyproject-nix = import ./vendor/pyproject.nix { inherit lib; }; + pyproject-nix = import ./vendor/pyproject.nix { inherit pkgs lib; }; # Name normalization - inherit (pyproject-nix.pypa) normalizePackageName; + inherit (pyproject-nix.lib.pypa) normalizePackageName; normalizePackageSet = lib.attrsets.mapAttrs' (name: value: lib.attrsets.nameValuePair (normalizePackageName name) value); # Map SPDX identifiers to license names @@ -172,7 +172,7 @@ lib.makeScope pkgs.newScope (self: { in lib.listToAttrs (lib.mapAttrsToList (n: v: { name = normalizePackageName n; value = v; }) lockfiles); - pep508Env = pyproject-nix.pep508.mkEnviron python; + pep508Env = pyproject-nix.lib.pep508.mkEnviron python; # Filter packages by their PEP508 markers & pyproject interpreter version partitions = @@ -181,9 +181,9 @@ lib.makeScope pkgs.newScope (self: { if pkgMeta ? marker then ( let - marker = pyproject-nix.pep508.parseMarkers pkgMeta.marker; + marker = pyproject-nix.lib.pep508.parseMarkers pkgMeta.marker; in - pyproject-nix.pep508.evalMarkers pep508Env marker + pyproject-nix.lib.pep508.evalMarkers pep508Env marker ) else true && isCompatible (poetryLib.getPythonVersion python) pkgMeta.python-versions; in lib.partition supportsPythonVersion poetryLock.package; diff --git a/editable.nix b/editable.nix index 5897212f9..a60aac020 100644 --- a/editable.nix +++ b/editable.nix @@ -6,7 +6,7 @@ , pyproject-nix }: let - name = pyproject-nix.pypa.normalizePackageName pyProject.tool.poetry.name; + name = pyproject-nix.lib.pypa.normalizePackageName pyProject.tool.poetry.name; # Just enough standard PKG-INFO fields for an editable installation pkgInfoFields = { diff --git a/lib.nix b/lib.nix index 821e49ae7..2aea27ab9 100644 --- a/lib.nix +++ b/lib.nix @@ -79,84 +79,6 @@ let else if lib.strings.hasInfix "manylinux_" f then { pkg = [ ml.manylinux2014 ]; str = "pep600"; } else { pkg = [ ]; str = null; }; - # Predict URL from the PyPI index. - # Args: - # pname: package name - # file: filename including extension - # hash: SRI hash - # kind: Language implementation and version tag - predictURLFromPypi = lib.makeOverridable ( - { pname, file, kind }: - "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}" - ); - - - # Fetch from the PyPI index. - # At first we try to fetch the predicated URL but if that fails we - # will use the Pypi API to determine the correct URL. - # Args: - # pname: package name - # file: filename including extension - # version: the version string of the dependency - # hash: SRI hash - # kind: Language implementation and version tag - fetchFromPypi = lib.makeOverridable ( - { pname, file, version, hash, kind, curlOpts ? "" }: - let - predictedURL = predictURLFromPypi { inherit pname file kind; }; - in - pkgs.stdenvNoCC.mkDerivation { - name = file; - nativeBuildInputs = [ - pkgs.buildPackages.curl - pkgs.buildPackages.jq - ]; - isWheel = lib.strings.hasSuffix "whl" file; - system = "builtin"; - - preferLocalBuild = true; - impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ - "NIX_CURL_FLAGS" - ]; - - inherit pname file version curlOpts predictedURL; - - builder = ./fetch-from-pypi.sh; - - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - outputHash = hash; - - passthru = { - urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl - }; - } - ); - - fetchFromLegacy = lib.makeOverridable ( - { python, pname, url, file, hash }: - let - pathParts = - builtins.filter - ({ prefix, path }: "NETRC" == prefix) - builtins.nixPath; - netrc_file = if (pathParts != [ ]) then (builtins.head pathParts).path else ""; - in - pkgs.runCommand file - { - nativeBuildInputs = [ python ]; - impureEnvVars = lib.fetchers.proxyImpureEnvVars; - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - outputHash = hash; - NETRC = netrc_file; - passthru.isWheel = lib.strings.hasSuffix "whl" file; - } '' - python ${./fetch_from_legacy.py} ${url} ${pname} ${file} - mv ${file} $out - '' - ); - getBuildSystemPkgs = { pythonPackages , pyProject @@ -206,8 +128,6 @@ let in { inherit - fetchFromPypi - fetchFromLegacy getManyLinuxDeps isCompatible readTOML diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index feab59fb8..47aa37e15 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -29,7 +29,7 @@ pythonPackages.callPackage }@args: let inherit (python) stdenv; - inherit (pyproject-nix.pypa) normalizePackageName; + inherit (pyproject-nix.lib.pypa) normalizePackageName; inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromLegacy fetchFromPypi; inherit (import ./pep425.nix { @@ -145,7 +145,7 @@ pythonPackages.callPackage pep508Markers = v.markers or ""; in compat constraints && (if pep508Markers == "" then true else - (pyproject-nix.pep508.evalMarkers + (pyproject-nix.lib.pep508.evalMarkers (pep508Env // { extra = { # All extras are always enabled @@ -153,7 +153,7 @@ pythonPackages.callPackage value = lib.attrNames extras; }; }) - (pyproject-nix.pep508.parseMarkers pep508Markers))) + (pyproject-nix.lib.pep508.parseMarkers pep508Markers))) ) dependencies ); @@ -215,15 +215,14 @@ pythonPackages.callPackage else if isFile then localDepPath else if isLegacy then - fetchFromLegacy + pyproject-nix.fetchers.fetchFromLegacy { pname = name; - inherit python; inherit (fileInfo) file hash; inherit (source) url; } else - fetchFromPypi { + pyproject-nix.fetchers.fetchFromPypi { pname = name; inherit (fileInfo) file hash kind; inherit version; diff --git a/pep425.nix b/pep425.nix index f5935e140..27cb07cea 100644 --- a/pep425.nix +++ b/pep425.nix @@ -1,7 +1,7 @@ { lib, stdenv, python, pyproject-nix, isLinux ? stdenv.isLinux }: let inherit (lib.strings) escapeRegex hasPrefix hasSuffix hasInfix splitString removeSuffix; - targetMachine = pyproject-nix.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name}; + targetMachine = pyproject-nix.lib.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name}; pythonVer = let diff --git a/vendor/pyproject.nix/default.nix b/vendor/pyproject.nix/default.nix index bcc49993e..99f6f25e4 100644 --- a/vendor/pyproject.nix/default.nix +++ b/vendor/pyproject.nix/default.nix @@ -1,22 +1,5 @@ -{ lib }: -let - inherit (builtins) mapAttrs; - inherit (lib) fix; -in - -fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) { - pip = ./pip.nix; - pypa = ./pypa.nix; - project = ./project.nix; - renderers = ./renderers.nix; - validators = ./validators.nix; - poetry = ./poetry.nix; - - pep427 = ./pep427.nix; - pep440 = ./pep440.nix; - pep508 = ./pep508.nix; - pep518 = ./pep518.nix; - pep599 = ./pep599.nix; - pep600 = ./pep600.nix; - pep621 = ./pep621.nix; -}) +{ pkgs, lib }: +{ + lib = import ./lib { inherit lib; }; + fetchers = import ./fetchers { inherit pkgs lib; }; +} diff --git a/vendor/pyproject.nix/fetchers/default.nix b/vendor/pyproject.nix/fetchers/default.nix new file mode 100644 index 000000000..1665a2e7c --- /dev/null +++ b/vendor/pyproject.nix/fetchers/default.nix @@ -0,0 +1,122 @@ +{ pkgs +, lib +, +}: +let + inherit (builtins) substring filter head nixPath; + inherit (lib) toLower; + + # Predict URL from the PyPI index. + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag + predictURLFromPypi = + { + # package name + pname + , # filename including extension + file + , # Language implementation and version tag + kind + , + }: "https://files.pythonhosted.org/packages/${kind}/${toLower (substring 0 1 file)}/${pname}/${file}"; +in +lib.mapAttrs (_: func: lib.makeOverridable func) { + /* + Fetch from the PyPI index. + + At first we try to fetch the predicated URL but if that fails we + will use the Pypi API to determine the correct URL. + + Type: fetchFromPypi :: AttrSet -> derivation + */ + fetchFromPypi = + { + # package name + pname + , # filename including extension + file + , # the version string of the dependency + version + , # SRI hash + hash + , # Language implementation and version tag + kind + , # Options to pass to `curl` + curlOpts ? "" + , + }: + let + predictedURL = predictURLFromPypi { inherit pname file kind; }; + in + pkgs.stdenvNoCC.mkDerivation { + name = file; + nativeBuildInputs = [ + pkgs.buildPackages.curl + pkgs.buildPackages.jq + ]; + isWheel = lib.strings.hasSuffix "whl" file; + system = "builtin"; + + preferLocalBuild = true; + impureEnvVars = + lib.fetchers.proxyImpureEnvVars + ++ [ + "NIX_CURL_FLAGS" + ]; + + inherit pname file version curlOpts predictedURL; + + builder = ./fetch-from-pypi.sh; + + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + + passthru = { + urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl + }; + }; + + /* + Fetch from the PyPI legacy API. + + Some repositories (such as Devpi) expose the Pypi legacy API (https://warehouse.pypa.io/api-reference/legacy.html). + + Type: fetchFromLegacy :: AttrSet -> derivation + */ + fetchFromLegacy = + { + # package name + pname + , # URL to package index + url + , # filename including extension + file + , # SRI hash + hash + , + }: + let + pathParts = filter ({ prefix, path }: "NETRC" == prefix) nixPath; # deadnix: skip + netrc_file = + if (pathParts != [ ]) + then (head pathParts).path + else ""; + in + pkgs.runCommand file + { + nativeBuildInputs = [ pkgs.buildPackages.python3 ]; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = hash; + NETRC = netrc_file; + passthru.isWheel = lib.strings.hasSuffix "whl" file; + } '' + python ${./fetch-from-legacy.py} ${url} ${pname} ${file} + mv ${file} $out + ''; +} diff --git a/fetch_from_legacy.py b/vendor/pyproject.nix/fetchers/fetch-from-legacy.py similarity index 85% rename from fetch_from_legacy.py rename to vendor/pyproject.nix/fetchers/fetch-from-legacy.py index 44ea092ac..b39ef5df4 100644 --- a/fetch_from_legacy.py +++ b/vendor/pyproject.nix/fetchers/fetch-from-legacy.py @@ -4,37 +4,41 @@ # Note it is not possible to use pip # https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12 -import os -import sys import netrc -from urllib.parse import urlparse, urlunparse -from html.parser import HTMLParser -import urllib.request +import os import shutil import ssl +import sys +import urllib.request +from html.parser import HTMLParser from os.path import normpath +from typing import Optional +from urllib.parse import urlparse, urlunparse # Parse the legacy index page to extract the href and package names class Pep503(HTMLParser): - def __init__(self): + def __init__(self) -> None: super().__init__() - self.sources = {} - self.url = None - self.name = None + self.sources: dict[str, str] = {} + self.url: Optional[str] = None + self.name: Optional[str] = None - def handle_data(self, data): + def handle_data(self, data: str) -> None: if self.url is not None: self.name = data - def handle_starttag(self, tag, attrs): + def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: if tag == "a": for name, value in attrs: if name == "href": self.url = value - def handle_endtag(self, tag): + def handle_endtag(self, tag: str) -> None: if self.url is not None: + if not self.name: + raise ValueError("Name not set") + self.sources[self.name] = self.url self.url = None @@ -45,14 +49,17 @@ def handle_endtag(self, tag): package_filename = sys.argv[3] # Parse username and password for this host from the netrc file if given. -username, password = None, None +username: Optional[str] = None +password: Optional[str] = None if os.environ["NETRC"]: netrc_obj = netrc.netrc(os.environ["NETRC"]) host = urlparse(index_url).netloc # Strip port number if present if ":" in host: host = host.split(":")[0] - username, _, password = netrc_obj.authenticators(host) + authenticators = netrc_obj.authenticators(host) + if authenticators: + username, _, password = authenticators print("Reading index %s" % index_url) diff --git a/fetch-from-pypi.sh b/vendor/pyproject.nix/fetchers/fetch-from-pypi.sh similarity index 78% rename from fetch-from-pypi.sh rename to vendor/pyproject.nix/fetchers/fetch-from-pypi.sh index e56dee684..e4137c6cd 100644 --- a/fetch-from-pypi.sh +++ b/vendor/pyproject.nix/fetchers/fetch-from-pypi.sh @@ -1,4 +1,7 @@ -source $stdenv/setup +#!/usr/bin/env bash + +# shellcheck disable=SC1091,SC2154 +source "$stdenv/setup" set -euo pipefail curl="curl \ @@ -16,9 +19,9 @@ curl="curl \ echo "Trying to fetch with predicted URL: $predictedURL" -$curl $predictedURL --output $out && exit 0 +$curl "$predictedURL" --output "$out" && exit 0 echo "Predicted URL '$predictedURL' failed, querying pypi.org" $curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url url=$(cat url) -$curl -k $url --output $out +$curl "$url" --output "$out" diff --git a/vendor/pyproject.nix/lib/default.nix b/vendor/pyproject.nix/lib/default.nix new file mode 100644 index 000000000..bcc49993e --- /dev/null +++ b/vendor/pyproject.nix/lib/default.nix @@ -0,0 +1,22 @@ +{ lib }: +let + inherit (builtins) mapAttrs; + inherit (lib) fix; +in + +fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) { + pip = ./pip.nix; + pypa = ./pypa.nix; + project = ./project.nix; + renderers = ./renderers.nix; + validators = ./validators.nix; + poetry = ./poetry.nix; + + pep427 = ./pep427.nix; + pep440 = ./pep440.nix; + pep508 = ./pep508.nix; + pep518 = ./pep518.nix; + pep599 = ./pep599.nix; + pep600 = ./pep600.nix; + pep621 = ./pep621.nix; +}) diff --git a/vendor/pyproject.nix/pep427.nix b/vendor/pyproject.nix/lib/pep427.nix similarity index 100% rename from vendor/pyproject.nix/pep427.nix rename to vendor/pyproject.nix/lib/pep427.nix diff --git a/vendor/pyproject.nix/pep440.nix b/vendor/pyproject.nix/lib/pep440.nix similarity index 99% rename from vendor/pyproject.nix/pep440.nix rename to vendor/pyproject.nix/lib/pep440.nix index 0b5f83428..3057ce43d 100644 --- a/vendor/pyproject.nix/pep440.nix +++ b/vendor/pyproject.nix/lib/pep440.nix @@ -151,7 +151,7 @@ fix (self: { */ parseVersionCond = cond: ( let - m = match " *([=>" = a: b: self.compareVersions a b > 0; "===" = throw "Arbitrary equality clause not supported"; + "" = _a: _b: true; }; }) diff --git a/vendor/pyproject.nix/pep508.nix b/vendor/pyproject.nix/lib/pep508.nix similarity index 98% rename from vendor/pyproject.nix/pep508.nix rename to vendor/pyproject.nix/lib/pep508.nix index 04b044771..40f75a18b 100644 --- a/vendor/pyproject.nix/pep508.nix +++ b/vendor/pyproject.nix/lib/pep508.nix @@ -3,7 +3,7 @@ let inherit (builtins) match elemAt split foldl' substring stringLength typeOf fromJSON isString head mapAttrs elem length; inherit (lib) stringToCharacters fix; - inherit (import ./util.nix { inherit lib; }) splitComma; + inherit (import ./util.nix { inherit lib; }) splitComma stripStr; re = { operators = "([=>= && < - else if c == "~" then [ - { - cond = ">="; - inherit version; - } - { - cond = "<"; - version = version // { - release = [ (head version.release + 1) ] ++ tail version.release; - }; - } - ] - # Desugar ^ into >= && < - else if c == "^" then [ - { - cond = ">="; - inherit version; - } - { - cond = "<"; - version = version // { - release = rewriteCaretRhs version.release; - }; - } - ] - # Versions without operators are exact matches, add operator according to PEP-440 - else [{ - cond = "=="; - inherit version; - }] - ); - # Normalized version of parseVersionCond' - parseVersionConds = s: flatten (map parseVersionCond' (splitComma s)); + parseVersionConds = s: flatten (map self.parseVersionCond (splitComma s)); dummyMarker = { type = "bool"; @@ -240,10 +195,64 @@ in build-systems = [ ]; # PEP-518 build-systems (List of parsed PEP-508 strings) } */ - # # Analogous to parseDependencies = pyproject: { dependencies = map parseDependency (normalizeDependendenciesToList (pyproject.tool.poetry.dependencies or { })); extras = mapAttrs (_: g: map parseDependency (normalizeDependendenciesToList g.dependencies)) pyproject.tool.poetry.group or { }; build-systems = pep518.parseBuildSystems pyproject; }; -} + + /* Parse a version conditional. + Supports additional non-standard operators `^` and `~` used by Poetry. + + Because some expressions desugar to multiple expressions parseVersionCond returns a list. + + Type: parseVersionCond :: string -> [ AttrSet ] + */ + parseVersionCond = cond: ( + let + m = match "^([~[:digit:]^])(.+)$" cond; + mAt = elemAt m; + c = mAt 0; + rest = mAt 1; + # Pad version before parsing as it's _much_ easier to reason about + # once they're the same length + version = pep440.parseVersion (lib.versions.pad 3 rest); + + # Count the number of segments in the input to use an an index in ~ rewriting + segments = length (filter (tok: typeOf tok == "string") (split "\\." rest)); + in + if m == null then [ (pep440.parseVersionCond cond) ] + # Desugar ~ into >= && < + else if c == "~" then [ + { + op = ">="; + inherit version; + } + { + op = "<"; + version = version // { + release = lib.imap0 (i: tok: if i >= segments - 1 then 0 else if i == segments - 2 then (tok + 1) else tok) version.release; + }; + } + ] + # Desugar ^ into >= && < + else if c == "^" then [ + { + op = ">="; + inherit version; + } + { + op = "<"; + version = version // { + release = rewriteCaretRhs version.release; + }; + } + ] + # Versions without operators are exact matches, add operator according to PEP-440 + else [{ + op = "=="; + inherit version; + }] + ); + +}) diff --git a/vendor/pyproject.nix/project.nix b/vendor/pyproject.nix/lib/project.nix similarity index 100% rename from vendor/pyproject.nix/project.nix rename to vendor/pyproject.nix/lib/project.nix diff --git a/vendor/pyproject.nix/pypa.nix b/vendor/pyproject.nix/lib/pypa.nix similarity index 100% rename from vendor/pyproject.nix/pypa.nix rename to vendor/pyproject.nix/lib/pypa.nix diff --git a/vendor/pyproject.nix/renderers.nix b/vendor/pyproject.nix/lib/renderers.nix similarity index 100% rename from vendor/pyproject.nix/renderers.nix rename to vendor/pyproject.nix/lib/renderers.nix diff --git a/vendor/pyproject.nix/util.nix b/vendor/pyproject.nix/lib/util.nix similarity index 59% rename from vendor/pyproject.nix/util.nix rename to vendor/pyproject.nix/lib/util.nix index 143d92338..2160d2f0a 100644 --- a/vendor/pyproject.nix/util.nix +++ b/vendor/pyproject.nix/lib/util.nix @@ -1,11 +1,19 @@ # Small utilities for internal reuse, not exposed externally { lib }: let - inherit (builtins) filter match split; + inherit (builtins) filter match split head; inherit (lib) isString; isEmptyStr = s: isString s && match " *" s == null; in { splitComma = s: if s == "" then [ ] else filter isEmptyStr (split " *, *" s); + + stripStr = s: + let + t = match "[\t ]*(.*[^\t ])[\t ]*" s; + in + if t == null + then "" + else head t; } diff --git a/vendor/pyproject.nix/validators.nix b/vendor/pyproject.nix/lib/validators.nix similarity index 100% rename from vendor/pyproject.nix/validators.nix rename to vendor/pyproject.nix/lib/validators.nix diff --git a/vendor/update.py b/vendor/update.py index 5e3591174..c498351f9 100755 --- a/vendor/update.py +++ b/vendor/update.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python3 +#!/usr/bin/env nix-shell +#! nix-shell -i python3 -p python3 import subprocess import shutil import json @@ -24,8 +25,14 @@ pass os.mkdir("pyproject.nix") + os.mkdir("pyproject.nix/lib") + shutil.copy(f"{store_path}/default.nix", f"pyproject.nix/default.nix") + + # Copy lib/ for filename in os.listdir(f"{store_path}/lib"): if filename.startswith("test") or not filename.endswith(".nix"): continue - shutil.copy(f"{store_path}/lib/{filename}", f"pyproject.nix/{filename}") + shutil.copy(f"{store_path}/lib/{filename}", f"pyproject.nix/lib/{filename}") + + shutil.copytree(f"{store_path}/fetchers", "pyproject.nix/fetchers")